From a5f7a5bb3b7f28ba7e4fa246775b29a0e5537255 Mon Sep 17 00:00:00 2001 From: Jinke Han Date: Tue, 8 Sep 2026 15:37:42 +0800 Subject: x86/kprobes: Fix crash when probing CS CALL instructions When using eBPF to probe CS CALL instructions within a function, a crash can be triggered. The eBPF tool probes offset 257 of the __hrtimer_run_queues() function: <__hrtimer_run_queues+249>: nopl 0x0(%rax,%rax,1) <__hrtimer_run_queues+254>: mov %r14,%rdi <__hrtimer_run_queues+257>: cs call <__x86_indirect_thunk_r12> <__hrtimer_run_queues+263>: mov %eax,%r12d <__hrtimer_run_queues+266>: xchg %ax,%ax <__hrtimer_run_queues+268>: mov %r13,%rdi Which triggers this crash: BUG: unable to handle page fault for address: 00000000000f41c9 #PF: supervisor write access in kernel mode #PF: error_code(0x0002) - not-present page PGD 0 P4D 0 Oops: 0002 [#1] SMP NOPTI CPU: 1 PID: 0 Comm: swapper/1 Kdump: loaded Tainted: P RIP: 0010:__hrtimer_run_queues+0x106/0x230 Note that __hrtimer_run_queues+0x106 is __hrtimer_run_queues+262, which is at the 6th byte of the above CS CALL instruction. Since the CS CALL instruction occupies 6 bytes, the exception occurred in the middle of that call instruction. The root cause is that when using eBPF tools to probe in the middle of a function, a kprobe with INT3 is used as the underlying implementation. During single-step emulation of the original CALL instruction, int3_emulate_call() assumes that the probed CALL instruction is 5 bytes long. However, the actual CS-prefixed CALL instruction occupies 6 bytes, so it constructs an incorrect exception return address. When the CPU returns from the kprobe handler, the next instruction to be executed is at the address of the last byte of that CS CALL instruction. Coincidentally, starting from that address, the CPU fetches and decodes a completely different instruction, which ultimately triggers a kernel crash. Fix the issue by using the actual instruction length obtained from the instruction decoder when constructing the exception return address, rather than relying on the hardcoded CALL_INSN_SIZE macro. [ mingo: Refined the changelog ] Fixes: 6256e668b7af ("x86/kprobes: Use int3 instead of debug trap for single-step") Suggested-by: Masami Hiramatsu (Google) Signed-off-by: Jinke Han Signed-off-by: Ingo Molnar Reviewed-by: Masami Hiramatsu (Google) Acked-by: Yafang Shao Acked-by: Borislav Petkov Cc: Peter Zijlstra Link: https://patch.msgid.link/20260908073742.GA10517@didi-ThinkCentre-M920t-N000 --- arch/sparc/Kbuild | 14 + arch/sparc/Kconfig | 468 ++++ arch/sparc/Kconfig.debug | 27 + arch/sparc/Makefile | 93 + arch/sparc/boot/.gitignore | 9 + arch/sparc/boot/Makefile | 72 + arch/sparc/boot/install.sh | 30 + arch/sparc/boot/piggyback.c | 265 +++ arch/sparc/configs/sparc32_defconfig | 93 + arch/sparc/configs/sparc64_defconfig | 229 ++ arch/sparc/crypto/Kconfig | 26 + arch/sparc/crypto/Makefile | 10 + arch/sparc/crypto/aes_glue.c | 344 +++ arch/sparc/crypto/camellia_asm.S | 563 +++++ arch/sparc/crypto/camellia_glue.c | 293 +++ arch/sparc/crypto/crop_devid.c | 15 + arch/sparc/include/asm/Kbuild | 8 + arch/sparc/include/asm/adi.h | 6 + arch/sparc/include/asm/adi_64.h | 46 + arch/sparc/include/asm/apb.h | 37 + arch/sparc/include/asm/asm-offsets.h | 1 + arch/sparc/include/asm/asm-prototypes.h | 33 + arch/sparc/include/asm/asm.h | 41 + arch/sparc/include/asm/asmmacro.h | 46 + arch/sparc/include/asm/atomic.h | 9 + arch/sparc/include/asm/atomic_32.h | 61 + arch/sparc/include/asm/atomic_64.h | 73 + arch/sparc/include/asm/auxio.h | 16 + arch/sparc/include/asm/auxio_32.h | 89 + arch/sparc/include/asm/auxio_64.h | 99 + arch/sparc/include/asm/backoff.h | 86 + arch/sparc/include/asm/barrier.h | 9 + arch/sparc/include/asm/barrier_32.h | 7 + arch/sparc/include/asm/barrier_64.h | 61 + arch/sparc/include/asm/bbc.h | 226 ++ arch/sparc/include/asm/bitext.h | 28 + arch/sparc/include/asm/bitops.h | 9 + arch/sparc/include/asm/bitops_32.h | 108 + arch/sparc/include/asm/bitops_64.h | 63 + arch/sparc/include/asm/btext.h | 7 + arch/sparc/include/asm/bug.h | 30 + arch/sparc/include/asm/cache.h | 26 + arch/sparc/include/asm/cacheflush.h | 13 + arch/sparc/include/asm/cacheflush_32.h | 63 + arch/sparc/include/asm/cacheflush_64.h | 83 + arch/sparc/include/asm/cachetlb_32.h | 30 + arch/sparc/include/asm/cachetype.h | 14 + arch/sparc/include/asm/chafsr.h | 242 ++ arch/sparc/include/asm/checksum.h | 12 + arch/sparc/include/asm/checksum_32.h | 210 ++ arch/sparc/include/asm/checksum_64.h | 144 ++ arch/sparc/include/asm/chmctrl.h | 184 ++ arch/sparc/include/asm/clock.h | 11 + arch/sparc/include/asm/clocksource.h | 8 + arch/sparc/include/asm/cmpxchg.h | 9 + arch/sparc/include/asm/cmpxchg_32.h | 78 + arch/sparc/include/asm/cmpxchg_64.h | 208 ++ arch/sparc/include/asm/compat.h | 160 ++ arch/sparc/include/asm/compat_signal.h | 24 + arch/sparc/include/asm/contregs.h | 32 + arch/sparc/include/asm/cpu_type.h | 29 + arch/sparc/include/asm/cpudata.h | 19 + arch/sparc/include/asm/cpudata_32.h | 32 + arch/sparc/include/asm/cpudata_64.h | 42 + arch/sparc/include/asm/current.h | 35 + arch/sparc/include/asm/dcr.h | 15 + arch/sparc/include/asm/dcu.h | 28 + arch/sparc/include/asm/delay.h | 9 + arch/sparc/include/asm/delay_32.h | 35 + arch/sparc/include/asm/delay_64.h | 18 + arch/sparc/include/asm/device.h | 29 + arch/sparc/include/asm/dma-mapping.h | 13 + arch/sparc/include/asm/dma.h | 92 + arch/sparc/include/asm/ebus_dma.h | 36 + arch/sparc/include/asm/ecc.h | 123 + arch/sparc/include/asm/eeprom.h | 9 + arch/sparc/include/asm/elf.h | 9 + arch/sparc/include/asm/elf_32.h | 132 ++ arch/sparc/include/asm/elf_64.h | 233 ++ arch/sparc/include/asm/estate.h | 50 + arch/sparc/include/asm/extable.h | 21 + arch/sparc/include/asm/fbio.h | 73 + arch/sparc/include/asm/fhc.h | 81 + arch/sparc/include/asm/floppy.h | 9 + arch/sparc/include/asm/floppy_32.h | 390 +++ arch/sparc/include/asm/floppy_64.h | 776 ++++++ arch/sparc/include/asm/fpumacro.h | 34 + arch/sparc/include/asm/ftrace.h | 34 + arch/sparc/include/asm/futex.h | 9 + arch/sparc/include/asm/futex_32.h | 6 + arch/sparc/include/asm/futex_64.h | 94 + arch/sparc/include/asm/hardirq.h | 9 + arch/sparc/include/asm/hardirq_32.h | 13 + arch/sparc/include/asm/hardirq_64.h | 19 + arch/sparc/include/asm/head.h | 9 + arch/sparc/include/asm/head_32.h | 84 + arch/sparc/include/asm/head_64.h | 81 + arch/sparc/include/asm/hibernate.h | 24 + arch/sparc/include/asm/highmem.h | 61 + arch/sparc/include/asm/hugetlb.h | 55 + arch/sparc/include/asm/hvtramp.h | 38 + arch/sparc/include/asm/hw_irq.h | 6 + arch/sparc/include/asm/hypervisor.h | 3527 ++++++++++++++++++++++++++++ arch/sparc/include/asm/idprom.h | 26 + arch/sparc/include/asm/intr_queue.h | 16 + arch/sparc/include/asm/io-unit.h | 59 + arch/sparc/include/asm/io.h | 24 + arch/sparc/include/asm/io_32.h | 153 ++ arch/sparc/include/asm/io_64.h | 474 ++++ arch/sparc/include/asm/ioctls.h | 14 + arch/sparc/include/asm/iommu-common.h | 53 + arch/sparc/include/asm/iommu.h | 9 + arch/sparc/include/asm/iommu_32.h | 122 + arch/sparc/include/asm/iommu_64.h | 93 + arch/sparc/include/asm/irq.h | 9 + arch/sparc/include/asm/irq_32.h | 24 + arch/sparc/include/asm/irq_64.h | 98 + arch/sparc/include/asm/irqflags.h | 9 + arch/sparc/include/asm/irqflags_32.h | 48 + arch/sparc/include/asm/irqflags_64.h | 98 + arch/sparc/include/asm/jump_label.h | 52 + arch/sparc/include/asm/kdebug.h | 9 + arch/sparc/include/asm/kdebug_32.h | 75 + arch/sparc/include/asm/kdebug_64.h | 24 + arch/sparc/include/asm/kgdb.h | 42 + arch/sparc/include/asm/kprobes.h | 55 + arch/sparc/include/asm/ldc.h | 149 ++ arch/sparc/include/asm/leon.h | 266 +++ arch/sparc/include/asm/leon_amba.h | 267 +++ arch/sparc/include/asm/leon_pci.h | 23 + arch/sparc/include/asm/lsu.h | 20 + arch/sparc/include/asm/machines.h | 51 + arch/sparc/include/asm/mbus.h | 97 + arch/sparc/include/asm/mc146818rtc.h | 14 + arch/sparc/include/asm/mc146818rtc_32.h | 30 + arch/sparc/include/asm/mc146818rtc_64.h | 29 + arch/sparc/include/asm/mdesc.h | 99 + arch/sparc/include/asm/memctrl.h | 10 + arch/sparc/include/asm/mman.h | 91 + arch/sparc/include/asm/mmu.h | 9 + arch/sparc/include/asm/mmu_32.h | 11 + arch/sparc/include/asm/mmu_64.h | 129 + arch/sparc/include/asm/mmu_context.h | 9 + arch/sparc/include/asm/mmu_context_32.h | 34 + arch/sparc/include/asm/mmu_context_64.h | 198 ++ arch/sparc/include/asm/mmzone.h | 14 + arch/sparc/include/asm/mxcc.h | 138 ++ arch/sparc/include/asm/nmi.h | 14 + arch/sparc/include/asm/ns87303.h | 118 + arch/sparc/include/asm/obio.h | 226 ++ arch/sparc/include/asm/opcodes.h | 100 + arch/sparc/include/asm/openprom.h | 280 +++ arch/sparc/include/asm/oplib.h | 9 + arch/sparc/include/asm/oplib_32.h | 184 ++ arch/sparc/include/asm/oplib_64.h | 253 ++ arch/sparc/include/asm/page.h | 10 + arch/sparc/include/asm/page_32.h | 138 ++ arch/sparc/include/asm/page_64.h | 161 ++ arch/sparc/include/asm/parport.h | 11 + arch/sparc/include/asm/parport_64.h | 255 ++ arch/sparc/include/asm/pbm.h | 48 + arch/sparc/include/asm/pci.h | 44 + arch/sparc/include/asm/pcic.h | 130 + arch/sparc/include/asm/pcr.h | 51 + arch/sparc/include/asm/percpu.h | 9 + arch/sparc/include/asm/percpu_32.h | 7 + arch/sparc/include/asm/percpu_64.h | 27 + arch/sparc/include/asm/perf_event.h | 30 + arch/sparc/include/asm/pgalloc.h | 9 + arch/sparc/include/asm/pgalloc_32.h | 77 + arch/sparc/include/asm/pgalloc_64.h | 118 + arch/sparc/include/asm/pgtable.h | 9 + arch/sparc/include/asm/pgtable_32.h | 426 ++++ arch/sparc/include/asm/pgtable_64.h | 1171 +++++++++ arch/sparc/include/asm/pgtsrmmu.h | 141 ++ arch/sparc/include/asm/pil.h | 31 + arch/sparc/include/asm/processor.h | 12 + arch/sparc/include/asm/processor_32.h | 98 + arch/sparc/include/asm/processor_64.h | 222 ++ arch/sparc/include/asm/prom.h | 59 + arch/sparc/include/asm/psr.h | 68 + arch/sparc/include/asm/ptrace.h | 162 ++ arch/sparc/include/asm/qrwlock.h | 8 + arch/sparc/include/asm/qspinlock.h | 8 + arch/sparc/include/asm/ross.h | 192 ++ arch/sparc/include/asm/sbi.h | 116 + arch/sparc/include/asm/scratchpad.h | 15 + arch/sparc/include/asm/seccomp.h | 11 + arch/sparc/include/asm/sections.h | 14 + arch/sparc/include/asm/setup.h | 70 + arch/sparc/include/asm/sfafsr.h | 83 + arch/sparc/include/asm/sfp-machine.h | 9 + arch/sparc/include/asm/sfp-machine_32.h | 212 ++ arch/sparc/include/asm/sfp-machine_64.h | 93 + arch/sparc/include/asm/shmparam.h | 9 + arch/sparc/include/asm/shmparam_32.h | 10 + arch/sparc/include/asm/shmparam_64.h | 11 + arch/sparc/include/asm/sigcontext.h | 109 + arch/sparc/include/asm/signal.h | 17 + arch/sparc/include/asm/smp.h | 9 + arch/sparc/include/asm/smp_32.h | 130 + arch/sparc/include/asm/smp_64.h | 84 + arch/sparc/include/asm/sparsemem.h | 14 + arch/sparc/include/asm/spinlock.h | 9 + arch/sparc/include/asm/spinlock_32.h | 188 ++ arch/sparc/include/asm/spinlock_64.h | 19 + arch/sparc/include/asm/spinlock_types.h | 25 + arch/sparc/include/asm/spitfire.h | 368 +++ arch/sparc/include/asm/stacktrace.h | 7 + arch/sparc/include/asm/starfire.h | 20 + arch/sparc/include/asm/string.h | 47 + arch/sparc/include/asm/string_32.h | 15 + arch/sparc/include/asm/string_64.h | 15 + arch/sparc/include/asm/sunbpp.h | 81 + arch/sparc/include/asm/swift.h | 107 + arch/sparc/include/asm/switch_to.h | 9 + arch/sparc/include/asm/switch_to_32.h | 107 + arch/sparc/include/asm/switch_to_64.h | 71 + arch/sparc/include/asm/syscall.h | 154 ++ arch/sparc/include/asm/syscalls.h | 12 + arch/sparc/include/asm/termbits.h | 9 + arch/sparc/include/asm/thread_info.h | 9 + arch/sparc/include/asm/thread_info_32.h | 130 + arch/sparc/include/asm/thread_info_64.h | 242 ++ arch/sparc/include/asm/timer.h | 9 + arch/sparc/include/asm/timer_32.h | 45 + arch/sparc/include/asm/timer_64.h | 98 + arch/sparc/include/asm/timex.h | 9 + arch/sparc/include/asm/timex_64.h | 16 + arch/sparc/include/asm/tlb.h | 9 + arch/sparc/include/asm/tlb_32.h | 7 + arch/sparc/include/asm/tlb_64.h | 38 + arch/sparc/include/asm/tlbflush.h | 9 + arch/sparc/include/asm/tlbflush_32.h | 25 + arch/sparc/include/asm/tlbflush_64.h | 68 + arch/sparc/include/asm/topology.h | 9 + arch/sparc/include/asm/topology_32.h | 7 + arch/sparc/include/asm/topology_64.h | 65 + arch/sparc/include/asm/trap_block.h | 217 ++ arch/sparc/include/asm/traps.h | 25 + arch/sparc/include/asm/tsb.h | 380 +++ arch/sparc/include/asm/tsunami.h | 65 + arch/sparc/include/asm/ttable.h | 694 ++++++ arch/sparc/include/asm/turbosparc.h | 126 + arch/sparc/include/asm/uaccess.h | 15 + arch/sparc/include/asm/uaccess_32.h | 221 ++ arch/sparc/include/asm/uaccess_64.h | 251 ++ arch/sparc/include/asm/unistd.h | 63 + arch/sparc/include/asm/upa.h | 110 + arch/sparc/include/asm/uprobes.h | 47 + arch/sparc/include/asm/user.h | 6 + arch/sparc/include/asm/vaddrs.h | 71 + arch/sparc/include/asm/vdso.h | 20 + arch/sparc/include/asm/vdso/clocksource.h | 10 + arch/sparc/include/asm/vdso/gettimeofday.h | 184 ++ arch/sparc/include/asm/vdso/processor.h | 41 + arch/sparc/include/asm/vdso/vsyscall.h | 10 + arch/sparc/include/asm/video.h | 47 + arch/sparc/include/asm/viking.h | 255 ++ arch/sparc/include/asm/vio.h | 508 ++++ arch/sparc/include/asm/visasm.h | 67 + arch/sparc/include/asm/vmalloc.h | 4 + arch/sparc/include/asm/winmacro.h | 132 ++ arch/sparc/include/uapi/asm/Kbuild | 3 + arch/sparc/include/uapi/asm/apc.h | 65 + arch/sparc/include/uapi/asm/asi.h | 303 +++ arch/sparc/include/uapi/asm/auxvec.h | 15 + arch/sparc/include/uapi/asm/bitsperlong.h | 14 + arch/sparc/include/uapi/asm/byteorder.h | 7 + arch/sparc/include/uapi/asm/display7seg.h | 80 + arch/sparc/include/uapi/asm/envctrl.h | 104 + arch/sparc/include/uapi/asm/errno.h | 122 + arch/sparc/include/uapi/asm/fbio.h | 260 ++ arch/sparc/include/uapi/asm/fcntl.h | 57 + arch/sparc/include/uapi/asm/ioctl.h | 68 + arch/sparc/include/uapi/asm/ioctls.h | 140 ++ arch/sparc/include/uapi/asm/ipcbuf.h | 35 + arch/sparc/include/uapi/asm/mman.h | 25 + arch/sparc/include/uapi/asm/msgbuf.h | 37 + arch/sparc/include/uapi/asm/openpromio.h | 68 + arch/sparc/include/uapi/asm/oradax.h | 79 + arch/sparc/include/uapi/asm/param.h | 8 + arch/sparc/include/uapi/asm/perfctr.h | 167 ++ arch/sparc/include/uapi/asm/poll.h | 13 + arch/sparc/include/uapi/asm/posix_types.h | 60 + arch/sparc/include/uapi/asm/psr.h | 48 + arch/sparc/include/uapi/asm/psrcompat.h | 46 + arch/sparc/include/uapi/asm/pstate.h | 116 + arch/sparc/include/uapi/asm/ptrace.h | 353 +++ arch/sparc/include/uapi/asm/resource.h | 31 + arch/sparc/include/uapi/asm/sembuf.h | 32 + arch/sparc/include/uapi/asm/setup.h | 16 + arch/sparc/include/uapi/asm/shmbuf.h | 51 + arch/sparc/include/uapi/asm/sigcontext.h | 5 + arch/sparc/include/uapi/asm/siginfo.h | 16 + arch/sparc/include/uapi/asm/signal.h | 181 ++ arch/sparc/include/uapi/asm/socket.h | 178 ++ arch/sparc/include/uapi/asm/stat.h | 108 + arch/sparc/include/uapi/asm/swab.h | 46 + arch/sparc/include/uapi/asm/termbits.h | 199 ++ arch/sparc/include/uapi/asm/termios.h | 53 + arch/sparc/include/uapi/asm/traps.h | 121 + arch/sparc/include/uapi/asm/ucontext.h | 3 + arch/sparc/include/uapi/asm/uctx.h | 72 + arch/sparc/include/uapi/asm/unistd.h | 33 + arch/sparc/include/uapi/asm/utrap.h | 52 + arch/sparc/include/uapi/asm/watchdog.h | 32 + arch/sparc/kernel/.gitignore | 2 + arch/sparc/kernel/Makefile | 120 + arch/sparc/kernel/adi_64.c | 396 ++++ arch/sparc/kernel/apc.c | 195 ++ arch/sparc/kernel/asm-offsets.c | 61 + arch/sparc/kernel/audit.c | 82 + arch/sparc/kernel/auxio_32.c | 139 ++ arch/sparc/kernel/auxio_64.c | 151 ++ arch/sparc/kernel/btext.c | 326 +++ arch/sparc/kernel/central.c | 271 +++ arch/sparc/kernel/cherrs.S | 576 +++++ arch/sparc/kernel/chmc.c | 864 +++++++ arch/sparc/kernel/compat_audit.c | 48 + arch/sparc/kernel/cpu.c | 556 +++++ arch/sparc/kernel/cpumap.c | 439 ++++ arch/sparc/kernel/cpumap.h | 17 + arch/sparc/kernel/devices.c | 138 ++ arch/sparc/kernel/ds.c | 1272 ++++++++++ arch/sparc/kernel/dtlb_miss.S | 40 + arch/sparc/kernel/dtlb_prot.S | 55 + arch/sparc/kernel/ebus.c | 257 ++ arch/sparc/kernel/entry.S | 1370 +++++++++++ arch/sparc/kernel/entry.h | 253 ++ arch/sparc/kernel/etrap_32.S | 281 +++ arch/sparc/kernel/etrap_64.S | 284 +++ arch/sparc/kernel/fpu_traps.S | 384 +++ arch/sparc/kernel/ftrace.c | 133 ++ arch/sparc/kernel/getsetcc.S | 25 + arch/sparc/kernel/head_32.S | 812 +++++++ arch/sparc/kernel/head_64.S | 969 ++++++++ arch/sparc/kernel/helpers.S | 66 + arch/sparc/kernel/hvapi.c | 205 ++ arch/sparc/kernel/hvcalls.S | 930 ++++++++ arch/sparc/kernel/hvtramp.S | 137 ++ arch/sparc/kernel/idprom.c | 99 + arch/sparc/kernel/iommu-common.c | 260 ++ arch/sparc/kernel/iommu.c | 776 ++++++ arch/sparc/kernel/iommu_common.h | 51 + arch/sparc/kernel/ioport.c | 344 +++ arch/sparc/kernel/irq.h | 102 + arch/sparc/kernel/irq_32.c | 363 +++ arch/sparc/kernel/irq_64.c | 1154 +++++++++ arch/sparc/kernel/itlb_miss.S | 40 + arch/sparc/kernel/ivec.S | 52 + arch/sparc/kernel/jump_label.c | 47 + arch/sparc/kernel/kernel.h | 197 ++ arch/sparc/kernel/kgdb_32.c | 172 ++ arch/sparc/kernel/kgdb_64.c | 201 ++ arch/sparc/kernel/kprobes.c | 489 ++++ arch/sparc/kernel/kstack.h | 84 + arch/sparc/kernel/ktlb.S | 270 +++ arch/sparc/kernel/ldc.c | 2428 +++++++++++++++++++ arch/sparc/kernel/led.c | 146 ++ arch/sparc/kernel/leon_kernel.c | 508 ++++ arch/sparc/kernel/leon_pci.c | 62 + arch/sparc/kernel/leon_pci_grpci1.c | 723 ++++++ arch/sparc/kernel/leon_pci_grpci2.c | 915 ++++++++ arch/sparc/kernel/leon_pmc.c | 98 + arch/sparc/kernel/leon_smp.c | 470 ++++ arch/sparc/kernel/mdesc.c | 1351 +++++++++++ arch/sparc/kernel/misctrap.S | 95 + arch/sparc/kernel/module.c | 194 ++ arch/sparc/kernel/nmi.c | 317 +++ arch/sparc/kernel/of_device_32.c | 433 ++++ arch/sparc/kernel/of_device_64.c | 725 ++++++ arch/sparc/kernel/of_device_common.c | 179 ++ arch/sparc/kernel/of_device_common.h | 37 + arch/sparc/kernel/pci.c | 1013 ++++++++ arch/sparc/kernel/pci_common.c | 547 +++++ arch/sparc/kernel/pci_fire.c | 524 +++++ arch/sparc/kernel/pci_impl.h | 191 ++ arch/sparc/kernel/pci_msi.c | 446 ++++ arch/sparc/kernel/pci_psycho.c | 620 +++++ arch/sparc/kernel/pci_sabre.c | 615 +++++ arch/sparc/kernel/pci_schizo.c | 1511 ++++++++++++ arch/sparc/kernel/pci_sun4v.c | 1359 +++++++++++ arch/sparc/kernel/pci_sun4v.h | 114 + arch/sparc/kernel/pci_sun4v_asm.S | 431 ++++ arch/sparc/kernel/pcic.c | 814 +++++++ arch/sparc/kernel/pcr.c | 373 +++ arch/sparc/kernel/perf_event.c | 1876 +++++++++++++++ arch/sparc/kernel/pmc.c | 100 + arch/sparc/kernel/power.c | 72 + arch/sparc/kernel/process.c | 134 ++ arch/sparc/kernel/process_32.c | 404 ++++ arch/sparc/kernel/process_64.c | 698 ++++++ arch/sparc/kernel/prom.h | 12 + arch/sparc/kernel/prom_32.c | 324 +++ arch/sparc/kernel/prom_64.c | 637 +++++ arch/sparc/kernel/prom_common.c | 161 ++ arch/sparc/kernel/prom_irqtrans.c | 844 +++++++ arch/sparc/kernel/psycho_common.c | 473 ++++ arch/sparc/kernel/psycho_common.h | 49 + arch/sparc/kernel/ptrace_32.c | 448 ++++ arch/sparc/kernel/ptrace_64.c | 1176 ++++++++++ arch/sparc/kernel/reboot.c | 55 + arch/sparc/kernel/rtrap_32.S | 265 +++ arch/sparc/kernel/rtrap_64.S | 378 +++ arch/sparc/kernel/sbus.c | 679 ++++++ arch/sparc/kernel/setup.c | 46 + arch/sparc/kernel/setup_32.c | 408 ++++ arch/sparc/kernel/setup_64.c | 697 ++++++ arch/sparc/kernel/signal32.c | 786 +++++++ arch/sparc/kernel/signal_32.c | 565 +++++ arch/sparc/kernel/signal_64.c | 595 +++++ arch/sparc/kernel/sigutil.h | 10 + arch/sparc/kernel/sigutil_32.c | 129 + arch/sparc/kernel/sigutil_64.c | 102 + arch/sparc/kernel/smp_32.c | 389 +++ arch/sparc/kernel/smp_64.c | 1563 ++++++++++++ arch/sparc/kernel/sparc_ksyms.c | 12 + arch/sparc/kernel/spiterrs.S | 241 ++ arch/sparc/kernel/sstate.c | 124 + arch/sparc/kernel/stacktrace.c | 89 + arch/sparc/kernel/starfire.c | 112 + arch/sparc/kernel/sun4d_irq.c | 519 ++++ arch/sparc/kernel/sun4d_smp.c | 415 ++++ arch/sparc/kernel/sun4m_irq.c | 478 ++++ arch/sparc/kernel/sun4m_smp.c | 275 +++ arch/sparc/kernel/sun4v_ivec.S | 357 +++ arch/sparc/kernel/sun4v_mcd.S | 17 + arch/sparc/kernel/sun4v_tlb_miss.S | 437 ++++ arch/sparc/kernel/sys32.S | 20 + arch/sparc/kernel/sys_sparc32.c | 237 ++ arch/sparc/kernel/sys_sparc_32.c | 231 ++ arch/sparc/kernel/sys_sparc_64.c | 723 ++++++ arch/sparc/kernel/syscalls.S | 309 +++ arch/sparc/kernel/syscalls/Makefile | 33 + arch/sparc/kernel/syscalls/syscall.tbl | 520 ++++ arch/sparc/kernel/sysfs.c | 265 +++ arch/sparc/kernel/systbls.h | 103 + arch/sparc/kernel/systbls_32.S | 18 + arch/sparc/kernel/systbls_64.S | 29 + arch/sparc/kernel/termios.c | 115 + arch/sparc/kernel/time_32.c | 356 +++ arch/sparc/kernel/time_64.c | 901 +++++++ arch/sparc/kernel/trampoline_32.S | 201 ++ arch/sparc/kernel/trampoline_64.S | 414 ++++ arch/sparc/kernel/traps_32.c | 395 ++++ arch/sparc/kernel/traps_64.c | 2925 +++++++++++++++++++++++ arch/sparc/kernel/tsb.S | 591 +++++ arch/sparc/kernel/ttable_32.S | 418 ++++ arch/sparc/kernel/ttable_64.S | 275 +++ arch/sparc/kernel/una_asm_32.S | 154 ++ arch/sparc/kernel/una_asm_64.S | 147 ++ arch/sparc/kernel/unaligned_32.c | 282 +++ arch/sparc/kernel/unaligned_64.c | 709 ++++++ arch/sparc/kernel/uprobes.c | 321 +++ arch/sparc/kernel/urtt_fill.S | 105 + arch/sparc/kernel/utrap.S | 29 + arch/sparc/kernel/vio.c | 574 +++++ arch/sparc/kernel/viohs.c | 860 +++++++ arch/sparc/kernel/visemul.c | 899 +++++++ arch/sparc/kernel/vmlinux.lds.S | 198 ++ arch/sparc/kernel/windows.c | 130 + arch/sparc/kernel/winfixup.S | 160 ++ arch/sparc/kernel/wof.S | 366 +++ arch/sparc/kernel/wuf.S | 313 +++ arch/sparc/lib/GENbzero.S | 157 ++ arch/sparc/lib/GENcopy_from_user.S | 31 + arch/sparc/lib/GENcopy_to_user.S | 35 + arch/sparc/lib/GENmemcpy.S | 142 ++ arch/sparc/lib/GENpage.S | 78 + arch/sparc/lib/GENpatch.S | 34 + arch/sparc/lib/M7copy_from_user.S | 40 + arch/sparc/lib/M7copy_to_user.S | 51 + arch/sparc/lib/M7memcpy.S | 923 ++++++++ arch/sparc/lib/M7memset.S | 352 +++ arch/sparc/lib/M7patch.S | 51 + arch/sparc/lib/Makefile | 55 + arch/sparc/lib/Memcpy_utils.S | 354 +++ arch/sparc/lib/NG2copy_from_user.S | 44 + arch/sparc/lib/NG2copy_to_user.S | 53 + arch/sparc/lib/NG2memcpy.S | 594 +++++ arch/sparc/lib/NG2patch.S | 34 + arch/sparc/lib/NG4clear_page.S | 30 + arch/sparc/lib/NG4copy_from_user.S | 39 + arch/sparc/lib/NG4copy_page.S | 58 + arch/sparc/lib/NG4copy_to_user.S | 48 + arch/sparc/lib/NG4fls.S | 30 + arch/sparc/lib/NG4memcpy.S | 386 +++ arch/sparc/lib/NG4memset.S | 107 + arch/sparc/lib/NG4patch.S | 64 + arch/sparc/lib/NGbzero.S | 161 ++ arch/sparc/lib/NGcopy_from_user.S | 33 + arch/sparc/lib/NGcopy_to_user.S | 36 + arch/sparc/lib/NGmemcpy.S | 516 ++++ arch/sparc/lib/NGpage.S | 138 ++ arch/sparc/lib/NGpatch.S | 34 + arch/sparc/lib/PeeCeeI.c | 212 ++ arch/sparc/lib/U1copy_from_user.S | 38 + arch/sparc/lib/U1copy_to_user.S | 38 + arch/sparc/lib/U1memcpy.S | 686 ++++++ arch/sparc/lib/U3copy_from_user.S | 27 + arch/sparc/lib/U3copy_to_user.S | 38 + arch/sparc/lib/U3memcpy.S | 520 ++++ arch/sparc/lib/U3patch.S | 34 + arch/sparc/lib/VISsave.S | 84 + arch/sparc/lib/ashldi3.S | 38 + arch/sparc/lib/ashrdi3.S | 40 + arch/sparc/lib/atomic32.c | 197 ++ arch/sparc/lib/atomic_64.S | 166 ++ arch/sparc/lib/bitext.c | 128 + arch/sparc/lib/bitops.S | 138 ++ arch/sparc/lib/blockops.S | 93 + arch/sparc/lib/bzero.S | 150 ++ arch/sparc/lib/checksum_32.S | 457 ++++ arch/sparc/lib/checksum_64.S | 177 ++ arch/sparc/lib/clear_page.S | 107 + arch/sparc/lib/copy_in_user.S | 110 + arch/sparc/lib/copy_page.S | 253 ++ arch/sparc/lib/copy_user.S | 394 ++++ arch/sparc/lib/csum_copy.S | 315 +++ arch/sparc/lib/csum_copy_from_user.S | 22 + arch/sparc/lib/csum_copy_to_user.S | 22 + arch/sparc/lib/divdi3.S | 271 +++ arch/sparc/lib/ffs.S | 88 + arch/sparc/lib/fls.S | 67 + arch/sparc/lib/fls64.S | 61 + arch/sparc/lib/hweight.S | 57 + arch/sparc/lib/iomap.c | 28 + arch/sparc/lib/ipcsum.S | 36 + arch/sparc/lib/libgcc.h | 19 + arch/sparc/lib/locks.S | 98 + arch/sparc/lib/lshrdi3.S | 30 + arch/sparc/lib/mcount.S | 127 + arch/sparc/lib/memcmp.S | 30 + arch/sparc/lib/memcpy.S | 463 ++++ arch/sparc/lib/memmove.S | 62 + arch/sparc/lib/memscan_32.S | 138 ++ arch/sparc/lib/memscan_64.S | 136 ++ arch/sparc/lib/memset.S | 197 ++ arch/sparc/lib/muldi3.S | 66 + arch/sparc/lib/multi3.S | 36 + arch/sparc/lib/strlen.S | 83 + arch/sparc/lib/strncmp_32.S | 121 + arch/sparc/lib/strncmp_64.S | 33 + arch/sparc/lib/udivdi3.S | 247 ++ arch/sparc/math-emu/Makefile | 9 + arch/sparc/math-emu/math_32.c | 515 ++++ arch/sparc/math-emu/math_64.c | 525 +++++ arch/sparc/math-emu/sfp-util_32.h | 116 + arch/sparc/math-emu/sfp-util_64.h | 121 + arch/sparc/mm/Makefile | 16 + arch/sparc/mm/execmem.c | 21 + arch/sparc/mm/fault_32.c | 382 +++ arch/sparc/mm/fault_64.c | 532 +++++ arch/sparc/mm/hugetlbpage.c | 317 +++ arch/sparc/mm/hypersparc.S | 414 ++++ arch/sparc/mm/init_32.c | 299 +++ arch/sparc/mm/init_64.c | 3149 +++++++++++++++++++++++++ arch/sparc/mm/init_64.h | 35 + arch/sparc/mm/io-unit.c | 288 +++ arch/sparc/mm/iommu.c | 457 ++++ arch/sparc/mm/leon_mm.c | 346 +++ arch/sparc/mm/mm_32.h | 20 + arch/sparc/mm/srmmu.c | 1773 ++++++++++++++ arch/sparc/mm/srmmu_access.S | 83 + arch/sparc/mm/swift.S | 256 ++ arch/sparc/mm/tlb.c | 314 +++ arch/sparc/mm/tsb.c | 635 +++++ arch/sparc/mm/tsunami.S | 132 ++ arch/sparc/mm/ultra.S | 1102 +++++++++ arch/sparc/mm/viking.S | 284 +++ arch/sparc/net/Makefile | 8 + arch/sparc/net/bpf_jit_32.h | 69 + arch/sparc/net/bpf_jit_64.h | 38 + arch/sparc/net/bpf_jit_asm_32.S | 202 ++ arch/sparc/net/bpf_jit_comp_32.c | 764 ++++++ arch/sparc/net/bpf_jit_comp_64.c | 1613 +++++++++++++ arch/sparc/power/Makefile | 4 + arch/sparc/power/hibernate.c | 41 + arch/sparc/power/hibernate_asm.S | 132 ++ arch/sparc/prom/Makefile | 16 + arch/sparc/prom/bootstr_32.c | 63 + arch/sparc/prom/bootstr_64.c | 44 + arch/sparc/prom/cif.S | 48 + arch/sparc/prom/console_32.c | 57 + arch/sparc/prom/console_64.c | 46 + arch/sparc/prom/init_32.c | 76 + arch/sparc/prom/init_64.c | 54 + arch/sparc/prom/memory.c | 88 + arch/sparc/prom/misc_32.c | 128 + arch/sparc/prom/misc_64.c | 447 ++++ arch/sparc/prom/mp.c | 44 + arch/sparc/prom/p1275.c | 55 + arch/sparc/prom/printf.c | 70 + arch/sparc/prom/ranges.c | 114 + arch/sparc/prom/tree_32.c | 310 +++ arch/sparc/prom/tree_64.c | 393 ++++ arch/sparc/vdso/.gitignore | 4 + arch/sparc/vdso/Makefile | 114 + arch/sparc/vdso/vclock_gettime.c | 63 + arch/sparc/vdso/vdso-layout.lds.S | 80 + arch/sparc/vdso/vdso-note.S | 12 + arch/sparc/vdso/vdso.lds.S | 25 + arch/sparc/vdso/vdso2c.c | 204 ++ arch/sparc/vdso/vdso2c.h | 99 + arch/sparc/vdso/vdso32/.gitignore | 2 + arch/sparc/vdso/vdso32/vclock_gettime.c | 22 + arch/sparc/vdso/vdso32/vdso-note.S | 12 + arch/sparc/vdso/vdso32/vdso32.lds.S | 28 + arch/sparc/vdso/vma.c | 211 ++ arch/sparc/video/Makefile | 3 + arch/sparc/video/video-common.c | 25 + 612 files changed, 125023 insertions(+) create mode 100644 arch/sparc/Kbuild create mode 100644 arch/sparc/Kconfig create mode 100644 arch/sparc/Kconfig.debug create mode 100644 arch/sparc/Makefile create mode 100644 arch/sparc/boot/.gitignore create mode 100644 arch/sparc/boot/Makefile create mode 100755 arch/sparc/boot/install.sh create mode 100644 arch/sparc/boot/piggyback.c create mode 100644 arch/sparc/configs/sparc32_defconfig create mode 100644 arch/sparc/configs/sparc64_defconfig create mode 100644 arch/sparc/crypto/Kconfig create mode 100644 arch/sparc/crypto/Makefile create mode 100644 arch/sparc/crypto/aes_glue.c create mode 100644 arch/sparc/crypto/camellia_asm.S create mode 100644 arch/sparc/crypto/camellia_glue.c create mode 100644 arch/sparc/crypto/crop_devid.c create mode 100644 arch/sparc/include/asm/Kbuild create mode 100644 arch/sparc/include/asm/adi.h create mode 100644 arch/sparc/include/asm/adi_64.h create mode 100644 arch/sparc/include/asm/apb.h create mode 100644 arch/sparc/include/asm/asm-offsets.h create mode 100644 arch/sparc/include/asm/asm-prototypes.h create mode 100644 arch/sparc/include/asm/asm.h create mode 100644 arch/sparc/include/asm/asmmacro.h create mode 100644 arch/sparc/include/asm/atomic.h create mode 100644 arch/sparc/include/asm/atomic_32.h create mode 100644 arch/sparc/include/asm/atomic_64.h create mode 100644 arch/sparc/include/asm/auxio.h create mode 100644 arch/sparc/include/asm/auxio_32.h create mode 100644 arch/sparc/include/asm/auxio_64.h create mode 100644 arch/sparc/include/asm/backoff.h create mode 100644 arch/sparc/include/asm/barrier.h create mode 100644 arch/sparc/include/asm/barrier_32.h create mode 100644 arch/sparc/include/asm/barrier_64.h create mode 100644 arch/sparc/include/asm/bbc.h create mode 100644 arch/sparc/include/asm/bitext.h create mode 100644 arch/sparc/include/asm/bitops.h create mode 100644 arch/sparc/include/asm/bitops_32.h create mode 100644 arch/sparc/include/asm/bitops_64.h create mode 100644 arch/sparc/include/asm/btext.h create mode 100644 arch/sparc/include/asm/bug.h create mode 100644 arch/sparc/include/asm/cache.h create mode 100644 arch/sparc/include/asm/cacheflush.h create mode 100644 arch/sparc/include/asm/cacheflush_32.h create mode 100644 arch/sparc/include/asm/cacheflush_64.h create mode 100644 arch/sparc/include/asm/cachetlb_32.h create mode 100644 arch/sparc/include/asm/cachetype.h create mode 100644 arch/sparc/include/asm/chafsr.h create mode 100644 arch/sparc/include/asm/checksum.h create mode 100644 arch/sparc/include/asm/checksum_32.h create mode 100644 arch/sparc/include/asm/checksum_64.h create mode 100644 arch/sparc/include/asm/chmctrl.h create mode 100644 arch/sparc/include/asm/clock.h create mode 100644 arch/sparc/include/asm/clocksource.h create mode 100644 arch/sparc/include/asm/cmpxchg.h create mode 100644 arch/sparc/include/asm/cmpxchg_32.h create mode 100644 arch/sparc/include/asm/cmpxchg_64.h create mode 100644 arch/sparc/include/asm/compat.h create mode 100644 arch/sparc/include/asm/compat_signal.h create mode 100644 arch/sparc/include/asm/contregs.h create mode 100644 arch/sparc/include/asm/cpu_type.h create mode 100644 arch/sparc/include/asm/cpudata.h create mode 100644 arch/sparc/include/asm/cpudata_32.h create mode 100644 arch/sparc/include/asm/cpudata_64.h create mode 100644 arch/sparc/include/asm/current.h create mode 100644 arch/sparc/include/asm/dcr.h create mode 100644 arch/sparc/include/asm/dcu.h create mode 100644 arch/sparc/include/asm/delay.h create mode 100644 arch/sparc/include/asm/delay_32.h create mode 100644 arch/sparc/include/asm/delay_64.h create mode 100644 arch/sparc/include/asm/device.h create mode 100644 arch/sparc/include/asm/dma-mapping.h create mode 100644 arch/sparc/include/asm/dma.h create mode 100644 arch/sparc/include/asm/ebus_dma.h create mode 100644 arch/sparc/include/asm/ecc.h create mode 100644 arch/sparc/include/asm/eeprom.h create mode 100644 arch/sparc/include/asm/elf.h create mode 100644 arch/sparc/include/asm/elf_32.h create mode 100644 arch/sparc/include/asm/elf_64.h create mode 100644 arch/sparc/include/asm/estate.h create mode 100644 arch/sparc/include/asm/extable.h create mode 100644 arch/sparc/include/asm/fbio.h create mode 100644 arch/sparc/include/asm/fhc.h create mode 100644 arch/sparc/include/asm/floppy.h create mode 100644 arch/sparc/include/asm/floppy_32.h create mode 100644 arch/sparc/include/asm/floppy_64.h create mode 100644 arch/sparc/include/asm/fpumacro.h create mode 100644 arch/sparc/include/asm/ftrace.h create mode 100644 arch/sparc/include/asm/futex.h create mode 100644 arch/sparc/include/asm/futex_32.h create mode 100644 arch/sparc/include/asm/futex_64.h create mode 100644 arch/sparc/include/asm/hardirq.h create mode 100644 arch/sparc/include/asm/hardirq_32.h create mode 100644 arch/sparc/include/asm/hardirq_64.h create mode 100644 arch/sparc/include/asm/head.h create mode 100644 arch/sparc/include/asm/head_32.h create mode 100644 arch/sparc/include/asm/head_64.h create mode 100644 arch/sparc/include/asm/hibernate.h create mode 100644 arch/sparc/include/asm/highmem.h create mode 100644 arch/sparc/include/asm/hugetlb.h create mode 100644 arch/sparc/include/asm/hvtramp.h create mode 100644 arch/sparc/include/asm/hw_irq.h create mode 100644 arch/sparc/include/asm/hypervisor.h create mode 100644 arch/sparc/include/asm/idprom.h create mode 100644 arch/sparc/include/asm/intr_queue.h create mode 100644 arch/sparc/include/asm/io-unit.h create mode 100644 arch/sparc/include/asm/io.h create mode 100644 arch/sparc/include/asm/io_32.h create mode 100644 arch/sparc/include/asm/io_64.h create mode 100644 arch/sparc/include/asm/ioctls.h create mode 100644 arch/sparc/include/asm/iommu-common.h create mode 100644 arch/sparc/include/asm/iommu.h create mode 100644 arch/sparc/include/asm/iommu_32.h create mode 100644 arch/sparc/include/asm/iommu_64.h create mode 100644 arch/sparc/include/asm/irq.h create mode 100644 arch/sparc/include/asm/irq_32.h create mode 100644 arch/sparc/include/asm/irq_64.h create mode 100644 arch/sparc/include/asm/irqflags.h create mode 100644 arch/sparc/include/asm/irqflags_32.h create mode 100644 arch/sparc/include/asm/irqflags_64.h create mode 100644 arch/sparc/include/asm/jump_label.h create mode 100644 arch/sparc/include/asm/kdebug.h create mode 100644 arch/sparc/include/asm/kdebug_32.h create mode 100644 arch/sparc/include/asm/kdebug_64.h create mode 100644 arch/sparc/include/asm/kgdb.h create mode 100644 arch/sparc/include/asm/kprobes.h create mode 100644 arch/sparc/include/asm/ldc.h create mode 100644 arch/sparc/include/asm/leon.h create mode 100644 arch/sparc/include/asm/leon_amba.h create mode 100644 arch/sparc/include/asm/leon_pci.h create mode 100644 arch/sparc/include/asm/lsu.h create mode 100644 arch/sparc/include/asm/machines.h create mode 100644 arch/sparc/include/asm/mbus.h create mode 100644 arch/sparc/include/asm/mc146818rtc.h create mode 100644 arch/sparc/include/asm/mc146818rtc_32.h create mode 100644 arch/sparc/include/asm/mc146818rtc_64.h create mode 100644 arch/sparc/include/asm/mdesc.h create mode 100644 arch/sparc/include/asm/memctrl.h create mode 100644 arch/sparc/include/asm/mman.h create mode 100644 arch/sparc/include/asm/mmu.h create mode 100644 arch/sparc/include/asm/mmu_32.h create mode 100644 arch/sparc/include/asm/mmu_64.h create mode 100644 arch/sparc/include/asm/mmu_context.h create mode 100644 arch/sparc/include/asm/mmu_context_32.h create mode 100644 arch/sparc/include/asm/mmu_context_64.h create mode 100644 arch/sparc/include/asm/mmzone.h create mode 100644 arch/sparc/include/asm/mxcc.h create mode 100644 arch/sparc/include/asm/nmi.h create mode 100644 arch/sparc/include/asm/ns87303.h create mode 100644 arch/sparc/include/asm/obio.h create mode 100644 arch/sparc/include/asm/opcodes.h create mode 100644 arch/sparc/include/asm/openprom.h create mode 100644 arch/sparc/include/asm/oplib.h create mode 100644 arch/sparc/include/asm/oplib_32.h create mode 100644 arch/sparc/include/asm/oplib_64.h create mode 100644 arch/sparc/include/asm/page.h create mode 100644 arch/sparc/include/asm/page_32.h create mode 100644 arch/sparc/include/asm/page_64.h create mode 100644 arch/sparc/include/asm/parport.h create mode 100644 arch/sparc/include/asm/parport_64.h create mode 100644 arch/sparc/include/asm/pbm.h create mode 100644 arch/sparc/include/asm/pci.h create mode 100644 arch/sparc/include/asm/pcic.h create mode 100644 arch/sparc/include/asm/pcr.h create mode 100644 arch/sparc/include/asm/percpu.h create mode 100644 arch/sparc/include/asm/percpu_32.h create mode 100644 arch/sparc/include/asm/percpu_64.h create mode 100644 arch/sparc/include/asm/perf_event.h create mode 100644 arch/sparc/include/asm/pgalloc.h create mode 100644 arch/sparc/include/asm/pgalloc_32.h create mode 100644 arch/sparc/include/asm/pgalloc_64.h create mode 100644 arch/sparc/include/asm/pgtable.h create mode 100644 arch/sparc/include/asm/pgtable_32.h create mode 100644 arch/sparc/include/asm/pgtable_64.h create mode 100644 arch/sparc/include/asm/pgtsrmmu.h create mode 100644 arch/sparc/include/asm/pil.h create mode 100644 arch/sparc/include/asm/processor.h create mode 100644 arch/sparc/include/asm/processor_32.h create mode 100644 arch/sparc/include/asm/processor_64.h create mode 100644 arch/sparc/include/asm/prom.h create mode 100644 arch/sparc/include/asm/psr.h create mode 100644 arch/sparc/include/asm/ptrace.h create mode 100644 arch/sparc/include/asm/qrwlock.h create mode 100644 arch/sparc/include/asm/qspinlock.h create mode 100644 arch/sparc/include/asm/ross.h create mode 100644 arch/sparc/include/asm/sbi.h create mode 100644 arch/sparc/include/asm/scratchpad.h create mode 100644 arch/sparc/include/asm/seccomp.h create mode 100644 arch/sparc/include/asm/sections.h create mode 100644 arch/sparc/include/asm/setup.h create mode 100644 arch/sparc/include/asm/sfafsr.h create mode 100644 arch/sparc/include/asm/sfp-machine.h create mode 100644 arch/sparc/include/asm/sfp-machine_32.h create mode 100644 arch/sparc/include/asm/sfp-machine_64.h create mode 100644 arch/sparc/include/asm/shmparam.h create mode 100644 arch/sparc/include/asm/shmparam_32.h create mode 100644 arch/sparc/include/asm/shmparam_64.h create mode 100644 arch/sparc/include/asm/sigcontext.h create mode 100644 arch/sparc/include/asm/signal.h create mode 100644 arch/sparc/include/asm/smp.h create mode 100644 arch/sparc/include/asm/smp_32.h create mode 100644 arch/sparc/include/asm/smp_64.h create mode 100644 arch/sparc/include/asm/sparsemem.h create mode 100644 arch/sparc/include/asm/spinlock.h create mode 100644 arch/sparc/include/asm/spinlock_32.h create mode 100644 arch/sparc/include/asm/spinlock_64.h create mode 100644 arch/sparc/include/asm/spinlock_types.h create mode 100644 arch/sparc/include/asm/spitfire.h create mode 100644 arch/sparc/include/asm/stacktrace.h create mode 100644 arch/sparc/include/asm/starfire.h create mode 100644 arch/sparc/include/asm/string.h create mode 100644 arch/sparc/include/asm/string_32.h create mode 100644 arch/sparc/include/asm/string_64.h create mode 100644 arch/sparc/include/asm/sunbpp.h create mode 100644 arch/sparc/include/asm/swift.h create mode 100644 arch/sparc/include/asm/switch_to.h create mode 100644 arch/sparc/include/asm/switch_to_32.h create mode 100644 arch/sparc/include/asm/switch_to_64.h create mode 100644 arch/sparc/include/asm/syscall.h create mode 100644 arch/sparc/include/asm/syscalls.h create mode 100644 arch/sparc/include/asm/termbits.h create mode 100644 arch/sparc/include/asm/thread_info.h create mode 100644 arch/sparc/include/asm/thread_info_32.h create mode 100644 arch/sparc/include/asm/thread_info_64.h create mode 100644 arch/sparc/include/asm/timer.h create mode 100644 arch/sparc/include/asm/timer_32.h create mode 100644 arch/sparc/include/asm/timer_64.h create mode 100644 arch/sparc/include/asm/timex.h create mode 100644 arch/sparc/include/asm/timex_64.h create mode 100644 arch/sparc/include/asm/tlb.h create mode 100644 arch/sparc/include/asm/tlb_32.h create mode 100644 arch/sparc/include/asm/tlb_64.h create mode 100644 arch/sparc/include/asm/tlbflush.h create mode 100644 arch/sparc/include/asm/tlbflush_32.h create mode 100644 arch/sparc/include/asm/tlbflush_64.h create mode 100644 arch/sparc/include/asm/topology.h create mode 100644 arch/sparc/include/asm/topology_32.h create mode 100644 arch/sparc/include/asm/topology_64.h create mode 100644 arch/sparc/include/asm/trap_block.h create mode 100644 arch/sparc/include/asm/traps.h create mode 100644 arch/sparc/include/asm/tsb.h create mode 100644 arch/sparc/include/asm/tsunami.h create mode 100644 arch/sparc/include/asm/ttable.h create mode 100644 arch/sparc/include/asm/turbosparc.h create mode 100644 arch/sparc/include/asm/uaccess.h create mode 100644 arch/sparc/include/asm/uaccess_32.h create mode 100644 arch/sparc/include/asm/uaccess_64.h create mode 100644 arch/sparc/include/asm/unistd.h create mode 100644 arch/sparc/include/asm/upa.h create mode 100644 arch/sparc/include/asm/uprobes.h create mode 100644 arch/sparc/include/asm/user.h create mode 100644 arch/sparc/include/asm/vaddrs.h create mode 100644 arch/sparc/include/asm/vdso.h create mode 100644 arch/sparc/include/asm/vdso/clocksource.h create mode 100644 arch/sparc/include/asm/vdso/gettimeofday.h create mode 100644 arch/sparc/include/asm/vdso/processor.h create mode 100644 arch/sparc/include/asm/vdso/vsyscall.h create mode 100644 arch/sparc/include/asm/video.h create mode 100644 arch/sparc/include/asm/viking.h create mode 100644 arch/sparc/include/asm/vio.h create mode 100644 arch/sparc/include/asm/visasm.h create mode 100644 arch/sparc/include/asm/vmalloc.h create mode 100644 arch/sparc/include/asm/winmacro.h create mode 100644 arch/sparc/include/uapi/asm/Kbuild create mode 100644 arch/sparc/include/uapi/asm/apc.h create mode 100644 arch/sparc/include/uapi/asm/asi.h create mode 100644 arch/sparc/include/uapi/asm/auxvec.h create mode 100644 arch/sparc/include/uapi/asm/bitsperlong.h create mode 100644 arch/sparc/include/uapi/asm/byteorder.h create mode 100644 arch/sparc/include/uapi/asm/display7seg.h create mode 100644 arch/sparc/include/uapi/asm/envctrl.h create mode 100644 arch/sparc/include/uapi/asm/errno.h create mode 100644 arch/sparc/include/uapi/asm/fbio.h create mode 100644 arch/sparc/include/uapi/asm/fcntl.h create mode 100644 arch/sparc/include/uapi/asm/ioctl.h create mode 100644 arch/sparc/include/uapi/asm/ioctls.h create mode 100644 arch/sparc/include/uapi/asm/ipcbuf.h create mode 100644 arch/sparc/include/uapi/asm/mman.h create mode 100644 arch/sparc/include/uapi/asm/msgbuf.h create mode 100644 arch/sparc/include/uapi/asm/openpromio.h create mode 100644 arch/sparc/include/uapi/asm/oradax.h create mode 100644 arch/sparc/include/uapi/asm/param.h create mode 100644 arch/sparc/include/uapi/asm/perfctr.h create mode 100644 arch/sparc/include/uapi/asm/poll.h create mode 100644 arch/sparc/include/uapi/asm/posix_types.h create mode 100644 arch/sparc/include/uapi/asm/psr.h create mode 100644 arch/sparc/include/uapi/asm/psrcompat.h create mode 100644 arch/sparc/include/uapi/asm/pstate.h create mode 100644 arch/sparc/include/uapi/asm/ptrace.h create mode 100644 arch/sparc/include/uapi/asm/resource.h create mode 100644 arch/sparc/include/uapi/asm/sembuf.h create mode 100644 arch/sparc/include/uapi/asm/setup.h create mode 100644 arch/sparc/include/uapi/asm/shmbuf.h create mode 100644 arch/sparc/include/uapi/asm/sigcontext.h create mode 100644 arch/sparc/include/uapi/asm/siginfo.h create mode 100644 arch/sparc/include/uapi/asm/signal.h create mode 100644 arch/sparc/include/uapi/asm/socket.h create mode 100644 arch/sparc/include/uapi/asm/stat.h create mode 100644 arch/sparc/include/uapi/asm/swab.h create mode 100644 arch/sparc/include/uapi/asm/termbits.h create mode 100644 arch/sparc/include/uapi/asm/termios.h create mode 100644 arch/sparc/include/uapi/asm/traps.h create mode 100644 arch/sparc/include/uapi/asm/ucontext.h create mode 100644 arch/sparc/include/uapi/asm/uctx.h create mode 100644 arch/sparc/include/uapi/asm/unistd.h create mode 100644 arch/sparc/include/uapi/asm/utrap.h create mode 100644 arch/sparc/include/uapi/asm/watchdog.h create mode 100644 arch/sparc/kernel/.gitignore create mode 100644 arch/sparc/kernel/Makefile create mode 100644 arch/sparc/kernel/adi_64.c create mode 100644 arch/sparc/kernel/apc.c create mode 100644 arch/sparc/kernel/asm-offsets.c create mode 100644 arch/sparc/kernel/audit.c create mode 100644 arch/sparc/kernel/auxio_32.c create mode 100644 arch/sparc/kernel/auxio_64.c create mode 100644 arch/sparc/kernel/btext.c create mode 100644 arch/sparc/kernel/central.c create mode 100644 arch/sparc/kernel/cherrs.S create mode 100644 arch/sparc/kernel/chmc.c create mode 100644 arch/sparc/kernel/compat_audit.c create mode 100644 arch/sparc/kernel/cpu.c create mode 100644 arch/sparc/kernel/cpumap.c create mode 100644 arch/sparc/kernel/cpumap.h create mode 100644 arch/sparc/kernel/devices.c create mode 100644 arch/sparc/kernel/ds.c create mode 100644 arch/sparc/kernel/dtlb_miss.S create mode 100644 arch/sparc/kernel/dtlb_prot.S create mode 100644 arch/sparc/kernel/ebus.c create mode 100644 arch/sparc/kernel/entry.S create mode 100644 arch/sparc/kernel/entry.h create mode 100644 arch/sparc/kernel/etrap_32.S create mode 100644 arch/sparc/kernel/etrap_64.S create mode 100644 arch/sparc/kernel/fpu_traps.S create mode 100644 arch/sparc/kernel/ftrace.c create mode 100644 arch/sparc/kernel/getsetcc.S create mode 100644 arch/sparc/kernel/head_32.S create mode 100644 arch/sparc/kernel/head_64.S create mode 100644 arch/sparc/kernel/helpers.S create mode 100644 arch/sparc/kernel/hvapi.c create mode 100644 arch/sparc/kernel/hvcalls.S create mode 100644 arch/sparc/kernel/hvtramp.S create mode 100644 arch/sparc/kernel/idprom.c create mode 100644 arch/sparc/kernel/iommu-common.c create mode 100644 arch/sparc/kernel/iommu.c create mode 100644 arch/sparc/kernel/iommu_common.h create mode 100644 arch/sparc/kernel/ioport.c create mode 100644 arch/sparc/kernel/irq.h create mode 100644 arch/sparc/kernel/irq_32.c create mode 100644 arch/sparc/kernel/irq_64.c create mode 100644 arch/sparc/kernel/itlb_miss.S create mode 100644 arch/sparc/kernel/ivec.S create mode 100644 arch/sparc/kernel/jump_label.c create mode 100644 arch/sparc/kernel/kernel.h create mode 100644 arch/sparc/kernel/kgdb_32.c create mode 100644 arch/sparc/kernel/kgdb_64.c create mode 100644 arch/sparc/kernel/kprobes.c create mode 100644 arch/sparc/kernel/kstack.h create mode 100644 arch/sparc/kernel/ktlb.S create mode 100644 arch/sparc/kernel/ldc.c create mode 100644 arch/sparc/kernel/led.c create mode 100644 arch/sparc/kernel/leon_kernel.c create mode 100644 arch/sparc/kernel/leon_pci.c create mode 100644 arch/sparc/kernel/leon_pci_grpci1.c create mode 100644 arch/sparc/kernel/leon_pci_grpci2.c create mode 100644 arch/sparc/kernel/leon_pmc.c create mode 100644 arch/sparc/kernel/leon_smp.c create mode 100644 arch/sparc/kernel/mdesc.c create mode 100644 arch/sparc/kernel/misctrap.S create mode 100644 arch/sparc/kernel/module.c create mode 100644 arch/sparc/kernel/nmi.c create mode 100644 arch/sparc/kernel/of_device_32.c create mode 100644 arch/sparc/kernel/of_device_64.c create mode 100644 arch/sparc/kernel/of_device_common.c create mode 100644 arch/sparc/kernel/of_device_common.h create mode 100644 arch/sparc/kernel/pci.c create mode 100644 arch/sparc/kernel/pci_common.c create mode 100644 arch/sparc/kernel/pci_fire.c create mode 100644 arch/sparc/kernel/pci_impl.h create mode 100644 arch/sparc/kernel/pci_msi.c create mode 100644 arch/sparc/kernel/pci_psycho.c create mode 100644 arch/sparc/kernel/pci_sabre.c create mode 100644 arch/sparc/kernel/pci_schizo.c create mode 100644 arch/sparc/kernel/pci_sun4v.c create mode 100644 arch/sparc/kernel/pci_sun4v.h create mode 100644 arch/sparc/kernel/pci_sun4v_asm.S create mode 100644 arch/sparc/kernel/pcic.c create mode 100644 arch/sparc/kernel/pcr.c create mode 100644 arch/sparc/kernel/perf_event.c create mode 100644 arch/sparc/kernel/pmc.c create mode 100644 arch/sparc/kernel/power.c create mode 100644 arch/sparc/kernel/process.c create mode 100644 arch/sparc/kernel/process_32.c create mode 100644 arch/sparc/kernel/process_64.c create mode 100644 arch/sparc/kernel/prom.h create mode 100644 arch/sparc/kernel/prom_32.c create mode 100644 arch/sparc/kernel/prom_64.c create mode 100644 arch/sparc/kernel/prom_common.c create mode 100644 arch/sparc/kernel/prom_irqtrans.c create mode 100644 arch/sparc/kernel/psycho_common.c create mode 100644 arch/sparc/kernel/psycho_common.h create mode 100644 arch/sparc/kernel/ptrace_32.c create mode 100644 arch/sparc/kernel/ptrace_64.c create mode 100644 arch/sparc/kernel/reboot.c create mode 100644 arch/sparc/kernel/rtrap_32.S create mode 100644 arch/sparc/kernel/rtrap_64.S create mode 100644 arch/sparc/kernel/sbus.c create mode 100644 arch/sparc/kernel/setup.c create mode 100644 arch/sparc/kernel/setup_32.c create mode 100644 arch/sparc/kernel/setup_64.c create mode 100644 arch/sparc/kernel/signal32.c create mode 100644 arch/sparc/kernel/signal_32.c create mode 100644 arch/sparc/kernel/signal_64.c create mode 100644 arch/sparc/kernel/sigutil.h create mode 100644 arch/sparc/kernel/sigutil_32.c create mode 100644 arch/sparc/kernel/sigutil_64.c create mode 100644 arch/sparc/kernel/smp_32.c create mode 100644 arch/sparc/kernel/smp_64.c create mode 100644 arch/sparc/kernel/sparc_ksyms.c create mode 100644 arch/sparc/kernel/spiterrs.S create mode 100644 arch/sparc/kernel/sstate.c create mode 100644 arch/sparc/kernel/stacktrace.c create mode 100644 arch/sparc/kernel/starfire.c create mode 100644 arch/sparc/kernel/sun4d_irq.c create mode 100644 arch/sparc/kernel/sun4d_smp.c create mode 100644 arch/sparc/kernel/sun4m_irq.c create mode 100644 arch/sparc/kernel/sun4m_smp.c create mode 100644 arch/sparc/kernel/sun4v_ivec.S create mode 100644 arch/sparc/kernel/sun4v_mcd.S create mode 100644 arch/sparc/kernel/sun4v_tlb_miss.S create mode 100644 arch/sparc/kernel/sys32.S create mode 100644 arch/sparc/kernel/sys_sparc32.c create mode 100644 arch/sparc/kernel/sys_sparc_32.c create mode 100644 arch/sparc/kernel/sys_sparc_64.c create mode 100644 arch/sparc/kernel/syscalls.S create mode 100644 arch/sparc/kernel/syscalls/Makefile create mode 100644 arch/sparc/kernel/syscalls/syscall.tbl create mode 100644 arch/sparc/kernel/sysfs.c create mode 100644 arch/sparc/kernel/systbls.h create mode 100644 arch/sparc/kernel/systbls_32.S create mode 100644 arch/sparc/kernel/systbls_64.S create mode 100644 arch/sparc/kernel/termios.c create mode 100644 arch/sparc/kernel/time_32.c create mode 100644 arch/sparc/kernel/time_64.c create mode 100644 arch/sparc/kernel/trampoline_32.S create mode 100644 arch/sparc/kernel/trampoline_64.S create mode 100644 arch/sparc/kernel/traps_32.c create mode 100644 arch/sparc/kernel/traps_64.c create mode 100644 arch/sparc/kernel/tsb.S create mode 100644 arch/sparc/kernel/ttable_32.S create mode 100644 arch/sparc/kernel/ttable_64.S create mode 100644 arch/sparc/kernel/una_asm_32.S create mode 100644 arch/sparc/kernel/una_asm_64.S create mode 100644 arch/sparc/kernel/unaligned_32.c create mode 100644 arch/sparc/kernel/unaligned_64.c create mode 100644 arch/sparc/kernel/uprobes.c create mode 100644 arch/sparc/kernel/urtt_fill.S create mode 100644 arch/sparc/kernel/utrap.S create mode 100644 arch/sparc/kernel/vio.c create mode 100644 arch/sparc/kernel/viohs.c create mode 100644 arch/sparc/kernel/visemul.c create mode 100644 arch/sparc/kernel/vmlinux.lds.S create mode 100644 arch/sparc/kernel/windows.c create mode 100644 arch/sparc/kernel/winfixup.S create mode 100644 arch/sparc/kernel/wof.S create mode 100644 arch/sparc/kernel/wuf.S create mode 100644 arch/sparc/lib/GENbzero.S create mode 100644 arch/sparc/lib/GENcopy_from_user.S create mode 100644 arch/sparc/lib/GENcopy_to_user.S create mode 100644 arch/sparc/lib/GENmemcpy.S create mode 100644 arch/sparc/lib/GENpage.S create mode 100644 arch/sparc/lib/GENpatch.S create mode 100644 arch/sparc/lib/M7copy_from_user.S create mode 100644 arch/sparc/lib/M7copy_to_user.S create mode 100644 arch/sparc/lib/M7memcpy.S create mode 100644 arch/sparc/lib/M7memset.S create mode 100644 arch/sparc/lib/M7patch.S create mode 100644 arch/sparc/lib/Makefile create mode 100644 arch/sparc/lib/Memcpy_utils.S create mode 100644 arch/sparc/lib/NG2copy_from_user.S create mode 100644 arch/sparc/lib/NG2copy_to_user.S create mode 100644 arch/sparc/lib/NG2memcpy.S create mode 100644 arch/sparc/lib/NG2patch.S create mode 100644 arch/sparc/lib/NG4clear_page.S create mode 100644 arch/sparc/lib/NG4copy_from_user.S create mode 100644 arch/sparc/lib/NG4copy_page.S create mode 100644 arch/sparc/lib/NG4copy_to_user.S create mode 100644 arch/sparc/lib/NG4fls.S create mode 100644 arch/sparc/lib/NG4memcpy.S create mode 100644 arch/sparc/lib/NG4memset.S create mode 100644 arch/sparc/lib/NG4patch.S create mode 100644 arch/sparc/lib/NGbzero.S create mode 100644 arch/sparc/lib/NGcopy_from_user.S create mode 100644 arch/sparc/lib/NGcopy_to_user.S create mode 100644 arch/sparc/lib/NGmemcpy.S create mode 100644 arch/sparc/lib/NGpage.S create mode 100644 arch/sparc/lib/NGpatch.S create mode 100644 arch/sparc/lib/PeeCeeI.c create mode 100644 arch/sparc/lib/U1copy_from_user.S create mode 100644 arch/sparc/lib/U1copy_to_user.S create mode 100644 arch/sparc/lib/U1memcpy.S create mode 100644 arch/sparc/lib/U3copy_from_user.S create mode 100644 arch/sparc/lib/U3copy_to_user.S create mode 100644 arch/sparc/lib/U3memcpy.S create mode 100644 arch/sparc/lib/U3patch.S create mode 100644 arch/sparc/lib/VISsave.S create mode 100644 arch/sparc/lib/ashldi3.S create mode 100644 arch/sparc/lib/ashrdi3.S create mode 100644 arch/sparc/lib/atomic32.c create mode 100644 arch/sparc/lib/atomic_64.S create mode 100644 arch/sparc/lib/bitext.c create mode 100644 arch/sparc/lib/bitops.S create mode 100644 arch/sparc/lib/blockops.S create mode 100644 arch/sparc/lib/bzero.S create mode 100644 arch/sparc/lib/checksum_32.S create mode 100644 arch/sparc/lib/checksum_64.S create mode 100644 arch/sparc/lib/clear_page.S create mode 100644 arch/sparc/lib/copy_in_user.S create mode 100644 arch/sparc/lib/copy_page.S create mode 100644 arch/sparc/lib/copy_user.S create mode 100644 arch/sparc/lib/csum_copy.S create mode 100644 arch/sparc/lib/csum_copy_from_user.S create mode 100644 arch/sparc/lib/csum_copy_to_user.S create mode 100644 arch/sparc/lib/divdi3.S create mode 100644 arch/sparc/lib/ffs.S create mode 100644 arch/sparc/lib/fls.S create mode 100644 arch/sparc/lib/fls64.S create mode 100644 arch/sparc/lib/hweight.S create mode 100644 arch/sparc/lib/iomap.c create mode 100644 arch/sparc/lib/ipcsum.S create mode 100644 arch/sparc/lib/libgcc.h create mode 100644 arch/sparc/lib/locks.S create mode 100644 arch/sparc/lib/lshrdi3.S create mode 100644 arch/sparc/lib/mcount.S create mode 100644 arch/sparc/lib/memcmp.S create mode 100644 arch/sparc/lib/memcpy.S create mode 100644 arch/sparc/lib/memmove.S create mode 100644 arch/sparc/lib/memscan_32.S create mode 100644 arch/sparc/lib/memscan_64.S create mode 100644 arch/sparc/lib/memset.S create mode 100644 arch/sparc/lib/muldi3.S create mode 100644 arch/sparc/lib/multi3.S create mode 100644 arch/sparc/lib/strlen.S create mode 100644 arch/sparc/lib/strncmp_32.S create mode 100644 arch/sparc/lib/strncmp_64.S create mode 100644 arch/sparc/lib/udivdi3.S create mode 100644 arch/sparc/math-emu/Makefile create mode 100644 arch/sparc/math-emu/math_32.c create mode 100644 arch/sparc/math-emu/math_64.c create mode 100644 arch/sparc/math-emu/sfp-util_32.h create mode 100644 arch/sparc/math-emu/sfp-util_64.h create mode 100644 arch/sparc/mm/Makefile create mode 100644 arch/sparc/mm/execmem.c create mode 100644 arch/sparc/mm/fault_32.c create mode 100644 arch/sparc/mm/fault_64.c create mode 100644 arch/sparc/mm/hugetlbpage.c create mode 100644 arch/sparc/mm/hypersparc.S create mode 100644 arch/sparc/mm/init_32.c create mode 100644 arch/sparc/mm/init_64.c create mode 100644 arch/sparc/mm/init_64.h create mode 100644 arch/sparc/mm/io-unit.c create mode 100644 arch/sparc/mm/iommu.c create mode 100644 arch/sparc/mm/leon_mm.c create mode 100644 arch/sparc/mm/mm_32.h create mode 100644 arch/sparc/mm/srmmu.c create mode 100644 arch/sparc/mm/srmmu_access.S create mode 100644 arch/sparc/mm/swift.S create mode 100644 arch/sparc/mm/tlb.c create mode 100644 arch/sparc/mm/tsb.c create mode 100644 arch/sparc/mm/tsunami.S create mode 100644 arch/sparc/mm/ultra.S create mode 100644 arch/sparc/mm/viking.S create mode 100644 arch/sparc/net/Makefile create mode 100644 arch/sparc/net/bpf_jit_32.h create mode 100644 arch/sparc/net/bpf_jit_64.h create mode 100644 arch/sparc/net/bpf_jit_asm_32.S create mode 100644 arch/sparc/net/bpf_jit_comp_32.c create mode 100644 arch/sparc/net/bpf_jit_comp_64.c create mode 100644 arch/sparc/power/Makefile create mode 100644 arch/sparc/power/hibernate.c create mode 100644 arch/sparc/power/hibernate_asm.S create mode 100644 arch/sparc/prom/Makefile create mode 100644 arch/sparc/prom/bootstr_32.c create mode 100644 arch/sparc/prom/bootstr_64.c create mode 100644 arch/sparc/prom/cif.S create mode 100644 arch/sparc/prom/console_32.c create mode 100644 arch/sparc/prom/console_64.c create mode 100644 arch/sparc/prom/init_32.c create mode 100644 arch/sparc/prom/init_64.c create mode 100644 arch/sparc/prom/memory.c create mode 100644 arch/sparc/prom/misc_32.c create mode 100644 arch/sparc/prom/misc_64.c create mode 100644 arch/sparc/prom/mp.c create mode 100644 arch/sparc/prom/p1275.c create mode 100644 arch/sparc/prom/printf.c create mode 100644 arch/sparc/prom/ranges.c create mode 100644 arch/sparc/prom/tree_32.c create mode 100644 arch/sparc/prom/tree_64.c create mode 100644 arch/sparc/vdso/.gitignore create mode 100644 arch/sparc/vdso/Makefile create mode 100644 arch/sparc/vdso/vclock_gettime.c create mode 100644 arch/sparc/vdso/vdso-layout.lds.S create mode 100644 arch/sparc/vdso/vdso-note.S create mode 100644 arch/sparc/vdso/vdso.lds.S create mode 100644 arch/sparc/vdso/vdso2c.c create mode 100644 arch/sparc/vdso/vdso2c.h create mode 100644 arch/sparc/vdso/vdso32/.gitignore create mode 100644 arch/sparc/vdso/vdso32/vclock_gettime.c create mode 100644 arch/sparc/vdso/vdso32/vdso-note.S create mode 100644 arch/sparc/vdso/vdso32/vdso32.lds.S create mode 100644 arch/sparc/vdso/vma.c create mode 100644 arch/sparc/video/Makefile create mode 100644 arch/sparc/video/video-common.c (limited to 'arch/sparc') diff --git a/arch/sparc/Kbuild b/arch/sparc/Kbuild new file mode 100644 index 000000000..71cb3d934 --- /dev/null +++ b/arch/sparc/Kbuild @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# core part of the sparc kernel +# + +obj-y += kernel/ +obj-y += mm/ +obj-y += math-emu/ +obj-y += net/ +obj-y += crypto/ +obj-$(CONFIG_SPARC64) += vdso/ + +# for cleaning +subdir- += boot diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig new file mode 100644 index 000000000..ab77d3f25 --- /dev/null +++ b/arch/sparc/Kconfig @@ -0,0 +1,468 @@ +# SPDX-License-Identifier: GPL-2.0-only +config 64BIT + bool "64-bit kernel" if "$(ARCH)" = "sparc" + default "$(ARCH)" = "sparc64" + help + SPARC is a family of RISC microprocessors designed and marketed by + Sun Microsystems, incorporated. They are very widely found in Sun + workstations and clones. + + Say yes to build a 64-bit kernel - formerly known as sparc64 + Say no to build a 32-bit kernel - formerly known as sparc + +config SPARC + bool + default y + select ARCH_HAS_CC_CAN_LINK + select ARCH_HAS_CPU_CACHE_ALIASING + select ARCH_HAS_DMA_OPS + select ARCH_MIGHT_HAVE_PC_PARPORT if SPARC64 && PCI + select ARCH_MIGHT_HAVE_PC_SERIO + select OF + select OF_PROMTREE + select HAVE_ASM_MODVERSIONS + select HAVE_ARCH_KGDB if !SMP || SPARC64 + select HAVE_ARCH_TRACEHOOK + select HAVE_ARCH_SECCOMP if SPARC64 + select HAVE_EXIT_THREAD + select HAVE_PCI + select SYSCTL_EXCEPTION_TRACE + select RTC_CLASS + select RTC_DRV_M48T59 + select RTC_SYSTOHC + select HAVE_ARCH_JUMP_LABEL if SPARC64 + select GENERIC_IRQ_SHOW + select ARCH_WANT_IPC_PARSE_VERSION + select GENERIC_PCI_IOMAP + select HAS_IOPORT + select HAVE_HARDLOCKUP_DETECTOR_SPARC64 if SPARC64 + select HAVE_CBPF_JIT if SPARC32 + select HAVE_EBPF_JIT if SPARC64 + select HAVE_DEBUG_BUGVERBOSE + select GENERIC_SMP_IDLE_THREAD + select MODULES_USE_ELF_RELA + select PCI_SYSCALL if PCI + select PCI_MSI_ARCH_FALLBACKS if PCI_MSI + select ODD_RT_SIGACTION + select OLD_SIGSUSPEND + select CPU_NO_EFFICIENT_FFS + select LOCKDEP_SMALL if LOCKDEP + select NEED_DMA_MAP_STATE + select NEED_SG_DMA_LENGTH + select TRACE_IRQFLAGS_SUPPORT + +config SPARC32 + def_bool !64BIT + select ARCH_32BIT_OFF_T + select ARCH_HAS_CPU_FINALIZE_INIT if !SMP + select ARCH_HAS_SYNC_DMA_FOR_CPU + select CLZ_TAB + select DMA_DIRECT_REMAP + select GENERIC_ATOMIC64 + select GENERIC_LIB_CMPDI2 + select GENERIC_LIB_UCMPDI2 + select HAVE_UID16 + select HAVE_PAGE_SIZE_4KB + select LOCK_MM_AND_FIND_VMA + select OLD_SIGACTION + select ZONE_DMA + +config SPARC64 + def_bool 64BIT + select ALTERNATE_USER_ADDRESS_SPACE + select ARCH_HAS_DELAY_TIMER + select HAVE_FUNCTION_TRACER + select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_KRETPROBES + select HAVE_KPROBES + select MMU_GATHER_RCU_TABLE_FREE if SMP + select HAVE_ARCH_TLB_REMOVE_TABLE if SMP + select MMU_GATHER_MERGE_VMAS + select MMU_GATHER_NO_FLUSH_CACHE + select HAVE_ARCH_TRANSPARENT_HUGEPAGE + select HAVE_DYNAMIC_FTRACE + select HAVE_PAGE_SIZE_8KB + select HAVE_SYSCALL_TRACEPOINTS + select HAVE_CONTEXT_TRACKING_USER + select HAVE_TIF_NOHZ + select HAVE_DEBUG_KMEMLEAK + select IOMMU_HELPER + select SPARSE_IRQ + select RTC_DRV_CMOS + select RTC_DRV_BQ4802 + select RTC_DRV_SUN4V + select RTC_DRV_STARFIRE + select HAVE_PERF_EVENTS + select PERF_USE_VMALLOC + select ARCH_HAVE_NMI_SAFE_CMPXCHG + select HAVE_C_RECORDMCOUNT + select HAVE_ARCH_AUDITSYSCALL + select ARCH_SUPPORTS_ATOMIC_RMW + select ARCH_SUPPORTS_DEBUG_PAGEALLOC + select ARCH_SUPPORTS_HUGETLBFS + select HAVE_NMI + select HAVE_REGS_AND_STACK_ACCESS_API + select ARCH_USE_QUEUED_RWLOCKS + select ARCH_USE_QUEUED_SPINLOCKS + select ARCH_HAS_PTE_SPECIAL + select PCI_DOMAINS if PCI + select ARCH_HAS_GIGANTIC_PAGE + select HAVE_SOFTIRQ_ON_OWN_STACK + select HAVE_SETUP_PER_CPU_AREA + select NEED_PER_CPU_EMBED_FIRST_CHUNK + select NEED_PER_CPU_PAGE_FIRST_CHUNK + select ARCH_SUPPORTS_SCHED_SMT if SMP + select ARCH_SUPPORTS_SCHED_MC if SMP + select ARCH_HAS_LAZY_MMU_MODE + select GENERIC_GETTIMEOFDAY + +config ARCH_PROC_KCORE_TEXT + def_bool y + +config CPU_BIG_ENDIAN + def_bool y + +config STACKTRACE_SUPPORT + bool + default y if SPARC64 + +config LOCKDEP_SUPPORT + bool + default y if SPARC64 + +config ARCH_HIBERNATION_POSSIBLE + def_bool y if SPARC64 + +config AUDIT_ARCH + bool + default y + +config MMU + bool + default y + +config HIGHMEM + bool + default y if SPARC32 + select KMAP_LOCAL + +config PGTABLE_LEVELS + default 4 if 64BIT + default 3 + +config ARCH_SUPPORTS_UPROBES + def_bool y if SPARC64 + +menu "Processor type and features" + +config SMP + bool "Symmetric multi-processing support" + help + This enables support for systems with more than one CPU. If you have + a system with only one CPU, say N. If you have a system with more + than one CPU, say Y. + + If you say N here, the kernel will run on uni- and multiprocessor + machines, but will use only one CPU of a multiprocessor machine. If + you say Y here, the kernel will run on many, but not all, + uniprocessor machines. On a uniprocessor machine, the kernel + will run faster if you say N here. + + People using multiprocessor machines who say Y here should also say + Y to "Enhanced Real Time Clock Support", below. The "Advanced Power + Management" code will be disabled if you say Y here. + + See also and the SMP-HOWTO + available at . + + If you don't know what to do here, say N. + +config NR_CPUS + int "Maximum number of CPUs" + depends on SMP + range 2 32 if SPARC32 + range 2 4096 if SPARC64 + default 32 if SPARC32 + default 4096 if SPARC64 + +source "kernel/Kconfig.hz" + +config GENERIC_HWEIGHT + bool + default y + +config GENERIC_CALIBRATE_DELAY + bool + default y + +config ARCH_MAY_HAVE_PC_FDC + bool + default y + +config EMULATED_CMPXCHG + bool + default y if SPARC32 + help + Sparc32 does not have a CAS instruction like sparc64. cmpxchg() + is emulated, and therefore it is not completely atomic. + +# Makefile helpers +config SPARC32_SMP + bool + default y + depends on SPARC32 && SMP + +config SPARC64_SMP + bool + default y + depends on SPARC64 && SMP + +config EARLYFB + bool "Support for early boot text console" + default y + depends on SPARC64 + select FONT_SUN8x16 + select FONT_SUPPORT + help + Say Y here to enable a faster early framebuffer boot console. + +config HOTPLUG_CPU + bool "Support for hot-pluggable CPUs" + depends on SPARC64 && SMP + help + Say Y here to experiment with turning CPUs off and on. CPUs + can be controlled through /sys/devices/system/cpu/cpu#. + Say N if you want to disable CPU hotplug. + +if SPARC64 +source "drivers/cpufreq/Kconfig" +endif + +config US3_MC + tristate "UltraSPARC-III Memory Controller driver" + depends on SPARC64 + default y + help + This adds a driver for the UltraSPARC-III memory controller. + Loading this driver allows exact mnemonic strings to be + printed in the event of a memory error, so that the faulty DIMM + on the motherboard can be matched to the error. + + If in doubt, say Y, as this information can be very useful. + +# Global things across all Sun machines. +config GENERIC_LOCKBREAK + bool + default y + depends on SPARC64 && SMP && PREEMPTION + +config NUMA + bool "NUMA support" + depends on SPARC64 && SMP + +config NODES_SHIFT + int "Maximum NUMA Nodes (as a power of 2)" + range 4 5 if SPARC64 + default "5" + depends on NUMA + help + Specify the maximum number of NUMA Nodes available on the target + system. Increases memory reserved to accommodate various tables. + +config ARCH_SPARSEMEM_ENABLE + def_bool y if SPARC64 + select SPARSEMEM_VMEMMAP_ENABLE + +config ARCH_SPARSEMEM_DEFAULT + def_bool y if SPARC64 + +config ARCH_FORCE_MAX_ORDER + int "Order of maximal physically contiguous allocations" + default "12" + help + The kernel page allocator limits the size of maximal physically + contiguous allocations. The limit is called MAX_PAGE_ORDER and it + defines the maximal power of two of number of pages that can be + allocated as a single contiguous block. This option allows + overriding the default setting when ability to allocate very + large blocks of physically contiguous memory is required. + + Don't change if unsure. + +if SPARC64 || COMPILE_TEST +source "kernel/power/Kconfig" +endif + +config CMDLINE_BOOL + bool "Default bootloader kernel arguments" + depends on SPARC64 + +config CMDLINE + string "Initial kernel command string" + depends on CMDLINE_BOOL + default "console=ttyS0,9600 root=/dev/sda1" + help + Say Y here if you want to be able to pass default arguments to + the kernel. This will be overridden by the bootloader, if you + use one (such as SILO). This is most useful if you want to boot + a kernel from TFTP, and want default options to be available + with having them passed on the command line. + + NOTE: This option WILL override the PROM bootargs setting! + +config SUN_PM + bool + default y if SPARC32 + help + Enable power management and CPU standby features on supported + SPARC platforms. + +config SPARC_LED + tristate "Sun4m LED driver" + depends on SPARC32 + help + This driver toggles the front-panel LED on sun4m systems + in a user-specifiable manner. Its state can be probed + by reading /proc/led and its blinking mode can be changed + via writes to /proc/led + +config SPARC_LEON + bool "Sparc Leon processor family" + depends on SPARC32 + select USB_EHCI_BIG_ENDIAN_MMIO + select USB_EHCI_BIG_ENDIAN_DESC + select USB_UHCI_BIG_ENDIAN_MMIO + select USB_UHCI_BIG_ENDIAN_DESC + help + If you say Y here if you are running on a SPARC-LEON processor. + The LEON processor is a synthesizable VHDL model of the + SPARC-v8 standard. LEON is part of the GRLIB collection of + IP cores that are distributed under GPL. GRLIB can be downloaded + from www.gaisler.com. You can download a sparc-linux cross-compilation + toolchain at www.gaisler.com. + +if SPARC_LEON +menu "U-Boot options" + +config UBOOT_LOAD_ADDR + hex "uImage Load Address" + default 0x40004000 + help + U-Boot kernel load address, the address in physical address space + where u-boot will place the Linux kernel before booting it. + This address is normally the base address of main memory + 0x4000. + +config UBOOT_FLASH_ADDR + hex "uImage.o Load Address" + default 0x00080000 + help + Optional setting only affecting the uImage.o ELF-image used to + download the uImage file to the target using a ELF-loader other than + U-Boot. It may for example be used to download an uImage to FLASH with + the GRMON utility before even starting u-boot. + +config UBOOT_ENTRY_ADDR + hex "uImage Entry Address" + default 0xf0004000 + help + Do not change this unless you know what you're doing. This is + hardcoded by the SPARC32 and LEON port. + + This is the virtual address u-boot jumps to when booting the Linux + Kernel. + +endmenu +endif + +endmenu + +menu "Bus options (PCI etc.)" +config SBUS + bool + default y + +config SBUSCHAR + bool + default y + +config SUN_LDOMS + bool "Sun Logical Domains support" + depends on SPARC64 + help + Say Y here is you want to support virtual devices via + Logical Domains. + +config PCIC_PCI + bool + depends on PCI && SPARC32 && !SPARC_LEON + default y + +config LEON_PCI + bool + depends on PCI && SPARC_LEON + default y + +config SPARC_GRPCI1 + bool "GRPCI Host Bridge Support" + depends on LEON_PCI + default y + help + Say Y here to include the GRPCI Host Bridge Driver. The GRPCI + PCI host controller is typically found in GRLIB SPARC32/LEON + systems. The driver has one property (all_pci_errors) controlled + from the bootloader that makes the GRPCI to generate interrupts + on detected PCI Parity and System errors. + +config SPARC_GRPCI2 + bool "GRPCI2 Host Bridge Support" + depends on LEON_PCI + default y + help + Say Y here to include the GRPCI2 Host Bridge Driver. + +config SUN_OPENPROMFS + tristate "Openprom tree appears in /proc/openprom" + help + If you say Y, the OpenPROM device tree will be available as a + virtual file system, which you can mount to /proc/openprom by "mount + -t openpromfs none /proc/openprom". + + To compile the /proc/openprom support as a module, choose M here: the + module will be called openpromfs. + + Only choose N if you know in advance that you will not need to modify + OpenPROM settings on the running system. + +# Makefile helpers +config SPARC64_PCI + bool + default y + depends on SPARC64 && PCI + +config SPARC64_PCI_MSI + bool + default y + depends on SPARC64_PCI && PCI_MSI + +endmenu + +config COMPAT + bool + depends on SPARC64 + depends on !LD_IS_LLD + default y + select HAVE_UID16 + select ARCH_WANT_OLD_COMPAT_IPC + select COMPAT_OLD_SIGACTION + +config ARCH_CC_CAN_LINK + bool + default $(cc_can_link_user,-m64) if 64BIT + default $(cc_can_link_user,-m32) + +config ARCH_USERFLAGS + string + default "-m64" if 64BIT + default "-m32" + +source "drivers/sbus/char/Kconfig" diff --git a/arch/sparc/Kconfig.debug b/arch/sparc/Kconfig.debug new file mode 100644 index 000000000..d475a056a --- /dev/null +++ b/arch/sparc/Kconfig.debug @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0 + +config DEBUG_DCFLUSH + bool "D-cache flush debugging" + depends on SPARC64 && DEBUG_KERNEL + +config MCOUNT + bool + depends on SPARC64 + depends on FUNCTION_TRACER + select ARCH_WANT_FRAME_POINTERS + select FRAME_POINTER + default y + +config HAVE_HARDLOCKUP_DETECTOR_SPARC64 + bool + depends on HAVE_NMI + select HARDLOCKUP_DETECTOR_SPARC64 + help + Sparc64 hardlockup detector is the last one developed before adding + the common infrastructure for handling hardlockup detectors. It is + always built. It does _not_ use the common command line parameters + and sysctl interface, except for /proc/sys/kernel/nmi_watchdog. + +config HARDLOCKUP_DETECTOR_SPARC64 + bool + depends on HAVE_HARDLOCKUP_DETECTOR_SPARC64 diff --git a/arch/sparc/Makefile b/arch/sparc/Makefile new file mode 100644 index 000000000..040007807 --- /dev/null +++ b/arch/sparc/Makefile @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# sparc/Makefile +# +# Makefile for the architecture dependent flags and dependencies on the +# Sparc and sparc64. +# +# Copyright (C) 1994,1996,1998 David S. Miller (davem@caip.rutgers.edu) +# Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) + +# We are not yet configured - so test on arch +ifeq ($(ARCH),sparc64) + KBUILD_DEFCONFIG := sparc64_defconfig +else + KBUILD_DEFCONFIG := sparc32_defconfig +endif + +ifeq ($(CONFIG_SPARC32),y) +##### +# sparc32 +# + +CHECKFLAGS += -D__sparc__ +KBUILD_LDFLAGS := -m elf32_sparc +export BITS := 32 +UTS_MACHINE := sparc + +# We are adding -Wa,-Av8 to KBUILD_CFLAGS to deal with a specs bug in some +# versions of gcc. Some gcc versions won't pass -Av8 to binutils when you +# give -mcpu=v8. This silently worked with older bintutils versions but +# does not any more. +KBUILD_CFLAGS += -m32 -mcpu=v8 -pipe -mno-fpu $(call cc-option,-fcall-used-g5) $(call cc-option,-fcall-used-g7) +KBUILD_CFLAGS += -Wa,-Av8 + +KBUILD_AFLAGS += -m32 -Wa,-Av8 + +else +##### +# sparc64 +# + +CHECKFLAGS += -D__sparc__ -D__sparc_v9__ -D__arch64__ +KBUILD_LDFLAGS := -m elf64_sparc +export BITS := 64 +UTS_MACHINE := sparc64 + +KBUILD_CFLAGS += -m64 -pipe -mno-fpu -mcpu=ultrasparc -mcmodel=medlow +KBUILD_CFLAGS += -ffixed-g4 -ffixed-g5 $(call cc-option,-fcall-used-g7) -Wno-sign-compare +KBUILD_CFLAGS += -Wa,--undeclared-regs +KBUILD_CFLAGS += $(call cc-option,-mtune=ultrasparc3) +KBUILD_AFLAGS += -m64 -mcpu=ultrasparc -Wa,--undeclared-regs + +ifeq ($(CONFIG_MCOUNT),y) + KBUILD_CFLAGS += -pg +endif + +endif + +libs-y += arch/sparc/prom/ +libs-y += arch/sparc/lib/ + +drivers-$(CONFIG_PM) += arch/sparc/power/ +drivers-$(CONFIG_VIDEO) += arch/sparc/video/ + +boot := arch/sparc/boot + +# Default target +all: zImage + +image zImage uImage tftpboot.img vmlinux.aout: vmlinux + $(Q)$(MAKE) $(build)=$(boot) $(boot)/$@ + +install: + $(call cmd,install) + +archheaders: + $(Q)$(MAKE) $(build)=arch/sparc/kernel/syscalls all + +vdso-install-$(CONFIG_SPARC64) += arch/sparc/vdso/vdso64.so.dbg +vdso-install-$(CONFIG_COMPAT) += arch/sparc/vdso/vdso32.so.dbg + +# This is the image used for packaging +KBUILD_IMAGE := $(boot)/zImage + +# Don't use tabs in echo arguments. +define archhelp + echo '* vmlinux - standard SPARC kernel' + echo ' image - kernel image ($(boot)/image)' + echo '* zImage - stripped/compressed kernel image ($(boot)/zImage)' + echo ' uImage - U-Boot SPARC32 Image (only for LEON)' + echo ' vmlinux.aout - a.out kernel for SPARC64' + echo ' tftpboot.img - image prepared for tftp' +endef diff --git a/arch/sparc/boot/.gitignore b/arch/sparc/boot/.gitignore new file mode 100644 index 000000000..f3d8569a2 --- /dev/null +++ b/arch/sparc/boot/.gitignore @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +btfix.S +btfixupprep +image +zImage +tftpboot.img +vmlinux.aout +piggyback + diff --git a/arch/sparc/boot/Makefile b/arch/sparc/boot/Makefile new file mode 100644 index 000000000..339c42d35 --- /dev/null +++ b/arch/sparc/boot/Makefile @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for the Sparc boot stuff. +# +# Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) +# Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz) + +ROOT_IMG := /usr/src/root.img +ELFTOAOUT := elftoaout + +hostprogs := piggyback +targets := tftpboot.img image zImage vmlinux.aout +clean-files := System.map + +quiet_cmd_elftoaout = ELFTOAOUT $@ + cmd_elftoaout = $(ELFTOAOUT) $(obj)/image -o $@ +quiet_cmd_piggy = PIGGY $@ + cmd_piggy = $(obj)/piggyback $(BITS) $@ System.map $(ROOT_IMG) +quiet_cmd_strip = STRIP $@ + cmd_strip = $(STRIP) -R .comment -R .note -K sun4u_init -K _end -K _start $< -o $@ + +ifeq ($(CONFIG_SPARC64),y) + +# Actual linking + +$(obj)/zImage: $(obj)/image FORCE + $(call if_changed,gzip) + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' + +$(obj)/vmlinux.aout: vmlinux FORCE + $(call if_changed,elftoaout) + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' +else + +$(obj)/zImage: $(obj)/image FORCE + $(call if_changed,strip) + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' + +# The following lines make a readable image for U-Boot. +# uImage - Binary file read by U-boot +# uImage.o - object file of uImage for loading with a +# flash programmer understanding ELF. + +OBJCOPYFLAGS_image.bin := -S -O binary -R .note -R .comment +$(obj)/image.bin: $(obj)/image FORCE + $(call if_changed,objcopy) + +$(obj)/image.gz: $(obj)/image.bin FORCE + $(call if_changed,gzip) + +UIMAGE_LOADADDR = $(CONFIG_UBOOT_LOAD_ADDR) +UIMAGE_ENTRYADDR = $(CONFIG_UBOOT_ENTRY_ADDR) +UIMAGE_COMPRESSION = gzip + +quiet_cmd_uimage.o = UIMAGE.O $@ + cmd_uimage.o = $(LD) -Tdata $(CONFIG_UBOOT_FLASH_ADDR) \ + -r -b binary $@ -o $@.o + +targets += uImage +$(obj)/uImage: $(obj)/image.gz FORCE + $(call if_changed,uimage) + $(call if_changed,uimage.o) + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' + +endif + +$(obj)/image: vmlinux FORCE + $(call if_changed,strip) + @$(kecho) 'Kernel: $@ is ready' '(#'$(or $(KBUILD_BUILD_VERSION),`cat .version`)')' + +$(obj)/tftpboot.img: $(obj)/image $(obj)/piggyback System.map $(ROOT_IMG) FORCE + $(call if_changed,elftoaout) + $(call if_changed,piggy) diff --git a/arch/sparc/boot/install.sh b/arch/sparc/boot/install.sh new file mode 100755 index 000000000..68de67c56 --- /dev/null +++ b/arch/sparc/boot/install.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# +# This file is subject to the terms and conditions of the GNU General Public +# License. See the file "COPYING" in the main directory of this archive +# for more details. +# +# Copyright (C) 1995 by Linus Torvalds +# +# Adapted from code in arch/i386/boot/Makefile by H. Peter Anvin +# +# "make install" script for SPARC architecture +# +# Arguments: +# $1 - kernel version +# $2 - kernel image file +# $3 - kernel map file +# $4 - default install path (blank if root directory) + +set -e + +if [ -f $4/vmlinuz ]; then + mv $4/vmlinuz $4/vmlinuz.old +fi + +if [ -f $4/System.map ]; then + mv $4/System.map $4/System.old +fi + +cat $2 > $4/vmlinuz +cp $3 $4/System.map diff --git a/arch/sparc/boot/piggyback.c b/arch/sparc/boot/piggyback.c new file mode 100644 index 000000000..6d74064ad --- /dev/null +++ b/arch/sparc/boot/piggyback.c @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + Simple utility to make a single-image install kernel with initial ramdisk + for Sparc tftpbooting without need to set up nfs. + + Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + Pete Zaitcev endian fixes for cross-compiles, 2000. + Copyright (C) 2011 Sam Ravnborg + + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* + * Note: run this on an a.out kernel (use elftoaout for it), + * as PROM looks for a.out image only. + */ + +#define AOUT_TEXT_OFFSET 32 + +static int is64bit = 0; + +/* align to power-of-two size */ +static int align(int n) +{ + if (is64bit) + return (n + 0x1fff) & ~0x1fff; + else + return (n + 0xfff) & ~0xfff; +} + +/* read two bytes as big endian */ +static unsigned short ld2(char *p) +{ + return (p[0] << 8) | p[1]; +} + +/* save 4 bytes as big endian */ +static void st4(char *p, unsigned int x) +{ + p[0] = x >> 24; + p[1] = x >> 16; + p[2] = x >> 8; + p[3] = x; +} + +static void die(const char *str) +{ + perror(str); + exit(1); +} + +static void usage(void) +{ + /* fs_img.gz is an image of initial ramdisk. */ + fprintf(stderr, "Usage: piggyback bits vmlinux.aout System.map fs_img.gz\n"); + fprintf(stderr, "\tKernel image will be modified in place.\n"); + exit(1); +} + +static int start_line(const char *line) +{ + if (strcmp(line + 10, " _start\n") == 0) + return 1; + else if (strcmp(line + 18, " _start\n") == 0) + return 1; + return 0; +} + +static int end_line(const char *line) +{ + if (strcmp(line + 10, " _end\n") == 0) + return 1; + else if (strcmp (line + 18, " _end\n") == 0) + return 1; + return 0; +} + +/* + * Find address for start and end in System.map. + * The file looks like this: + * f0004000 ... _start + * f0379f79 ... _end + * 1234567890123456 + * ^coloumn 1 + * There is support for 64 bit addresses too. + * + * Return 0 if either start or end is not found + */ +static int get_start_end(const char *filename, unsigned int *start, + unsigned int *end) +{ + FILE *map; + char buffer[1024]; + + *start = 0; + *end = 0; + map = fopen(filename, "r"); + if (!map) + die(filename); + while (fgets(buffer, 1024, map)) { + if (start_line(buffer)) + *start = strtoul(buffer, NULL, 16); + else if (end_line(buffer)) + *end = strtoul(buffer, NULL, 16); + } + fclose (map); + + if (*start == 0 || *end == 0) + return 0; + + return 1; +} + +#define LOOKBACK (128 * 4) +#define BUFSIZE 1024 +/* + * Find the HdrS entry from head_32/head_64. + * We check if it is at the beginning of the file (sparc64 case) + * and if not we search for it. + * When we search do so in steps of 4 as HdrS is on a 4-byte aligned + * address (it is on same alignment as sparc instructions) + * Return the offset to the HdrS entry (as off_t) + */ +static off_t get_hdrs_offset(int kernelfd, const char *filename) +{ + char buffer[BUFSIZE]; + off_t offset; + int i; + + if (lseek(kernelfd, 0, SEEK_SET) < 0) + die("lseek"); + if (read(kernelfd, buffer, BUFSIZE) != BUFSIZE) + die(filename); + + if (buffer[40] == 'H' && buffer[41] == 'd' && + buffer[42] == 'r' && buffer[43] == 'S') { + return 40; + } else { + /* Find the gokernel label */ + /* Decode offset from branch instruction */ + offset = ld2(buffer + AOUT_TEXT_OFFSET + 2) << 2; + /* Go back 512 bytes so we do not miss HdrS */ + offset -= LOOKBACK; + /* skip a.out header */ + offset += AOUT_TEXT_OFFSET; + if (offset < 0) { + errno = -EINVAL; + die("Calculated a negative offset, probably elftoaout generated an invalid image. Did you use a recent elftoaout ?"); + } + if (lseek(kernelfd, offset, SEEK_SET) < 0) + die("lseek"); + if (read(kernelfd, buffer, BUFSIZE) != BUFSIZE) + die(filename); + + for (i = 0; i < LOOKBACK; i += 4) { + if (buffer[i + 0] == 'H' && buffer[i + 1] == 'd' && + buffer[i + 2] == 'r' && buffer[i + 3] == 'S') { + return offset + i; + } + } + } + fprintf (stderr, "Couldn't find headers signature in %s\n", filename); + exit(1); +} + +int main(int argc,char **argv) +{ + static char aout_magic[] = { 0x01, 0x03, 0x01, 0x07 }; + char buffer[1024]; + unsigned int i, start, end; + off_t offset; + struct stat s; + int image, tail; + + if (argc != 5) + usage(); + if (strcmp(argv[1], "64") == 0) + is64bit = 1; + if (stat (argv[4], &s) < 0) + die(argv[4]); + + if (!get_start_end(argv[3], &start, &end)) { + fprintf(stderr, "Could not determine start and end from %s\n", + argv[3]); + exit(1); + } + if ((image = open(argv[2], O_RDWR)) < 0) + die(argv[2]); + if (read(image, buffer, 512) != 512) + die(argv[2]); + if (memcmp(buffer, aout_magic, 4) != 0) { + fprintf (stderr, "Not a.out. Don't blame me.\n"); + exit(1); + } + /* + * We need to fill in values for + * sparc_ramdisk_image + sparc_ramdisk_size + * To locate these symbols search for the "HdrS" text which appear + * in the image a little before the gokernel symbol. + * See definition of these in init_32.S + */ + + offset = get_hdrs_offset(image, argv[2]); + /* skip HdrS + LINUX_VERSION_CODE + HdrS version */ + offset += 10; + + if (lseek(image, offset, 0) < 0) + die("lseek"); + + /* + * root_flags = 0 + * root_dev = 1 (RAMDISK_MAJOR) + * ram_flags = 0 + * sparc_ramdisk_image = "PAGE aligned address after _end") + * sparc_ramdisk_size = size of image + */ + st4(buffer, 0); + st4(buffer + 4, 0x01000000); + st4(buffer + 8, align(end + 32)); + st4(buffer + 12, s.st_size); + + if (write(image, buffer + 2, 14) != 14) + die(argv[2]); + + /* For sparc64 update a_text and clear a_data + a_bss */ + if (is64bit) + { + if (lseek(image, 4, 0) < 0) + die("lseek"); + /* a_text */ + st4(buffer, align(end + 32 + 8191) - (start & ~0x3fffffUL) + + s.st_size); + /* a_data */ + st4(buffer + 4, 0); + /* a_bss */ + st4(buffer + 8, 0); + if (write(image, buffer, 12) != 12) + die(argv[2]); + } + + /* seek page aligned boundary in the image file and add boot image */ + if (lseek(image, AOUT_TEXT_OFFSET - start + align(end + 32), 0) < 0) + die("lseek"); + if ((tail = open(argv[4], O_RDONLY)) < 0) + die(argv[4]); + while ((i = read(tail, buffer, 1024)) > 0) + if (write(image, buffer, i) != i) + die(argv[2]); + if (close(image) < 0) + die("close"); + if (close(tail) < 0) + die("close"); + return 0; +} diff --git a/arch/sparc/configs/sparc32_defconfig b/arch/sparc/configs/sparc32_defconfig new file mode 100644 index 000000000..d5579217f --- /dev/null +++ b/arch/sparc/configs/sparc32_defconfig @@ -0,0 +1,93 @@ +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_LOG_BUF_SHIFT=14 +CONFIG_SYSFS_DEPRECATED_V2=y +CONFIG_BLK_DEV_INITRD=y +# CONFIG_CC_OPTIMIZE_FOR_SIZE is not set +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +# CONFIG_BLK_DEV_BSG is not set +CONFIG_PCI=y +CONFIG_SUN_OPENPROMFS=m +CONFIG_BINFMT_MISC=m +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_XFRM_USER=m +CONFIG_NET_KEY=m +CONFIG_INET=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_INET_AH=y +CONFIG_INET_ESP=y +CONFIG_INET_IPCOMP=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_TUNNEL=m +CONFIG_NET_PKTGEN=m +CONFIG_BLK_DEV_LOOP=m +CONFIG_BLK_DEV_CRYPTOLOOP=m +CONFIG_BLK_DEV_RAM=y +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR=m +CONFIG_CHR_DEV_SG=m +CONFIG_SCSI_QLOGICPTI=m +CONFIG_SCSI_SUNESP=y +CONFIG_NETDEVICES=y +CONFIG_DUMMY=m +CONFIG_NET_ETHERNET=y +CONFIG_MII=m +CONFIG_SUNLANCE=y +CONFIG_HAPPYMEAL=m +CONFIG_SUNBMAC=m +CONFIG_SUNQE=m +# CONFIG_WLAN is not set +CONFIG_INPUT_JOYDEV=m +CONFIG_INPUT_EVDEV=m +CONFIG_INPUT_EVBUG=m +CONFIG_KEYBOARD_ATKBD=m +CONFIG_KEYBOARD_SUNKBD=m +CONFIG_MOUSE_PS2=m +CONFIG_MOUSE_SERIAL=m +CONFIG_SERIO=m +# CONFIG_SERIO_I8042 is not set +CONFIG_SERIAL_SUNZILOG=y +CONFIG_SERIAL_SUNZILOG_CONSOLE=y +CONFIG_SERIAL_SUNSU=y +CONFIG_SERIAL_SUNSU_CONSOLE=y +CONFIG_SPI=y +CONFIG_SPI_XILINX=m +CONFIG_SPI_XILINX_PLTFM=m +CONFIG_SUN_OPENPROMIO=m +CONFIG_EXT2_FS=y +CONFIG_EXT2_FS_XATTR=y +CONFIG_EXT2_FS_POSIX_ACL=y +CONFIG_EXT2_FS_SECURITY=y +CONFIG_AUTOFS_FS=m +CONFIG_ISO9660_FS=m +CONFIG_PROC_KCORE=y +CONFIG_ROMFS_FS=m +CONFIG_NFS_FS=y +CONFIG_ROOT_NFS=y +CONFIG_RPCSEC_GSS_KRB5=m +CONFIG_NLS=y +CONFIG_DEBUG_KERNEL=y +CONFIG_DETECT_HUNG_TASK=y +# CONFIG_SCHED_DEBUG is not set +CONFIG_KGDB=y +CONFIG_KGDB_TESTS=y +CONFIG_CRYPTO_NULL=m +CONFIG_CRYPTO_ECB=m +CONFIG_CRYPTO_MD4=y +CONFIG_CRYPTO_SHA256=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_AES=m +CONFIG_CRYPTO_ARC4=m +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_TWOFISH=m +# CONFIG_CRYPTO_HW is not set diff --git a/arch/sparc/configs/sparc64_defconfig b/arch/sparc/configs/sparc64_defconfig new file mode 100644 index 000000000..34a6a31e6 --- /dev/null +++ b/arch/sparc/configs/sparc64_defconfig @@ -0,0 +1,229 @@ +CONFIG_64BIT=y +# CONFIG_LOCALVERSION_AUTO is not set +CONFIG_SYSVIPC=y +CONFIG_POSIX_MQUEUE=y +CONFIG_LOG_BUF_SHIFT=18 +CONFIG_BLK_DEV_INITRD=y +CONFIG_PERF_EVENTS=y +# CONFIG_COMPAT_BRK is not set +CONFIG_PROFILING=y +CONFIG_KPROBES=y +CONFIG_MODULES=y +CONFIG_MODULE_UNLOAD=y +CONFIG_MODULE_FORCE_UNLOAD=y +CONFIG_MODVERSIONS=y +CONFIG_MODULE_SRCVERSION_ALL=y +CONFIG_SMP=y +CONFIG_HZ_100=y +CONFIG_HOTPLUG_CPU=y +CONFIG_NO_HZ=y +CONFIG_HIGH_RES_TIMERS=y +CONFIG_NUMA=y +CONFIG_DEFAULT_MMAP_MIN_ADDR=8192 +CONFIG_PREEMPT_VOLUNTARY=y +CONFIG_SUN_LDOMS=y +CONFIG_PCI=y +CONFIG_PCI_MSI=y +CONFIG_SUN_OPENPROMFS=m +CONFIG_BINFMT_MISC=m +CONFIG_NET=y +CONFIG_PACKET=y +CONFIG_UNIX=y +CONFIG_XFRM_USER=m +CONFIG_NET_KEY=m +CONFIG_NET_KEY_MIGRATE=y +CONFIG_INET=y +CONFIG_IP_MULTICAST=y +CONFIG_NET_IPIP=m +CONFIG_NET_IPGRE=m +CONFIG_NET_IPGRE_BROADCAST=y +CONFIG_IP_MROUTE=y +CONFIG_IP_PIMSM_V1=y +CONFIG_IP_PIMSM_V2=y +CONFIG_ARPD=y +CONFIG_SYN_COOKIES=y +CONFIG_INET_AH=y +CONFIG_INET_ESP=y +CONFIG_INET_IPCOMP=y +CONFIG_IPV6_ROUTER_PREF=y +CONFIG_IPV6_ROUTE_INFO=y +CONFIG_IPV6_OPTIMISTIC_DAD=y +CONFIG_INET6_AH=m +CONFIG_INET6_ESP=m +CONFIG_INET6_IPCOMP=m +CONFIG_IPV6_TUNNEL=m +CONFIG_VLAN_8021Q=m +CONFIG_NET_PKTGEN=m +CONFIG_NET_TCPPROBE=m +# CONFIG_PREVENT_FIRMWARE_BUILD is not set +CONFIG_CONNECTOR=m +CONFIG_BLK_DEV_LOOP=m +CONFIG_BLK_DEV_CRYPTOLOOP=m +CONFIG_BLK_DEV_NBD=m +CONFIG_ATA_OVER_ETH=m +CONFIG_SUNVDC=m +CONFIG_ATA=y +CONFIG_PATA_ALI=y +CONFIG_RAID_ATTRS=m +CONFIG_SCSI=y +CONFIG_BLK_DEV_SD=y +CONFIG_BLK_DEV_SR=m +CONFIG_CHR_DEV_SG=m +CONFIG_SCSI_MULTI_LUN=y +CONFIG_SCSI_CONSTANTS=y +CONFIG_SCSI_SPI_ATTRS=y +CONFIG_SCSI_FC_ATTRS=y +CONFIG_MD=y +CONFIG_BLK_DEV_MD=m +CONFIG_MD_LINEAR=m +CONFIG_MD_RAID0=m +CONFIG_MD_RAID1=m +CONFIG_MD_RAID10=m +CONFIG_MD_RAID456=m +CONFIG_MD_MULTIPATH=m +CONFIG_BLK_DEV_DM=m +CONFIG_DM_CRYPT=m +CONFIG_DM_SNAPSHOT=m +CONFIG_DM_MIRROR=m +CONFIG_DM_ZERO=m +CONFIG_NETDEVICES=y +CONFIG_NET_ETHERNET=y +CONFIG_MII=m +CONFIG_SUNLANCE=m +CONFIG_HAPPYMEAL=y +CONFIG_SUNGEM=m +CONFIG_SUNVNET=m +CONFIG_LDMVSW=m +CONFIG_NET_PCI=y +CONFIG_E1000=m +CONFIG_E1000E=m +CONFIG_TIGON3=m +CONFIG_BNX2=m +CONFIG_NIU=m +# CONFIG_WLAN is not set +CONFIG_PPP=m +CONFIG_PPP_MULTILINK=y +CONFIG_PPP_FILTER=y +CONFIG_PPP_ASYNC=m +CONFIG_PPP_SYNC_TTY=m +CONFIG_PPP_DEFLATE=m +CONFIG_PPP_BSDCOMP=m +CONFIG_PPP_MPPE=m +CONFIG_PPPOE=m +CONFIG_INPUT_EVDEV=y +CONFIG_KEYBOARD_LKKBD=m +CONFIG_KEYBOARD_SUNKBD=y +CONFIG_MOUSE_SERIAL=y +CONFIG_INPUT_MISC=y +CONFIG_INPUT_SPARCSPKR=y +# CONFIG_SERIO_SERPORT is not set +CONFIG_SERIO_PCIPS2=m +CONFIG_SERIO_RAW=m +CONFIG_SERIAL_SUNSU=y +CONFIG_SERIAL_SUNSU_CONSOLE=y +CONFIG_SERIAL_SUNSAB=y +CONFIG_SERIAL_SUNSAB_CONSOLE=y +CONFIG_SERIAL_SUNHV=y +# CONFIG_LEGACY_PTYS is not set +CONFIG_FB=y +CONFIG_FB_TILEBLITTING=y +CONFIG_FB_SBUS=y +CONFIG_FB_CG6=y +CONFIG_FB_FFB=y +CONFIG_FB_XVR500=y +CONFIG_FB_XVR2500=y +CONFIG_FB_XVR1000=y +CONFIG_FB_RADEON=y +# CONFIG_FB_RADEON_BACKLIGHT is not set +CONFIG_FB_ATY=y +CONFIG_FB_ATY_GX=y +# CONFIG_FB_ATY_BACKLIGHT is not set +CONFIG_FRAMEBUFFER_CONSOLE=y +CONFIG_FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y +CONFIG_FONTS=y +CONFIG_FONT_SUN8x16=y +CONFIG_LOGO=y +# CONFIG_LOGO_LINUX_MONO is not set +# CONFIG_LOGO_LINUX_VGA16 is not set +# CONFIG_LOGO_LINUX_CLUT224 is not set +CONFIG_SOUND=m +CONFIG_SND=m +CONFIG_SND_SEQUENCER=m +CONFIG_SND_SEQ_DUMMY=m +CONFIG_SND_MIXER_OSS=m +CONFIG_SND_PCM_OSS=m +CONFIG_SND_SEQUENCER_OSS=y +CONFIG_SND_DUMMY=m +CONFIG_SND_VIRMIDI=m +CONFIG_SND_MTPAV=m +CONFIG_SND_ALI5451=m +CONFIG_SND_SUN_CS4231=m +CONFIG_USB_HIDDEV=y +CONFIG_HID_DRAGONRISE=y +CONFIG_HID_GYRATION=y +CONFIG_HID_TWINHAN=y +CONFIG_HID_NTRIG=y +CONFIG_HID_ORTEK=y +CONFIG_HID_PANTHERLORD=y +CONFIG_HID_PETALYNX=y +CONFIG_HID_SAMSUNG=y +CONFIG_HID_SONY=y +CONFIG_HID_SUNPLUS=y +CONFIG_HID_GREENASIA=y +CONFIG_HID_SMARTJOYPLUS=y +CONFIG_HID_TOPSEED=y +CONFIG_HID_THRUSTMASTER=y +CONFIG_HID_ZEROPLUS=y +CONFIG_USB=y +CONFIG_USB_EHCI_HCD=m +# CONFIG_USB_EHCI_TT_NEWSCHED is not set +CONFIG_USB_OHCI_HCD=y +CONFIG_USB_UHCI_HCD=m +CONFIG_USB_STORAGE=m +CONFIG_SUN_OPENPROMIO=y +CONFIG_EXT2_FS=y +CONFIG_EXT2_FS_XATTR=y +CONFIG_EXT2_FS_POSIX_ACL=y +CONFIG_EXT2_FS_SECURITY=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_PROC_KCORE=y +CONFIG_TMPFS=y +CONFIG_HUGETLBFS=y +CONFIG_PRINTK_TIME=y +CONFIG_MAGIC_SYSRQ=y +CONFIG_DEBUG_KERNEL=y +CONFIG_LOCKUP_DETECTOR=y +CONFIG_DETECT_HUNG_TASK=y +# CONFIG_SCHED_DEBUG is not set +CONFIG_SCHEDSTATS=y +CONFIG_BLK_DEV_IO_TRACE=y +CONFIG_UPROBE_EVENTS=y +CONFIG_KEYS=y +CONFIG_CRYPTO_NULL=m +CONFIG_CRYPTO_BENCHMARK=m +CONFIG_CRYPTO_LRW=m +CONFIG_CRYPTO_XTS=m +CONFIG_CRYPTO_XCBC=y +CONFIG_CRYPTO_MD4=y +CONFIG_CRYPTO_SHA256=m +CONFIG_CRYPTO_SHA512=m +CONFIG_CRYPTO_TGR192=m +CONFIG_CRYPTO_WP512=m +CONFIG_CRYPTO_AES=m +CONFIG_CRYPTO_ANUBIS=m +CONFIG_CRYPTO_BLOWFISH=m +CONFIG_CRYPTO_CAMELLIA=m +CONFIG_CRYPTO_CAST5=m +CONFIG_CRYPTO_CAST6=m +CONFIG_CRYPTO_KHAZAD=m +CONFIG_CRYPTO_SEED=m +CONFIG_CRYPTO_SERPENT=m +CONFIG_CRYPTO_TEA=m +CONFIG_CRYPTO_TWOFISH=m +CONFIG_VCC=m +CONFIG_PATA_CMD64X=y +CONFIG_IP_PNP=y +CONFIG_IP_PNP_DHCP=y +CONFIG_DEVTMPFS=y diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig new file mode 100644 index 000000000..8db3f6eea --- /dev/null +++ b/arch/sparc/crypto/Kconfig @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Accelerated Cryptographic Algorithms for CPU (sparc64)" + +config CRYPTO_AES_SPARC64 + tristate "Ciphers: AES, modes: ECB, CBC, CTR" + depends on SPARC64 + select CRYPTO_LIB_AES + select CRYPTO_SKCIPHER + help + Length-preseving ciphers: AES with ECB, CBC, and CTR modes + + Architecture: sparc64 using crypto instructions + +config CRYPTO_CAMELLIA_SPARC64 + tristate "Ciphers: Camellia, modes: ECB, CBC" + depends on SPARC64 + select CRYPTO_ALGAPI + select CRYPTO_SKCIPHER + help + Block ciphers: Camellia cipher algorithms + Length-preserving ciphers: Camellia with ECB and CBC modes + + Architecture: sparc64 + +endmenu diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile new file mode 100644 index 000000000..ab4a7765b --- /dev/null +++ b/arch/sparc/crypto/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Arch-specific CryptoAPI modules. +# + +obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o +obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o + +aes-sparc64-y := aes_glue.o +camellia-sparc64-y := camellia_asm.o camellia_glue.o diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c new file mode 100644 index 000000000..661561837 --- /dev/null +++ b/arch/sparc/crypto/aes_glue.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Glue code for AES encryption optimized for sparc64 crypto opcodes. + * + * This is based largely upon arch/x86/crypto/aesni-intel_glue.c + * + * Copyright (C) 2008, Intel Corp. + * Author: Huang Ying + * + * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD + * interface for 64-bit kernels. + * Authors: Adrian Hoban + * Gabriele Paoloni + * Tadeusz Struk (tadeusz.struk@intel.com) + * Aidan O'Mahony (aidan.o.mahony@intel.com) + * Copyright (c) 2010, Intel Corporation. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +struct aes_ops { + void (*load_encrypt_keys)(const u64 *key); + void (*load_decrypt_keys)(const u64 *key); + void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len); + void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len); + void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); + void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); + void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output, + unsigned int len, u64 *iv); +}; + +struct crypto_sparc64_aes_ctx { + struct aes_ops *ops; + u64 key[AES_MAX_KEYLENGTH / sizeof(u64)]; + u32 key_length; + u32 expanded_key_length; +}; + +static struct aes_ops aes128_ops = { + .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128, + .load_decrypt_keys = aes_sparc64_load_decrypt_keys_128, + .ecb_encrypt = aes_sparc64_ecb_encrypt_128, + .ecb_decrypt = aes_sparc64_ecb_decrypt_128, + .cbc_encrypt = aes_sparc64_cbc_encrypt_128, + .cbc_decrypt = aes_sparc64_cbc_decrypt_128, + .ctr_crypt = aes_sparc64_ctr_crypt_128, +}; + +static struct aes_ops aes192_ops = { + .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192, + .load_decrypt_keys = aes_sparc64_load_decrypt_keys_192, + .ecb_encrypt = aes_sparc64_ecb_encrypt_192, + .ecb_decrypt = aes_sparc64_ecb_decrypt_192, + .cbc_encrypt = aes_sparc64_cbc_encrypt_192, + .cbc_decrypt = aes_sparc64_cbc_decrypt_192, + .ctr_crypt = aes_sparc64_ctr_crypt_192, +}; + +static struct aes_ops aes256_ops = { + .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256, + .load_decrypt_keys = aes_sparc64_load_decrypt_keys_256, + .ecb_encrypt = aes_sparc64_ecb_encrypt_256, + .ecb_decrypt = aes_sparc64_ecb_decrypt_256, + .cbc_encrypt = aes_sparc64_cbc_encrypt_256, + .cbc_decrypt = aes_sparc64_cbc_decrypt_256, + .ctr_crypt = aes_sparc64_ctr_crypt_256, +}; + +static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) +{ + struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + + switch (key_len) { + case AES_KEYSIZE_128: + ctx->expanded_key_length = 0xb0; + ctx->ops = &aes128_ops; + break; + + case AES_KEYSIZE_192: + ctx->expanded_key_length = 0xd0; + ctx->ops = &aes192_ops; + break; + + case AES_KEYSIZE_256: + ctx->expanded_key_length = 0xf0; + ctx->ops = &aes256_ops; + break; + + default: + return -EINVAL; + } + + aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len); + ctx->key_length = key_len; + + return 0; +} + +static int ecb_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; + + ctx->ops->load_encrypt_keys(&ctx->key[0]); + while ((nbytes = walk.nbytes) != 0) { + ctx->ops->ecb_encrypt(&ctx->key[0], walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE)); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); + } + fprs_write(0); + return err; +} + +static int ecb_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + const u64 *key_end; + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; + + ctx->ops->load_decrypt_keys(&ctx->key[0]); + key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; + while ((nbytes = walk.nbytes) != 0) { + ctx->ops->ecb_decrypt(key_end, walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE)); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); + } + fprs_write(0); + + return err; +} + +static int cbc_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; + + ctx->ops->load_encrypt_keys(&ctx->key[0]); + while ((nbytes = walk.nbytes) != 0) { + ctx->ops->cbc_encrypt(&ctx->key[0], walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + walk.iv); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); + } + fprs_write(0); + return err; +} + +static int cbc_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + const u64 *key_end; + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; + + ctx->ops->load_decrypt_keys(&ctx->key[0]); + key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)]; + while ((nbytes = walk.nbytes) != 0) { + ctx->ops->cbc_decrypt(key_end, walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + walk.iv); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); + } + fprs_write(0); + + return err; +} + +static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx, + struct skcipher_walk *walk) +{ + u8 *ctrblk = walk->iv; + u64 keystream[AES_BLOCK_SIZE / sizeof(u64)]; + const u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; + unsigned int nbytes = walk->nbytes; + + ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk, + keystream, AES_BLOCK_SIZE); + crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes); + crypto_inc(ctrblk, AES_BLOCK_SIZE); +} + +static int ctr_crypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; + int err; + + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; + + ctx->ops->load_encrypt_keys(&ctx->key[0]); + while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { + ctx->ops->ctr_crypt(&ctx->key[0], walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + walk.iv); + err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); + } + if (walk.nbytes) { + ctr_crypt_final(ctx, &walk); + err = skcipher_walk_done(&walk, 0); + } + fprs_write(0); + return err; +} + +static struct skcipher_alg skcipher_algs[] = { + { + .base.cra_name = "ecb(aes)", + .base.cra_driver_name = "ecb-aes-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = aes_set_key_skcipher, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "cbc-aes-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aes_set_key_skcipher, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "ctr-aes-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = aes_set_key_skcipher, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + .chunksize = AES_BLOCK_SIZE, + } +}; + +static bool __init sparc64_has_aes_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_AES)) + return false; + + return true; +} + +static int __init aes_sparc64_mod_init(void) +{ + if (!sparc64_has_aes_opcode()) { + pr_info("sparc64 aes opcodes not available.\n"); + return -ENODEV; + } + pr_info("Using sparc64 aes opcodes optimized AES implementation\n"); + return crypto_register_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); +} + +static void __exit aes_sparc64_mod_fini(void) +{ + crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); +} + +module_init(aes_sparc64_mod_init); +module_exit(aes_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated"); + +MODULE_ALIAS_CRYPTO("aes"); + +#include "crop_devid.c" diff --git a/arch/sparc/crypto/camellia_asm.S b/arch/sparc/crypto/camellia_asm.S new file mode 100644 index 000000000..8471b346e --- /dev/null +++ b/arch/sparc/crypto/camellia_asm.S @@ -0,0 +1,563 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include + +#define CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \ + CAMELLIA_F(KEY_BASE + 0, I1, I0, I1) \ + CAMELLIA_F(KEY_BASE + 2, I0, I1, I0) \ + CAMELLIA_F(KEY_BASE + 4, I1, I0, I1) \ + CAMELLIA_F(KEY_BASE + 6, I0, I1, I0) \ + CAMELLIA_F(KEY_BASE + 8, I1, I0, I1) \ + CAMELLIA_F(KEY_BASE + 10, I0, I1, I0) + +#define CAMELLIA_6ROUNDS_FL_FLI(KEY_BASE, I0, I1) \ + CAMELLIA_6ROUNDS(KEY_BASE, I0, I1) \ + CAMELLIA_FL(KEY_BASE + 12, I0, I0) \ + CAMELLIA_FLI(KEY_BASE + 14, I1, I1) + + .data + + .align 8 +SIGMA: .xword 0xA09E667F3BCC908B + .xword 0xB67AE8584CAA73B2 + .xword 0xC6EF372FE94F82BE + .xword 0x54FF53A5F1D36F1C + .xword 0x10E527FADE682D1D + .xword 0xB05688C2B3E6C1FD + + .text + + .align 32 +ENTRY(camellia_sparc64_key_expand) + /* %o0=in_key, %o1=encrypt_key, %o2=key_len, %o3=decrypt_key */ + VISEntry + ld [%o0 + 0x00], %f0 ! i0, k[0] + ld [%o0 + 0x04], %f1 ! i1, k[1] + ld [%o0 + 0x08], %f2 ! i2, k[2] + ld [%o0 + 0x0c], %f3 ! i3, k[3] + std %f0, [%o1 + 0x00] ! k[0, 1] + fsrc2 %f0, %f28 + std %f2, [%o1 + 0x08] ! k[2, 3] + cmp %o2, 16 + be 10f + fsrc2 %f2, %f30 + + ld [%o0 + 0x10], %f0 + ld [%o0 + 0x14], %f1 + std %f0, [%o1 + 0x20] ! k[8, 9] + cmp %o2, 24 + fone %f10 + be,a 1f + fxor %f10, %f0, %f2 + ld [%o0 + 0x18], %f2 + ld [%o0 + 0x1c], %f3 +1: + std %f2, [%o1 + 0x28] ! k[10, 11] + fxor %f28, %f0, %f0 + fxor %f30, %f2, %f2 + +10: + sethi %hi(SIGMA), %g3 + or %g3, %lo(SIGMA), %g3 + ldd [%g3 + 0x00], %f16 + ldd [%g3 + 0x08], %f18 + ldd [%g3 + 0x10], %f20 + ldd [%g3 + 0x18], %f22 + ldd [%g3 + 0x20], %f24 + ldd [%g3 + 0x28], %f26 + CAMELLIA_F(16, 2, 0, 2) + CAMELLIA_F(18, 0, 2, 0) + fxor %f28, %f0, %f0 + fxor %f30, %f2, %f2 + CAMELLIA_F(20, 2, 0, 2) + CAMELLIA_F(22, 0, 2, 0) + +#define ROTL128(S01, S23, TMP1, TMP2, N) \ + srlx S01, (64 - N), TMP1; \ + sllx S01, N, S01; \ + srlx S23, (64 - N), TMP2; \ + sllx S23, N, S23; \ + or S01, TMP2, S01; \ + or S23, TMP1, S23 + + cmp %o2, 16 + bne 1f + nop + /* 128-bit key */ + std %f0, [%o1 + 0x10] ! k[ 4, 5] + std %f2, [%o1 + 0x18] ! k[ 6, 7] + MOVDTOX_F0_O4 + MOVDTOX_F2_O5 + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x30] ! k[12, 13] + stx %o5, [%o1 + 0x38] ! k[14, 15] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x40] ! k[16, 17] + stx %o5, [%o1 + 0x48] ! k[18, 19] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x60] ! k[24, 25] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x70] ! k[28, 29] + stx %o5, [%o1 + 0x78] ! k[30, 31] + ROTL128(%o4, %o5, %g2, %g3, 34) + stx %o4, [%o1 + 0xa0] ! k[40, 41] + stx %o5, [%o1 + 0xa8] ! k[42, 43] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0xc0] ! k[48, 49] + stx %o5, [%o1 + 0xc8] ! k[50, 51] + + ldx [%o1 + 0x00], %o4 ! k[ 0, 1] + ldx [%o1 + 0x08], %o5 ! k[ 2, 3] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x20] ! k[ 8, 9] + stx %o5, [%o1 + 0x28] ! k[10, 11] + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0x50] ! k[20, 21] + stx %o5, [%o1 + 0x58] ! k[22, 23] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o5, [%o1 + 0x68] ! k[26, 27] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0x80] ! k[32, 33] + stx %o5, [%o1 + 0x88] ! k[34, 35] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0x90] ! k[36, 37] + stx %o5, [%o1 + 0x98] ! k[38, 39] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0xb0] ! k[44, 45] + stx %o5, [%o1 + 0xb8] ! k[46, 47] + + ba,pt %xcc, 2f + mov (3 * 16 * 4), %o0 + +1: + /* 192-bit or 256-bit key */ + std %f0, [%o1 + 0x30] ! k[12, 13] + std %f2, [%o1 + 0x38] ! k[14, 15] + ldd [%o1 + 0x20], %f4 ! k[ 8, 9] + ldd [%o1 + 0x28], %f6 ! k[10, 11] + fxor %f0, %f4, %f0 + fxor %f2, %f6, %f2 + CAMELLIA_F(24, 2, 0, 2) + CAMELLIA_F(26, 0, 2, 0) + std %f0, [%o1 + 0x10] ! k[ 4, 5] + std %f2, [%o1 + 0x18] ! k[ 6, 7] + MOVDTOX_F0_O4 + MOVDTOX_F2_O5 + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0x50] ! k[20, 21] + stx %o5, [%o1 + 0x58] ! k[22, 23] + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0xa0] ! k[40, 41] + stx %o5, [%o1 + 0xa8] ! k[42, 43] + ROTL128(%o4, %o5, %g2, %g3, 51) + stx %o4, [%o1 + 0x100] ! k[64, 65] + stx %o5, [%o1 + 0x108] ! k[66, 67] + ldx [%o1 + 0x20], %o4 ! k[ 8, 9] + ldx [%o1 + 0x28], %o5 ! k[10, 11] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x20] ! k[ 8, 9] + stx %o5, [%o1 + 0x28] ! k[10, 11] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x40] ! k[16, 17] + stx %o5, [%o1 + 0x48] ! k[18, 19] + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0x90] ! k[36, 37] + stx %o5, [%o1 + 0x98] ! k[38, 39] + ROTL128(%o4, %o5, %g2, %g3, 34) + stx %o4, [%o1 + 0xd0] ! k[52, 53] + stx %o5, [%o1 + 0xd8] ! k[54, 55] + ldx [%o1 + 0x30], %o4 ! k[12, 13] + ldx [%o1 + 0x38], %o5 ! k[14, 15] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x30] ! k[12, 13] + stx %o5, [%o1 + 0x38] ! k[14, 15] + ROTL128(%o4, %o5, %g2, %g3, 30) + stx %o4, [%o1 + 0x70] ! k[28, 29] + stx %o5, [%o1 + 0x78] ! k[30, 31] + srlx %o4, 32, %g2 + srlx %o5, 32, %g3 + stw %o4, [%o1 + 0xc0] ! k[48] + stw %g3, [%o1 + 0xc4] ! k[49] + stw %o5, [%o1 + 0xc8] ! k[50] + stw %g2, [%o1 + 0xcc] ! k[51] + ROTL128(%o4, %o5, %g2, %g3, 49) + stx %o4, [%o1 + 0xe0] ! k[56, 57] + stx %o5, [%o1 + 0xe8] ! k[58, 59] + ldx [%o1 + 0x00], %o4 ! k[ 0, 1] + ldx [%o1 + 0x08], %o5 ! k[ 2, 3] + ROTL128(%o4, %o5, %g2, %g3, 45) + stx %o4, [%o1 + 0x60] ! k[24, 25] + stx %o5, [%o1 + 0x68] ! k[26, 27] + ROTL128(%o4, %o5, %g2, %g3, 15) + stx %o4, [%o1 + 0x80] ! k[32, 33] + stx %o5, [%o1 + 0x88] ! k[34, 35] + ROTL128(%o4, %o5, %g2, %g3, 17) + stx %o4, [%o1 + 0xb0] ! k[44, 45] + stx %o5, [%o1 + 0xb8] ! k[46, 47] + ROTL128(%o4, %o5, %g2, %g3, 34) + stx %o4, [%o1 + 0xf0] ! k[60, 61] + stx %o5, [%o1 + 0xf8] ! k[62, 63] + mov (4 * 16 * 4), %o0 +2: + add %o1, %o0, %o1 + ldd [%o1 + 0x00], %f0 + ldd [%o1 + 0x08], %f2 + std %f0, [%o3 + 0x00] + std %f2, [%o3 + 0x08] + add %o3, 0x10, %o3 +1: + sub %o1, (16 * 4), %o1 + ldd [%o1 + 0x38], %f0 + ldd [%o1 + 0x30], %f2 + ldd [%o1 + 0x28], %f4 + ldd [%o1 + 0x20], %f6 + ldd [%o1 + 0x18], %f8 + ldd [%o1 + 0x10], %f10 + std %f0, [%o3 + 0x00] + std %f2, [%o3 + 0x08] + std %f4, [%o3 + 0x10] + std %f6, [%o3 + 0x18] + std %f8, [%o3 + 0x20] + std %f10, [%o3 + 0x28] + + ldd [%o1 + 0x08], %f0 + ldd [%o1 + 0x00], %f2 + std %f0, [%o3 + 0x30] + std %f2, [%o3 + 0x38] + subcc %o0, (16 * 4), %o0 + bne,pt %icc, 1b + add %o3, (16 * 4), %o3 + + std %f2, [%o3 - 0x10] + std %f0, [%o3 - 0x08] + + retl + VISExit +ENDPROC(camellia_sparc64_key_expand) + + .align 32 +ENTRY(camellia_sparc64_crypt) + /* %o0=key, %o1=input, %o2=output, %o3=key_len */ + VISEntry + + ld [%o1 + 0x00], %f0 + ld [%o1 + 0x04], %f1 + ld [%o1 + 0x08], %f2 + ld [%o1 + 0x0c], %f3 + + ldd [%o0 + 0x00], %f4 + ldd [%o0 + 0x08], %f6 + + cmp %o3, 16 + fxor %f4, %f0, %f0 + be 1f + fxor %f6, %f2, %f2 + + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + ldd [%o0 + 0x30], %f16 + ldd [%o0 + 0x38], %f18 + ldd [%o0 + 0x40], %f20 + ldd [%o0 + 0x48], %f22 + add %o0, 0x40, %o0 + + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + +1: + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + ldd [%o0 + 0x30], %f16 + ldd [%o0 + 0x38], %f18 + ldd [%o0 + 0x40], %f20 + ldd [%o0 + 0x48], %f22 + ldd [%o0 + 0x50], %f24 + ldd [%o0 + 0x58], %f26 + ldd [%o0 + 0x60], %f28 + ldd [%o0 + 0x68], %f30 + ldd [%o0 + 0x70], %f32 + ldd [%o0 + 0x78], %f34 + ldd [%o0 + 0x80], %f36 + ldd [%o0 + 0x88], %f38 + ldd [%o0 + 0x90], %f40 + ldd [%o0 + 0x98], %f42 + ldd [%o0 + 0xa0], %f44 + ldd [%o0 + 0xa8], %f46 + ldd [%o0 + 0xb0], %f48 + ldd [%o0 + 0xb8], %f50 + ldd [%o0 + 0xc0], %f52 + ldd [%o0 + 0xc8], %f54 + + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS(40, 0, 2) + fxor %f52, %f2, %f2 + fxor %f54, %f0, %f0 + + st %f2, [%o2 + 0x00] + st %f3, [%o2 + 0x04] + st %f0, [%o2 + 0x08] + st %f1, [%o2 + 0x0c] + + retl + VISExit +ENDPROC(camellia_sparc64_crypt) + + .align 32 +ENTRY(camellia_sparc64_load_keys) + /* %o0=key, %o1=key_len */ + VISEntry + ldd [%o0 + 0x00], %f4 + ldd [%o0 + 0x08], %f6 + ldd [%o0 + 0x10], %f8 + ldd [%o0 + 0x18], %f10 + ldd [%o0 + 0x20], %f12 + ldd [%o0 + 0x28], %f14 + ldd [%o0 + 0x30], %f16 + ldd [%o0 + 0x38], %f18 + ldd [%o0 + 0x40], %f20 + ldd [%o0 + 0x48], %f22 + ldd [%o0 + 0x50], %f24 + ldd [%o0 + 0x58], %f26 + ldd [%o0 + 0x60], %f28 + ldd [%o0 + 0x68], %f30 + ldd [%o0 + 0x70], %f32 + ldd [%o0 + 0x78], %f34 + ldd [%o0 + 0x80], %f36 + ldd [%o0 + 0x88], %f38 + ldd [%o0 + 0x90], %f40 + ldd [%o0 + 0x98], %f42 + ldd [%o0 + 0xa0], %f44 + ldd [%o0 + 0xa8], %f46 + ldd [%o0 + 0xb0], %f48 + ldd [%o0 + 0xb8], %f50 + ldd [%o0 + 0xc0], %f52 + retl + ldd [%o0 + 0xc8], %f54 +ENDPROC(camellia_sparc64_load_keys) + + .align 32 +ENTRY(camellia_sparc64_ecb_crypt_3_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key */ +1: ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + add %o0, 0x10, %o0 + fxor %f4, %f0, %f0 + fxor %f6, %f2, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS(40, 0, 2) + fxor %f52, %f2, %f2 + fxor %f54, %f0, %f0 + std %f2, [%o1 + 0x00] + std %f0, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + retl + nop +ENDPROC(camellia_sparc64_ecb_crypt_3_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_ecb_crypt_4_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key */ +1: ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + add %o0, 0x10, %o0 + fxor %f4, %f0, %f0 + fxor %f6, %f2, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + ldd [%o3 + 0xd0], %f8 + ldd [%o3 + 0xd8], %f10 + ldd [%o3 + 0xe0], %f12 + ldd [%o3 + 0xe8], %f14 + ldd [%o3 + 0xf0], %f16 + ldd [%o3 + 0xf8], %f18 + ldd [%o3 + 0x100], %f20 + ldd [%o3 + 0x108], %f22 + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2) + CAMELLIA_F(8, 2, 0, 2) + CAMELLIA_F(10, 0, 2, 0) + ldd [%o3 + 0x10], %f8 + ldd [%o3 + 0x18], %f10 + CAMELLIA_F(12, 2, 0, 2) + CAMELLIA_F(14, 0, 2, 0) + ldd [%o3 + 0x20], %f12 + ldd [%o3 + 0x28], %f14 + CAMELLIA_F(16, 2, 0, 2) + CAMELLIA_F(18, 0, 2, 0) + ldd [%o3 + 0x30], %f16 + ldd [%o3 + 0x38], %f18 + fxor %f20, %f2, %f2 + fxor %f22, %f0, %f0 + ldd [%o3 + 0x40], %f20 + ldd [%o3 + 0x48], %f22 + std %f2, [%o1 + 0x00] + std %f0, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + retl + nop +ENDPROC(camellia_sparc64_ecb_crypt_4_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_cbc_encrypt_3_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */ + ldd [%o4 + 0x00], %f60 + ldd [%o4 + 0x08], %f62 +1: ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + add %o0, 0x10, %o0 + fxor %f60, %f0, %f0 + fxor %f62, %f2, %f2 + fxor %f4, %f0, %f0 + fxor %f6, %f2, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS(40, 0, 2) + fxor %f52, %f2, %f60 + fxor %f54, %f0, %f62 + std %f60, [%o1 + 0x00] + std %f62, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + std %f60, [%o4 + 0x00] + retl + std %f62, [%o4 + 0x08] +ENDPROC(camellia_sparc64_cbc_encrypt_3_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_cbc_encrypt_4_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */ + ldd [%o4 + 0x00], %f60 + ldd [%o4 + 0x08], %f62 +1: ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + add %o0, 0x10, %o0 + fxor %f60, %f0, %f0 + fxor %f62, %f2, %f2 + fxor %f4, %f0, %f0 + fxor %f6, %f2, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + ldd [%o3 + 0xd0], %f8 + ldd [%o3 + 0xd8], %f10 + ldd [%o3 + 0xe0], %f12 + ldd [%o3 + 0xe8], %f14 + ldd [%o3 + 0xf0], %f16 + ldd [%o3 + 0xf8], %f18 + ldd [%o3 + 0x100], %f20 + ldd [%o3 + 0x108], %f22 + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2) + CAMELLIA_F(8, 2, 0, 2) + CAMELLIA_F(10, 0, 2, 0) + ldd [%o3 + 0x10], %f8 + ldd [%o3 + 0x18], %f10 + CAMELLIA_F(12, 2, 0, 2) + CAMELLIA_F(14, 0, 2, 0) + ldd [%o3 + 0x20], %f12 + ldd [%o3 + 0x28], %f14 + CAMELLIA_F(16, 2, 0, 2) + CAMELLIA_F(18, 0, 2, 0) + ldd [%o3 + 0x30], %f16 + ldd [%o3 + 0x38], %f18 + fxor %f20, %f2, %f60 + fxor %f22, %f0, %f62 + ldd [%o3 + 0x40], %f20 + ldd [%o3 + 0x48], %f22 + std %f60, [%o1 + 0x00] + std %f62, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + std %f60, [%o4 + 0x00] + retl + std %f62, [%o4 + 0x08] +ENDPROC(camellia_sparc64_cbc_encrypt_4_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_cbc_decrypt_3_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */ + ldd [%o4 + 0x00], %f60 + ldd [%o4 + 0x08], %f62 +1: ldd [%o0 + 0x00], %f56 + ldd [%o0 + 0x08], %f58 + add %o0, 0x10, %o0 + fxor %f4, %f56, %f0 + fxor %f6, %f58, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS(40, 0, 2) + fxor %f52, %f2, %f2 + fxor %f54, %f0, %f0 + fxor %f60, %f2, %f2 + fxor %f62, %f0, %f0 + fsrc2 %f56, %f60 + fsrc2 %f58, %f62 + std %f2, [%o1 + 0x00] + std %f0, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + std %f60, [%o4 + 0x00] + retl + std %f62, [%o4 + 0x08] +ENDPROC(camellia_sparc64_cbc_decrypt_3_grand_rounds) + + .align 32 +ENTRY(camellia_sparc64_cbc_decrypt_4_grand_rounds) + /* %o0=input, %o1=output, %o2=len, %o3=key, %o4=IV */ + ldd [%o4 + 0x00], %f60 + ldd [%o4 + 0x08], %f62 +1: ldd [%o0 + 0x00], %f56 + ldd [%o0 + 0x08], %f58 + add %o0, 0x10, %o0 + fxor %f4, %f56, %f0 + fxor %f6, %f58, %f2 + CAMELLIA_6ROUNDS_FL_FLI( 8, 0, 2) + ldd [%o3 + 0xd0], %f8 + ldd [%o3 + 0xd8], %f10 + ldd [%o3 + 0xe0], %f12 + ldd [%o3 + 0xe8], %f14 + ldd [%o3 + 0xf0], %f16 + ldd [%o3 + 0xf8], %f18 + ldd [%o3 + 0x100], %f20 + ldd [%o3 + 0x108], %f22 + CAMELLIA_6ROUNDS_FL_FLI(24, 0, 2) + CAMELLIA_6ROUNDS_FL_FLI(40, 0, 2) + CAMELLIA_F(8, 2, 0, 2) + CAMELLIA_F(10, 0, 2, 0) + ldd [%o3 + 0x10], %f8 + ldd [%o3 + 0x18], %f10 + CAMELLIA_F(12, 2, 0, 2) + CAMELLIA_F(14, 0, 2, 0) + ldd [%o3 + 0x20], %f12 + ldd [%o3 + 0x28], %f14 + CAMELLIA_F(16, 2, 0, 2) + CAMELLIA_F(18, 0, 2, 0) + ldd [%o3 + 0x30], %f16 + ldd [%o3 + 0x38], %f18 + fxor %f20, %f2, %f2 + fxor %f22, %f0, %f0 + ldd [%o3 + 0x40], %f20 + ldd [%o3 + 0x48], %f22 + fxor %f60, %f2, %f2 + fxor %f62, %f0, %f0 + fsrc2 %f56, %f60 + fsrc2 %f58, %f62 + std %f2, [%o1 + 0x00] + std %f0, [%o1 + 0x08] + subcc %o2, 0x10, %o2 + bne,pt %icc, 1b + add %o1, 0x10, %o1 + std %f60, [%o4 + 0x00] + retl + std %f62, [%o4 + 0x08] +ENDPROC(camellia_sparc64_cbc_decrypt_4_grand_rounds) diff --git a/arch/sparc/crypto/camellia_glue.c b/arch/sparc/crypto/camellia_glue.c new file mode 100644 index 000000000..e7a1e1c42 --- /dev/null +++ b/arch/sparc/crypto/camellia_glue.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Glue code for CAMELLIA encryption optimized for sparc64 crypto opcodes. + * + * Copyright (C) 2012 David S. Miller + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define CAMELLIA_MIN_KEY_SIZE 16 +#define CAMELLIA_MAX_KEY_SIZE 32 +#define CAMELLIA_BLOCK_SIZE 16 +#define CAMELLIA_TABLE_BYTE_LEN 272 + +struct camellia_sparc64_ctx { + u64 encrypt_key[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)]; + u64 decrypt_key[CAMELLIA_TABLE_BYTE_LEN / sizeof(u64)]; + int key_len; +}; + +extern void camellia_sparc64_key_expand(const u32 *in_key, u64 *encrypt_key, + unsigned int key_len, u64 *decrypt_key); + +static int camellia_set_key(struct crypto_tfm *tfm, const u8 *_in_key, + unsigned int key_len) +{ + struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + const u32 *in_key = (const u32 *) _in_key; + + if (key_len != 16 && key_len != 24 && key_len != 32) + return -EINVAL; + + ctx->key_len = key_len; + + camellia_sparc64_key_expand(in_key, &ctx->encrypt_key[0], + key_len, &ctx->decrypt_key[0]); + return 0; +} + +static int camellia_set_key_skcipher(struct crypto_skcipher *tfm, + const u8 *in_key, unsigned int key_len) +{ + return camellia_set_key(crypto_skcipher_tfm(tfm), in_key, key_len); +} + +extern void camellia_sparc64_crypt(const u64 *key, const u32 *input, + u32 *output, unsigned int key_len); + +static void camellia_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + + camellia_sparc64_crypt(&ctx->encrypt_key[0], + (const u32 *) src, + (u32 *) dst, ctx->key_len); +} + +static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) +{ + struct camellia_sparc64_ctx *ctx = crypto_tfm_ctx(tfm); + + camellia_sparc64_crypt(&ctx->decrypt_key[0], + (const u32 *) src, + (u32 *) dst, ctx->key_len); +} + +extern void camellia_sparc64_load_keys(const u64 *key, unsigned int key_len); + +typedef void ecb_crypt_op(const u64 *input, u64 *output, unsigned int len, + const u64 *key); + +extern ecb_crypt_op camellia_sparc64_ecb_crypt_3_grand_rounds; +extern ecb_crypt_op camellia_sparc64_ecb_crypt_4_grand_rounds; + +static int __ecb_crypt(struct skcipher_request *req, bool encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct camellia_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + ecb_crypt_op *op; + const u64 *key; + unsigned int nbytes; + int err; + + op = camellia_sparc64_ecb_crypt_3_grand_rounds; + if (ctx->key_len != 16) + op = camellia_sparc64_ecb_crypt_4_grand_rounds; + + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; + + if (encrypt) + key = &ctx->encrypt_key[0]; + else + key = &ctx->decrypt_key[0]; + camellia_sparc64_load_keys(key, ctx->key_len); + while ((nbytes = walk.nbytes) != 0) { + op(walk.src.virt.addr, walk.dst.virt.addr, + round_down(nbytes, CAMELLIA_BLOCK_SIZE), key); + err = skcipher_walk_done(&walk, nbytes % CAMELLIA_BLOCK_SIZE); + } + fprs_write(0); + return err; +} + +static int ecb_encrypt(struct skcipher_request *req) +{ + return __ecb_crypt(req, true); +} + +static int ecb_decrypt(struct skcipher_request *req) +{ + return __ecb_crypt(req, false); +} + +typedef void cbc_crypt_op(const u64 *input, u64 *output, unsigned int len, + const u64 *key, u64 *iv); + +extern cbc_crypt_op camellia_sparc64_cbc_encrypt_3_grand_rounds; +extern cbc_crypt_op camellia_sparc64_cbc_encrypt_4_grand_rounds; +extern cbc_crypt_op camellia_sparc64_cbc_decrypt_3_grand_rounds; +extern cbc_crypt_op camellia_sparc64_cbc_decrypt_4_grand_rounds; + +static int cbc_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct camellia_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + cbc_crypt_op *op; + const u64 *key; + unsigned int nbytes; + int err; + + op = camellia_sparc64_cbc_encrypt_3_grand_rounds; + if (ctx->key_len != 16) + op = camellia_sparc64_cbc_encrypt_4_grand_rounds; + + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; + + key = &ctx->encrypt_key[0]; + camellia_sparc64_load_keys(key, ctx->key_len); + while ((nbytes = walk.nbytes) != 0) { + op(walk.src.virt.addr, walk.dst.virt.addr, + round_down(nbytes, CAMELLIA_BLOCK_SIZE), key, walk.iv); + err = skcipher_walk_done(&walk, nbytes % CAMELLIA_BLOCK_SIZE); + } + fprs_write(0); + return err; +} + +static int cbc_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct camellia_sparc64_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + cbc_crypt_op *op; + const u64 *key; + unsigned int nbytes; + int err; + + op = camellia_sparc64_cbc_decrypt_3_grand_rounds; + if (ctx->key_len != 16) + op = camellia_sparc64_cbc_decrypt_4_grand_rounds; + + err = skcipher_walk_virt(&walk, req, true); + if (err) + return err; + + key = &ctx->decrypt_key[0]; + camellia_sparc64_load_keys(key, ctx->key_len); + while ((nbytes = walk.nbytes) != 0) { + op(walk.src.virt.addr, walk.dst.virt.addr, + round_down(nbytes, CAMELLIA_BLOCK_SIZE), key, walk.iv); + err = skcipher_walk_done(&walk, nbytes % CAMELLIA_BLOCK_SIZE); + } + fprs_write(0); + return err; +} + +static struct crypto_alg cipher_alg = { + .cra_name = "camellia", + .cra_driver_name = "camellia-sparc64", + .cra_priority = SPARC_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_CIPHER, + .cra_blocksize = CAMELLIA_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct camellia_sparc64_ctx), + .cra_alignmask = 3, + .cra_module = THIS_MODULE, + .cra_u = { + .cipher = { + .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE, + .cia_max_keysize = CAMELLIA_MAX_KEY_SIZE, + .cia_setkey = camellia_set_key, + .cia_encrypt = camellia_encrypt, + .cia_decrypt = camellia_decrypt + } + } +}; + +static struct skcipher_alg skcipher_algs[] = { + { + .base.cra_name = "ecb(camellia)", + .base.cra_driver_name = "ecb-camellia-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct camellia_sparc64_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .setkey = camellia_set_key_skcipher, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, { + .base.cra_name = "cbc(camellia)", + .base.cra_driver_name = "cbc-camellia-sparc64", + .base.cra_priority = SPARC_CR_OPCODE_PRIORITY, + .base.cra_blocksize = CAMELLIA_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct camellia_sparc64_ctx), + .base.cra_alignmask = 7, + .base.cra_module = THIS_MODULE, + .min_keysize = CAMELLIA_MIN_KEY_SIZE, + .max_keysize = CAMELLIA_MAX_KEY_SIZE, + .ivsize = CAMELLIA_BLOCK_SIZE, + .setkey = camellia_set_key_skcipher, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + } +}; + +static bool __init sparc64_has_camellia_opcode(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return false; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_CAMELLIA)) + return false; + + return true; +} + +static int __init camellia_sparc64_mod_init(void) +{ + int err; + + if (!sparc64_has_camellia_opcode()) { + pr_info("sparc64 camellia opcodes not available.\n"); + return -ENODEV; + } + pr_info("Using sparc64 camellia opcodes optimized CAMELLIA implementation\n"); + err = crypto_register_alg(&cipher_alg); + if (err) + return err; + err = crypto_register_skciphers(skcipher_algs, + ARRAY_SIZE(skcipher_algs)); + if (err) + crypto_unregister_alg(&cipher_alg); + return err; +} + +static void __exit camellia_sparc64_mod_fini(void) +{ + crypto_unregister_alg(&cipher_alg); + crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs)); +} + +module_init(camellia_sparc64_mod_init); +module_exit(camellia_sparc64_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Camellia Cipher Algorithm, sparc64 camellia opcode accelerated"); + +MODULE_ALIAS_CRYPTO("camellia"); + +#include "crop_devid.c" diff --git a/arch/sparc/crypto/crop_devid.c b/arch/sparc/crypto/crop_devid.c new file mode 100644 index 000000000..79a059829 --- /dev/null +++ b/arch/sparc/crypto/crop_devid.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +/* This is a dummy device table linked into all of the crypto + * opcode drivers. It serves to trigger the module autoloading + * mechanisms in userspace which scan the OF device tree and + * load any modules which have device table entries that + * match OF device nodes. + */ +static const struct of_device_id crypto_opcode_match[] = { + { .name = "cpu", .compatible = "sun4v", }, + {}, +}; +MODULE_DEVICE_TABLE(of, crypto_opcode_match); diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild new file mode 100644 index 000000000..49c6bb326 --- /dev/null +++ b/arch/sparc/include/asm/Kbuild @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0 +generated-y += syscall_table_32.h +generated-y += syscall_table_64.h +generic-y += agp.h +generic-y += kvm_para.h +generic-y += mcs_spinlock.h +generic-y += ring_buffer.h +generic-y += text-patching.h diff --git a/arch/sparc/include/asm/adi.h b/arch/sparc/include/asm/adi.h new file mode 100644 index 000000000..acad0d04e --- /dev/null +++ b/arch/sparc/include/asm/adi.h @@ -0,0 +1,6 @@ +#ifndef ___ASM_SPARC_ADI_H +#define ___ASM_SPARC_ADI_H +#if defined(__sparc__) && defined(__arch64__) +#include +#endif +#endif diff --git a/arch/sparc/include/asm/adi_64.h b/arch/sparc/include/asm/adi_64.h new file mode 100644 index 000000000..0c066fdab --- /dev/null +++ b/arch/sparc/include/asm/adi_64.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* adi_64.h: ADI related data structures + * + * Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved. + * Author: Khalid Aziz (khalid.aziz@oracle.com) + */ +#ifndef __ASM_SPARC64_ADI_H +#define __ASM_SPARC64_ADI_H + +#include + +#ifndef __ASSEMBLER__ + +struct adi_caps { + __u64 blksz; + __u64 nbits; + __u64 ue_on_adi; +}; + +struct adi_config { + bool enabled; + struct adi_caps caps; +}; + +extern struct adi_config adi_state; + +extern void mdesc_adi_init(void); + +static inline bool adi_capable(void) +{ + return adi_state.enabled; +} + +static inline unsigned long adi_blksize(void) +{ + return adi_state.caps.blksz; +} + +static inline unsigned long adi_nbits(void) +{ + return adi_state.caps.nbits; +} + +#endif /* __ASSEMBLER__ */ + +#endif /* !(__ASM_SPARC64_ADI_H) */ diff --git a/arch/sparc/include/asm/apb.h b/arch/sparc/include/asm/apb.h new file mode 100644 index 000000000..b1dfb1a99 --- /dev/null +++ b/arch/sparc/include/asm/apb.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * apb.h: Advanced PCI Bridge Configuration Registers and Bits + * + * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be) + */ + +#ifndef _SPARC64_APB_H +#define _SPARC64_APB_H + +#define APB_TICK_REGISTER 0xb0 +#define APB_INT_ACK 0xb8 +#define APB_PRIMARY_MASTER_RETRY_LIMIT 0xc0 +#define APB_DMA_ASFR 0xc8 +#define APB_DMA_AFAR 0xd0 +#define APB_PIO_TARGET_RETRY_LIMIT 0xd8 +#define APB_PIO_TARGET_LATENCY_TIMER 0xd9 +#define APB_DMA_TARGET_RETRY_LIMIT 0xda +#define APB_DMA_TARGET_LATENCY_TIMER 0xdb +#define APB_SECONDARY_MASTER_RETRY_LIMIT 0xdc +#define APB_SECONDARY_CONTROL 0xdd +#define APB_IO_ADDRESS_MAP 0xde +#define APB_MEM_ADDRESS_MAP 0xdf + +#define APB_PCI_CONTROL_LOW 0xe0 +# define APB_PCI_CTL_LOW_ARB_PARK (1 << 21) +# define APB_PCI_CTL_LOW_ERRINT_EN (1 << 8) + +#define APB_PCI_CONTROL_HIGH 0xe4 +# define APB_PCI_CTL_HIGH_SERR (1 << 2) +# define APB_PCI_CTL_HIGH_ARBITER_EN (1 << 0) + +#define APB_PIO_ASFR 0xe8 +#define APB_PIO_AFAR 0xf0 +#define APB_DIAG_REGISTER 0xf8 + +#endif /* !(_SPARC64_APB_H) */ diff --git a/arch/sparc/include/asm/asm-offsets.h b/arch/sparc/include/asm/asm-offsets.h new file mode 100644 index 000000000..d370ee36a --- /dev/null +++ b/arch/sparc/include/asm/asm-offsets.h @@ -0,0 +1 @@ +#include diff --git a/arch/sparc/include/asm/asm-prototypes.h b/arch/sparc/include/asm/asm-prototypes.h new file mode 100644 index 000000000..a39a24c53 --- /dev/null +++ b/arch/sparc/include/asm/asm-prototypes.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +void *__memscan_zero(void *, size_t); +void *__memscan_generic(void *, int, size_t); +void *__bzero(void *, size_t); +void VISenter(void); /* Dummy prototype to supress warning */ +#undef memcpy +#undef memset +void *memcpy(void *dest, const void *src, size_t n); +void *memset(void *s, int c, size_t n); +typedef int TItype __attribute__((mode(TI))); +TItype __multi3(TItype a, TItype b); +void _mcount(void); +void mcount(void); + +s64 __ashldi3(s64, int); +s64 __lshrdi3(s64, int); +s64 __ashrdi3(s64, int); diff --git a/arch/sparc/include/asm/asm.h b/arch/sparc/include/asm/asm.h new file mode 100644 index 000000000..eaed0117a --- /dev/null +++ b/arch/sparc/include/asm/asm.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_ASM_H +#define _SPARC_ASM_H + +/* Macros to assist the sharing of assembler code between 32-bit and + * 64-bit sparc. + */ + +#ifdef CONFIG_SPARC64 +#define BRANCH32(TYPE, PREDICT, DEST) \ + TYPE,PREDICT %icc, DEST +#define BRANCH32_ANNUL(TYPE, PREDICT, DEST) \ + TYPE,a,PREDICT %icc, DEST +#define BRANCH_REG_ZERO(PREDICT, REG, DEST) \ + brz,PREDICT REG, DEST +#define BRANCH_REG_ZERO_ANNUL(PREDICT, REG, DEST) \ + brz,a,PREDICT REG, DEST +#define BRANCH_REG_NOT_ZERO(PREDICT, REG, DEST) \ + brnz,PREDICT REG, DEST +#define BRANCH_REG_NOT_ZERO_ANNUL(PREDICT, REG, DEST) \ + brnz,a,PREDICT REG, DEST +#else +#define BRANCH32(TYPE, PREDICT, DEST) \ + TYPE DEST +#define BRANCH32_ANNUL(TYPE, PREDICT, DEST) \ + TYPE,a DEST +#define BRANCH_REG_ZERO(PREDICT, REG, DEST) \ + cmp REG, 0; \ + be DEST +#define BRANCH_REG_ZERO_ANNUL(PREDICT, REG, DEST) \ + cmp REG, 0; \ + be,a DEST +#define BRANCH_REG_NOT_ZERO(PREDICT, REG, DEST) \ + cmp REG, 0; \ + bne DEST +#define BRANCH_REG_NOT_ZERO_ANNUL(PREDICT, REG, DEST) \ + cmp REG, 0; \ + bne,a DEST +#endif + +#endif /* _SPARC_ASM_H */ diff --git a/arch/sparc/include/asm/asmmacro.h b/arch/sparc/include/asm/asmmacro.h new file mode 100644 index 000000000..49aaf6f3b --- /dev/null +++ b/arch/sparc/include/asm/asmmacro.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* asmmacro.h: Assembler macros. + * + * Copyright (C) 1996 David S. Miller (davem@caipfs.rutgers.edu) + */ + +#ifndef _SPARC_ASMMACRO_H +#define _SPARC_ASMMACRO_H + +/* All trap entry points _must_ begin with this macro or else you + * lose. It makes sure the kernel has a proper window so that + * c-code can be called. + */ +#define SAVE_ALL_HEAD \ + sethi %hi(trap_setup), %l4; \ + jmpl %l4 + %lo(trap_setup), %l6; +#define SAVE_ALL \ + SAVE_ALL_HEAD \ + nop; + +/* All traps low-level code here must end with this macro. */ +#define RESTORE_ALL b ret_trap_entry; clr %l6; + +/* Support for run-time patching of single instructions. + * This is used to handle the differences in the ASI for + * MMUREGS for LEON and SUN. + * + * Sample: + * LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %o0 + * SUN_PI_(lda [%g0] ASI_M_MMUREGS, %o0 + * PI == Patch Instruction + * + * For LEON we will use the first variant, + * and for all other we will use the SUN variant. + * The order is important. + */ +#define LEON_PI(...) \ +662: __VA_ARGS__ + +#define SUN_PI_(...) \ + .section .leon_1insn_patch, "ax"; \ + .word 662b; \ + __VA_ARGS__; \ + .previous + +#endif /* !(_SPARC_ASMMACRO_H) */ diff --git a/arch/sparc/include/asm/atomic.h b/arch/sparc/include/asm/atomic.h new file mode 100644 index 000000000..425151cc4 --- /dev/null +++ b/arch/sparc/include/asm/atomic.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_ATOMIC_H +#define ___ASM_SPARC_ATOMIC_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h new file mode 100644 index 000000000..60ce2fe57 --- /dev/null +++ b/arch/sparc/include/asm/atomic_32.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* atomic.h: These still suck, but the I-cache hit rate is higher. + * + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) + * Copyright (C) 2000 Anton Blanchard (anton@linuxcare.com.au) + * Copyright (C) 2007 Kyle McMartin (kyle@parisc-linux.org) + * + * Additions by Keith M Wesolowski (wesolows@foobazco.org) based + * on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf . + */ + +#ifndef __ARCH_SPARC_ATOMIC__ +#define __ARCH_SPARC_ATOMIC__ + +#include + +#include +#include +#include + +int arch_atomic_add_return(int, atomic_t *); +#define arch_atomic_add_return arch_atomic_add_return + +int arch_atomic_fetch_add(int, atomic_t *); +#define arch_atomic_fetch_add arch_atomic_fetch_add + +int arch_atomic_fetch_and(int, atomic_t *); +#define arch_atomic_fetch_and arch_atomic_fetch_and + +int arch_atomic_fetch_or(int, atomic_t *); +#define arch_atomic_fetch_or arch_atomic_fetch_or + +int arch_atomic_fetch_xor(int, atomic_t *); +#define arch_atomic_fetch_xor arch_atomic_fetch_xor + +int arch_atomic_cmpxchg(atomic_t *, int, int); +#define arch_atomic_cmpxchg arch_atomic_cmpxchg + +int arch_atomic_xchg(atomic_t *, int); +#define arch_atomic_xchg arch_atomic_xchg + +int arch_atomic_fetch_add_unless(atomic_t *, int, int); +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless + +void arch_atomic_set(atomic_t *, int); + +#define arch_atomic_set_release(v, i) arch_atomic_set((v), (i)) + +#define arch_atomic_read(v) READ_ONCE((v)->counter) + +#define arch_atomic_add(i, v) ((void)arch_atomic_add_return( (int)(i), (v))) +#define arch_atomic_sub(i, v) ((void)arch_atomic_add_return(-(int)(i), (v))) + +#define arch_atomic_and(i, v) ((void)arch_atomic_fetch_and((i), (v))) +#define arch_atomic_or(i, v) ((void)arch_atomic_fetch_or((i), (v))) +#define arch_atomic_xor(i, v) ((void)arch_atomic_fetch_xor((i), (v))) + +#define arch_atomic_sub_return(i, v) (arch_atomic_add_return(-(int)(i), (v))) +#define arch_atomic_fetch_sub(i, v) (arch_atomic_fetch_add (-(int)(i), (v))) + +#endif /* !(__ARCH_SPARC_ATOMIC__) */ diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h new file mode 100644 index 000000000..a5e9c3760 --- /dev/null +++ b/arch/sparc/include/asm/atomic_64.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* atomic.h: Thankfully the V9 is at least reasonable for this + * stuff. + * + * Copyright (C) 1996, 1997, 2000, 2012 David S. Miller (davem@redhat.com) + */ + +#ifndef __ARCH_SPARC64_ATOMIC__ +#define __ARCH_SPARC64_ATOMIC__ + +#include +#include +#include + +#define ATOMIC64_INIT(i) { (i) } + +#define arch_atomic_read(v) READ_ONCE((v)->counter) +#define arch_atomic64_read(v) READ_ONCE((v)->counter) + +#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic64_set(v, i) WRITE_ONCE(((v)->counter), (i)) + +#define ATOMIC_OP(op) \ +void arch_atomic_##op(int, atomic_t *); \ +void arch_atomic64_##op(s64, atomic64_t *); + +#define ATOMIC_OP_RETURN(op) \ +int arch_atomic_##op##_return(int, atomic_t *); \ +s64 arch_atomic64_##op##_return(s64, atomic64_t *); + +#define ATOMIC_FETCH_OP(op) \ +int arch_atomic_fetch_##op(int, atomic_t *); \ +s64 arch_atomic64_fetch_##op(s64, atomic64_t *); + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(add) +ATOMIC_OPS(sub) + +#define arch_atomic_add_return arch_atomic_add_return +#define arch_atomic_sub_return arch_atomic_sub_return +#define arch_atomic_fetch_add arch_atomic_fetch_add +#define arch_atomic_fetch_sub arch_atomic_fetch_sub + +#define arch_atomic64_add_return arch_atomic64_add_return +#define arch_atomic64_sub_return arch_atomic64_sub_return +#define arch_atomic64_fetch_add arch_atomic64_fetch_add +#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub + +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) + +#define arch_atomic_fetch_and arch_atomic_fetch_and +#define arch_atomic_fetch_or arch_atomic_fetch_or +#define arch_atomic_fetch_xor arch_atomic_fetch_xor + +#define arch_atomic64_fetch_and arch_atomic64_fetch_and +#define arch_atomic64_fetch_or arch_atomic64_fetch_or +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +s64 arch_atomic64_dec_if_positive(atomic64_t *v); +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive + +#endif /* !(__ARCH_SPARC64_ATOMIC__) */ diff --git a/arch/sparc/include/asm/auxio.h b/arch/sparc/include/asm/auxio.h new file mode 100644 index 000000000..d0a933ed0 --- /dev/null +++ b/arch/sparc/include/asm/auxio.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_AUXIO_H +#define ___ASM_SPARC_AUXIO_H + +#ifndef __ASSEMBLER__ + +extern void __iomem *auxio_register; + +#endif /* ifndef __ASSEMBLER__ */ + +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/auxio_32.h b/arch/sparc/include/asm/auxio_32.h new file mode 100644 index 000000000..db58fa28d --- /dev/null +++ b/arch/sparc/include/asm/auxio_32.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * auxio.h: Definitions and code for the Auxiliary I/O register. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ +#ifndef _SPARC_AUXIO_H +#define _SPARC_AUXIO_H + +#include + +/* This register is an unsigned char in IO space. It does two things. + * First, it is used to control the front panel LED light on machines + * that have it (good for testing entry points to trap handlers and irq's) + * Secondly, it controls various floppy drive parameters. + */ +#define AUXIO_ORMEIN 0xf0 /* All writes must set these bits. */ +#define AUXIO_ORMEIN4M 0xc0 /* sun4m - All writes must set these bits. */ +#define AUXIO_FLPY_DENS 0x20 /* Floppy density, high if set. Read only. */ +#define AUXIO_FLPY_DCHG 0x10 /* A disk change occurred. Read only. */ +#define AUXIO_EDGE_ON 0x10 /* sun4m - On means Jumper block is in. */ +#define AUXIO_FLPY_DSEL 0x08 /* Drive select/start-motor. Write only. */ +#define AUXIO_LINK_TEST 0x08 /* sun4m - On means TPE Carrier detect. */ + +/* Set the following to one, then zero, after doing a pseudo DMA transfer. */ +#define AUXIO_FLPY_TCNT 0x04 /* Floppy terminal count. Write only. */ + +/* Set the following to zero to eject the floppy. */ +#define AUXIO_FLPY_EJCT 0x02 /* Eject floppy disk. Write only. */ +#define AUXIO_LED 0x01 /* On if set, off if unset. Read/Write */ + +#ifndef __ASSEMBLER__ + +/* + * NOTE: these routines are implementation dependent-- + * understand the hardware you are querying! + */ +void set_auxio(unsigned char bits_on, unsigned char bits_off); +unsigned char get_auxio(void); /* .../asm/floppy.h */ + +/* + * The following routines are provided for driver-compatibility + * with sparc64 (primarily sunlance.c) + */ + +#define AUXIO_LTE_ON 1 +#define AUXIO_LTE_OFF 0 + +/* auxio_set_lte - Set Link Test Enable (TPE Link Detect) + * + * on - AUXIO_LTE_ON or AUXIO_LTE_OFF + */ +#define auxio_set_lte(on) \ +do { \ + if(on) { \ + set_auxio(AUXIO_LINK_TEST, 0); \ + } else { \ + set_auxio(0, AUXIO_LINK_TEST); \ + } \ +} while (0) + +#define AUXIO_LED_ON 1 +#define AUXIO_LED_OFF 0 + +/* auxio_set_led - Set system front panel LED + * + * on - AUXIO_LED_ON or AUXIO_LED_OFF + */ +#define auxio_set_led(on) \ +do { \ + if(on) { \ + set_auxio(AUXIO_LED, 0); \ + } else { \ + set_auxio(0, AUXIO_LED); \ + } \ +} while (0) + +#endif /* !(__ASSEMBLER__) */ + + +/* AUXIO2 (Power Off Control) */ +extern volatile u8 __iomem *auxio_power_register; + +#define AUXIO_POWER_DETECT_FAILURE 32 +#define AUXIO_POWER_CLEAR_FAILURE 2 +#define AUXIO_POWER_OFF 1 + + +#endif /* !(_SPARC_AUXIO_H) */ diff --git a/arch/sparc/include/asm/auxio_64.h b/arch/sparc/include/asm/auxio_64.h new file mode 100644 index 000000000..8a4ae07da --- /dev/null +++ b/arch/sparc/include/asm/auxio_64.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * auxio.h: Definitions and code for the Auxiliary I/O registers. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * + * Refactoring for unified NCR/PCIO support 2002 Eric Brower (ebrower@usa.net) + */ +#ifndef _SPARC64_AUXIO_H +#define _SPARC64_AUXIO_H + +/* AUXIO implementations: + * sbus-based NCR89C105 "Slavio" + * LED/Floppy (AUX1) register + * Power (AUX2) register + * + * ebus-based auxio on PCIO + * LED Auxio Register + * Power Auxio Register + * + * Register definitions from NCR _NCR89C105 Chip Specification_ + * + * SLAVIO AUX1 @ 0x1900000 + * ------------------------------------------------- + * | (R) | (R) | D | (R) | E | M | T | L | + * ------------------------------------------------- + * (R) - bit 7:6,4 are reserved and should be masked in s/w + * D - Floppy Density Sense (1=high density) R/O + * E - Link Test Enable, directly reflected on AT&T 7213 LTE pin + * M - Monitor/Mouse Mux, directly reflected on MON_MSE_MUX pin + * T - Terminal Count: sends TC pulse to 82077 floppy controller + * L - System LED on front panel (0=off, 1=on) + */ +#define AUXIO_AUX1_MASK 0xc0 /* Mask bits */ +#define AUXIO_AUX1_FDENS 0x20 /* Floppy Density Sense */ +#define AUXIO_AUX1_LTE 0x08 /* Link Test Enable */ +#define AUXIO_AUX1_MMUX 0x04 /* Monitor/Mouse Mux */ +#define AUXIO_AUX1_FTCNT 0x02 /* Terminal Count, */ +#define AUXIO_AUX1_LED 0x01 /* System LED */ + +/* SLAVIO AUX2 @ 0x1910000 + * ------------------------------------------------- + * | (R) | (R) | D | (R) | (R) | (R) | C | F | + * ------------------------------------------------- + * (R) - bits 7:6,4:2 are reserved and should be masked in s/w + * D - Power Failure Detect (1=power fail) + * C - Clear Power Failure Detect Int (1=clear) + * F - Power Off (1=power off) + */ +#define AUXIO_AUX2_MASK 0xdc /* Mask Bits */ +#define AUXIO_AUX2_PFAILDET 0x20 /* Power Fail Detect */ +#define AUXIO_AUX2_PFAILCLR 0x02 /* Clear Pwr Fail Det Intr */ +#define AUXIO_AUX2_PWR_OFF 0x01 /* Power Off */ + +/* Register definitions from Sun Microsystems _PCIO_ p/n 802-7837 + * + * PCIO LED Auxio @ 0x726000 + * ------------------------------------------------- + * | 31:1 Unused | LED | + * ------------------------------------------------- + * Bits 31:1 unused + * LED - System LED on front panel (0=off, 1=on) + */ +#define AUXIO_PCIO_LED 0x01 /* System LED */ + +/* PCIO Power Auxio @ 0x724000 + * ------------------------------------------------- + * | 31:2 Unused | CPO | SPO | + * ------------------------------------------------- + * Bits 31:2 unused + * CPO - Courtesy Power Off (1=off) + * SPO - System Power Off (1=off) + */ +#define AUXIO_PCIO_CPWR_OFF 0x02 /* Courtesy Power Off */ +#define AUXIO_PCIO_SPWR_OFF 0x01 /* System Power Off */ + +#ifndef __ASSEMBLER__ + +#define AUXIO_LTE_ON 1 +#define AUXIO_LTE_OFF 0 + +/* auxio_set_lte - Set Link Test Enable (TPE Link Detect) + * + * on - AUXIO_LTE_ON or AUXIO_LTE_OFF + */ +void auxio_set_lte(int on); + +#define AUXIO_LED_ON 1 +#define AUXIO_LED_OFF 0 + +/* auxio_set_led - Set system front panel LED + * + * on - AUXIO_LED_ON or AUXIO_LED_OFF + */ +void auxio_set_led(int on); + +#endif /* ifndef __ASSEMBLER__ */ + +#endif /* !(_SPARC64_AUXIO_H) */ diff --git a/arch/sparc/include/asm/backoff.h b/arch/sparc/include/asm/backoff.h new file mode 100644 index 000000000..597a22953 --- /dev/null +++ b/arch/sparc/include/asm/backoff.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_BACKOFF_H +#define _SPARC64_BACKOFF_H + +/* The macros in this file implement an exponential backoff facility + * for atomic operations. + * + * When multiple threads compete on an atomic operation, it is + * possible for one thread to be continually denied a successful + * completion of the compare-and-swap instruction. Heavily + * threaded cpu implementations like Niagara can compound this + * problem even further. + * + * When an atomic operation fails and needs to be retried, we spin a + * certain number of times. At each subsequent failure of the same + * operation we double the spin count, realizing an exponential + * backoff. + * + * When we spin, we try to use an operation that will cause the + * current cpu strand to block, and therefore make the core fully + * available to any other runnable strands. There are two + * options, based upon cpu capabilities. + * + * On all cpus prior to SPARC-T4 we do three dummy reads of the + * condition code register. Each read blocks the strand for something + * between 40 and 50 cpu cycles. + * + * For SPARC-T4 and later we have a special "pause" instruction + * available. This is implemented using writes to register %asr27. + * The cpu will block the number of cycles written into the register, + * unless a disrupting trap happens first. SPARC-T4 specifically + * implements pause with a granularity of 8 cycles. Each strand has + * an internal pause counter which decrements every 8 cycles. So the + * chip shifts the %asr27 value down by 3 bits, and writes the result + * into the pause counter. If a value smaller than 8 is written, the + * chip blocks for 1 cycle. + * + * To achieve the same amount of backoff as the three %ccr reads give + * on earlier chips, we shift the backoff value up by 7 bits. (Three + * %ccr reads block for about 128 cycles, 1 << 7 == 128) We write the + * whole amount we want to block into the pause register, rather than + * loop writing 128 each time. + */ + +#define BACKOFF_LIMIT (4 * 1024) + +#ifdef CONFIG_SMP + +#define BACKOFF_SETUP(reg) \ + mov 1, reg + +#define BACKOFF_LABEL(spin_label, continue_label) \ + spin_label + +#define BACKOFF_SPIN(reg, tmp, label) \ + mov reg, tmp; \ +88: rd %ccr, %g0; \ + rd %ccr, %g0; \ + rd %ccr, %g0; \ + .section .pause_3insn_patch,"ax";\ + .word 88b; \ + sllx tmp, 7, tmp; \ + wr tmp, 0, %asr27; \ + clr tmp; \ + .previous; \ + brnz,pt tmp, 88b; \ + sub tmp, 1, tmp; \ + set BACKOFF_LIMIT, tmp; \ + cmp reg, tmp; \ + bg,pn %xcc, label; \ + nop; \ + ba,pt %xcc, label; \ + sllx reg, 1, reg; + +#else + +#define BACKOFF_SETUP(reg) + +#define BACKOFF_LABEL(spin_label, continue_label) \ + continue_label + +#define BACKOFF_SPIN(reg, tmp, label) + +#endif + +#endif /* _SPARC64_BACKOFF_H */ diff --git a/arch/sparc/include/asm/barrier.h b/arch/sparc/include/asm/barrier.h new file mode 100644 index 000000000..c10b4164b --- /dev/null +++ b/arch/sparc/include/asm/barrier.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_BARRIER_H +#define ___ASM_SPARC_BARRIER_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/barrier_32.h b/arch/sparc/include/asm/barrier_32.h new file mode 100644 index 000000000..304f9c3f7 --- /dev/null +++ b/arch/sparc/include/asm/barrier_32.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_BARRIER_H +#define __SPARC_BARRIER_H + +#include + +#endif /* !(__SPARC_BARRIER_H) */ diff --git a/arch/sparc/include/asm/barrier_64.h b/arch/sparc/include/asm/barrier_64.h new file mode 100644 index 000000000..9fb148bd3 --- /dev/null +++ b/arch/sparc/include/asm/barrier_64.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC64_BARRIER_H +#define __SPARC64_BARRIER_H + +/* These are here in an effort to more fully work around Spitfire Errata + * #51. Essentially, if a memory barrier occurs soon after a mispredicted + * branch, the chip can stop executing instructions until a trap occurs. + * Therefore, if interrupts are disabled, the chip can hang forever. + * + * It used to be believed that the memory barrier had to be right in the + * delay slot, but a case has been traced recently wherein the memory barrier + * was one instruction after the branch delay slot and the chip still hung. + * The offending sequence was the following in sym_wakeup_done() of the + * sym53c8xx_2 driver: + * + * call sym_ccb_from_dsa, 0 + * movge %icc, 0, %l0 + * brz,pn %o0, .LL1303 + * mov %o0, %l2 + * membar #LoadLoad + * + * The branch has to be mispredicted for the bug to occur. Therefore, we put + * the memory barrier explicitly into a "branch always, predicted taken" + * delay slot to avoid the problem case. + */ +#define membar_safe(type) \ +do { __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" \ + " membar " type "\n" \ + "1:\n" \ + : : : "memory"); \ +} while (0) + +/* The kernel always executes in TSO memory model these days, + * and furthermore most sparc64 chips implement more stringent + * memory ordering than required by the specifications. + */ +#define mb() membar_safe("#StoreLoad") +#define rmb() __asm__ __volatile__("":::"memory") +#define wmb() __asm__ __volatile__("":::"memory") + +#define __smp_store_release(p, v) \ +do { \ + compiletime_assert_atomic_type(*p); \ + barrier(); \ + WRITE_ONCE(*p, v); \ +} while (0) + +#define __smp_load_acquire(p) \ +({ \ + typeof(*p) ___p1 = READ_ONCE(*p); \ + compiletime_assert_atomic_type(*p); \ + barrier(); \ + ___p1; \ +}) + +#define __smp_mb__before_atomic() barrier() +#define __smp_mb__after_atomic() barrier() + +#include + +#endif /* !(__SPARC64_BARRIER_H) */ diff --git a/arch/sparc/include/asm/bbc.h b/arch/sparc/include/asm/bbc.h new file mode 100644 index 000000000..00a4f1161 --- /dev/null +++ b/arch/sparc/include/asm/bbc.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * bbc.h: Defines for BootBus Controller found on UltraSPARC-III + * systems. + * + * Copyright (C) 2000 David S. Miller (davem@redhat.com) + */ + +#ifndef _SPARC64_BBC_H +#define _SPARC64_BBC_H + +/* Register sizes are indicated by "B" (Byte, 1-byte), + * "H" (Half-word, 2 bytes), "W" (Word, 4 bytes) or + * "Q" (Quad, 8 bytes) inside brackets. + */ + +#define BBC_AID 0x00 /* [B] Agent ID */ +#define BBC_DEVP 0x01 /* [B] Device Present */ +#define BBC_ARB 0x02 /* [B] Arbitration */ +#define BBC_QUIESCE 0x03 /* [B] Quiesce */ +#define BBC_WDACTION 0x04 /* [B] Watchdog Action */ +#define BBC_SPG 0x06 /* [B] Soft POR Gen */ +#define BBC_SXG 0x07 /* [B] Soft XIR Gen */ +#define BBC_PSRC 0x08 /* [W] POR Source */ +#define BBC_XSRC 0x0c /* [B] XIR Source */ +#define BBC_CSC 0x0d /* [B] Clock Synthesizers Control*/ +#define BBC_ES_CTRL 0x0e /* [H] Energy Star Control */ +#define BBC_ES_ACT 0x10 /* [W] E* Assert Change Time */ +#define BBC_ES_DACT 0x14 /* [B] E* De-Assert Change Time */ +#define BBC_ES_DABT 0x15 /* [B] E* De-Assert Bypass Time */ +#define BBC_ES_ABT 0x16 /* [H] E* Assert Bypass Time */ +#define BBC_ES_PST 0x18 /* [W] E* PLL Settle Time */ +#define BBC_ES_FSL 0x1c /* [W] E* Frequency Switch Latency*/ +#define BBC_EBUST 0x20 /* [Q] EBUS Timing */ +#define BBC_JTAG_CMD 0x28 /* [W] JTAG+ Command */ +#define BBC_JTAG_CTRL 0x2c /* [B] JTAG+ Control */ +#define BBC_I2C_SEL 0x2d /* [B] I2C Selection */ +#define BBC_I2C_0_S1 0x2e /* [B] I2C ctrlr-0 reg S1 */ +#define BBC_I2C_0_S0 0x2f /* [B] I2C ctrlr-0 regs S0,S0',S2,S3*/ +#define BBC_I2C_1_S1 0x30 /* [B] I2C ctrlr-1 reg S1 */ +#define BBC_I2C_1_S0 0x31 /* [B] I2C ctrlr-1 regs S0,S0',S2,S3*/ +#define BBC_KBD_BEEP 0x32 /* [B] Keyboard Beep */ +#define BBC_KBD_BCNT 0x34 /* [W] Keyboard Beep Counter */ + +#define BBC_REGS_SIZE 0x40 + +/* There is a 2K scratch ram area at offset 0x80000 but I doubt + * we will use it for anything. + */ + +/* Agent ID register. This register shows the Safari Agent ID + * for the processors. The value returned depends upon which + * cpu is reading the register. + */ +#define BBC_AID_ID 0x07 /* Safari ID */ +#define BBC_AID_RESV 0xf8 /* Reserved */ + +/* Device Present register. One can determine which cpus are actually + * present in the machine by interrogating this register. + */ +#define BBC_DEVP_CPU0 0x01 /* Processor 0 present */ +#define BBC_DEVP_CPU1 0x02 /* Processor 1 present */ +#define BBC_DEVP_CPU2 0x04 /* Processor 2 present */ +#define BBC_DEVP_CPU3 0x08 /* Processor 3 present */ +#define BBC_DEVP_RESV 0xf0 /* Reserved */ + +/* Arbitration register. This register is used to block access to + * the BBC from a particular cpu. + */ +#define BBC_ARB_CPU0 0x01 /* Enable cpu 0 BBC arbitratrion */ +#define BBC_ARB_CPU1 0x02 /* Enable cpu 1 BBC arbitratrion */ +#define BBC_ARB_CPU2 0x04 /* Enable cpu 2 BBC arbitratrion */ +#define BBC_ARB_CPU3 0x08 /* Enable cpu 3 BBC arbitratrion */ +#define BBC_ARB_RESV 0xf0 /* Reserved */ + +/* Quiesce register. Bus and BBC segments for cpus can be disabled + * with this register, ie. for hot plugging. + */ +#define BBC_QUIESCE_S02 0x01 /* Quiesce Safari segment for cpu 0 and 2 */ +#define BBC_QUIESCE_S13 0x02 /* Quiesce Safari segment for cpu 1 and 3 */ +#define BBC_QUIESCE_B02 0x04 /* Quiesce BBC segment for cpu 0 and 2 */ +#define BBC_QUIESCE_B13 0x08 /* Quiesce BBC segment for cpu 1 and 3 */ +#define BBC_QUIESCE_FD0 0x10 /* Disable Fatal_Error[0] reporting */ +#define BBC_QUIESCE_FD1 0x20 /* Disable Fatal_Error[1] reporting */ +#define BBC_QUIESCE_FD2 0x40 /* Disable Fatal_Error[2] reporting */ +#define BBC_QUIESCE_FD3 0x80 /* Disable Fatal_Error[3] reporting */ + +/* Watchdog Action register. When the watchdog device timer expires + * a line is enabled to the BBC. The action BBC takes when this line + * is asserted can be controlled by this regiser. + */ +#define BBC_WDACTION_RST 0x01 /* When set, watchdog causes system reset. + * When clear, BBC ignores watchdog signal. + */ +#define BBC_WDACTION_RESV 0xfe /* Reserved */ + +/* Soft_POR_GEN register. The POR (Power On Reset) signal may be asserted + * for specific processors or all processors via this register. + */ +#define BBC_SPG_CPU0 0x01 /* Assert POR for processor 0 */ +#define BBC_SPG_CPU1 0x02 /* Assert POR for processor 1 */ +#define BBC_SPG_CPU2 0x04 /* Assert POR for processor 2 */ +#define BBC_SPG_CPU3 0x08 /* Assert POR for processor 3 */ +#define BBC_SPG_CPUALL 0x10 /* Reset all processors and reset + * the entire system. + */ +#define BBC_SPG_RESV 0xe0 /* Reserved */ + +/* Soft_XIR_GEN register. The XIR (eXternally Initiated Reset) signal + * may be asserted to specific processors via this register. + */ +#define BBC_SXG_CPU0 0x01 /* Assert XIR for processor 0 */ +#define BBC_SXG_CPU1 0x02 /* Assert XIR for processor 1 */ +#define BBC_SXG_CPU2 0x04 /* Assert XIR for processor 2 */ +#define BBC_SXG_CPU3 0x08 /* Assert XIR for processor 3 */ +#define BBC_SXG_RESV 0xf0 /* Reserved */ + +/* POR Source register. One may identify the cause of the most recent + * reset by reading this register. + */ +#define BBC_PSRC_SPG0 0x0001 /* CPU 0 reset via BBC_SPG register */ +#define BBC_PSRC_SPG1 0x0002 /* CPU 1 reset via BBC_SPG register */ +#define BBC_PSRC_SPG2 0x0004 /* CPU 2 reset via BBC_SPG register */ +#define BBC_PSRC_SPG3 0x0008 /* CPU 3 reset via BBC_SPG register */ +#define BBC_PSRC_SPGSYS 0x0010 /* System reset via BBC_SPG register */ +#define BBC_PSRC_JTAG 0x0020 /* System reset via JTAG+ */ +#define BBC_PSRC_BUTTON 0x0040 /* System reset via push-button dongle */ +#define BBC_PSRC_PWRUP 0x0080 /* System reset via power-up */ +#define BBC_PSRC_FE0 0x0100 /* CPU 0 reported Fatal_Error */ +#define BBC_PSRC_FE1 0x0200 /* CPU 1 reported Fatal_Error */ +#define BBC_PSRC_FE2 0x0400 /* CPU 2 reported Fatal_Error */ +#define BBC_PSRC_FE3 0x0800 /* CPU 3 reported Fatal_Error */ +#define BBC_PSRC_FE4 0x1000 /* Schizo reported Fatal_Error */ +#define BBC_PSRC_FE5 0x2000 /* Safari device 5 reported Fatal_Error */ +#define BBC_PSRC_FE6 0x4000 /* CPMS reported Fatal_Error */ +#define BBC_PSRC_SYNTH 0x8000 /* System reset when on-board clock synthesizers + * were updated. + */ +#define BBC_PSRC_WDT 0x10000 /* System reset via Super I/O watchdog */ +#define BBC_PSRC_RSC 0x20000 /* System reset via RSC remote monitoring + * device + */ + +/* XIR Source register. The source of an XIR event sent to a processor may + * be determined via this register. + */ +#define BBC_XSRC_SXG0 0x01 /* CPU 0 received XIR via Soft_XIR_GEN reg */ +#define BBC_XSRC_SXG1 0x02 /* CPU 1 received XIR via Soft_XIR_GEN reg */ +#define BBC_XSRC_SXG2 0x04 /* CPU 2 received XIR via Soft_XIR_GEN reg */ +#define BBC_XSRC_SXG3 0x08 /* CPU 3 received XIR via Soft_XIR_GEN reg */ +#define BBC_XSRC_JTAG 0x10 /* All CPUs received XIR via JTAG+ */ +#define BBC_XSRC_W_OR_B 0x20 /* All CPUs received XIR either because: + * a) Super I/O watchdog fired, or + * b) XIR push button was activated + */ +#define BBC_XSRC_RESV 0xc0 /* Reserved */ + +/* Clock Synthesizers Control register. This register provides the big-bang + * programming interface to the two clock synthesizers of the machine. + */ +#define BBC_CSC_SLOAD 0x01 /* Directly connected to S_LOAD pins */ +#define BBC_CSC_SDATA 0x02 /* Directly connected to S_DATA pins */ +#define BBC_CSC_SCLOCK 0x04 /* Directly connected to S_CLOCK pins */ +#define BBC_CSC_RESV 0x78 /* Reserved */ +#define BBC_CSC_RST 0x80 /* Generate system reset when S_LOAD==1 */ + +/* Energy Star Control register. This register is used to generate the + * clock frequency change trigger to the main system devices (Schizo and + * the processors). The transition occurs when bits in this register + * go from 0 to 1, only one bit must be set at once else no action + * occurs. Basically the sequence of events is: + * a) Choose new frequency: full, 1/2 or 1/32 + * b) Program this desired frequency into the cpus and Schizo. + * c) Set the same value in this register. + * d) 16 system clocks later, clear this register. + */ +#define BBC_ES_CTRL_1_1 0x01 /* Full frequency */ +#define BBC_ES_CTRL_1_2 0x02 /* 1/2 frequency */ +#define BBC_ES_CTRL_1_32 0x20 /* 1/32 frequency */ +#define BBC_ES_RESV 0xdc /* Reserved */ + +/* Energy Star Assert Change Time register. This determines the number + * of BBC clock cycles (which is half the system frequency) between + * the detection of FREEZE_ACK being asserted and the assertion of + * the CLK_CHANGE_L[2:0] signals. + */ +#define BBC_ES_ACT_VAL 0xff + +/* Energy Star Assert Bypass Time register. This determines the number + * of BBC clock cycles (which is half the system frequency) between + * the assertion of the CLK_CHANGE_L[2:0] signals and the assertion of + * the ESTAR_PLL_BYPASS signal. + */ +#define BBC_ES_ABT_VAL 0xffff + +/* Energy Star PLL Settle Time register. This determines the number of + * BBC clock cycles (which is half the system frequency) between the + * de-assertion of CLK_CHANGE_L[2:0] and the de-assertion of the FREEZE_L + * signal. + */ +#define BBC_ES_PST_VAL 0xffffffff + +/* Energy Star Frequency Switch Latency register. This is the number of + * BBC clocks between the de-assertion of CLK_CHANGE_L[2:0] and the first + * edge of the Safari clock at the new frequency. + */ +#define BBC_ES_FSL_VAL 0xffffffff + +/* Keyboard Beep control register. This is a simple enabler for the audio + * beep sound. + */ +#define BBC_KBD_BEEP_ENABLE 0x01 /* Enable beep */ +#define BBC_KBD_BEEP_RESV 0xfe /* Reserved */ + +/* Keyboard Beep Counter register. There is a free-running counter inside + * the BBC which runs at half the system clock. The bit set in this register + * determines when the audio sound is generated. So for example if bit + * 10 is set, the audio beep will oscillate at 1/(2**12). The keyboard beep + * generator automatically selects a different bit to use if the system clock + * is changed via Energy Star. + */ +#define BBC_KBD_BCNT_BITS 0x0007fc00 +#define BBC_KBC_BCNT_RESV 0xfff803ff + +#endif /* _SPARC64_BBC_H */ + diff --git a/arch/sparc/include/asm/bitext.h b/arch/sparc/include/asm/bitext.h new file mode 100644 index 000000000..2c2a2d85e --- /dev/null +++ b/arch/sparc/include/asm/bitext.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * bitext.h: Bit string operations on the sparc, specific to architecture. + * + * Copyright 2002 Pete Zaitcev + */ + +#ifndef _SPARC_BITEXT_H +#define _SPARC_BITEXT_H + +#include + +struct bit_map { + spinlock_t lock; + unsigned long *map; + int size; + int used; + int last_off; + int last_size; + int first_free; + int num_colors; +}; + +int bit_map_string_get(struct bit_map *t, int len, int align); +void bit_map_clear(struct bit_map *t, int offset, int len); +void bit_map_init(struct bit_map *t, unsigned long *map, int size); + +#endif /* defined(_SPARC_BITEXT_H) */ diff --git a/arch/sparc/include/asm/bitops.h b/arch/sparc/include/asm/bitops.h new file mode 100644 index 000000000..4c431d274 --- /dev/null +++ b/arch/sparc/include/asm/bitops.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_BITOPS_H +#define ___ASM_SPARC_BITOPS_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/bitops_32.h b/arch/sparc/include/asm/bitops_32.h new file mode 100644 index 000000000..3448c191b --- /dev/null +++ b/arch/sparc/include/asm/bitops_32.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * bitops.h: Bit string operations on the Sparc. + * + * Copyright 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright 1996 Eddie C. Dost (ecd@skynet.be) + * Copyright 2001 Anton Blanchard (anton@samba.org) + */ + +#ifndef _SPARC_BITOPS_H +#define _SPARC_BITOPS_H + +#include +#include + +#ifdef __KERNEL__ + +#ifndef _LINUX_BITOPS_H +#error only can be included directly +#endif + +unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask); +unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask); + +/* + * Set bit 'nr' in 32-bit quantity at address 'addr' where bit '0' + * is in the highest of the four bytes and bit '31' is the high bit + * within the first byte. Sparc is BIG-Endian. Unless noted otherwise + * all bit-ops return 0 if bit was previously clear and != 0 otherwise. + */ +static inline int test_and_set_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long *ADDR, mask; + + ADDR = ((unsigned long *) addr) + (nr >> 5); + mask = 1 << (nr & 31); + + return sp32___set_bit(ADDR, mask) != 0; +} + +static inline void set_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long *ADDR, mask; + + ADDR = ((unsigned long *) addr) + (nr >> 5); + mask = 1 << (nr & 31); + + (void) sp32___set_bit(ADDR, mask); +} + +static inline int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long *ADDR, mask; + + ADDR = ((unsigned long *) addr) + (nr >> 5); + mask = 1 << (nr & 31); + + return sp32___clear_bit(ADDR, mask) != 0; +} + +static inline void clear_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long *ADDR, mask; + + ADDR = ((unsigned long *) addr) + (nr >> 5); + mask = 1 << (nr & 31); + + (void) sp32___clear_bit(ADDR, mask); +} + +static inline int test_and_change_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long *ADDR, mask; + + ADDR = ((unsigned long *) addr) + (nr >> 5); + mask = 1 << (nr & 31); + + return sp32___change_bit(ADDR, mask) != 0; +} + +static inline void change_bit(unsigned long nr, volatile unsigned long *addr) +{ + unsigned long *ADDR, mask; + + ADDR = ((unsigned long *) addr) + (nr >> 5); + mask = 1 << (nr & 31); + + (void) sp32___change_bit(ADDR, mask); +} + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif /* __KERNEL__ */ + +#endif /* defined(_SPARC_BITOPS_H) */ diff --git a/arch/sparc/include/asm/bitops_64.h b/arch/sparc/include/asm/bitops_64.h new file mode 100644 index 000000000..2c7d33b3e --- /dev/null +++ b/arch/sparc/include/asm/bitops_64.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * bitops.h: Bit string operations on the V9. + * + * Copyright 1996, 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC64_BITOPS_H +#define _SPARC64_BITOPS_H + +#ifndef _LINUX_BITOPS_H +#error only can be included directly +#endif + +#include +#include +#include + +int test_and_set_bit(unsigned long nr, volatile unsigned long *addr); +int test_and_clear_bit(unsigned long nr, volatile unsigned long *addr); +int test_and_change_bit(unsigned long nr, volatile unsigned long *addr); +void set_bit(unsigned long nr, volatile unsigned long *addr); +void clear_bit(unsigned long nr, volatile unsigned long *addr); +void change_bit(unsigned long nr, volatile unsigned long *addr); + +int __attribute_const__ fls(unsigned int word); +int __attribute_const__ __fls(unsigned long word); + +#include + +#include + +#ifdef __KERNEL__ + +int __attribute_const__ ffs(int x); +unsigned long __attribute_const__ __ffs(unsigned long); + +#include +#include + +/* + * hweightN: returns the hamming weight (i.e. the number + * of bits set) of a N-bit word + */ + +unsigned long __arch_hweight64(__u64 w); +unsigned int __arch_hweight32(unsigned int w); +unsigned int __arch_hweight16(unsigned int w); +unsigned int __arch_hweight8(unsigned int w); + +#include +#include +#endif /* __KERNEL__ */ + +#ifdef __KERNEL__ + +#include + +#include + +#endif /* __KERNEL__ */ + +#endif /* defined(_SPARC64_BITOPS_H) */ diff --git a/arch/sparc/include/asm/btext.h b/arch/sparc/include/asm/btext.h new file mode 100644 index 000000000..01cdc4265 --- /dev/null +++ b/arch/sparc/include/asm/btext.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_BTEXT_H +#define _SPARC_BTEXT_H + +int btext_find_display(void); + +#endif /* _SPARC_BTEXT_H */ diff --git a/arch/sparc/include/asm/bug.h b/arch/sparc/include/asm/bug.h new file mode 100644 index 000000000..ea53e418f --- /dev/null +++ b/arch/sparc/include/asm/bug.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_BUG_H +#define _SPARC_BUG_H + +#ifdef CONFIG_BUG +#include + +#ifdef CONFIG_DEBUG_BUGVERBOSE +void do_BUG(const char *file, int line); +#define BUG() do { \ + do_BUG(__FILE__, __LINE__); \ + barrier_before_unreachable(); \ + __builtin_trap(); \ +} while (0) +#else +#define BUG() do { \ + barrier_before_unreachable(); \ + __builtin_trap(); \ +} while (0) +#endif + +#define HAVE_ARCH_BUG +#endif + +#include + +struct pt_regs; +void __noreturn die_if_kernel(char *str, struct pt_regs *regs); + +#endif diff --git a/arch/sparc/include/asm/cache.h b/arch/sparc/include/asm/cache.h new file mode 100644 index 000000000..e62fd0e72 --- /dev/null +++ b/arch/sparc/include/asm/cache.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* cache.h: Cache specific code for the Sparc. These include flushing + * and direct tag/data line access. + * + * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) + */ + +#ifndef _SPARC_CACHE_H +#define _SPARC_CACHE_H + +#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) + +#define L1_CACHE_SHIFT 5 +#define L1_CACHE_BYTES 32 + +#ifdef CONFIG_SPARC32 +#define SMP_CACHE_BYTES_SHIFT 5 +#else +#define SMP_CACHE_BYTES_SHIFT 6 +#endif + +#define SMP_CACHE_BYTES (1 << SMP_CACHE_BYTES_SHIFT) + +#define __read_mostly __section(".data..read_mostly") + +#endif /* !(_SPARC_CACHE_H) */ diff --git a/arch/sparc/include/asm/cacheflush.h b/arch/sparc/include/asm/cacheflush.h new file mode 100644 index 000000000..881ac76ea --- /dev/null +++ b/arch/sparc/include/asm/cacheflush.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_CACHEFLUSH_H +#define ___ASM_SPARC_CACHEFLUSH_H + +/* flush addr - to allow use of self-modifying code */ +#define flushi(addr) __asm__ __volatile__ ("flush %0" : : "r" (addr) : "memory") + +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/cacheflush_32.h b/arch/sparc/include/asm/cacheflush_32.h new file mode 100644 index 000000000..9fee0ccfc --- /dev/null +++ b/arch/sparc/include/asm/cacheflush_32.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_CACHEFLUSH_H +#define _SPARC_CACHEFLUSH_H + +#include +#include + +#define flush_cache_all() \ + sparc32_cachetlb_ops->cache_all() +#define flush_cache_mm(mm) \ + sparc32_cachetlb_ops->cache_mm(mm) +#define flush_cache_dup_mm(mm) \ + sparc32_cachetlb_ops->cache_mm(mm) +#define flush_cache_range(vma,start,end) \ + sparc32_cachetlb_ops->cache_range(vma, start, end) +#define flush_cache_page(vma,addr,pfn) \ + sparc32_cachetlb_ops->cache_page(vma, addr) +#define flush_icache_range(start, end) do { } while (0) + +#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ + do { \ + flush_cache_page(vma, vaddr, page_to_pfn(page));\ + memcpy(dst, src, len); \ + } while (0) +#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ + do { \ + flush_cache_page(vma, vaddr, page_to_pfn(page));\ + memcpy(dst, src, len); \ + } while (0) + +#define __flush_page_to_ram(addr) \ + sparc32_cachetlb_ops->page_to_ram(addr) +#define flush_sig_insns(mm,insn_addr) \ + sparc32_cachetlb_ops->sig_insns(mm, insn_addr) +#define flush_page_for_dma(addr) \ + sparc32_cachetlb_ops->page_for_dma(addr) + +void sparc_flush_page_to_ram(struct page *page); +void sparc_flush_folio_to_ram(struct folio *folio); + +#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 +#define flush_dcache_folio(folio) sparc_flush_folio_to_ram(folio) +static inline void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); +} +#define flush_dcache_mmap_lock(mapping) do { } while (0) +#define flush_dcache_mmap_unlock(mapping) do { } while (0) + +#define flush_cache_vmap(start, end) flush_cache_all() +#define flush_cache_vmap_early(start, end) do { } while (0) +#define flush_cache_vunmap(start, end) flush_cache_all() + +/* When a context switch happens we must flush all user windows so that + * the windows of the current process are flushed onto its stack. This + * way the windows are all clean for the next process and the stack + * frames are up to date. + */ +void flush_user_windows(void); +void kill_user_windows(void); +void flushw_all(void); + +#endif /* _SPARC_CACHEFLUSH_H */ diff --git a/arch/sparc/include/asm/cacheflush_64.h b/arch/sparc/include/asm/cacheflush_64.h new file mode 100644 index 000000000..06092572c --- /dev/null +++ b/arch/sparc/include/asm/cacheflush_64.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_CACHEFLUSH_H +#define _SPARC64_CACHEFLUSH_H + +#include + +#ifndef __ASSEMBLER__ + +#include + +/* Cache flush operations. */ +#define flushw_all() __asm__ __volatile__("flushw") + +void __flushw_user(void); +#define flushw_user() __flushw_user() + +#define flush_user_windows flushw_user +#define flush_register_windows flushw_all + +/* These are the same regardless of whether this is an SMP kernel or not. */ +#define flush_cache_mm(__mm) \ + do { if ((__mm) == current->mm) flushw_user(); } while(0) +#define flush_cache_dup_mm(mm) flush_cache_mm(mm) +#define flush_cache_range(vma, start, end) \ + flush_cache_mm((vma)->vm_mm) +#define flush_cache_page(vma, page, pfn) \ + flush_cache_mm((vma)->vm_mm) + +/* + * On spitfire, the icache doesn't snoop local stores and we don't + * use block commit stores (which invalidate icache lines) during + * module load, so we need this. + */ +void flush_icache_range(unsigned long start, unsigned long end); +void __flush_icache_page(unsigned long); + +void __flush_dcache_page(void *addr, int flush_icache); +void flush_dcache_folio_impl(struct folio *folio); +#ifdef CONFIG_SMP +void smp_flush_dcache_folio_impl(struct folio *folio, int cpu); +void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio); +#else +#define smp_flush_dcache_folio_impl(folio, cpu) flush_dcache_folio_impl(folio) +#define flush_dcache_folio_all(mm, folio) flush_dcache_folio_impl(folio) +#endif + +void __flush_dcache_range(unsigned long start, unsigned long end); +#define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 +void flush_dcache_folio(struct folio *folio); +#define flush_dcache_folio flush_dcache_folio +static inline void flush_dcache_page(struct page *page) +{ + flush_dcache_folio(page_folio(page)); +} + +void flush_ptrace_access(struct vm_area_struct *, struct page *, + unsigned long uaddr, void *kaddr, + unsigned long len, int write); + +#define copy_to_user_page(vma, page, vaddr, dst, src, len) \ + do { \ + flush_cache_page(vma, vaddr, page_to_pfn(page)); \ + memcpy(dst, src, len); \ + flush_ptrace_access(vma, page, vaddr, src, len, 0); \ + } while (0) + +#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ + do { \ + flush_cache_page(vma, vaddr, page_to_pfn(page)); \ + memcpy(dst, src, len); \ + flush_ptrace_access(vma, page, vaddr, dst, len, 1); \ + } while (0) + +#define flush_dcache_mmap_lock(mapping) do { } while (0) +#define flush_dcache_mmap_unlock(mapping) do { } while (0) + +#define flush_cache_vmap(start, end) do { } while (0) +#define flush_cache_vmap_early(start, end) do { } while (0) +#define flush_cache_vunmap(start, end) do { } while (0) + +#endif /* !__ASSEMBLER__ */ + +#endif /* _SPARC64_CACHEFLUSH_H */ diff --git a/arch/sparc/include/asm/cachetlb_32.h b/arch/sparc/include/asm/cachetlb_32.h new file mode 100644 index 000000000..534da70c6 --- /dev/null +++ b/arch/sparc/include/asm/cachetlb_32.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_CACHETLB_H +#define _SPARC_CACHETLB_H + +struct mm_struct; +struct vm_area_struct; + +struct sparc32_cachetlb_ops { + void (*cache_all)(void); + void (*cache_mm)(struct mm_struct *); + void (*cache_range)(struct vm_area_struct *, unsigned long, + unsigned long); + void (*cache_page)(struct vm_area_struct *, unsigned long); + + void (*tlb_all)(void); + void (*tlb_mm)(struct mm_struct *); + void (*tlb_range)(struct vm_area_struct *, unsigned long, + unsigned long); + void (*tlb_page)(struct vm_area_struct *, unsigned long); + + void (*page_to_ram)(unsigned long); + void (*sig_insns)(struct mm_struct *, unsigned long); + void (*page_for_dma)(unsigned long); +}; +extern const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops; +#ifdef CONFIG_SMP +extern const struct sparc32_cachetlb_ops *local_ops; +#endif + +#endif /* SPARC_CACHETLB_H */ diff --git a/arch/sparc/include/asm/cachetype.h b/arch/sparc/include/asm/cachetype.h new file mode 100644 index 000000000..caf1c0045 --- /dev/null +++ b/arch/sparc/include/asm/cachetype.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SPARC_CACHETYPE_H +#define __ASM_SPARC_CACHETYPE_H + +#include + +#ifdef CONFIG_SPARC32 +extern int vac_cache_size; +#define cpu_dcache_is_aliasing() (vac_cache_size > PAGE_SIZE) +#else +#define cpu_dcache_is_aliasing() (L1DCACHE_SIZE > PAGE_SIZE) +#endif + +#endif diff --git a/arch/sparc/include/asm/chafsr.h b/arch/sparc/include/asm/chafsr.h new file mode 100644 index 000000000..01540eca1 --- /dev/null +++ b/arch/sparc/include/asm/chafsr.h @@ -0,0 +1,242 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_CHAFSR_H +#define _SPARC64_CHAFSR_H + +/* Cheetah Asynchronous Fault Status register, ASI=0x4C VA<63:0>=0x0 */ + +/* Comments indicate which processor variants on which the bit definition + * is valid. Codes are: + * ch --> cheetah + * ch+ --> cheetah plus + * jp --> jalapeno + */ + +/* All bits of this register except M_SYNDROME and E_SYNDROME are + * read, write 1 to clear. M_SYNDROME and E_SYNDROME are read-only. + */ + +/* Software bit set by linux trap handlers to indicate that the trap was + * signalled at %tl >= 1. + */ +#define CHAFSR_TL1 (1UL << 63UL) /* n/a */ + +/* Unmapped error from system bus for prefetch queue or + * store queue read operation + */ +#define CHPAFSR_DTO (1UL << 59UL) /* ch+ */ + +/* Bus error from system bus for prefetch queue or store queue + * read operation + */ +#define CHPAFSR_DBERR (1UL << 58UL) /* ch+ */ + +/* Hardware corrected E-cache Tag ECC error */ +#define CHPAFSR_THCE (1UL << 57UL) /* ch+ */ +/* System interface protocol error, hw timeout caused */ +#define JPAFSR_JETO (1UL << 57UL) /* jp */ + +/* SW handled correctable E-cache Tag ECC error */ +#define CHPAFSR_TSCE (1UL << 56UL) /* ch+ */ +/* Parity error on system snoop results */ +#define JPAFSR_SCE (1UL << 56UL) /* jp */ + +/* Uncorrectable E-cache Tag ECC error */ +#define CHPAFSR_TUE (1UL << 55UL) /* ch+ */ +/* System interface protocol error, illegal command detected */ +#define JPAFSR_JEIC (1UL << 55UL) /* jp */ + +/* Uncorrectable system bus data ECC error due to prefetch + * or store fill request + */ +#define CHPAFSR_DUE (1UL << 54UL) /* ch+ */ +/* System interface protocol error, illegal ADTYPE detected */ +#define JPAFSR_JEIT (1UL << 54UL) /* jp */ + +/* Multiple errors of the same type have occurred. This bit is set when + * an uncorrectable error or a SW correctable error occurs and the status + * bit to report that error is already set. When multiple errors of + * different types are indicated by setting multiple status bits. + * + * This bit is not set if multiple HW corrected errors with the same + * status bit occur, only uncorrectable and SW correctable ones have + * this behavior. + * + * This bit is not set when multiple ECC errors happen within a single + * 64-byte system bus transaction. Only the first ECC error in a 16-byte + * subunit will be logged. All errors in subsequent 16-byte subunits + * from the same 64-byte transaction are ignored. + */ +#define CHAFSR_ME (1UL << 53UL) /* ch,ch+,jp */ + +/* Privileged state error has occurred. This is a capture of PSTATE.PRIV + * at the time the error is detected. + */ +#define CHAFSR_PRIV (1UL << 52UL) /* ch,ch+,jp */ + +/* The following bits 51 (CHAFSR_PERR) to 33 (CHAFSR_CE) are sticky error + * bits and record the most recently detected errors. Bits accumulate + * errors that have been detected since the last write to clear the bit. + */ + +/* System interface protocol error. The processor asserts its' ERROR + * pin when this event occurs and it also logs a specific cause code + * into a JTAG scannable flop. + */ +#define CHAFSR_PERR (1UL << 51UL) /* ch,ch+,jp */ + +/* Internal processor error. The processor asserts its' ERROR + * pin when this event occurs and it also logs a specific cause code + * into a JTAG scannable flop. + */ +#define CHAFSR_IERR (1UL << 50UL) /* ch,ch+,jp */ + +/* System request parity error on incoming address */ +#define CHAFSR_ISAP (1UL << 49UL) /* ch,ch+,jp */ + +/* HW Corrected system bus MTAG ECC error */ +#define CHAFSR_EMC (1UL << 48UL) /* ch,ch+ */ +/* Parity error on L2 cache tag SRAM */ +#define JPAFSR_ETP (1UL << 48UL) /* jp */ + +/* Uncorrectable system bus MTAG ECC error */ +#define CHAFSR_EMU (1UL << 47UL) /* ch,ch+ */ +/* Out of range memory error has occurred */ +#define JPAFSR_OM (1UL << 47UL) /* jp */ + +/* HW Corrected system bus data ECC error for read of interrupt vector */ +#define CHAFSR_IVC (1UL << 46UL) /* ch,ch+ */ +/* Error due to unsupported store */ +#define JPAFSR_UMS (1UL << 46UL) /* jp */ + +/* Uncorrectable system bus data ECC error for read of interrupt vector */ +#define CHAFSR_IVU (1UL << 45UL) /* ch,ch+,jp */ + +/* Unmapped error from system bus */ +#define CHAFSR_TO (1UL << 44UL) /* ch,ch+,jp */ + +/* Bus error response from system bus */ +#define CHAFSR_BERR (1UL << 43UL) /* ch,ch+,jp */ + +/* SW Correctable E-cache ECC error for instruction fetch or data access + * other than block load. + */ +#define CHAFSR_UCC (1UL << 42UL) /* ch,ch+,jp */ + +/* Uncorrectable E-cache ECC error for instruction fetch or data access + * other than block load. + */ +#define CHAFSR_UCU (1UL << 41UL) /* ch,ch+,jp */ + +/* Copyout HW Corrected ECC error */ +#define CHAFSR_CPC (1UL << 40UL) /* ch,ch+,jp */ + +/* Copyout Uncorrectable ECC error */ +#define CHAFSR_CPU (1UL << 39UL) /* ch,ch+,jp */ + +/* HW Corrected ECC error from E-cache for writeback */ +#define CHAFSR_WDC (1UL << 38UL) /* ch,ch+,jp */ + +/* Uncorrectable ECC error from E-cache for writeback */ +#define CHAFSR_WDU (1UL << 37UL) /* ch,ch+,jp */ + +/* HW Corrected ECC error from E-cache for store merge or block load */ +#define CHAFSR_EDC (1UL << 36UL) /* ch,ch+,jp */ + +/* Uncorrectable ECC error from E-cache for store merge or block load */ +#define CHAFSR_EDU (1UL << 35UL) /* ch,ch+,jp */ + +/* Uncorrectable system bus data ECC error for read of memory or I/O */ +#define CHAFSR_UE (1UL << 34UL) /* ch,ch+,jp */ + +/* HW Corrected system bus data ECC error for read of memory or I/O */ +#define CHAFSR_CE (1UL << 33UL) /* ch,ch+,jp */ + +/* Uncorrectable ECC error from remote cache/memory */ +#define JPAFSR_RUE (1UL << 32UL) /* jp */ + +/* Correctable ECC error from remote cache/memory */ +#define JPAFSR_RCE (1UL << 31UL) /* jp */ + +/* JBUS parity error on returned read data */ +#define JPAFSR_BP (1UL << 30UL) /* jp */ + +/* JBUS parity error on data for writeback or block store */ +#define JPAFSR_WBP (1UL << 29UL) /* jp */ + +/* Foreign read to DRAM incurring correctable ECC error */ +#define JPAFSR_FRC (1UL << 28UL) /* jp */ + +/* Foreign read to DRAM incurring uncorrectable ECC error */ +#define JPAFSR_FRU (1UL << 27UL) /* jp */ + +#define CHAFSR_ERRORS (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP | CHAFSR_EMC | \ + CHAFSR_EMU | CHAFSR_IVC | CHAFSR_IVU | CHAFSR_TO | \ + CHAFSR_BERR | CHAFSR_UCC | CHAFSR_UCU | CHAFSR_CPC | \ + CHAFSR_CPU | CHAFSR_WDC | CHAFSR_WDU | CHAFSR_EDC | \ + CHAFSR_EDU | CHAFSR_UE | CHAFSR_CE) +#define CHPAFSR_ERRORS (CHPAFSR_DTO | CHPAFSR_DBERR | CHPAFSR_THCE | \ + CHPAFSR_TSCE | CHPAFSR_TUE | CHPAFSR_DUE | \ + CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP | CHAFSR_EMC | \ + CHAFSR_EMU | CHAFSR_IVC | CHAFSR_IVU | CHAFSR_TO | \ + CHAFSR_BERR | CHAFSR_UCC | CHAFSR_UCU | CHAFSR_CPC | \ + CHAFSR_CPU | CHAFSR_WDC | CHAFSR_WDU | CHAFSR_EDC | \ + CHAFSR_EDU | CHAFSR_UE | CHAFSR_CE) +#define JPAFSR_ERRORS (JPAFSR_JETO | JPAFSR_SCE | JPAFSR_JEIC | \ + JPAFSR_JEIT | CHAFSR_PERR | CHAFSR_IERR | \ + CHAFSR_ISAP | JPAFSR_ETP | JPAFSR_OM | \ + JPAFSR_UMS | CHAFSR_IVU | CHAFSR_TO | \ + CHAFSR_BERR | CHAFSR_UCC | CHAFSR_UCU | \ + CHAFSR_CPC | CHAFSR_CPU | CHAFSR_WDC | \ + CHAFSR_WDU | CHAFSR_EDC | CHAFSR_EDU | \ + CHAFSR_UE | CHAFSR_CE | JPAFSR_RUE | \ + JPAFSR_RCE | JPAFSR_BP | JPAFSR_WBP | \ + JPAFSR_FRC | JPAFSR_FRU) + +/* Active JBUS request signal when error occurred */ +#define JPAFSR_JBREQ (0x7UL << 24UL) /* jp */ +#define JPAFSR_JBREQ_SHIFT 24UL + +/* L2 cache way information */ +#define JPAFSR_ETW (0x3UL << 22UL) /* jp */ +#define JPAFSR_ETW_SHIFT 22UL + +/* System bus MTAG ECC syndrome. This field captures the status of the + * first occurrence of the highest-priority error according to the M_SYND + * overwrite policy. After the AFSR sticky bit, corresponding to the error + * for which the M_SYND is reported, is cleared, the contents of the M_SYND + * field will be unchanged by will be unfrozen for further error capture. + */ +#define CHAFSR_M_SYNDROME (0xfUL << 16UL) /* ch,ch+,jp */ +#define CHAFSR_M_SYNDROME_SHIFT 16UL + +/* Agenid Id of the foreign device causing the UE/CE errors */ +#define JPAFSR_AID (0x1fUL << 9UL) /* jp */ +#define JPAFSR_AID_SHIFT 9UL + +/* System bus or E-cache data ECC syndrome. This field captures the status + * of the first occurrence of the highest-priority error according to the + * E_SYND overwrite policy. After the AFSR sticky bit, corresponding to the + * error for which the E_SYND is reported, is cleare, the contents of the E_SYND + * field will be unchanged but will be unfrozen for further error capture. + */ +#define CHAFSR_E_SYNDROME (0x1ffUL << 0UL) /* ch,ch+,jp */ +#define CHAFSR_E_SYNDROME_SHIFT 0UL + +/* The AFSR must be explicitly cleared by software, it is not cleared automatically + * by a read. Writes to bits <51:33> with bits set will clear the corresponding + * bits in the AFSR. Bits associated with disrupting traps must be cleared before + * interrupts are re-enabled to prevent multiple traps for the same error. I.e. + * PSTATE.IE and AFSR bits control delivery of disrupting traps. + * + * Since there is only one AFAR, when multiple events have been logged by the + * bits in the AFSR, at most one of these events will have its status captured + * in the AFAR. The highest priority of those event bits will get AFAR logging. + * The AFAR will be unlocked and available to capture the address of another event + * as soon as the one bit in AFSR that corresponds to the event logged in AFAR is + * cleared. For example, if AFSR.CE is detected, then AFSR.UE (which overwrites + * the AFAR), and AFSR.UE is cleared by not AFSR.CE, then the AFAR will be unlocked + * and ready for another event, even though AFSR.CE is still set. The same rules + * also apply to the M_SYNDROME and E_SYNDROME fields of the AFSR. + */ + +#endif /* _SPARC64_CHAFSR_H */ diff --git a/arch/sparc/include/asm/checksum.h b/arch/sparc/include/asm/checksum.h new file mode 100644 index 000000000..f2ac13323 --- /dev/null +++ b/arch/sparc/include/asm/checksum.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_CHECKSUM_H +#define ___ASM_SPARC_CHECKSUM_H +#define _HAVE_ARCH_CSUM_AND_COPY +#define _HAVE_ARCH_COPY_AND_CSUM_FROM_USER +#define HAVE_CSUM_COPY_USER +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/checksum_32.h b/arch/sparc/include/asm/checksum_32.h new file mode 100644 index 000000000..ce11e0ad8 --- /dev/null +++ b/arch/sparc/include/asm/checksum_32.h @@ -0,0 +1,210 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_CHECKSUM_H +#define __SPARC_CHECKSUM_H + +/* checksum.h: IP/UDP/TCP checksum routines on the Sparc. + * + * Copyright(C) 1995 Linus Torvalds + * Copyright(C) 1995 Miguel de Icaza + * Copyright(C) 1996 David S. Miller + * Copyright(C) 1996 Eddie C. Dost + * Copyright(C) 1997 Jakub Jelinek + * + * derived from: + * Alpha checksum c-code + * ix86 inline assembly + * RFC1071 Computing the Internet Checksum + */ + +#include +#include + +/* computes the checksum of a memory block at buff, length len, + * and adds in "sum" (32-bit) + * + * returns a 32-bit number suitable for feeding into itself + * or csum_tcpudp_magic + * + * this function must be called with even lengths, except + * for the last fragment, which may be odd + * + * it's best to have buff aligned on a 32-bit boundary + */ +__wsum csum_partial(const void *buff, int len, __wsum sum); + +/* the same as csum_partial, but copies from fs:src while it + * checksums + * + * here even more important to align src and dst on a 32-bit (or even + * better 64-bit) boundary + */ + +unsigned int __csum_partial_copy_sparc_generic (const unsigned char *, unsigned char *); + +static inline __wsum +csum_partial_copy_nocheck(const void *src, void *dst, int len) +{ + register unsigned int ret asm("o0") = (unsigned int)src; + register char *d asm("o1") = dst; + register int l asm("g1") = len; + + __asm__ __volatile__ ( + "call __csum_partial_copy_sparc_generic\n\t" + " mov -1, %%g7\n" + : "=&r" (ret), "=&r" (d), "=&r" (l) + : "0" (ret), "1" (d), "2" (l) + : "o2", "o3", "o4", "o5", "o7", + "g2", "g3", "g4", "g5", "g7", + "memory", "cc"); + return (__force __wsum)ret; +} + +static inline __wsum +csum_and_copy_from_user(const void __user *src, void *dst, int len) +{ + if (unlikely(!access_ok(src, len))) + return 0; + return csum_partial_copy_nocheck((__force void *)src, dst, len); +} + +static inline __wsum +csum_and_copy_to_user(const void *src, void __user *dst, int len) +{ + if (!access_ok(dst, len)) + return 0; + return csum_partial_copy_nocheck(src, (__force void *)dst, len); +} + +/* ihl is always 5 or greater, almost always is 5, and iph is word aligned + * the majority of the time. + */ +static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) +{ + __sum16 sum; + + /* Note: We must read %2 before we touch %0 for the first time, + * because GCC can legitimately use the same register for + * both operands. + */ + __asm__ __volatile__("sub\t%2, 4, %%g4\n\t" + "ld\t[%1 + 0x00], %0\n\t" + "ld\t[%1 + 0x04], %%g2\n\t" + "ld\t[%1 + 0x08], %%g3\n\t" + "addcc\t%%g2, %0, %0\n\t" + "addxcc\t%%g3, %0, %0\n\t" + "ld\t[%1 + 0x0c], %%g2\n\t" + "ld\t[%1 + 0x10], %%g3\n\t" + "addxcc\t%%g2, %0, %0\n\t" + "addx\t%0, %%g0, %0\n" + "1:\taddcc\t%%g3, %0, %0\n\t" + "add\t%1, 4, %1\n\t" + "addxcc\t%0, %%g0, %0\n\t" + "subcc\t%%g4, 1, %%g4\n\t" + "be,a\t2f\n\t" + "sll\t%0, 16, %%g2\n\t" + "b\t1b\n\t" + "ld\t[%1 + 0x10], %%g3\n" + "2:\taddcc\t%0, %%g2, %%g2\n\t" + "srl\t%%g2, 16, %0\n\t" + "addx\t%0, %%g0, %0\n\t" + "xnor\t%%g0, %0, %0" + : "=r" (sum), "=&r" (iph) + : "r" (ihl), "1" (iph) + : "g2", "g3", "g4", "cc", "memory"); + return sum; +} + +/* Fold a partial checksum without adding pseudo headers. */ +static inline __sum16 csum_fold(__wsum sum) +{ + unsigned int tmp; + + __asm__ __volatile__("addcc\t%0, %1, %1\n\t" + "srl\t%1, 16, %1\n\t" + "addx\t%1, %%g0, %1\n\t" + "xnor\t%%g0, %1, %0" + : "=&r" (sum), "=r" (tmp) + : "0" (sum), "1" ((__force u32)sum<<16) + : "cc"); + return (__force __sum16)sum; +} + +static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, + __wsum sum) +{ + __asm__ __volatile__("addcc\t%1, %0, %0\n\t" + "addxcc\t%2, %0, %0\n\t" + "addxcc\t%3, %0, %0\n\t" + "addx\t%0, %%g0, %0\n\t" + : "=r" (sum), "=r" (saddr) + : "r" (daddr), "r" (proto + len), "0" (sum), + "1" (saddr) + : "cc"); + return sum; +} + +/* + * computes the checksum of the TCP/UDP pseudo-header + * returns a 16-bit checksum, already complemented + */ +static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, + __wsum sum) +{ + return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum)); +} + +#define _HAVE_ARCH_IPV6_CSUM + +static inline __sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, __u8 proto, __wsum sum) +{ + __asm__ __volatile__ ( + "addcc %3, %4, %%g4\n\t" + "addxcc %5, %%g4, %%g4\n\t" + "ld [%2 + 0x0c], %%g2\n\t" + "ld [%2 + 0x08], %%g3\n\t" + "addxcc %%g2, %%g4, %%g4\n\t" + "ld [%2 + 0x04], %%g2\n\t" + "addxcc %%g3, %%g4, %%g4\n\t" + "ld [%2 + 0x00], %%g3\n\t" + "addxcc %%g2, %%g4, %%g4\n\t" + "ld [%1 + 0x0c], %%g2\n\t" + "addxcc %%g3, %%g4, %%g4\n\t" + "ld [%1 + 0x08], %%g3\n\t" + "addxcc %%g2, %%g4, %%g4\n\t" + "ld [%1 + 0x04], %%g2\n\t" + "addxcc %%g3, %%g4, %%g4\n\t" + "ld [%1 + 0x00], %%g3\n\t" + "addxcc %%g2, %%g4, %%g4\n\t" + "addxcc %%g3, %%g4, %0\n\t" + "addx 0, %0, %0\n" + : "=&r" (sum) + : "r" (saddr), "r" (daddr), + "r"(htonl(len)), "r"(htonl(proto)), "r"(sum) + : "g2", "g3", "g4", "cc"); + + return csum_fold(sum); +} + +/* this routine is used for miscellaneous IP-like checksums, mainly in icmp.c */ +static inline __sum16 ip_compute_csum(const void *buff, int len) +{ + return csum_fold(csum_partial(buff, len, 0)); +} + +#define HAVE_ARCH_CSUM_ADD +static inline __wsum csum_add(__wsum csum, __wsum addend) +{ + __asm__ __volatile__( + "addcc %0, %1, %0\n" + "addx %0, %%g0, %0" + : "=r" (csum) + : "r" (addend), "0" (csum)); + + return csum; +} + +#endif /* !(__SPARC_CHECKSUM_H) */ diff --git a/arch/sparc/include/asm/checksum_64.h b/arch/sparc/include/asm/checksum_64.h new file mode 100644 index 000000000..d6b59461e --- /dev/null +++ b/arch/sparc/include/asm/checksum_64.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC64_CHECKSUM_H +#define __SPARC64_CHECKSUM_H + +/* checksum.h: IP/UDP/TCP checksum routines on the V9. + * + * Copyright(C) 1995 Linus Torvalds + * Copyright(C) 1995 Miguel de Icaza + * Copyright(C) 1996 David S. Miller + * Copyright(C) 1996 Eddie C. Dost + * Copyright(C) 1997 Jakub Jelinek + * + * derived from: + * Alpha checksum c-code + * ix86 inline assembly + * RFC1071 Computing the Internet Checksum + */ + +#include +#include + +/* computes the checksum of a memory block at buff, length len, + * and adds in "sum" (32-bit) + * + * returns a 32-bit number suitable for feeding into itself + * or csum_tcpudp_magic + * + * this function must be called with even lengths, except + * for the last fragment, which may be odd + * + * it's best to have buff aligned on a 32-bit boundary + */ +__wsum csum_partial(const void * buff, int len, __wsum sum); + +/* the same as csum_partial, but copies from user space while it + * checksums + * + * here even more important to align src and dst on a 32-bit (or even + * better 64-bit) boundary + */ +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len); +__wsum csum_and_copy_from_user(const void __user *src, void *dst, int len); +__wsum csum_and_copy_to_user(const void *src, void __user *dst, int len); + +/* ihl is always 5 or greater, almost always is 5, and iph is word aligned + * the majority of the time. + */ +__sum16 ip_fast_csum(const void *iph, unsigned int ihl); + +/* Fold a partial checksum without adding pseudo headers. */ +static inline __sum16 csum_fold(__wsum sum) +{ + unsigned int tmp; + + __asm__ __volatile__( +" addcc %0, %1, %1\n" +" srl %1, 16, %1\n" +" addc %1, %%g0, %1\n" +" xnor %%g0, %1, %0\n" + : "=&r" (sum), "=r" (tmp) + : "0" (sum), "1" ((__force u32)sum<<16) + : "cc"); + return (__force __sum16)sum; +} + +static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, + __wsum sum) +{ + __asm__ __volatile__( +" addcc %1, %0, %0\n" +" addccc %2, %0, %0\n" +" addccc %3, %0, %0\n" +" addc %0, %%g0, %0\n" + : "=r" (sum), "=r" (saddr) + : "r" (daddr), "r" (proto + len), "0" (sum), "1" (saddr) + : "cc"); + return sum; +} + +/* + * computes the checksum of the TCP/UDP pseudo-header + * returns a 16-bit checksum, already complemented + */ +static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr, + __u32 len, __u8 proto, + __wsum sum) +{ + return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum)); +} + +#define _HAVE_ARCH_IPV6_CSUM + +static inline __sum16 csum_ipv6_magic(const struct in6_addr *saddr, + const struct in6_addr *daddr, + __u32 len, __u8 proto, __wsum sum) +{ + __asm__ __volatile__ ( +" addcc %3, %4, %%g7\n" +" addccc %5, %%g7, %%g7\n" +" lduw [%2 + 0x0c], %%g2\n" +" lduw [%2 + 0x08], %%g3\n" +" addccc %%g2, %%g7, %%g7\n" +" lduw [%2 + 0x04], %%g2\n" +" addccc %%g3, %%g7, %%g7\n" +" lduw [%2 + 0x00], %%g3\n" +" addccc %%g2, %%g7, %%g7\n" +" lduw [%1 + 0x0c], %%g2\n" +" addccc %%g3, %%g7, %%g7\n" +" lduw [%1 + 0x08], %%g3\n" +" addccc %%g2, %%g7, %%g7\n" +" lduw [%1 + 0x04], %%g2\n" +" addccc %%g3, %%g7, %%g7\n" +" lduw [%1 + 0x00], %%g3\n" +" addccc %%g2, %%g7, %%g7\n" +" addccc %%g3, %%g7, %0\n" +" addc 0, %0, %0\n" + : "=&r" (sum) + : "r" (saddr), "r" (daddr), "r"(htonl(len)), + "r"(htonl(proto)), "r"(sum) + : "g2", "g3", "g7", "cc"); + + return csum_fold(sum); +} + +/* this routine is used for miscellaneous IP-like checksums, mainly in icmp.c */ +static inline __sum16 ip_compute_csum(const void *buff, int len) +{ + return csum_fold(csum_partial(buff, len, 0)); +} + +#define HAVE_ARCH_CSUM_ADD +static inline __wsum csum_add(__wsum csum, __wsum addend) +{ + __asm__ __volatile__( + "addcc %0, %1, %0\n" + "addx %0, %%g0, %0" + : "=r" (csum) + : "r" (addend), "0" (csum)); + + return csum; +} + +#endif /* !(__SPARC64_CHECKSUM_H) */ diff --git a/arch/sparc/include/asm/chmctrl.h b/arch/sparc/include/asm/chmctrl.h new file mode 100644 index 000000000..96f043cb2 --- /dev/null +++ b/arch/sparc/include/asm/chmctrl.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_CHMCTRL_H +#define _SPARC64_CHMCTRL_H + +/* Cheetah memory controller programmable registers. */ +#define CHMCTRL_TCTRL1 0x00 /* Memory Timing Control I */ +#define CHMCTRL_TCTRL2 0x08 /* Memory Timing Control II */ +#define CHMCTRL_TCTRL3 0x38 /* Memory Timing Control III */ +#define CHMCTRL_TCTRL4 0x40 /* Memory Timing Control IV */ +#define CHMCTRL_DECODE1 0x10 /* Memory Address Decode I */ +#define CHMCTRL_DECODE2 0x18 /* Memory Address Decode II */ +#define CHMCTRL_DECODE3 0x20 /* Memory Address Decode III */ +#define CHMCTRL_DECODE4 0x28 /* Memory Address Decode IV */ +#define CHMCTRL_MACTRL 0x30 /* Memory Address Control */ + +/* Memory Timing Control I */ +#define TCTRL1_SDRAMCTL_DLY 0xf000000000000000UL +#define TCTRL1_SDRAMCTL_DLY_SHIFT 60 +#define TCTRL1_SDRAMCLK_DLY 0x0e00000000000000UL +#define TCTRL1_SDRAMCLK_DLY_SHIFT 57 +#define TCTRL1_R 0x0100000000000000UL +#define TCTRL1_R_SHIFT 56 +#define TCTRL1_AUTORFR_CYCLE 0x00fe000000000000UL +#define TCTRL1_AUTORFR_CYCLE_SHIFT 49 +#define TCTRL1_RD_WAIT 0x0001f00000000000UL +#define TCTRL1_RD_WAIT_SHIFT 44 +#define TCTRL1_PC_CYCLE 0x00000fc000000000UL +#define TCTRL1_PC_CYCLE_SHIFT 38 +#define TCTRL1_WR_MORE_RAS_PW 0x0000003f00000000UL +#define TCTRL1_WR_MORE_RAS_PW_SHIFT 32 +#define TCTRL1_RD_MORE_RAW_PW 0x00000000fc000000UL +#define TCTRL1_RD_MORE_RAS_PW_SHIFT 26 +#define TCTRL1_ACT_WR_DLY 0x0000000003f00000UL +#define TCTRL1_ACT_WR_DLY_SHIFT 20 +#define TCTRL1_ACT_RD_DLY 0x00000000000fc000UL +#define TCTRL1_ACT_RD_DLY_SHIFT 14 +#define TCTRL1_BANK_PRESENT 0x0000000000003000UL +#define TCTRL1_BANK_PRESENT_SHIFT 12 +#define TCTRL1_RFR_INT 0x0000000000000ff8UL +#define TCTRL1_RFR_INT_SHIFT 3 +#define TCTRL1_SET_MODE_REG 0x0000000000000004UL +#define TCTRL1_SET_MODE_REG_SHIFT 2 +#define TCTRL1_RFR_ENABLE 0x0000000000000002UL +#define TCTRL1_RFR_ENABLE_SHIFT 1 +#define TCTRL1_PRECHG_ALL 0x0000000000000001UL +#define TCTRL1_PRECHG_ALL_SHIFT 0 + +/* Memory Timing Control II */ +#define TCTRL2_WR_MSEL_DLY 0xfc00000000000000UL +#define TCTRL2_WR_MSEL_DLY_SHIFT 58 +#define TCTRL2_RD_MSEL_DLY 0x03f0000000000000UL +#define TCTRL2_RD_MSEL_DLY_SHIFT 52 +#define TCTRL2_WRDATA_THLD 0x000c000000000000UL +#define TCTRL2_WRDATA_THLD_SHIFT 50 +#define TCTRL2_RDWR_RD_TI_DLY 0x0003f00000000000UL +#define TCTRL2_RDWR_RD_TI_DLY_SHIFT 44 +#define TCTRL2_AUTOPRECHG_ENBL 0x0000080000000000UL +#define TCTRL2_AUTOPRECHG_ENBL_SHIFT 43 +#define TCTRL2_RDWR_PI_MORE_DLY 0x000007c000000000UL +#define TCTRL2_RDWR_PI_MORE_DLY_SHIFT 38 +#define TCTRL2_RDWR_1_DLY 0x0000003f00000000UL +#define TCTRL2_RDWR_1_DLY_SHIFT 32 +#define TCTRL2_WRWR_PI_MORE_DLY 0x00000000f8000000UL +#define TCTRL2_WRWR_PI_MORE_DLY_SHIFT 27 +#define TCTRL2_WRWR_1_DLY 0x0000000007e00000UL +#define TCTRL2_WRWR_1_DLY_SHIFT 21 +#define TCTRL2_RDWR_RD_PI_MORE_DLY 0x00000000001f0000UL +#define TCTRL2_RDWR_RD_PI_MORE_DLY_SHIFT 16 +#define TCTRL2_R 0x0000000000008000UL +#define TCTRL2_R_SHIFT 15 +#define TCTRL2_SDRAM_MODE_REG_DATA 0x0000000000007fffUL +#define TCTRL2_SDRAM_MODE_REG_DATA_SHIFT 0 + +/* Memory Timing Control III */ +#define TCTRL3_SDRAM_CTL_DLY 0xf000000000000000UL +#define TCTRL3_SDRAM_CTL_DLY_SHIFT 60 +#define TCTRL3_SDRAM_CLK_DLY 0x0e00000000000000UL +#define TCTRL3_SDRAM_CLK_DLY_SHIFT 57 +#define TCTRL3_R 0x0100000000000000UL +#define TCTRL3_R_SHIFT 56 +#define TCTRL3_AUTO_RFR_CYCLE 0x00fe000000000000UL +#define TCTRL3_AUTO_RFR_CYCLE_SHIFT 49 +#define TCTRL3_RD_WAIT 0x0001f00000000000UL +#define TCTRL3_RD_WAIT_SHIFT 44 +#define TCTRL3_PC_CYCLE 0x00000fc000000000UL +#define TCTRL3_PC_CYCLE_SHIFT 38 +#define TCTRL3_WR_MORE_RAW_PW 0x0000003f00000000UL +#define TCTRL3_WR_MORE_RAW_PW_SHIFT 32 +#define TCTRL3_RD_MORE_RAW_PW 0x00000000fc000000UL +#define TCTRL3_RD_MORE_RAW_PW_SHIFT 26 +#define TCTRL3_ACT_WR_DLY 0x0000000003f00000UL +#define TCTRL3_ACT_WR_DLY_SHIFT 20 +#define TCTRL3_ACT_RD_DLY 0x00000000000fc000UL +#define TCTRL3_ACT_RD_DLY_SHIFT 14 +#define TCTRL3_BANK_PRESENT 0x0000000000003000UL +#define TCTRL3_BANK_PRESENT_SHIFT 12 +#define TCTRL3_RFR_INT 0x0000000000000ff8UL +#define TCTRL3_RFR_INT_SHIFT 3 +#define TCTRL3_SET_MODE_REG 0x0000000000000004UL +#define TCTRL3_SET_MODE_REG_SHIFT 2 +#define TCTRL3_RFR_ENABLE 0x0000000000000002UL +#define TCTRL3_RFR_ENABLE_SHIFT 1 +#define TCTRL3_PRECHG_ALL 0x0000000000000001UL +#define TCTRL3_PRECHG_ALL_SHIFT 0 + +/* Memory Timing Control IV */ +#define TCTRL4_WR_MSEL_DLY 0xfc00000000000000UL +#define TCTRL4_WR_MSEL_DLY_SHIFT 58 +#define TCTRL4_RD_MSEL_DLY 0x03f0000000000000UL +#define TCTRL4_RD_MSEL_DLY_SHIFT 52 +#define TCTRL4_WRDATA_THLD 0x000c000000000000UL +#define TCTRL4_WRDATA_THLD_SHIFT 50 +#define TCTRL4_RDWR_RD_RI_DLY 0x0003f00000000000UL +#define TCTRL4_RDWR_RD_RI_DLY_SHIFT 44 +#define TCTRL4_AUTO_PRECHG_ENBL 0x0000080000000000UL +#define TCTRL4_AUTO_PRECHG_ENBL_SHIFT 43 +#define TCTRL4_RD_WR_PI_MORE_DLY 0x000007c000000000UL +#define TCTRL4_RD_WR_PI_MORE_DLY_SHIFT 38 +#define TCTRL4_RD_WR_TI_DLY 0x0000003f00000000UL +#define TCTRL4_RD_WR_TI_DLY_SHIFT 32 +#define TCTRL4_WR_WR_PI_MORE_DLY 0x00000000f8000000UL +#define TCTRL4_WR_WR_PI_MORE_DLY_SHIFT 27 +#define TCTRL4_WR_WR_TI_DLY 0x0000000007e00000UL +#define TCTRL4_WR_WR_TI_DLY_SHIFT 21 +#define TCTRL4_RDWR_RD_PI_MORE_DLY 0x00000000001f000UL0 +#define TCTRL4_RDWR_RD_PI_MORE_DLY_SHIFT 16 +#define TCTRL4_R 0x0000000000008000UL +#define TCTRL4_R_SHIFT 15 +#define TCTRL4_SDRAM_MODE_REG_DATA 0x0000000000007fffUL +#define TCTRL4_SDRAM_MODE_REG_DATA_SHIFT 0 + +/* All 4 memory address decoding registers have the + * same layout. + */ +#define MEM_DECODE_VALID 0x8000000000000000UL /* Valid */ +#define MEM_DECODE_VALID_SHIFT 63 +#define MEM_DECODE_UK 0x001ffe0000000000UL /* Upper mask */ +#define MEM_DECODE_UK_SHIFT 41 +#define MEM_DECODE_UM 0x0000001ffff00000UL /* Upper match */ +#define MEM_DECODE_UM_SHIFT 20 +#define MEM_DECODE_LK 0x000000000003c000UL /* Lower mask */ +#define MEM_DECODE_LK_SHIFT 14 +#define MEM_DECODE_LM 0x0000000000000f00UL /* Lower match */ +#define MEM_DECODE_LM_SHIFT 8 + +#define PA_UPPER_BITS 0x000007fffc000000UL +#define PA_UPPER_BITS_SHIFT 26 +#define PA_LOWER_BITS 0x00000000000003c0UL +#define PA_LOWER_BITS_SHIFT 6 + +#define MACTRL_R0 0x8000000000000000UL +#define MACTRL_R0_SHIFT 63 +#define MACTRL_ADDR_LE_PW 0x7000000000000000UL +#define MACTRL_ADDR_LE_PW_SHIFT 60 +#define MACTRL_CMD_PW 0x0f00000000000000UL +#define MACTRL_CMD_PW_SHIFT 56 +#define MACTRL_HALF_MODE_WR_MSEL_DLY 0x00fc000000000000UL +#define MACTRL_HALF_MODE_WR_MSEL_DLY_SHIFT 50 +#define MACTRL_HALF_MODE_RD_MSEL_DLY 0x0003f00000000000UL +#define MACTRL_HALF_MODE_RD_MSEL_DLY_SHIFT 44 +#define MACTRL_HALF_MODE_SDRAM_CTL_DLY 0x00000f0000000000UL +#define MACTRL_HALF_MODE_SDRAM_CTL_DLY_SHIFT 40 +#define MACTRL_HALF_MODE_SDRAM_CLK_DLY 0x000000e000000000UL +#define MACTRL_HALF_MODE_SDRAM_CLK_DLY_SHIFT 37 +#define MACTRL_R1 0x0000001000000000UL +#define MACTRL_R1_SHIFT 36 +#define MACTRL_BANKSEL_N_ROWADDR_SIZE_B3 0x0000000f00000000UL +#define MACTRL_BANKSEL_N_ROWADDR_SIZE_B3_SHIFT 32 +#define MACTRL_ENC_INTLV_B3 0x00000000f8000000UL +#define MACTRL_ENC_INTLV_B3_SHIFT 27 +#define MACTRL_BANKSEL_N_ROWADDR_SIZE_B2 0x0000000007800000UL +#define MACTRL_BANKSEL_N_ROWADDR_SIZE_B2_SHIFT 23 +#define MACTRL_ENC_INTLV_B2 0x00000000007c0000UL +#define MACTRL_ENC_INTLV_B2_SHIFT 18 +#define MACTRL_BANKSEL_N_ROWADDR_SIZE_B1 0x000000000003c000UL +#define MACTRL_BANKSEL_N_ROWADDR_SIZE_B1_SHIFT 14 +#define MACTRL_ENC_INTLV_B1 0x0000000000003e00UL +#define MACTRL_ENC_INTLV_B1_SHIFT 9 +#define MACTRL_BANKSEL_N_ROWADDR_SIZE_B0 0x00000000000001e0UL +#define MACTRL_BANKSEL_N_ROWADDR_SIZE_B0_SHIFT 5 +#define MACTRL_ENC_INTLV_B0 0x000000000000001fUL +#define MACTRL_ENC_INTLV_B0_SHIFT 0 + +#endif /* _SPARC64_CHMCTRL_H */ diff --git a/arch/sparc/include/asm/clock.h b/arch/sparc/include/asm/clock.h new file mode 100644 index 000000000..2cf99dade --- /dev/null +++ b/arch/sparc/include/asm/clock.h @@ -0,0 +1,11 @@ +/* + * clock.h: Definitions for clock operations on the Sparc. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ +#ifndef _SPARC_CLOCK_H +#define _SPARC_CLOCK_H + +/* Foo for now. */ + +#endif /* !(_SPARC_CLOCK_H) */ diff --git a/arch/sparc/include/asm/clocksource.h b/arch/sparc/include/asm/clocksource.h new file mode 100644 index 000000000..68303ad26 --- /dev/null +++ b/arch/sparc/include/asm/clocksource.h @@ -0,0 +1,8 @@ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#ifndef _ASM_SPARC_CLOCKSOURCE_H +#define _ASM_SPARC_CLOCKSOURCE_H + +#endif /* _ASM_SPARC_CLOCKSOURCE_H */ diff --git a/arch/sparc/include/asm/cmpxchg.h b/arch/sparc/include/asm/cmpxchg.h new file mode 100644 index 000000000..b7955c6b9 --- /dev/null +++ b/arch/sparc/include/asm/cmpxchg.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_CMPXCHG_H +#define ___ASM_SPARC_CMPXCHG_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/cmpxchg_32.h b/arch/sparc/include/asm/cmpxchg_32.h new file mode 100644 index 000000000..8c1a3ca34 --- /dev/null +++ b/arch/sparc/include/asm/cmpxchg_32.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* 32-bit atomic xchg() and cmpxchg() definitions. + * + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) + * Copyright (C) 2000 Anton Blanchard (anton@linuxcare.com.au) + * Copyright (C) 2007 Kyle McMartin (kyle@parisc-linux.org) + * + * Additions by Keith M Wesolowski (wesolows@foobazco.org) based + * on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf . + */ + +#ifndef __ARCH_SPARC_CMPXCHG__ +#define __ARCH_SPARC_CMPXCHG__ + +unsigned long __xchg_u32(volatile u32 *m, u32 new); +void __xchg_called_with_bad_pointer(void); + +static __always_inline unsigned long __arch_xchg(unsigned long x, __volatile__ void * ptr, int size) +{ + switch (size) { + case 4: + return __xchg_u32(ptr, x); + } + __xchg_called_with_bad_pointer(); + return x; +} + +#define arch_xchg(ptr,x) ({(__typeof__(*(ptr)))__arch_xchg((unsigned long)(x),(ptr),sizeof(*(ptr)));}) + +/* Emulate cmpxchg() the same way we emulate atomics, + * by hashing the object address and indexing into an array + * of spinlocks to get a bit of performance... + * + * See arch/sparc/lib/atomic32.c for implementation. + * + * Cribbed from + */ + +/* bug catcher for when unsupported size is used - won't link */ +void __cmpxchg_called_with_bad_pointer(void); +u8 __cmpxchg_u8(volatile u8 *m, u8 old, u8 new_); +u16 __cmpxchg_u16(volatile u16 *m, u16 old, u16 new_); +u32 __cmpxchg_u32(volatile u32 *m, u32 old, u32 new_); + +/* don't worry...optimizer will get rid of most of this */ +static inline unsigned long +__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new_, int size) +{ + return + size == 1 ? __cmpxchg_u8(ptr, old, new_) : + size == 2 ? __cmpxchg_u16(ptr, old, new_) : + size == 4 ? __cmpxchg_u32(ptr, old, new_) : + (__cmpxchg_called_with_bad_pointer(), old); +} + +#define arch_cmpxchg(ptr, o, n) \ +({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ + (unsigned long)_n_, sizeof(*(ptr))); \ +}) + +u64 __cmpxchg_u64(volatile u64 *ptr, u64 old, u64 new); +#define arch_cmpxchg64(ptr, old, new) __cmpxchg_u64(ptr, old, new) + +#include + +/* + * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make + * them available. + */ +#define arch_cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), (unsigned long)(o),\ + (unsigned long)(n), sizeof(*(ptr)))) +#define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) + +#endif /* __ARCH_SPARC_CMPXCHG__ */ diff --git a/arch/sparc/include/asm/cmpxchg_64.h b/arch/sparc/include/asm/cmpxchg_64.h new file mode 100644 index 000000000..3de25262c --- /dev/null +++ b/arch/sparc/include/asm/cmpxchg_64.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* 64-bit atomic xchg() and cmpxchg() definitions. + * + * Copyright (C) 1996, 1997, 2000 David S. Miller (davem@redhat.com) + */ + +#ifndef __ARCH_SPARC64_CMPXCHG__ +#define __ARCH_SPARC64_CMPXCHG__ + +static inline unsigned long +__cmpxchg_u32(volatile int *m, int old, int new) +{ + __asm__ __volatile__("cas [%2], %3, %0" + : "=&r" (new) + : "0" (new), "r" (m), "r" (old) + : "memory"); + + return new; +} + +static inline unsigned long xchg32(__volatile__ unsigned int *m, unsigned int val) +{ + unsigned long tmp1, tmp2; + + __asm__ __volatile__( +" mov %0, %1\n" +"1: lduw [%4], %2\n" +" cas [%4], %2, %0\n" +" cmp %2, %0\n" +" bne,a,pn %%icc, 1b\n" +" mov %1, %0\n" + : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2) + : "0" (val), "r" (m) + : "cc", "memory"); + return val; +} + +static inline unsigned long xchg64(__volatile__ unsigned long *m, unsigned long val) +{ + unsigned long tmp1, tmp2; + + __asm__ __volatile__( +" mov %0, %1\n" +"1: ldx [%4], %2\n" +" casx [%4], %2, %0\n" +" cmp %2, %0\n" +" bne,a,pn %%xcc, 1b\n" +" mov %1, %0\n" + : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2) + : "0" (val), "r" (m) + : "cc", "memory"); + return val; +} + +#define arch_xchg(ptr,x) \ +({ __typeof__(*(ptr)) __ret; \ + __ret = (__typeof__(*(ptr))) \ + __arch_xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \ + __ret; \ +}) + +void __xchg_called_with_bad_pointer(void); + +/* + * Use 4 byte cas instruction to achieve 2 byte xchg. Main logic + * here is to get the bit shift of the byte we are interested in. + * The XOR is handy for reversing the bits for big-endian byte order. + */ +static inline unsigned long +xchg16(__volatile__ unsigned short *m, unsigned short val) +{ + unsigned long maddr = (unsigned long)m; + int bit_shift = (((unsigned long)m & 2) ^ 2) << 3; + unsigned int mask = 0xffff << bit_shift; + unsigned int *ptr = (unsigned int *) (maddr & ~2); + unsigned int old32, new32, load32; + + /* Read the old value */ + load32 = *ptr; + + do { + old32 = load32; + new32 = (load32 & (~mask)) | val << bit_shift; + load32 = __cmpxchg_u32(ptr, old32, new32); + } while (load32 != old32); + + return (load32 & mask) >> bit_shift; +} + +static __always_inline unsigned long +__arch_xchg(unsigned long x, __volatile__ void * ptr, int size) +{ + switch (size) { + case 2: + return xchg16(ptr, x); + case 4: + return xchg32(ptr, x); + case 8: + return xchg64(ptr, x); + } + __xchg_called_with_bad_pointer(); + return x; +} + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + */ + +#include + + +static inline unsigned long +__cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new) +{ + __asm__ __volatile__("casx [%2], %3, %0" + : "=&r" (new) + : "0" (new), "r" (m), "r" (old) + : "memory"); + + return new; +} + +/* + * Use 4 byte cas instruction to achieve 1 byte cmpxchg. Main logic + * here is to get the bit shift of the byte we are interested in. + * The XOR is handy for reversing the bits for big-endian byte order + */ +static inline unsigned long +__cmpxchg_u8(volatile unsigned char *m, unsigned char old, unsigned char new) +{ + unsigned long maddr = (unsigned long)m; + int bit_shift = (((unsigned long)m & 3) ^ 3) << 3; + unsigned int mask = 0xff << bit_shift; + unsigned int *ptr = (unsigned int *) (maddr & ~3); + unsigned int old32, new32, load; + unsigned int load32 = *ptr; + + do { + new32 = (load32 & ~mask) | (new << bit_shift); + old32 = (load32 & ~mask) | (old << bit_shift); + load32 = __cmpxchg_u32(ptr, old32, new32); + if (load32 == old32) + return old; + load = (load32 & mask) >> bit_shift; + } while (load == old); + + return load; +} + +/* This function doesn't exist, so you'll get a linker error + if something tries to do an invalid cmpxchg(). */ +void __cmpxchg_called_with_bad_pointer(void); + +static inline unsigned long +__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) +{ + switch (size) { + case 1: + return __cmpxchg_u8(ptr, old, new); + case 4: + return __cmpxchg_u32(ptr, old, new); + case 8: + return __cmpxchg_u64(ptr, old, new); + } + __cmpxchg_called_with_bad_pointer(); + return old; +} + +#define arch_cmpxchg(ptr,o,n) \ + ({ \ + __typeof__(*(ptr)) _o_ = (o); \ + __typeof__(*(ptr)) _n_ = (n); \ + (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ + (unsigned long)_n_, sizeof(*(ptr))); \ + }) + +/* + * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make + * them available. + */ + +static inline unsigned long __cmpxchg_local(volatile void *ptr, + unsigned long old, + unsigned long new, int size) +{ + switch (size) { + case 4: + case 8: return __cmpxchg(ptr, old, new, size); + default: + return __generic_cmpxchg_local(ptr, old, new, size); + } + + return old; +} + +#define arch_cmpxchg_local(ptr, o, n) \ + ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ + (unsigned long)(n), sizeof(*(ptr)))) +#define arch_cmpxchg64_local(ptr, o, n) \ + ({ \ + BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ + arch_cmpxchg_local((ptr), (o), (n)); \ + }) +#define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) + +#endif /* __ARCH_SPARC64_CMPXCHG__ */ diff --git a/arch/sparc/include/asm/compat.h b/arch/sparc/include/asm/compat.h new file mode 100644 index 000000000..e4382d2ef --- /dev/null +++ b/arch/sparc/include/asm/compat.h @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC64_COMPAT_H +#define _ASM_SPARC64_COMPAT_H +/* + * Architecture specific compatibility types + */ +#include + +#define compat_mode_t compat_mode_t +typedef u16 compat_mode_t; + +#define __compat_uid_t __compat_uid_t +typedef u16 __compat_uid_t; +typedef u16 __compat_gid_t; + +#define compat_dev_t compat_dev_t +typedef u16 compat_dev_t; + +#define compat_ipc_pid_t compat_ipc_pid_t +typedef u16 compat_ipc_pid_t; + +#define compat_ipc64_perm compat_ipc64_perm + +#define COMPAT_RLIM_INFINITY 0x7fffffff + +#include + +#define COMPAT_UTS_MACHINE "sparc\0\0" + +typedef s16 compat_nlink_t; + +struct compat_stat { + compat_dev_t st_dev; + compat_ino_t st_ino; + compat_mode_t st_mode; + compat_nlink_t st_nlink; + __compat_uid_t st_uid; + __compat_gid_t st_gid; + compat_dev_t st_rdev; + compat_off_t st_size; + old_time32_t st_atime; + compat_ulong_t st_atime_nsec; + old_time32_t st_mtime; + compat_ulong_t st_mtime_nsec; + old_time32_t st_ctime; + compat_ulong_t st_ctime_nsec; + compat_off_t st_blksize; + compat_off_t st_blocks; + u32 __unused4[2]; +}; + +struct compat_stat64 { + unsigned long long st_dev; + + unsigned long long st_ino; + + unsigned int st_mode; + unsigned int st_nlink; + + unsigned int st_uid; + unsigned int st_gid; + + unsigned long long st_rdev; + + unsigned char __pad3[8]; + + long long st_size; + unsigned int st_blksize; + + unsigned char __pad4[8]; + unsigned int st_blocks; + + unsigned int st_atime; + unsigned int st_atime_nsec; + + unsigned int st_mtime; + unsigned int st_mtime_nsec; + + unsigned int st_ctime; + unsigned int st_ctime_nsec; + + unsigned int __unused4; + unsigned int __unused5; +}; + +#define __ARCH_COMPAT_FLOCK_PAD short __unused; + +struct compat_ipc64_perm { + compat_key_t key; + __compat_uid32_t uid; + __compat_gid32_t gid; + __compat_uid32_t cuid; + __compat_gid32_t cgid; + unsigned short __pad1; + compat_mode_t mode; + unsigned short __pad2; + unsigned short seq; + unsigned long __unused1; /* yes they really are 64bit pads */ + unsigned long __unused2; +}; + +struct compat_semid64_ds { + struct compat_ipc64_perm sem_perm; + unsigned int sem_otime_high; + unsigned int sem_otime; + unsigned int sem_ctime_high; + unsigned int sem_ctime; + u32 sem_nsems; + u32 __unused1; + u32 __unused2; +}; + +struct compat_msqid64_ds { + struct compat_ipc64_perm msg_perm; + unsigned int msg_stime_high; + unsigned int msg_stime; + unsigned int msg_rtime_high; + unsigned int msg_rtime; + unsigned int msg_ctime_high; + unsigned int msg_ctime; + unsigned int msg_cbytes; + unsigned int msg_qnum; + unsigned int msg_qbytes; + compat_pid_t msg_lspid; + compat_pid_t msg_lrpid; + unsigned int __unused1; + unsigned int __unused2; +}; + +struct compat_shmid64_ds { + struct compat_ipc64_perm shm_perm; + unsigned int shm_atime_high; + unsigned int shm_atime; + unsigned int shm_dtime_high; + unsigned int shm_dtime; + unsigned int shm_ctime_high; + unsigned int shm_ctime; + compat_size_t shm_segsz; + compat_pid_t shm_cpid; + compat_pid_t shm_lpid; + unsigned int shm_nattch; + unsigned int __unused1; + unsigned int __unused2; +}; + +#ifdef CONFIG_COMPAT +static inline int is_compat_task(void) +{ + return test_thread_flag(TIF_32BIT); +} + +static inline bool in_compat_syscall(void) +{ + /* Vector 0x110 is LINUX_32BIT_SYSCALL_TRAP */ + return pt_regs_trap_type(current_pt_regs()) == 0x110; +} +#define in_compat_syscall in_compat_syscall +#endif + +#endif /* _ASM_SPARC64_COMPAT_H */ diff --git a/arch/sparc/include/asm/compat_signal.h b/arch/sparc/include/asm/compat_signal.h new file mode 100644 index 000000000..e5f7a7f28 --- /dev/null +++ b/arch/sparc/include/asm/compat_signal.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _COMPAT_SIGNAL_H +#define _COMPAT_SIGNAL_H + +#include +#include + +#ifdef CONFIG_COMPAT +struct __new_sigaction32 { + unsigned int sa_handler; + unsigned int sa_flags; + unsigned int sa_restorer; /* not used by Linux/SPARC yet */ + compat_sigset_t sa_mask; +}; + +struct __old_sigaction32 { + unsigned int sa_handler; + compat_old_sigset_t sa_mask; + unsigned int sa_flags; + unsigned int sa_restorer; /* not used by Linux/SPARC yet */ +}; +#endif + +#endif /* !(_COMPAT_SIGNAL_H) */ diff --git a/arch/sparc/include/asm/contregs.h b/arch/sparc/include/asm/contregs.h new file mode 100644 index 000000000..4df56a6ba --- /dev/null +++ b/arch/sparc/include/asm/contregs.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_CONTREGS_H +#define _SPARC_CONTREGS_H + +/* contregs.h: Addresses of registers in the ASI_CONTROL alternate address + * space. These are for the mmu's context register, etc. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +/* s=Swift, h=Ross_HyperSPARC, v=TI_Viking, t=Tsunami, r=Ross_Cypress */ +#define AC_M_PCR 0x0000 /* shv Processor Control Reg */ +#define AC_M_CTPR 0x0100 /* shv Context Table Pointer Reg */ +#define AC_M_CXR 0x0200 /* shv Context Register */ +#define AC_M_SFSR 0x0300 /* shv Synchronous Fault Status Reg */ +#define AC_M_SFAR 0x0400 /* shv Synchronous Fault Address Reg */ +#define AC_M_AFSR 0x0500 /* hv Asynchronous Fault Status Reg */ +#define AC_M_AFAR 0x0600 /* hv Asynchronous Fault Address Reg */ +#define AC_M_RESET 0x0700 /* hv Reset Reg */ +#define AC_M_RPR 0x1000 /* hv Root Pointer Reg */ +#define AC_M_TSUTRCR 0x1000 /* s TLB Replacement Ctrl Reg */ +#define AC_M_IAPTP 0x1100 /* hv Instruction Access PTP */ +#define AC_M_DAPTP 0x1200 /* hv Data Access PTP */ +#define AC_M_ITR 0x1300 /* hv Index Tag Register */ +#define AC_M_TRCR 0x1400 /* hv TLB Replacement Control Reg */ +#define AC_M_SFSRX 0x1300 /* s Synch Fault Status Reg prim */ +#define AC_M_SFARX 0x1400 /* s Synch Fault Address Reg prim */ +#define AC_M_RPR1 0x1500 /* h Root Pointer Reg (entry 2) */ +#define AC_M_IAPTP1 0x1600 /* h Instruction Access PTP (entry 2) */ +#define AC_M_DAPTP1 0x1700 /* h Data Access PTP (entry 2) */ + +#endif /* _SPARC_CONTREGS_H */ diff --git a/arch/sparc/include/asm/cpu_type.h b/arch/sparc/include/asm/cpu_type.h new file mode 100644 index 000000000..2b5979985 --- /dev/null +++ b/arch/sparc/include/asm/cpu_type.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_CPU_TYPE_H +#define __ASM_CPU_TYPE_H + +/* + * Sparc (general) CPU types + */ +enum sparc_cpu { + sun4m = 0x00, + sun4d = 0x01, + sun4e = 0x02, + sun4u = 0x03, /* V8 ploos ploos */ + sun_unknown = 0x04, + ap1000 = 0x05, /* almost a sun4m */ + sparc_leon = 0x06, /* Leon SoC */ +}; + +#ifdef CONFIG_SPARC32 +extern enum sparc_cpu sparc_cpu_model; + +#define SUN4M_NCPUS 4 /* Architectural limit of sun4m. */ + +#else + +#define sparc_cpu_model sun4u + +#endif + +#endif /* __ASM_CPU_TYPE_H */ diff --git a/arch/sparc/include/asm/cpudata.h b/arch/sparc/include/asm/cpudata.h new file mode 100644 index 000000000..67022a153 --- /dev/null +++ b/arch/sparc/include/asm/cpudata.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_CPUDATA_H +#define ___ASM_SPARC_CPUDATA_H + +#ifndef __ASSEMBLER__ + +#include +#include + +extern const struct seq_operations cpuinfo_op; + +#endif /* !(__ASSEMBLER__) */ + +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/cpudata_32.h b/arch/sparc/include/asm/cpudata_32.h new file mode 100644 index 000000000..895d0964b --- /dev/null +++ b/arch/sparc/include/asm/cpudata_32.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* cpudata.h: Per-cpu parameters. + * + * Copyright (C) 2004 Keith M Wesolowski (wesolows@foobazco.org) + * + * Based on include/asm/cpudata.h and Linux 2.4 smp.h + * both (C) David S. Miller. + */ + +#ifndef _SPARC_CPUDATA_H +#define _SPARC_CPUDATA_H + +#include + +typedef struct { + unsigned long udelay_val; + unsigned long clock_tick; + unsigned int counter; +#ifdef CONFIG_SMP + unsigned int irq_resched_count; + unsigned int irq_call_count; +#endif + int prom_node; + int mid; + int next; +} cpuinfo_sparc; + +DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data); +#define cpu_data(__cpu) per_cpu(__cpu_data, (__cpu)) +#define local_cpu_data() (*this_cpu_ptr(&__cpu_data)) + +#endif /* _SPARC_CPUDATA_H */ diff --git a/arch/sparc/include/asm/cpudata_64.h b/arch/sparc/include/asm/cpudata_64.h new file mode 100644 index 000000000..056b3c0e7 --- /dev/null +++ b/arch/sparc/include/asm/cpudata_64.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* cpudata.h: Per-cpu parameters. + * + * Copyright (C) 2003, 2005, 2006 David S. Miller (davem@davemloft.net) + */ + +#ifndef _SPARC64_CPUDATA_H +#define _SPARC64_CPUDATA_H + +#ifndef __ASSEMBLER__ + +typedef struct { + /* Dcache line 1 */ + unsigned int __softirq_pending; /* must be 1st, see rtrap.S */ + unsigned int __nmi_count; + unsigned long clock_tick; /* %tick's per second */ + unsigned long __pad; + unsigned int irq0_irqs; + unsigned int __pad2; + + /* Dcache line 2, rarely used */ + unsigned int dcache_size; + unsigned int dcache_line_size; + unsigned int icache_size; + unsigned int icache_line_size; + unsigned int ecache_size; + unsigned int ecache_line_size; + unsigned short sock_id; /* physical package */ + unsigned short core_id; + unsigned short max_cache_id; /* groupings of highest shared cache */ + signed short proc_id; /* strand (aka HW thread) id */ +} cpuinfo_sparc; + +DECLARE_PER_CPU(cpuinfo_sparc, __cpu_data); +#define cpu_data(__cpu) per_cpu(__cpu_data, (__cpu)) +#define local_cpu_data() (*this_cpu_ptr(&__cpu_data)) + +#endif /* !(__ASSEMBLER__) */ + +#include + +#endif /* _SPARC64_CPUDATA_H */ diff --git a/arch/sparc/include/asm/current.h b/arch/sparc/include/asm/current.h new file mode 100644 index 000000000..c68312a89 --- /dev/null +++ b/arch/sparc/include/asm/current.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* include/asm/current.h + * + * Copyright (C) 1999 IBM Deutschland Entwicklung GmbH, IBM Corporation + * Copyright (C) 2002 Pete Zaitcev (zaitcev@yahoo.com) + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + * + * Derived from "include/asm-s390/current.h" by + * Martin Schwidefsky (schwidefsky@de.ibm.com) + * Derived from "include/asm-i386/current.h" +*/ +#ifndef _SPARC_CURRENT_H +#define _SPARC_CURRENT_H + +#include + +#ifdef CONFIG_SPARC64 +register struct task_struct *current asm("g4"); +#endif + +#ifdef CONFIG_SPARC32 +/* We might want to consider using %g4 like sparc64 to shave a few cycles. + * + * Two stage process (inline + #define) for type-checking. + * We also obfuscate get_current() to check if anyone used that by mistake. + */ +struct task_struct; +static inline struct task_struct *__get_current(void) +{ + return current_thread_info()->task; +} +#define current __get_current() +#endif + +#endif /* !(_SPARC_CURRENT_H) */ diff --git a/arch/sparc/include/asm/dcr.h b/arch/sparc/include/asm/dcr.h new file mode 100644 index 000000000..c810e1b42 --- /dev/null +++ b/arch/sparc/include/asm/dcr.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_DCR_H +#define _SPARC64_DCR_H + +/* UltraSparc-III/III+ Dispatch Control Register, ASR 0x12 */ +#define DCR_DPE 0x0000000000001000 /* III+: D$ Parity Error Enable */ +#define DCR_OBS 0x0000000000000fc0 /* Observability Bus Controls */ +#define DCR_BPE 0x0000000000000020 /* Branch Predict Enable */ +#define DCR_RPE 0x0000000000000010 /* Return Address Prediction Enable */ +#define DCR_SI 0x0000000000000008 /* Single Instruction Disable */ +#define DCR_IPE 0x0000000000000004 /* III+: I$ Parity Error Enable */ +#define DCR_IFPOE 0x0000000000000002 /* IRQ FP Operation Enable */ +#define DCR_MS 0x0000000000000001 /* Multi-Scalar dispatch */ + +#endif /* _SPARC64_DCR_H */ diff --git a/arch/sparc/include/asm/dcu.h b/arch/sparc/include/asm/dcu.h new file mode 100644 index 000000000..93f325071 --- /dev/null +++ b/arch/sparc/include/asm/dcu.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_DCU_H +#define _SPARC64_DCU_H + +#include + +/* UltraSparc-III Data Cache Unit Control Register */ +#define DCU_CP _AC(0x0002000000000000,UL) /* Phys Cache Enable w/o mmu */ +#define DCU_CV _AC(0x0001000000000000,UL) /* Virt Cache Enable w/o mmu */ +#define DCU_ME _AC(0x0000800000000000,UL) /* NC-store Merging Enable */ +#define DCU_RE _AC(0x0000400000000000,UL) /* RAW bypass Enable */ +#define DCU_PE _AC(0x0000200000000000,UL) /* PCache Enable */ +#define DCU_HPE _AC(0x0000100000000000,UL) /* HW prefetch Enable */ +#define DCU_SPE _AC(0x0000080000000000,UL) /* SW prefetch Enable */ +#define DCU_SL _AC(0x0000040000000000,UL) /* Secondary ld-steering Enab*/ +#define DCU_WE _AC(0x0000020000000000,UL) /* WCache enable */ +#define DCU_PM _AC(0x000001fe00000000,UL) /* PA Watchpoint Byte Mask */ +#define DCU_VM _AC(0x00000001fe000000,UL) /* VA Watchpoint Byte Mask */ +#define DCU_PR _AC(0x0000000001000000,UL) /* PA Watchpoint Read Enable */ +#define DCU_PW _AC(0x0000000000800000,UL) /* PA Watchpoint Write Enable*/ +#define DCU_VR _AC(0x0000000000400000,UL) /* VA Watchpoint Read Enable */ +#define DCU_VW _AC(0x0000000000200000,UL) /* VA Watchpoint Write Enable*/ +#define DCU_DM _AC(0x0000000000000008,UL) /* DMMU Enable */ +#define DCU_IM _AC(0x0000000000000004,UL) /* IMMU Enable */ +#define DCU_DC _AC(0x0000000000000002,UL) /* Data Cache Enable */ +#define DCU_IC _AC(0x0000000000000001,UL) /* Instruction Cache Enable */ + +#endif /* _SPARC64_DCU_H */ diff --git a/arch/sparc/include/asm/delay.h b/arch/sparc/include/asm/delay.h new file mode 100644 index 000000000..c96af9b91 --- /dev/null +++ b/arch/sparc/include/asm/delay.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_DELAY_H +#define ___ASM_SPARC_DELAY_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/delay_32.h b/arch/sparc/include/asm/delay_32.h new file mode 100644 index 000000000..0e6dfe857 --- /dev/null +++ b/arch/sparc/include/asm/delay_32.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * delay.h: Linux delay routines on the Sparc. + * + * Copyright (C) 1994 David S. Miller (davem@caip.rutgers.edu). + */ + +#ifndef __SPARC_DELAY_H +#define __SPARC_DELAY_H + +#include + +static inline void __delay(unsigned long loops) +{ + __asm__ __volatile__("cmp %0, 0\n\t" + "1: bne 1b\n\t" + "subcc %0, 1, %0\n" : + "=&r" (loops) : + "0" (loops) : + "cc"); +} + +/* This is too messy with inline asm on the Sparc. */ +void __udelay(unsigned long usecs, unsigned long lpj); +void __ndelay(unsigned long nsecs, unsigned long lpj); + +#ifdef CONFIG_SMP +#define __udelay_val cpu_data(smp_processor_id()).udelay_val +#else /* SMP */ +#define __udelay_val loops_per_jiffy +#endif /* SMP */ +#define udelay(__usecs) __udelay(__usecs, __udelay_val) +#define ndelay(__nsecs) __ndelay(__nsecs, __udelay_val) + +#endif /* defined(__SPARC_DELAY_H) */ diff --git a/arch/sparc/include/asm/delay_64.h b/arch/sparc/include/asm/delay_64.h new file mode 100644 index 000000000..5de5b5f23 --- /dev/null +++ b/arch/sparc/include/asm/delay_64.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* delay.h: Linux delay routines on sparc64. + * + * Copyright (C) 1996, 2004, 2007 David S. Miller (davem@davemloft.net). + */ + +#ifndef _SPARC64_DELAY_H +#define _SPARC64_DELAY_H + +#ifndef __ASSEMBLER__ + +void __delay(unsigned long loops); +void udelay(unsigned long usecs); +#define mdelay(n) udelay((n) * 1000) + +#endif /* !__ASSEMBLER__ */ + +#endif /* _SPARC64_DELAY_H */ diff --git a/arch/sparc/include/asm/device.h b/arch/sparc/include/asm/device.h new file mode 100644 index 000000000..a797d5e86 --- /dev/null +++ b/arch/sparc/include/asm/device.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Arch specific extensions to struct device + */ +#ifndef _ASM_SPARC_DEVICE_H +#define _ASM_SPARC_DEVICE_H + +#include + +struct device_node; +struct platform_device; + +struct dev_archdata { + void *iommu; + void *stc; + void *host_controller; + struct platform_device *op; + int numa_node; +}; + +void of_propagate_archdata(struct platform_device *bus); + +struct pdev_archdata { + struct resource resource[PROMREG_MAX]; + unsigned int irqs[PROMINTR_MAX]; + int num_irqs; +}; + +#endif /* _ASM_SPARC_DEVICE_H */ diff --git a/arch/sparc/include/asm/dma-mapping.h b/arch/sparc/include/asm/dma-mapping.h new file mode 100644 index 000000000..55c12fc2b --- /dev/null +++ b/arch/sparc/include/asm/dma-mapping.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_DMA_MAPPING_H +#define ___ASM_SPARC_DMA_MAPPING_H + +extern const struct dma_map_ops *dma_ops; + +static inline const struct dma_map_ops *get_arch_dma_ops(void) +{ + /* sparc32 uses per-device dma_ops */ + return IS_ENABLED(CONFIG_SPARC64) ? dma_ops : NULL; +} + +#endif diff --git a/arch/sparc/include/asm/dma.h b/arch/sparc/include/asm/dma.h new file mode 100644 index 000000000..08043f35b --- /dev/null +++ b/arch/sparc/include/asm/dma.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC_DMA_H +#define _ASM_SPARC_DMA_H + +/* These are irrelevant for Sparc DMA, but we leave it in so that + * things can compile. + */ +#define MAX_DMA_CHANNELS 8 +#define DMA_MODE_READ 1 +#define DMA_MODE_WRITE 2 +#define MAX_DMA_ADDRESS (~0UL) + +/* Useful constants */ +#define SIZE_16MB (16*1024*1024) +#define SIZE_64K (64*1024) + +/* SBUS DMA controller reg offsets */ +#define DMA_CSR 0x00UL /* rw DMA control/status register 0x00 */ +#define DMA_ADDR 0x04UL /* rw DMA transfer address register 0x04 */ +#define DMA_COUNT 0x08UL /* rw DMA transfer count register 0x08 */ +#define DMA_TEST 0x0cUL /* rw DMA test/debug register 0x0c */ + +/* Fields in the cond_reg register */ +/* First, the version identification bits */ +#define DMA_DEVICE_ID 0xf0000000 /* Device identification bits */ +#define DMA_VERS0 0x00000000 /* Sunray DMA version */ +#define DMA_ESCV1 0x40000000 /* DMA ESC Version 1 */ +#define DMA_VERS1 0x80000000 /* DMA rev 1 */ +#define DMA_VERS2 0xa0000000 /* DMA rev 2 */ +#define DMA_VERHME 0xb0000000 /* DMA hme gate array */ +#define DMA_VERSPLUS 0x90000000 /* DMA rev 1 PLUS */ + +#define DMA_HNDL_INTR 0x00000001 /* An IRQ needs to be handled */ +#define DMA_HNDL_ERROR 0x00000002 /* We need to take an error */ +#define DMA_FIFO_ISDRAIN 0x0000000c /* The DMA FIFO is draining */ +#define DMA_INT_ENAB 0x00000010 /* Turn on interrupts */ +#define DMA_FIFO_INV 0x00000020 /* Invalidate the FIFO */ +#define DMA_ACC_SZ_ERR 0x00000040 /* The access size was bad */ +#define DMA_FIFO_STDRAIN 0x00000040 /* DMA_VERS1 Drain the FIFO */ +#define DMA_RST_SCSI 0x00000080 /* Reset the SCSI controller */ +#define DMA_RST_ENET DMA_RST_SCSI /* Reset the ENET controller */ +#define DMA_ST_WRITE 0x00000100 /* write from device to memory */ +#define DMA_ENABLE 0x00000200 /* Fire up DMA, handle requests */ +#define DMA_PEND_READ 0x00000400 /* DMA_VERS1/0/PLUS Pending Read */ +#define DMA_ESC_BURST 0x00000800 /* 1=16byte 0=32byte */ +#define DMA_READ_AHEAD 0x00001800 /* DMA read ahead partial longword */ +#define DMA_DSBL_RD_DRN 0x00001000 /* No EC drain on slave reads */ +#define DMA_BCNT_ENAB 0x00002000 /* If on, use the byte counter */ +#define DMA_TERM_CNTR 0x00004000 /* Terminal counter */ +#define DMA_SCSI_SBUS64 0x00008000 /* HME: Enable 64-bit SBUS mode. */ +#define DMA_CSR_DISAB 0x00010000 /* No FIFO drains during csr */ +#define DMA_SCSI_DISAB 0x00020000 /* No FIFO drains during reg */ +#define DMA_DSBL_WR_INV 0x00020000 /* No EC inval. on slave writes */ +#define DMA_ADD_ENABLE 0x00040000 /* Special ESC DVMA optimization */ +#define DMA_E_BURSTS 0x000c0000 /* ENET: SBUS r/w burst mask */ +#define DMA_E_BURST32 0x00040000 /* ENET: SBUS 32 byte r/w burst */ +#define DMA_E_BURST16 0x00000000 /* ENET: SBUS 16 byte r/w burst */ +#define DMA_BRST_SZ 0x000c0000 /* SCSI: SBUS r/w burst size */ +#define DMA_BRST64 0x000c0000 /* SCSI: 64byte bursts (HME on UltraSparc only) */ +#define DMA_BRST32 0x00040000 /* SCSI: 32byte bursts */ +#define DMA_BRST16 0x00000000 /* SCSI: 16byte bursts */ +#define DMA_BRST0 0x00080000 /* SCSI: no bursts (non-HME gate arrays) */ +#define DMA_ADDR_DISAB 0x00100000 /* No FIFO drains during addr */ +#define DMA_2CLKS 0x00200000 /* Each transfer = 2 clock ticks */ +#define DMA_3CLKS 0x00400000 /* Each transfer = 3 clock ticks */ +#define DMA_EN_ENETAUI DMA_3CLKS /* Put lance into AUI-cable mode */ +#define DMA_CNTR_DISAB 0x00800000 /* No IRQ when DMA_TERM_CNTR set */ +#define DMA_AUTO_NADDR 0x01000000 /* Use "auto nxt addr" feature */ +#define DMA_SCSI_ON 0x02000000 /* Enable SCSI dma */ +#define DMA_PARITY_OFF 0x02000000 /* HME: disable parity checking */ +#define DMA_LOADED_ADDR 0x04000000 /* Address has been loaded */ +#define DMA_LOADED_NADDR 0x08000000 /* Next address has been loaded */ +#define DMA_RESET_FAS366 0x08000000 /* HME: Assert RESET to FAS366 */ + +/* Values describing the burst-size property from the PROM */ +#define DMA_BURST1 0x01 +#define DMA_BURST2 0x02 +#define DMA_BURST4 0x04 +#define DMA_BURST8 0x08 +#define DMA_BURST16 0x10 +#define DMA_BURST32 0x20 +#define DMA_BURST64 0x40 +#define DMA_BURSTBITS 0x7f + +#ifdef CONFIG_SPARC32 +struct device; + +unsigned long sparc_dma_alloc_resource(struct device *dev, size_t len); +bool sparc_dma_free_resource(void *cpu_addr, size_t size); +#endif + +#endif /* !(_ASM_SPARC_DMA_H) */ diff --git a/arch/sparc/include/asm/ebus_dma.h b/arch/sparc/include/asm/ebus_dma.h new file mode 100644 index 000000000..75563ed09 --- /dev/null +++ b/arch/sparc/include/asm/ebus_dma.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SPARC_EBUS_DMA_H +#define __ASM_SPARC_EBUS_DMA_H + +struct ebus_dma_info { + spinlock_t lock; + void __iomem *regs; + + unsigned int flags; +#define EBUS_DMA_FLAG_USE_EBDMA_HANDLER 0x00000001 +#define EBUS_DMA_FLAG_TCI_DISABLE 0x00000002 + + /* These are only valid is EBUS_DMA_FLAG_USE_EBDMA_HANDLER is + * set. + */ + void (*callback)(struct ebus_dma_info *p, int event, void *cookie); + void *client_cookie; + unsigned int irq; +#define EBUS_DMA_EVENT_ERROR 1 +#define EBUS_DMA_EVENT_DMA 2 +#define EBUS_DMA_EVENT_DEVICE 4 + + unsigned char name[64]; +}; + +int ebus_dma_register(struct ebus_dma_info *p); +int ebus_dma_irq_enable(struct ebus_dma_info *p, int on); +void ebus_dma_unregister(struct ebus_dma_info *p); +int ebus_dma_request(struct ebus_dma_info *p, dma_addr_t bus_addr, + size_t len); +void ebus_dma_prepare(struct ebus_dma_info *p, int write); +unsigned int ebus_dma_residue(struct ebus_dma_info *p); +unsigned int ebus_dma_addr(struct ebus_dma_info *p); +void ebus_dma_enable(struct ebus_dma_info *p, int on); + +#endif /* __ASM_SPARC_EBUS_DMA_H */ diff --git a/arch/sparc/include/asm/ecc.h b/arch/sparc/include/asm/ecc.h new file mode 100644 index 000000000..3d5edee36 --- /dev/null +++ b/arch/sparc/include/asm/ecc.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ecc.h: Definitions and defines for the external cache/memory + * controller on the sun4m. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_ECC_H +#define _SPARC_ECC_H + +/* These registers are accessed through the SRMMU passthrough ASI 0x20 */ +#define ECC_ENABLE 0x00000000 /* ECC enable register */ +#define ECC_FSTATUS 0x00000008 /* ECC fault status register */ +#define ECC_FADDR 0x00000010 /* ECC fault address register */ +#define ECC_DIGNOSTIC 0x00000018 /* ECC diagnostics register */ +#define ECC_MBAENAB 0x00000020 /* MBus arbiter enable register */ +#define ECC_DMESG 0x00001000 /* Diagnostic message passing area */ + +/* ECC MBus Arbiter Enable register: + * + * ---------------------------------------- + * | |SBUS|MOD3|MOD2|MOD1|RSV| + * ---------------------------------------- + * 31 5 4 3 2 1 0 + * + * SBUS: Enable MBus Arbiter on the SBus 0=off 1=on + * MOD3: Enable MBus Arbiter on MBus module 3 0=off 1=on + * MOD2: Enable MBus Arbiter on MBus module 2 0=off 1=on + * MOD1: Enable MBus Arbiter on MBus module 1 0=off 1=on + */ + +#define ECC_MBAE_SBUS 0x00000010 +#define ECC_MBAE_MOD3 0x00000008 +#define ECC_MBAE_MOD2 0x00000004 +#define ECC_MBAE_MOD1 0x00000002 + +/* ECC Fault Control Register layout: + * + * ----------------------------- + * | RESV | ECHECK | EINT | + * ----------------------------- + * 31 2 1 0 + * + * ECHECK: Enable ECC checking. 0=off 1=on + * EINT: Enable Interrupts for correctable errors. 0=off 1=on + */ +#define ECC_FCR_CHECK 0x00000002 +#define ECC_FCR_INTENAB 0x00000001 + +/* ECC Fault Address Register Zero layout: + * + * ----------------------------------------------------- + * | MID | S | RSV | VA | BM |AT| C| SZ |TYP| PADDR | + * ----------------------------------------------------- + * 31-28 27 26-22 21-14 13 12 11 10-8 7-4 3-0 + * + * MID: ModuleID of the faulting processor. ie. who did it? + * S: Supervisor/Privileged access? 0=no 1=yes + * VA: Bits 19-12 of the virtual faulting address, these are the + * superset bits in the virtual cache and can be used for + * a flush operation if necessary. + * BM: Boot mode? 0=no 1=yes This is just like the SRMMU boot + * mode bit. + * AT: Did this fault happen during an atomic instruction? 0=no + * 1=yes. This means either an 'ldstub' or 'swap' instruction + * was in progress (but not finished) when this fault happened. + * This indicated whether the bus was locked when the fault + * occurred. + * C: Did the pte for this access indicate that it was cacheable? + * 0=no 1=yes + * SZ: The size of the transaction. + * TYP: The transaction type. + * PADDR: Bits 35-32 of the physical address for the fault. + */ +#define ECC_FADDR0_MIDMASK 0xf0000000 +#define ECC_FADDR0_S 0x08000000 +#define ECC_FADDR0_VADDR 0x003fc000 +#define ECC_FADDR0_BMODE 0x00002000 +#define ECC_FADDR0_ATOMIC 0x00001000 +#define ECC_FADDR0_CACHE 0x00000800 +#define ECC_FADDR0_SIZE 0x00000700 +#define ECC_FADDR0_TYPE 0x000000f0 +#define ECC_FADDR0_PADDR 0x0000000f + +/* ECC Fault Address Register One layout: + * + * ------------------------------------- + * | Physical Address 31-0 | + * ------------------------------------- + * 31 0 + * + * You get the upper 4 bits of the physical address from the + * PADDR field in ECC Fault Address Zero register. + */ + +/* ECC Fault Status Register layout: + * + * ---------------------------------------------- + * | RESV|C2E|MULT|SYNDROME|DWORD|UNC|TIMEO|BS|C| + * ---------------------------------------------- + * 31-18 17 16 15-8 7-4 3 2 1 0 + * + * C2E: A C2 graphics error occurred. 0=no 1=yes (SS10 only) + * MULT: Multiple errors occurred ;-O 0=no 1=prom_panic(yes) + * SYNDROME: Controller is mentally unstable. + * DWORD: + * UNC: Uncorrectable error. 0=no 1=yes + * TIMEO: Timeout occurred. 0=no 1=yes + * BS: C2 graphics bad slot access. 0=no 1=yes (SS10 only) + * C: Correctable error? 0=no 1=yes + */ + +#define ECC_FSR_C2ERR 0x00020000 +#define ECC_FSR_MULT 0x00010000 +#define ECC_FSR_SYND 0x0000ff00 +#define ECC_FSR_DWORD 0x000000f0 +#define ECC_FSR_UNC 0x00000008 +#define ECC_FSR_TIMEO 0x00000004 +#define ECC_FSR_BADSLOT 0x00000002 +#define ECC_FSR_C 0x00000001 + +#endif /* !(_SPARC_ECC_H) */ diff --git a/arch/sparc/include/asm/eeprom.h b/arch/sparc/include/asm/eeprom.h new file mode 100644 index 000000000..e17beeceb --- /dev/null +++ b/arch/sparc/include/asm/eeprom.h @@ -0,0 +1,9 @@ +/* + * eeprom.h: Definitions for the Sun eeprom. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +/* The EEPROM and the Mostek Mk48t02 use the same IO address space + * for their registers/data areas. The IDPROM lives here too. + */ diff --git a/arch/sparc/include/asm/elf.h b/arch/sparc/include/asm/elf.h new file mode 100644 index 000000000..bbfb4b002 --- /dev/null +++ b/arch/sparc/include/asm/elf.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_ELF_H +#define ___ASM_SPARC_ELF_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/elf_32.h b/arch/sparc/include/asm/elf_32.h new file mode 100644 index 000000000..37a6016c9 --- /dev/null +++ b/arch/sparc/include/asm/elf_32.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASMSPARC_ELF_H +#define __ASMSPARC_ELF_H + +/* + * ELF register definitions.. + */ + +#include + +/* + * Sparc section types + */ +#define STT_REGISTER 13 + +/* + * Sparc ELF relocation types + */ +#define R_SPARC_NONE 0 +#define R_SPARC_8 1 +#define R_SPARC_16 2 +#define R_SPARC_32 3 +#define R_SPARC_DISP8 4 +#define R_SPARC_DISP16 5 +#define R_SPARC_DISP32 6 +#define R_SPARC_WDISP30 7 +#define R_SPARC_WDISP22 8 +#define R_SPARC_HI22 9 +#define R_SPARC_22 10 +#define R_SPARC_13 11 +#define R_SPARC_LO10 12 +#define R_SPARC_GOT10 13 +#define R_SPARC_GOT13 14 +#define R_SPARC_GOT22 15 +#define R_SPARC_PC10 16 +#define R_SPARC_PC22 17 +#define R_SPARC_WPLT30 18 +#define R_SPARC_COPY 19 +#define R_SPARC_GLOB_DAT 20 +#define R_SPARC_JMP_SLOT 21 +#define R_SPARC_RELATIVE 22 +#define R_SPARC_UA32 23 +#define R_SPARC_PLT32 24 +#define R_SPARC_HIPLT22 25 +#define R_SPARC_LOPLT10 26 +#define R_SPARC_PCPLT32 27 +#define R_SPARC_PCPLT22 28 +#define R_SPARC_PCPLT10 29 +#define R_SPARC_10 30 +#define R_SPARC_11 31 +#define R_SPARC_64 32 +#define R_SPARC_OLO10 33 +#define R_SPARC_WDISP16 40 +#define R_SPARC_WDISP19 41 +#define R_SPARC_7 43 +#define R_SPARC_5 44 +#define R_SPARC_6 45 + +/* Bits present in AT_HWCAP, primarily for Sparc32. */ + +#define HWCAP_SPARC_FLUSH 1 /* CPU supports flush instruction. */ +#define HWCAP_SPARC_STBAR 2 +#define HWCAP_SPARC_SWAP 4 +#define HWCAP_SPARC_MULDIV 8 +#define HWCAP_SPARC_V9 16 +#define HWCAP_SPARC_ULTRA3 32 + +#define CORE_DUMP_USE_REGSET + +/* Format is: + * G0 --> G7 + * O0 --> O7 + * L0 --> L7 + * I0 --> I7 + * PSR, PC, nPC, Y, WIM, TBR + */ +typedef unsigned long elf_greg_t; +#define ELF_NGREG 38 +typedef elf_greg_t elf_gregset_t[ELF_NGREG]; + +typedef struct { + union { + unsigned long pr_regs[32]; + double pr_dregs[16]; + } pr_fr; + unsigned long __unused; + unsigned long pr_fsr; + unsigned char pr_qcnt; + unsigned char pr_q_entrysize; + unsigned char pr_en; + unsigned int pr_q[64]; +} elf_fpregset_t; + +#include + +/* + * This is used to ensure we don't load something for the wrong architecture. + */ +#define elf_check_arch(x) ((x)->e_machine == EM_SPARC) + +/* + * These are used to set parameters in the core dumps. + */ +#define ELF_ARCH EM_SPARC +#define ELF_CLASS ELFCLASS32 +#define ELF_DATA ELFDATA2MSB + +#define ELF_EXEC_PAGESIZE 4096 + + +/* This is the location that an ET_DYN program is loaded if exec'ed. Typical + use of this is to invoke "./ld.so someprog" to test out a new version of + the loader. We need to make sure that it is out of the way of the program + that it will "exec", and that there is sufficient room for the brk. */ + +#define ELF_ET_DYN_BASE (TASK_UNMAPPED_BASE) + +/* This yields a mask that user programs can use to figure out what + instruction set this cpu supports. This can NOT be done in userspace + on Sparc. */ + +/* Most sun4m's have them all. */ +#define ELF_HWCAP (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | \ + HWCAP_SPARC_SWAP | HWCAP_SPARC_MULDIV) + +/* This yields a string that ld.so will use to load implementation + specific libraries for optimization. This is more specific in + intent than poking at uname or /proc/cpuinfo. */ + +#define ELF_PLATFORM (NULL) + +#endif /* !(__ASMSPARC_ELF_H) */ diff --git a/arch/sparc/include/asm/elf_64.h b/arch/sparc/include/asm/elf_64.h new file mode 100644 index 000000000..694ed081c --- /dev/null +++ b/arch/sparc/include/asm/elf_64.h @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SPARC64_ELF_H +#define __ASM_SPARC64_ELF_H + +/* + * ELF register definitions.. + */ + +#include +#include +#include +#include + +/* + * Sparc section types + */ +#define STT_REGISTER 13 + +/* + * Sparc ELF relocation types + */ +#define R_SPARC_NONE 0 +#define R_SPARC_8 1 +#define R_SPARC_16 2 +#define R_SPARC_32 3 +#define R_SPARC_DISP8 4 +#define R_SPARC_DISP16 5 +#define R_SPARC_DISP32 6 +#define R_SPARC_WDISP30 7 +#define R_SPARC_WDISP22 8 +#define R_SPARC_HI22 9 +#define R_SPARC_22 10 +#define R_SPARC_13 11 +#define R_SPARC_LO10 12 +#define R_SPARC_GOT10 13 +#define R_SPARC_GOT13 14 +#define R_SPARC_GOT22 15 +#define R_SPARC_PC10 16 +#define R_SPARC_PC22 17 +#define R_SPARC_WPLT30 18 +#define R_SPARC_COPY 19 +#define R_SPARC_GLOB_DAT 20 +#define R_SPARC_JMP_SLOT 21 +#define R_SPARC_RELATIVE 22 +#define R_SPARC_UA32 23 +#define R_SPARC_PLT32 24 +#define R_SPARC_HIPLT22 25 +#define R_SPARC_LOPLT10 26 +#define R_SPARC_PCPLT32 27 +#define R_SPARC_PCPLT22 28 +#define R_SPARC_PCPLT10 29 +#define R_SPARC_10 30 +#define R_SPARC_11 31 +#define R_SPARC_64 32 +#define R_SPARC_OLO10 33 +#define R_SPARC_WDISP16 40 +#define R_SPARC_WDISP19 41 +#define R_SPARC_7 43 +#define R_SPARC_5 44 +#define R_SPARC_6 45 +#define R_SPARC_UA64 54 + +/* Bits present in AT_HWCAP, primarily for Sparc32. */ +#define HWCAP_SPARC_FLUSH 0x00000001 +#define HWCAP_SPARC_STBAR 0x00000002 +#define HWCAP_SPARC_SWAP 0x00000004 +#define HWCAP_SPARC_MULDIV 0x00000008 +#define HWCAP_SPARC_V9 0x00000010 +#define HWCAP_SPARC_ULTRA3 0x00000020 +#define HWCAP_SPARC_BLKINIT 0x00000040 +#define HWCAP_SPARC_N2 0x00000080 + +/* Solaris compatible AT_HWCAP bits. */ +#define AV_SPARC_MUL32 0x00000100 /* 32x32 multiply is efficient */ +#define AV_SPARC_DIV32 0x00000200 /* 32x32 divide is efficient */ +#define AV_SPARC_FSMULD 0x00000400 /* 'fsmuld' is efficient */ +#define AV_SPARC_V8PLUS 0x00000800 /* v9 insn available to 32bit */ +#define AV_SPARC_POPC 0x00001000 /* 'popc' is efficient */ +#define AV_SPARC_VIS 0x00002000 /* VIS insns available */ +#define AV_SPARC_VIS2 0x00004000 /* VIS2 insns available */ +#define AV_SPARC_ASI_BLK_INIT 0x00008000 /* block init ASIs available */ +#define AV_SPARC_FMAF 0x00010000 /* fused multiply-add */ +#define AV_SPARC_VIS3 0x00020000 /* VIS3 insns available */ +#define AV_SPARC_HPC 0x00040000 /* HPC insns available */ +#define AV_SPARC_RANDOM 0x00080000 /* 'random' insn available */ +#define AV_SPARC_TRANS 0x00100000 /* transaction insns available */ +#define AV_SPARC_FJFMAU 0x00200000 /* unfused multiply-add */ +#define AV_SPARC_IMA 0x00400000 /* integer multiply-add */ +#define AV_SPARC_ASI_CACHE_SPARING \ + 0x00800000 /* cache sparing ASIs available */ +#define AV_SPARC_PAUSE 0x01000000 /* PAUSE available */ +#define AV_SPARC_CBCOND 0x02000000 /* CBCOND insns available */ + +/* Solaris decided to enumerate every single crypto instruction type + * in the AT_HWCAP bits. This is wasteful, since if crypto is present, + * you still need to look in the CFR register to see if the opcode is + * really available. So we simply advertise only "crypto" support. + */ +#define HWCAP_SPARC_CRYPTO 0x04000000 /* CRYPTO insns available */ +#define HWCAP_SPARC_ADI 0x08000000 /* ADI available */ + +#define CORE_DUMP_USE_REGSET + +/* + * These are used to set parameters in the core dumps. + */ +#define ELF_ARCH EM_SPARCV9 +#define ELF_CLASS ELFCLASS64 +#define ELF_DATA ELFDATA2MSB + +/* Format of 64-bit elf_gregset_t is: + * G0 --> G7 + * O0 --> O7 + * L0 --> L7 + * I0 --> I7 + * TSTATE + * TPC + * TNPC + * Y + */ +typedef unsigned long elf_greg_t; +#define ELF_NGREG 36 +typedef elf_greg_t elf_gregset_t[ELF_NGREG]; + +typedef struct { + unsigned long pr_regs[32]; + unsigned long pr_fsr; + unsigned long pr_gsr; + unsigned long pr_fprs; +} elf_fpregset_t; + +/* Format of 32-bit elf_gregset_t is: + * G0 --> G7 + * O0 --> O7 + * L0 --> L7 + * I0 --> I7 + * PSR, PC, nPC, Y, WIM, TBR + */ +typedef unsigned int compat_elf_greg_t; +#define COMPAT_ELF_NGREG 38 +typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG]; + +typedef struct { + union { + unsigned int pr_regs[32]; + unsigned long pr_dregs[16]; + } pr_fr; + unsigned int __unused; + unsigned int pr_fsr; + unsigned char pr_qcnt; + unsigned char pr_q_entrysize; + unsigned char pr_en; + unsigned int pr_q[64]; +} compat_elf_fpregset_t; + +/* UltraSparc extensions. Still unused, but will be eventually. */ +typedef struct { + unsigned int pr_type; + unsigned int pr_align; + union { + struct { + union { + unsigned int pr_regs[32]; + unsigned long pr_dregs[16]; + long double pr_qregs[8]; + } pr_xfr; + } pr_v8p; + unsigned int pr_xfsr; + unsigned int pr_fprs; + unsigned int pr_xg[8]; + unsigned int pr_xo[8]; + unsigned long pr_tstate; + unsigned int pr_filler[8]; + } pr_un; +} elf_xregset_t; + +/* + * This is used to ensure we don't load something for the wrong architecture. + */ +#define elf_check_arch(x) ((x)->e_machine == ELF_ARCH) +#define compat_elf_check_arch(x) ((x)->e_machine == EM_SPARC || \ + (x)->e_machine == EM_SPARC32PLUS) +#define compat_start_thread start_thread32 + +#define ELF_EXEC_PAGESIZE PAGE_SIZE + +/* This is the location that an ET_DYN program is loaded if exec'ed. Typical + use of this is to invoke "./ld.so someprog" to test out a new version of + the loader. We need to make sure that it is out of the way of the program + that it will "exec", and that there is sufficient room for the brk. */ + +#define ELF_ET_DYN_BASE 0x0000010000000000UL +#define COMPAT_ELF_ET_DYN_BASE 0x0000000070000000UL + +extern unsigned long sparc64_elf_hwcap; +#define ELF_HWCAP sparc64_elf_hwcap + +/* This yields a string that ld.so will use to load implementation + specific libraries for optimization. This is more specific in + intent than poking at uname or /proc/cpuinfo. */ + +#define ELF_PLATFORM (NULL) + +#define SET_PERSONALITY(ex) \ +do { if ((ex).e_ident[EI_CLASS] == ELFCLASS32) \ + set_thread_flag(TIF_32BIT); \ + else \ + clear_thread_flag(TIF_32BIT); \ + /* flush_thread will update pgd cache */ \ + if (personality(current->personality) != PER_LINUX32) \ + set_personality(PER_LINUX | \ + (current->personality & (~PER_MASK))); \ +} while (0) + +extern unsigned int vdso_enabled; + +#define ARCH_DLINFO \ +do { \ + extern struct adi_config adi_state; \ + if (vdso_enabled) \ + NEW_AUX_ENT(AT_SYSINFO_EHDR, \ + (unsigned long)current->mm->context.vdso); \ + NEW_AUX_ENT(AT_ADI_BLKSZ, adi_state.caps.blksz); \ + NEW_AUX_ENT(AT_ADI_NBITS, adi_state.caps.nbits); \ + NEW_AUX_ENT(AT_ADI_UEONADI, adi_state.caps.ue_on_adi); \ +} while (0) + +struct linux_binprm; + +#define ARCH_HAS_SETUP_ADDITIONAL_PAGES 1 +extern int arch_setup_additional_pages(struct linux_binprm *bprm, + int uses_interp); +#endif /* !(__ASM_SPARC64_ELF_H) */ diff --git a/arch/sparc/include/asm/estate.h b/arch/sparc/include/asm/estate.h new file mode 100644 index 000000000..e5e0f84ac --- /dev/null +++ b/arch/sparc/include/asm/estate.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_ESTATE_H +#define _SPARC64_ESTATE_H + +/* UltraSPARC-III E-cache Error Enable */ +#define ESTATE_ERROR_FMT 0x0000000000040000 /* Force MTAG ECC */ +#define ESTATE_ERROR_FMESS 0x000000000003c000 /* Forced MTAG ECC val */ +#define ESTATE_ERROR_FMD 0x0000000000002000 /* Force DATA ECC */ +#define ESTATE_ERROR_FDECC 0x0000000000001ff0 /* Forced DATA ECC val */ +#define ESTATE_ERROR_UCEEN 0x0000000000000008 /* See below */ +#define ESTATE_ERROR_NCEEN 0x0000000000000002 /* See below */ +#define ESTATE_ERROR_CEEN 0x0000000000000001 /* See below */ + +/* UCEEN enables the fast_ECC_error trap for: 1) software correctable E-cache + * errors 2) uncorrectable E-cache errors. Such events only occur on reads + * of the E-cache by the local processor for: 1) data loads 2) instruction + * fetches 3) atomic operations. Such events _cannot_ occur for: 1) merge + * 2) writeback 2) copyout. The AFSR bits associated with these traps are + * UCC and UCU. + */ + +/* NCEEN enables instruction_access_error, data_access_error, and ECC_error traps + * for uncorrectable ECC errors and system errors. + * + * Uncorrectable system bus data error or MTAG ECC error, system bus TimeOUT, + * or system bus BusERR: + * 1) As the result of an instruction fetch, will generate instruction_access_error + * 2) As the result of a load etc. will generate data_access_error. + * 3) As the result of store merge completion, writeback, or copyout will + * generate a disrupting ECC_error trap. + * 4) As the result of such errors on instruction vector fetch can generate any + * of the 3 trap types. + * + * The AFSR bits associated with these traps are EMU, EDU, WDU, CPU, IVU, UE, + * BERR, and TO. + */ + +/* CEEN enables the ECC_error trap for hardware corrected ECC errors. System bus + * reads resulting in a hardware corrected data or MTAG ECC error will generate an + * ECC_error disrupting trap with this bit enabled. + * + * This same trap will also be generated when a hardware corrected ECC error results + * during store merge, writeback, and copyout operations. + */ + +/* In general, if the trap enable bits above are disabled the AFSR bits will still + * log the events even though the trap will not be generated by the processor. + */ + +#endif /* _SPARC64_ESTATE_H */ diff --git a/arch/sparc/include/asm/extable.h b/arch/sparc/include/asm/extable.h new file mode 100644 index 000000000..554a9dc37 --- /dev/null +++ b/arch/sparc/include/asm/extable.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_EXTABLE_H +#define __ASM_EXTABLE_H +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry { + unsigned int insn, fixup; +}; + +#endif diff --git a/arch/sparc/include/asm/fbio.h b/arch/sparc/include/asm/fbio.h new file mode 100644 index 000000000..02654cb95 --- /dev/null +++ b/arch/sparc/include/asm/fbio.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_FBIO_H +#define __LINUX_FBIO_H + +#include + +#define FBIOPUTCMAP_SPARC _IOW('F', 3, struct fbcmap) +#define FBIOGETCMAP_SPARC _IOW('F', 4, struct fbcmap) +/* Addresses on the fd of a cgsix that are mappable */ +#define CG6_FBC 0x70000000 +#define CG6_TEC 0x70001000 +#define CG6_BTREGS 0x70002000 +#define CG6_FHC 0x70004000 +#define CG6_THC 0x70005000 +#define CG6_ROM 0x70006000 +#define CG6_RAM 0x70016000 +#define CG6_DHC 0x80000000 + +#define CG3_MMAP_OFFSET 0x4000000 + +/* Addresses on the fd of a tcx that are mappable */ +#define TCX_RAM8BIT 0x00000000 +#define TCX_RAM24BIT 0x01000000 +#define TCX_UNK3 0x10000000 +#define TCX_UNK4 0x20000000 +#define TCX_CONTROLPLANE 0x28000000 +#define TCX_UNK6 0x30000000 +#define TCX_UNK7 0x38000000 +#define TCX_TEC 0x70000000 +#define TCX_BTREGS 0x70002000 +#define TCX_THC 0x70004000 +#define TCX_DHC 0x70008000 +#define TCX_ALT 0x7000a000 +#define TCX_SYNC 0x7000e000 +#define TCX_UNK2 0x70010000 + +/* CG14 definitions */ + +/* Offsets into the OBIO space: */ +#define CG14_REGS 0 /* registers */ +#define CG14_CURSORREGS 0x1000 /* cursor registers */ +#define CG14_DACREGS 0x2000 /* DAC registers */ +#define CG14_XLUT 0x3000 /* X Look Up Table -- ??? */ +#define CG14_CLUT1 0x4000 /* Color Look Up Table */ +#define CG14_CLUT2 0x5000 /* Color Look Up Table */ +#define CG14_CLUT3 0x6000 /* Color Look Up Table */ +#define CG14_AUTO 0xf000 + +struct fbcmap32 { + int index; /* first element (0 origin) */ + int count; + u32 red; + u32 green; + u32 blue; +}; + +#define FBIOPUTCMAP32 _IOW('F', 3, struct fbcmap32) +#define FBIOGETCMAP32 _IOW('F', 4, struct fbcmap32) + +struct fbcursor32 { + short set; /* what to set, choose from the list above */ + short enable; /* cursor on/off */ + struct fbcurpos pos; /* cursor position */ + struct fbcurpos hot; /* cursor hot spot */ + struct fbcmap32 cmap; /* color map info */ + struct fbcurpos size; /* cursor bit map size */ + u32 image; /* cursor image bits */ + u32 mask; /* cursor mask bits */ +}; + +#define FBIOSCURSOR32 _IOW('F', 24, struct fbcursor32) +#define FBIOGCURSOR32 _IOW('F', 25, struct fbcursor32) +#endif /* __LINUX_FBIO_H */ diff --git a/arch/sparc/include/asm/fhc.h b/arch/sparc/include/asm/fhc.h new file mode 100644 index 000000000..0627fa126 --- /dev/null +++ b/arch/sparc/include/asm/fhc.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* fhc.h: FHC and Clock board register definitions. + * + * Copyright (C) 1997, 1999 David S. Miller (davem@redhat.com) + */ + +#ifndef _SPARC64_FHC_H +#define _SPARC64_FHC_H + +/* Clock board register offsets. */ +#define CLOCK_CTRL 0x00UL /* Main control */ +#define CLOCK_STAT1 0x10UL /* Status one */ +#define CLOCK_STAT2 0x20UL /* Status two */ +#define CLOCK_PWRSTAT 0x30UL /* Power status */ +#define CLOCK_PWRPRES 0x40UL /* Power presence */ +#define CLOCK_TEMP 0x50UL /* Temperature */ +#define CLOCK_IRQDIAG 0x60UL /* IRQ diagnostics */ +#define CLOCK_PWRSTAT2 0x70UL /* Power status two */ + +#define CLOCK_CTRL_LLED 0x04 /* Left LED, 0 == on */ +#define CLOCK_CTRL_MLED 0x02 /* Mid LED, 1 == on */ +#define CLOCK_CTRL_RLED 0x01 /* RIght LED, 1 == on */ + +/* Firehose controller register offsets */ +#define FHC_PREGS_ID 0x00UL /* FHC ID */ +#define FHC_ID_VERS 0xf0000000 /* Version of this FHC */ +#define FHC_ID_PARTID 0x0ffff000 /* Part ID code (0x0f9f == FHC) */ +#define FHC_ID_MANUF 0x0000007e /* Manufacturer (0x3e == SUN's JEDEC)*/ +#define FHC_ID_RESV 0x00000001 /* Read as one */ +#define FHC_PREGS_RCS 0x10UL /* FHC Reset Control/Status Register */ +#define FHC_RCS_POR 0x80000000 /* Last reset was a power cycle */ +#define FHC_RCS_SPOR 0x40000000 /* Last reset was sw power on reset */ +#define FHC_RCS_SXIR 0x20000000 /* Last reset was sw XIR reset */ +#define FHC_RCS_BPOR 0x10000000 /* Last reset was due to POR button */ +#define FHC_RCS_BXIR 0x08000000 /* Last reset was due to XIR button */ +#define FHC_RCS_WEVENT 0x04000000 /* CPU reset was due to wakeup event */ +#define FHC_RCS_CFATAL 0x02000000 /* Centerplane Fatal Error signalled */ +#define FHC_RCS_FENAB 0x01000000 /* Fatal errors elicit system reset */ +#define FHC_PREGS_CTRL 0x20UL /* FHC Control Register */ +#define FHC_CONTROL_ICS 0x00100000 /* Ignore Centerplane Signals */ +#define FHC_CONTROL_FRST 0x00080000 /* Fatal Error Reset Enable */ +#define FHC_CONTROL_LFAT 0x00040000 /* AC/DC signalled a local error */ +#define FHC_CONTROL_SLINE 0x00010000 /* Firmware Synchronization Line */ +#define FHC_CONTROL_DCD 0x00008000 /* DC-->DC Converter Disable */ +#define FHC_CONTROL_POFF 0x00004000 /* AC/DC Controller PLL Disable */ +#define FHC_CONTROL_FOFF 0x00002000 /* FHC Controller PLL Disable */ +#define FHC_CONTROL_AOFF 0x00001000 /* CPU A SRAM/SBD Low Power Mode */ +#define FHC_CONTROL_BOFF 0x00000800 /* CPU B SRAM/SBD Low Power Mode */ +#define FHC_CONTROL_PSOFF 0x00000400 /* Turns off this FHC's power supply */ +#define FHC_CONTROL_IXIST 0x00000200 /* 0=FHC tells clock board it exists */ +#define FHC_CONTROL_XMSTR 0x00000100 /* 1=Causes this FHC to be XIR master*/ +#define FHC_CONTROL_LLED 0x00000040 /* 0=Left LED ON */ +#define FHC_CONTROL_MLED 0x00000020 /* 1=Middle LED ON */ +#define FHC_CONTROL_RLED 0x00000010 /* 1=Right LED */ +#define FHC_CONTROL_BPINS 0x00000003 /* Spare Bidirectional Pins */ +#define FHC_PREGS_BSR 0x30UL /* FHC Board Status Register */ +#define FHC_BSR_DA64 0x00040000 /* Port A: 0=128bit 1=64bit data path */ +#define FHC_BSR_DB64 0x00020000 /* Port B: 0=128bit 1=64bit data path */ +#define FHC_BSR_BID 0x0001e000 /* Board ID */ +#define FHC_BSR_SA 0x00001c00 /* Port A UPA Speed (from the pins) */ +#define FHC_BSR_SB 0x00000380 /* Port B UPA Speed (from the pins) */ +#define FHC_BSR_NDIAG 0x00000040 /* Not in Diag Mode */ +#define FHC_BSR_NTBED 0x00000020 /* Not in TestBED Mode */ +#define FHC_BSR_NIA 0x0000001c /* Jumper, bit 18 in PROM space */ +#define FHC_BSR_SI 0x00000001 /* Spare input pin value */ +#define FHC_PREGS_ECC 0x40UL /* FHC ECC Control Register (16 bits) */ +#define FHC_PREGS_JCTRL 0xf0UL /* FHC JTAG Control Register */ +#define FHC_JTAG_CTRL_MENAB 0x80000000 /* Indicates this is JTAG Master */ +#define FHC_JTAG_CTRL_MNONE 0x40000000 /* Indicates no JTAG Master present */ +#define FHC_PREGS_JCMD 0x100UL /* FHC JTAG Command Register */ +#define FHC_IREG_IGN 0x00UL /* This FHC's IGN */ +#define FHC_FFREGS_IMAP 0x00UL /* FHC Fanfail IMAP */ +#define FHC_FFREGS_ICLR 0x10UL /* FHC Fanfail ICLR */ +#define FHC_SREGS_IMAP 0x00UL /* FHC System IMAP */ +#define FHC_SREGS_ICLR 0x10UL /* FHC System ICLR */ +#define FHC_UREGS_IMAP 0x00UL /* FHC Uart IMAP */ +#define FHC_UREGS_ICLR 0x10UL /* FHC Uart ICLR */ +#define FHC_TREGS_IMAP 0x00UL /* FHC TOD IMAP */ +#define FHC_TREGS_ICLR 0x10UL /* FHC TOD ICLR */ + +#endif /* !(_SPARC64_FHC_H) */ diff --git a/arch/sparc/include/asm/floppy.h b/arch/sparc/include/asm/floppy.h new file mode 100644 index 000000000..4b315802e --- /dev/null +++ b/arch/sparc/include/asm/floppy.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_FLOPPY_H +#define ___ASM_SPARC_FLOPPY_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/floppy_32.h b/arch/sparc/include/asm/floppy_32.h new file mode 100644 index 000000000..7251d1fed --- /dev/null +++ b/arch/sparc/include/asm/floppy_32.h @@ -0,0 +1,390 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* asm/floppy.h: Sparc specific parts of the Floppy driver. + * + * Copyright (C) 1995 David S. Miller (davem@davemloft.net) + */ + +#ifndef __ASM_SPARC_FLOPPY_H +#define __ASM_SPARC_FLOPPY_H + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/* We don't need no stinkin' I/O port allocation crap. */ +#undef release_region +#undef request_region +#define release_region(X, Y) do { } while(0) +#define request_region(X, Y, Z) (1) + +/* References: + * 1) Netbsd Sun floppy driver. + * 2) NCR 82077 controller manual + * 3) Intel 82077 controller manual + */ +struct sun_flpy_controller { + volatile unsigned char status_82072; /* Main Status reg. */ +#define dcr_82072 status_82072 /* Digital Control reg. */ +#define status1_82077 status_82072 /* Auxiliary Status reg. 1 */ + + volatile unsigned char data_82072; /* Data fifo. */ +#define status2_82077 data_82072 /* Auxiliary Status reg. 2 */ + + volatile unsigned char dor_82077; /* Digital Output reg. */ + volatile unsigned char tapectl_82077; /* What the? Tape control reg? */ + + volatile unsigned char status_82077; /* Main Status Register. */ +#define drs_82077 status_82077 /* Digital Rate Select reg. */ + + volatile unsigned char data_82077; /* Data fifo. */ + volatile unsigned char ___unused; + volatile unsigned char dir_82077; /* Digital Input reg. */ +#define dcr_82077 dir_82077 /* Config Control reg. */ +}; + +/* You'll only ever find one controller on a SparcStation anyways. */ +static struct sun_flpy_controller *sun_fdc = NULL; + +struct sun_floppy_ops { + unsigned char (*fd_inb)(int port); + void (*fd_outb)(unsigned char value, int port); +}; + +static struct sun_floppy_ops sun_fdops; + +#define fd_inb(base, reg) sun_fdops.fd_inb(reg) +#define fd_outb(value, base, reg) sun_fdops.fd_outb(value, reg) +#define fd_enable_dma() sun_fd_enable_dma() +#define fd_disable_dma() sun_fd_disable_dma() +#define fd_request_dma() (0) /* nothing... */ +#define fd_free_dma() /* nothing... */ +#define fd_clear_dma_ff() /* nothing... */ +#define fd_set_dma_mode(mode) sun_fd_set_dma_mode(mode) +#define fd_set_dma_addr(addr) sun_fd_set_dma_addr(addr) +#define fd_set_dma_count(count) sun_fd_set_dma_count(count) +#define fd_enable_irq() /* nothing... */ +#define fd_disable_irq() /* nothing... */ +#define fd_request_irq() sun_fd_request_irq() +#define fd_free_irq() /* nothing... */ +#if 0 /* P3: added by Alain, these cause a MMU corruption. 19960524 XXX */ +#define fd_dma_mem_alloc(size) ((unsigned long) vmalloc(size)) +#define fd_dma_mem_free(addr,size) (vfree((void *)(addr))) +#endif + +/* XXX This isn't really correct. XXX */ +#define get_dma_residue(x) (0) + +#define FLOPPY0_TYPE 4 +#define FLOPPY1_TYPE 0 + +/* Super paranoid... */ +#undef HAVE_DISABLE_HLT + +/* Here is where we catch the floppy driver trying to initialize, + * therefore this is where we call the PROM device tree probing + * routine etc. on the Sparc. + */ +#define FDC1 sun_floppy_init() + +#define N_FDC 1 +#define N_DRIVE 8 + +/* Routines unique to each controller type on a Sun. */ +static void sun_set_dor(unsigned char value, int fdc_82077) +{ + if (fdc_82077) + sun_fdc->dor_82077 = value; +} + +static unsigned char sun_read_dir(void) +{ + return sun_fdc->dir_82077; +} + +static unsigned char sun_82072_fd_inb(int port) +{ + udelay(5); + switch (port) { + default: + printk("floppy: Asked to read unknown port %d\n", port); + panic("floppy: Port bolixed."); + case FD_STATUS: + return sun_fdc->status_82072 & ~STATUS_DMA; + case FD_DATA: + return sun_fdc->data_82072; + case FD_DIR: + return sun_read_dir(); + } + panic("sun_82072_fd_inb: How did I get here?"); +} + +static void sun_82072_fd_outb(unsigned char value, int port) +{ + udelay(5); + switch (port) { + default: + printk("floppy: Asked to write to unknown port %d\n", port); + panic("floppy: Port bolixed."); + case FD_DOR: + sun_set_dor(value, 0); + break; + case FD_DATA: + sun_fdc->data_82072 = value; + break; + case FD_DCR: + sun_fdc->dcr_82072 = value; + break; + case FD_DSR: + sun_fdc->status_82072 = value; + break; + } + return; +} + +static unsigned char sun_82077_fd_inb(int port) +{ + udelay(5); + switch (port) { + default: + printk("floppy: Asked to read unknown port %d\n", port); + panic("floppy: Port bolixed."); + case FD_SRA: + return sun_fdc->status1_82077; + case FD_SRB: + return sun_fdc->status2_82077; + case FD_DOR: + return sun_fdc->dor_82077; + case FD_TDR: + return sun_fdc->tapectl_82077; + case FD_STATUS: + return sun_fdc->status_82077 & ~STATUS_DMA; + case FD_DATA: + return sun_fdc->data_82077; + case FD_DIR: + return sun_read_dir(); + } + panic("sun_82077_fd_inb: How did I get here?"); +} + +static void sun_82077_fd_outb(unsigned char value, int port) +{ + udelay(5); + switch (port) { + default: + printk("floppy: Asked to write to unknown port %d\n", port); + panic("floppy: Port bolixed."); + case FD_DOR: + sun_set_dor(value, 1); + break; + case FD_DATA: + sun_fdc->data_82077 = value; + break; + case FD_DCR: + sun_fdc->dcr_82077 = value; + break; + case FD_DSR: + sun_fdc->status_82077 = value; + break; + case FD_TDR: + sun_fdc->tapectl_82077 = value; + break; + } + return; +} + +/* For pseudo-dma (Sun floppy drives have no real DMA available to + * them so we must eat the data fifo bytes directly ourselves) we have + * three state variables. doing_pdma tells our inline low-level + * assembly floppy interrupt entry point whether it should sit and eat + * bytes from the fifo or just transfer control up to the higher level + * floppy interrupt c-code. I tried very hard but I could not get the + * pseudo-dma to work in c-code without getting many overruns and + * underruns. If non-zero, doing_pdma encodes the direction of + * the transfer for debugging. 1=read 2=write + */ + +/* Common routines to all controller types on the Sparc. */ +static inline void virtual_dma_init(void) +{ + /* nothing... */ +} + +static inline void sun_fd_disable_dma(void) +{ + doing_pdma = 0; + pdma_base = NULL; +} + +static inline void sun_fd_set_dma_mode(int mode) +{ + switch(mode) { + case DMA_MODE_READ: + doing_pdma = 1; + break; + case DMA_MODE_WRITE: + doing_pdma = 2; + break; + default: + printk("Unknown dma mode %d\n", mode); + panic("floppy: Giving up..."); + } +} + +static inline void sun_fd_set_dma_addr(char *buffer) +{ + pdma_vaddr = buffer; +} + +static inline void sun_fd_set_dma_count(int length) +{ + pdma_size = length; +} + +static inline void sun_fd_enable_dma(void) +{ + pdma_base = pdma_vaddr; + pdma_areasize = pdma_size; +} + +int sparc_floppy_request_irq(unsigned int irq, irq_handler_t irq_handler); + +static int sun_fd_request_irq(void) +{ + static int once = 0; + + if (!once) { + once = 1; + return sparc_floppy_request_irq(FLOPPY_IRQ, floppy_interrupt); + } else { + return 0; + } +} + +static struct linux_prom_registers fd_regs[2]; + +static int sun_floppy_init(void) +{ + struct platform_device *op; + struct device_node *dp; + struct resource r; + char state[128]; + phandle fd_node; + phandle tnode; + int num_regs; + + use_virtual_dma = 1; + + /* Forget it if we aren't on a machine that could possibly + * ever have a floppy drive. + */ + if (sparc_cpu_model != sun4m) { + /* We certainly don't have a floppy controller. */ + goto no_sun_fdc; + } + /* Well, try to find one. */ + tnode = prom_getchild(prom_root_node); + fd_node = prom_searchsiblings(tnode, "obio"); + if (fd_node != 0) { + tnode = prom_getchild(fd_node); + fd_node = prom_searchsiblings(tnode, "SUNW,fdtwo"); + } else { + fd_node = prom_searchsiblings(tnode, "fd"); + } + if (fd_node == 0) { + goto no_sun_fdc; + } + + /* The sun4m lets us know if the controller is actually usable. */ + if (prom_getproperty(fd_node, "status", state, sizeof(state)) != -1) { + if(!strcmp(state, "disabled")) { + goto no_sun_fdc; + } + } + num_regs = prom_getproperty(fd_node, "reg", (char *) fd_regs, sizeof(fd_regs)); + num_regs = (num_regs / sizeof(fd_regs[0])); + prom_apply_obio_ranges(fd_regs, num_regs); + memset(&r, 0, sizeof(r)); + r.flags = fd_regs[0].which_io; + r.start = fd_regs[0].phys_addr; + sun_fdc = of_ioremap(&r, 0, fd_regs[0].reg_size, "floppy"); + + /* Look up irq in platform_device. + * We try "SUNW,fdtwo" and "fd" + */ + op = NULL; + for_each_node_by_name(dp, "SUNW,fdtwo") { + op = of_find_device_by_node(dp); + if (op) + break; + } + if (!op) { + for_each_node_by_name(dp, "fd") { + op = of_find_device_by_node(dp); + if (op) + break; + } + } + if (!op) + goto no_sun_fdc; + + FLOPPY_IRQ = op->archdata.irqs[0]; + + /* Last minute sanity check... */ + if (sun_fdc->status_82072 == 0xff) { + sun_fdc = NULL; + goto no_sun_fdc; + } + + sun_fdops.fd_inb = sun_82077_fd_inb; + sun_fdops.fd_outb = sun_82077_fd_outb; + fdc_status = &sun_fdc->status_82077; + + if (sun_fdc->dor_82077 == 0x80) { + sun_fdc->dor_82077 = 0x02; + if (sun_fdc->dor_82077 == 0x80) { + sun_fdops.fd_inb = sun_82072_fd_inb; + sun_fdops.fd_outb = sun_82072_fd_outb; + fdc_status = &sun_fdc->status_82072; + } + } + + /* Success... */ + allowed_drive_mask = 0x01; + return (int) sun_fdc; + +no_sun_fdc: + return -1; +} + +static int sparc_eject(void) +{ + set_dor(0x00, 0xff, 0x90); + udelay(500); + set_dor(0x00, 0x6f, 0x00); + udelay(500); + return 0; +} + +#define fd_eject(drive) sparc_eject() + +#define EXTRA_FLOPPY_PARAMS + +static DEFINE_SPINLOCK(dma_spin_lock); + +#define claim_dma_lock() \ +({ unsigned long flags; \ + spin_lock_irqsave(&dma_spin_lock, flags); \ + flags; \ +}) + +#define release_dma_lock(__flags) \ + spin_unlock_irqrestore(&dma_spin_lock, __flags); + +#endif /* !(__ASM_SPARC_FLOPPY_H) */ diff --git a/arch/sparc/include/asm/floppy_64.h b/arch/sparc/include/asm/floppy_64.h new file mode 100644 index 000000000..d1bb0f133 --- /dev/null +++ b/arch/sparc/include/asm/floppy_64.h @@ -0,0 +1,776 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* floppy.h: Sparc specific parts of the Floppy driver. + * + * Copyright (C) 1996, 2007, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * + * Ultra/PCI support added: Sep 1997 Eddie C. Dost (ecd@skynet.be) + */ + +#ifndef __ASM_SPARC64_FLOPPY_H +#define __ASM_SPARC64_FLOPPY_H + +#include +#include +#include +#include + +#include + +/* + * Define this to enable exchanging drive 0 and 1 if only drive 1 is + * probed on PCI machines. + */ +#undef PCI_FDC_SWAP_DRIVES + + +/* References: + * 1) Netbsd Sun floppy driver. + * 2) NCR 82077 controller manual + * 3) Intel 82077 controller manual + */ +struct sun_flpy_controller { + volatile unsigned char status1_82077; /* Auxiliary Status reg. 1 */ + volatile unsigned char status2_82077; /* Auxiliary Status reg. 2 */ + volatile unsigned char dor_82077; /* Digital Output reg. */ + volatile unsigned char tapectl_82077; /* Tape Control reg */ + volatile unsigned char status_82077; /* Main Status Register. */ +#define drs_82077 status_82077 /* Digital Rate Select reg. */ + volatile unsigned char data_82077; /* Data fifo. */ + volatile unsigned char ___unused; + volatile unsigned char dir_82077; /* Digital Input reg. */ +#define dcr_82077 dir_82077 /* Config Control reg. */ +}; + +/* You'll only ever find one controller on an Ultra anyways. */ +static struct sun_flpy_controller *sun_fdc = (struct sun_flpy_controller *)-1; +unsigned long fdc_status; +static struct platform_device *floppy_op = NULL; + +struct sun_floppy_ops { + unsigned char (*fd_inb) (unsigned long port, unsigned int reg); + void (*fd_outb) (unsigned char value, unsigned long base, + unsigned int reg); + void (*fd_enable_dma) (void); + void (*fd_disable_dma) (void); + void (*fd_set_dma_mode) (int); + void (*fd_set_dma_addr) (char *); + void (*fd_set_dma_count) (int); + unsigned int (*get_dma_residue) (void); + int (*fd_request_irq) (void); + void (*fd_free_irq) (void); + int (*fd_eject) (int); +}; + +static struct sun_floppy_ops sun_fdops; + +#define fd_inb(base, reg) sun_fdops.fd_inb(base, reg) +#define fd_outb(value, base, reg) sun_fdops.fd_outb(value, base, reg) +#define fd_enable_dma() sun_fdops.fd_enable_dma() +#define fd_disable_dma() sun_fdops.fd_disable_dma() +#define fd_request_dma() (0) /* nothing... */ +#define fd_free_dma() /* nothing... */ +#define fd_clear_dma_ff() /* nothing... */ +#define fd_set_dma_mode(mode) sun_fdops.fd_set_dma_mode(mode) +#define fd_set_dma_addr(addr) sun_fdops.fd_set_dma_addr(addr) +#define fd_set_dma_count(count) sun_fdops.fd_set_dma_count(count) +#define get_dma_residue(x) sun_fdops.get_dma_residue() +#define fd_request_irq() sun_fdops.fd_request_irq() +#define fd_free_irq() sun_fdops.fd_free_irq() +#define fd_eject(drive) sun_fdops.fd_eject(drive) + +/* Super paranoid... */ +#undef HAVE_DISABLE_HLT + +static int sun_floppy_types[2] = { 0, 0 }; + +/* Here is where we catch the floppy driver trying to initialize, + * therefore this is where we call the PROM device tree probing + * routine etc. on the Sparc. + */ +#define FLOPPY0_TYPE sun_floppy_init() +#define FLOPPY1_TYPE sun_floppy_types[1] + +#define FDC1 ((unsigned long)sun_fdc) + +#define N_FDC 1 +#define N_DRIVE 8 + +static unsigned char sun_82077_fd_inb(unsigned long base, unsigned int reg) +{ + udelay(5); + switch (reg) { + default: + printk("floppy: Asked to read unknown port %x\n", reg); + panic("floppy: Port bolixed."); + case FD_STATUS: + return sbus_readb(&sun_fdc->status_82077) & ~STATUS_DMA; + case FD_DATA: + return sbus_readb(&sun_fdc->data_82077); + case FD_DIR: + /* XXX: Is DCL on 0x80 in sun4m? */ + return sbus_readb(&sun_fdc->dir_82077); + } + panic("sun_82072_fd_inb: How did I get here?"); +} + +static void sun_82077_fd_outb(unsigned char value, unsigned long base, + unsigned int reg) +{ + udelay(5); + switch (reg) { + default: + printk("floppy: Asked to write to unknown port %x\n", reg); + panic("floppy: Port bolixed."); + case FD_DOR: + /* Happily, the 82077 has a real DOR register. */ + sbus_writeb(value, &sun_fdc->dor_82077); + break; + case FD_DATA: + sbus_writeb(value, &sun_fdc->data_82077); + break; + case FD_DCR: + sbus_writeb(value, &sun_fdc->dcr_82077); + break; + case FD_DSR: + sbus_writeb(value, &sun_fdc->status_82077); + break; + } + return; +} + +/* For pseudo-dma (Sun floppy drives have no real DMA available to + * them so we must eat the data fifo bytes directly ourselves) we have + * three state variables. doing_pdma tells our inline low-level + * assembly floppy interrupt entry point whether it should sit and eat + * bytes from the fifo or just transfer control up to the higher level + * floppy interrupt c-code. I tried very hard but I could not get the + * pseudo-dma to work in c-code without getting many overruns and + * underruns. If non-zero, doing_pdma encodes the direction of + * the transfer for debugging. 1=read 2=write + */ +unsigned char *pdma_vaddr; +unsigned long pdma_size; +volatile int doing_pdma = 0; + +/* This is software state */ +char *pdma_base = NULL; +unsigned long pdma_areasize; + +/* Common routines to all controller types on the Sparc. */ +static void sun_fd_disable_dma(void) +{ + doing_pdma = 0; + pdma_base = NULL; +} + +static void sun_fd_set_dma_mode(int mode) +{ + switch(mode) { + case DMA_MODE_READ: + doing_pdma = 1; + break; + case DMA_MODE_WRITE: + doing_pdma = 2; + break; + default: + printk("Unknown dma mode %d\n", mode); + panic("floppy: Giving up..."); + } +} + +static void sun_fd_set_dma_addr(char *buffer) +{ + pdma_vaddr = buffer; +} + +static void sun_fd_set_dma_count(int length) +{ + pdma_size = length; +} + +static void sun_fd_enable_dma(void) +{ + pdma_base = pdma_vaddr; + pdma_areasize = pdma_size; +} + +static irqreturn_t sparc_floppy_irq(int irq, void *dev_cookie) +{ + if (likely(doing_pdma)) { + void __iomem *stat = (void __iomem *) fdc_status; + unsigned char *vaddr = pdma_vaddr; + unsigned long size = pdma_size; + u8 val; + + while (size) { + val = readb(stat); + if (unlikely(!(val & 0x80))) { + pdma_vaddr = vaddr; + pdma_size = size; + return IRQ_HANDLED; + } + if (unlikely(!(val & 0x20))) { + pdma_vaddr = vaddr; + pdma_size = size; + doing_pdma = 0; + goto main_interrupt; + } + if (val & 0x40) { + /* read */ + *vaddr++ = readb(stat + 1); + } else { + unsigned char data = *vaddr++; + + /* write */ + writeb(data, stat + 1); + } + size--; + } + + pdma_vaddr = vaddr; + pdma_size = size; + + /* Send Terminal Count pulse to floppy controller. */ + val = readb(auxio_register); + val |= AUXIO_AUX1_FTCNT; + writeb(val, auxio_register); + val &= ~AUXIO_AUX1_FTCNT; + writeb(val, auxio_register); + + doing_pdma = 0; + } + +main_interrupt: + return floppy_interrupt(irq, dev_cookie); +} + +static int sun_fd_request_irq(void) +{ + static int once = 0; + int error; + + if(!once) { + once = 1; + + error = request_irq(FLOPPY_IRQ, sparc_floppy_irq, + 0, "floppy", NULL); + + return ((error == 0) ? 0 : -1); + } + return 0; +} + +static void sun_fd_free_irq(void) +{ +} + +static unsigned int sun_get_dma_residue(void) +{ + /* XXX This isn't really correct. XXX */ + return 0; +} + +static int sun_fd_eject(int drive) +{ + set_dor(0x00, 0xff, 0x90); + udelay(500); + set_dor(0x00, 0x6f, 0x00); + udelay(500); + return 0; +} + +#include +#include + +static struct ebus_dma_info sun_pci_fd_ebus_dma; +static struct device *sun_floppy_dev; +static int sun_pci_broken_drive = -1; + +struct sun_pci_dma_op { + unsigned int addr; + int len; + int direction; + char *buf; +}; +static struct sun_pci_dma_op sun_pci_dma_current = { -1U, 0, 0, NULL}; +static struct sun_pci_dma_op sun_pci_dma_pending = { -1U, 0, 0, NULL}; + +irqreturn_t floppy_interrupt(int irq, void *dev_id); + +static unsigned char sun_pci_fd_inb(unsigned long base, unsigned int reg) +{ + udelay(5); + return inb(base + reg); +} + +static void sun_pci_fd_outb(unsigned char val, unsigned long base, + unsigned int reg) +{ + udelay(5); + outb(val, base + reg); +} + +static void sun_pci_fd_broken_outb(unsigned char val, unsigned long base, + unsigned int reg) +{ + udelay(5); + /* + * XXX: Due to SUN's broken floppy connector on AX and AXi + * we need to turn on MOTOR_0 also, if the floppy is + * jumpered to DS1 (like most PC floppies are). I hope + * this does not hurt correct hardware like the AXmp. + * (Eddie, Sep 12 1998). + */ + if (reg == FD_DOR) { + if (((val & 0x03) == sun_pci_broken_drive) && (val & 0x20)) { + val |= 0x10; + } + } + outb(val, base + reg); +} + +#ifdef PCI_FDC_SWAP_DRIVES +static void sun_pci_fd_lde_broken_outb(unsigned char val, unsigned long base, + unsigned int reg) +{ + udelay(5); + /* + * XXX: Due to SUN's broken floppy connector on AX and AXi + * we need to turn on MOTOR_0 also, if the floppy is + * jumpered to DS1 (like most PC floppies are). I hope + * this does not hurt correct hardware like the AXmp. + * (Eddie, Sep 12 1998). + */ + if (reg == FD_DOR) { + if (((val & 0x03) == sun_pci_broken_drive) && (val & 0x10)) { + val &= ~(0x03); + val |= 0x21; + } + } + outb(val, base + reg); +} +#endif /* PCI_FDC_SWAP_DRIVES */ + +static void sun_pci_fd_enable_dma(void) +{ + BUG_ON((NULL == sun_pci_dma_pending.buf) || + (0 == sun_pci_dma_pending.len) || + (0 == sun_pci_dma_pending.direction)); + + sun_pci_dma_current.buf = sun_pci_dma_pending.buf; + sun_pci_dma_current.len = sun_pci_dma_pending.len; + sun_pci_dma_current.direction = sun_pci_dma_pending.direction; + + sun_pci_dma_pending.buf = NULL; + sun_pci_dma_pending.len = 0; + sun_pci_dma_pending.direction = 0; + sun_pci_dma_pending.addr = -1U; + + sun_pci_dma_current.addr = + dma_map_single(sun_floppy_dev, + sun_pci_dma_current.buf, + sun_pci_dma_current.len, + sun_pci_dma_current.direction); + + ebus_dma_enable(&sun_pci_fd_ebus_dma, 1); + + if (ebus_dma_request(&sun_pci_fd_ebus_dma, + sun_pci_dma_current.addr, + sun_pci_dma_current.len)) + BUG(); +} + +static void sun_pci_fd_disable_dma(void) +{ + ebus_dma_enable(&sun_pci_fd_ebus_dma, 0); + if (sun_pci_dma_current.addr != -1U) + dma_unmap_single(sun_floppy_dev, + sun_pci_dma_current.addr, + sun_pci_dma_current.len, + sun_pci_dma_current.direction); + sun_pci_dma_current.addr = -1U; +} + +static void sun_pci_fd_set_dma_mode(int mode) +{ + if (mode == DMA_MODE_WRITE) + sun_pci_dma_pending.direction = DMA_TO_DEVICE; + else + sun_pci_dma_pending.direction = DMA_FROM_DEVICE; + + ebus_dma_prepare(&sun_pci_fd_ebus_dma, mode != DMA_MODE_WRITE); +} + +static void sun_pci_fd_set_dma_count(int length) +{ + sun_pci_dma_pending.len = length; +} + +static void sun_pci_fd_set_dma_addr(char *buffer) +{ + sun_pci_dma_pending.buf = buffer; +} + +static unsigned int sun_pci_get_dma_residue(void) +{ + return ebus_dma_residue(&sun_pci_fd_ebus_dma); +} + +static int sun_pci_fd_request_irq(void) +{ + return ebus_dma_irq_enable(&sun_pci_fd_ebus_dma, 1); +} + +static void sun_pci_fd_free_irq(void) +{ + ebus_dma_irq_enable(&sun_pci_fd_ebus_dma, 0); +} + +static int sun_pci_fd_eject(int drive) +{ + return -EINVAL; +} + +static void sun_pci_fd_dma_callback(struct ebus_dma_info *p, int event, + void *cookie) +{ + floppy_interrupt(0, NULL); +} + +/* + * Floppy probing, we'd like to use /dev/fd0 for a single Floppy on PCI, + * even if this is configured using DS1, thus looks like /dev/fd1 with + * the cabling used in Ultras. + */ +#define DOR (port + 2) +#define MSR (port + 4) +#define FIFO (port + 5) + +static void sun_pci_fd_out_byte(unsigned long port, unsigned char val, + unsigned long reg) +{ + unsigned char status; + int timeout = 1000; + + while (!((status = inb(MSR)) & 0x80) && --timeout) + udelay(100); + outb(val, reg); +} + +static unsigned char sun_pci_fd_sensei(unsigned long port) +{ + unsigned char result[2] = { 0x70, 0x00 }; + unsigned char status; + int i = 0; + + sun_pci_fd_out_byte(port, 0x08, FIFO); + do { + int timeout = 1000; + + while (!((status = inb(MSR)) & 0x80) && --timeout) + udelay(100); + + if (!timeout) + break; + + if ((status & 0xf0) == 0xd0) + result[i++] = inb(FIFO); + else + break; + } while (i < 2); + + return result[0]; +} + +static void sun_pci_fd_reset(unsigned long port) +{ + unsigned char mask = 0x00; + unsigned char status; + int timeout = 10000; + + outb(0x80, MSR); + do { + status = sun_pci_fd_sensei(port); + if ((status & 0xc0) == 0xc0) + mask |= 1 << (status & 0x03); + else + udelay(100); + } while ((mask != 0x0f) && --timeout); +} + +static int sun_pci_fd_test_drive(unsigned long port, int drive) +{ + unsigned char status, data; + int timeout = 1000; + int ready; + + sun_pci_fd_reset(port); + + data = (0x10 << drive) | 0x0c | drive; + sun_pci_fd_out_byte(port, data, DOR); + + sun_pci_fd_out_byte(port, 0x07, FIFO); + sun_pci_fd_out_byte(port, drive & 0x03, FIFO); + + do { + udelay(100); + status = sun_pci_fd_sensei(port); + } while (((status & 0xc0) == 0x80) && --timeout); + + if (!timeout) + ready = 0; + else + ready = (status & 0x10) ? 0 : 1; + + sun_pci_fd_reset(port); + return ready; +} +#undef FIFO +#undef MSR +#undef DOR + +static int __init ebus_fdthree_p(struct device_node *dp) +{ + if (of_node_name_eq(dp, "fdthree")) + return 1; + if (of_node_name_eq(dp, "floppy")) { + const char *compat; + + compat = of_get_property(dp, "compatible", NULL); + if (compat && !strcmp(compat, "fdthree")) + return 1; + } + return 0; +} + +static unsigned long __init sun_floppy_init(void) +{ + static int initialized = 0; + struct device_node *dp; + struct platform_device *op; + const char *prop; + char state[128]; + + if (initialized) + return sun_floppy_types[0]; + initialized = 1; + + op = NULL; + + for_each_node_by_name(dp, "SUNW,fdtwo") { + if (!of_node_name_eq(dp->parent, "sbus")) + continue; + op = of_find_device_by_node(dp); + if (op) + break; + } + if (op) { + floppy_op = op; + FLOPPY_IRQ = op->archdata.irqs[0]; + } else { + struct device_node *ebus_dp; + void __iomem *auxio_reg; + const char *state_prop; + unsigned long config; + + dp = NULL; + for_each_node_by_name(ebus_dp, "ebus") { + for (dp = ebus_dp->child; dp; dp = dp->sibling) { + if (ebus_fdthree_p(dp)) + goto found_fdthree; + } + } + found_fdthree: + if (!dp) + return 0; + + op = of_find_device_by_node(dp); + if (!op) + return 0; + + state_prop = of_get_property(op->dev.of_node, "status", NULL); + if (state_prop && !strncmp(state_prop, "disabled", 8)) + return 0; + + FLOPPY_IRQ = op->archdata.irqs[0]; + + /* Make sure the high density bit is set, some systems + * (most notably Ultra5/Ultra10) come up with it clear. + */ + auxio_reg = (void __iomem *) op->resource[2].start; + writel(readl(auxio_reg)|0x2, auxio_reg); + + sun_floppy_dev = &op->dev; + + spin_lock_init(&sun_pci_fd_ebus_dma.lock); + + /* XXX ioremap */ + sun_pci_fd_ebus_dma.regs = (void __iomem *) + op->resource[1].start; + if (!sun_pci_fd_ebus_dma.regs) + return 0; + + sun_pci_fd_ebus_dma.flags = (EBUS_DMA_FLAG_USE_EBDMA_HANDLER | + EBUS_DMA_FLAG_TCI_DISABLE); + sun_pci_fd_ebus_dma.callback = sun_pci_fd_dma_callback; + sun_pci_fd_ebus_dma.client_cookie = NULL; + sun_pci_fd_ebus_dma.irq = FLOPPY_IRQ; + strscpy(sun_pci_fd_ebus_dma.name, "floppy"); + if (ebus_dma_register(&sun_pci_fd_ebus_dma)) + return 0; + + /* XXX ioremap */ + sun_fdc = (struct sun_flpy_controller *) op->resource[0].start; + + sun_fdops.fd_inb = sun_pci_fd_inb; + sun_fdops.fd_outb = sun_pci_fd_outb; + + can_use_virtual_dma = use_virtual_dma = 0; + sun_fdops.fd_enable_dma = sun_pci_fd_enable_dma; + sun_fdops.fd_disable_dma = sun_pci_fd_disable_dma; + sun_fdops.fd_set_dma_mode = sun_pci_fd_set_dma_mode; + sun_fdops.fd_set_dma_addr = sun_pci_fd_set_dma_addr; + sun_fdops.fd_set_dma_count = sun_pci_fd_set_dma_count; + sun_fdops.get_dma_residue = sun_pci_get_dma_residue; + + sun_fdops.fd_request_irq = sun_pci_fd_request_irq; + sun_fdops.fd_free_irq = sun_pci_fd_free_irq; + + sun_fdops.fd_eject = sun_pci_fd_eject; + + fdc_status = (unsigned long) &sun_fdc->status_82077; + + /* + * XXX: Find out on which machines this is really needed. + */ + if (1) { + sun_pci_broken_drive = 1; + sun_fdops.fd_outb = sun_pci_fd_broken_outb; + } + + allowed_drive_mask = 0; + if (sun_pci_fd_test_drive((unsigned long)sun_fdc, 0)) + sun_floppy_types[0] = 4; + if (sun_pci_fd_test_drive((unsigned long)sun_fdc, 1)) + sun_floppy_types[1] = 4; + + /* + * Find NS87303 SuperIO config registers (through ecpp). + */ + config = 0; + for (dp = ebus_dp->child; dp; dp = dp->sibling) { + if (of_node_name_eq(dp, "ecpp")) { + struct platform_device *ecpp_op; + + ecpp_op = of_find_device_by_node(dp); + if (ecpp_op) + config = ecpp_op->resource[1].start; + goto config_done; + } + } + config_done: + + /* + * Sanity check, is this really the NS87303? + */ + switch (config & 0x3ff) { + case 0x02e: + case 0x15c: + case 0x26e: + case 0x398: + break; + default: + config = 0; + } + + if (!config) + return sun_floppy_types[0]; + + /* Enable PC-AT mode. */ + ns87303_modify(config, ASC, 0, 0xc0); + +#ifdef PCI_FDC_SWAP_DRIVES + /* + * If only Floppy 1 is present, swap drives. + */ + if (!sun_floppy_types[0] && sun_floppy_types[1]) { + /* + * Set the drive exchange bit in FCR on NS87303, + * make sure other bits are sane before doing so. + */ + ns87303_modify(config, FER, FER_EDM, 0); + ns87303_modify(config, ASC, ASC_DRV2_SEL, 0); + ns87303_modify(config, FCR, 0, FCR_LDE); + + swap(sun_floppy_types[0], sun_floppy_types[1]); + + if (sun_pci_broken_drive != -1) { + sun_pci_broken_drive = 1 - sun_pci_broken_drive; + sun_fdops.fd_outb = sun_pci_fd_lde_broken_outb; + } + } +#endif /* PCI_FDC_SWAP_DRIVES */ + + return sun_floppy_types[0]; + } + prop = of_get_property(op->dev.of_node, "status", NULL); + if (prop && !strncmp(state, "disabled", 8)) + return 0; + + /* + * We cannot do of_ioremap here: it does request_region, + * which the generic floppy driver tries to do once again. + * But we must use the sdev resource values as they have + * had parent ranges applied. + */ + sun_fdc = (struct sun_flpy_controller *) + (op->resource[0].start + + ((op->resource[0].flags & 0x1ffUL) << 32UL)); + + /* Last minute sanity check... */ + if (sbus_readb(&sun_fdc->status1_82077) == 0xff) { + sun_fdc = (struct sun_flpy_controller *)-1; + return 0; + } + + sun_fdops.fd_inb = sun_82077_fd_inb; + sun_fdops.fd_outb = sun_82077_fd_outb; + + can_use_virtual_dma = use_virtual_dma = 1; + sun_fdops.fd_enable_dma = sun_fd_enable_dma; + sun_fdops.fd_disable_dma = sun_fd_disable_dma; + sun_fdops.fd_set_dma_mode = sun_fd_set_dma_mode; + sun_fdops.fd_set_dma_addr = sun_fd_set_dma_addr; + sun_fdops.fd_set_dma_count = sun_fd_set_dma_count; + sun_fdops.get_dma_residue = sun_get_dma_residue; + + sun_fdops.fd_request_irq = sun_fd_request_irq; + sun_fdops.fd_free_irq = sun_fd_free_irq; + + sun_fdops.fd_eject = sun_fd_eject; + + fdc_status = (unsigned long) &sun_fdc->status_82077; + + /* Success... */ + allowed_drive_mask = 0x01; + sun_floppy_types[0] = 4; + sun_floppy_types[1] = 0; + + return sun_floppy_types[0]; +} + +#define EXTRA_FLOPPY_PARAMS + +static DEFINE_SPINLOCK(dma_spin_lock); + +#define claim_dma_lock() \ +({ unsigned long flags; \ + spin_lock_irqsave(&dma_spin_lock, flags); \ + flags; \ +}) + +#define release_dma_lock(__flags) \ + spin_unlock_irqrestore(&dma_spin_lock, __flags); + +#endif /* !(__ASM_SPARC64_FLOPPY_H) */ diff --git a/arch/sparc/include/asm/fpumacro.h b/arch/sparc/include/asm/fpumacro.h new file mode 100644 index 000000000..bc378df97 --- /dev/null +++ b/arch/sparc/include/asm/fpumacro.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* fpumacro.h: FPU related macros. + * + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC64_FPUMACRO_H +#define _SPARC64_FPUMACRO_H + +#include +#include + +struct fpustate { + u32 regs[64]; +}; + +#define FPUSTATE (struct fpustate *)(current_thread_info()->fpregs) + +static inline unsigned long fprs_read(void) +{ + unsigned long retval; + + __asm__ __volatile__("rd %%fprs, %0" : "=r" (retval)); + + return retval; +} + +static inline void fprs_write(unsigned long val) +{ + __asm__ __volatile__("wr %0, 0x0, %%fprs" : : "r" (val)); +} + +#endif /* !(_SPARC64_FPUMACRO_H) */ diff --git a/arch/sparc/include/asm/ftrace.h b/arch/sparc/include/asm/ftrace.h new file mode 100644 index 000000000..f7c903619 --- /dev/null +++ b/arch/sparc/include/asm/ftrace.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC64_FTRACE +#define _ASM_SPARC64_FTRACE + +#ifdef CONFIG_MCOUNT +#define MCOUNT_ADDR ((unsigned long)(_mcount)) +#define MCOUNT_INSN_SIZE 4 /* sizeof mcount call */ + +#ifndef __ASSEMBLER__ +void _mcount(void); +#endif + +#endif /* CONFIG_MCOUNT */ + +#if defined(CONFIG_SPARC64) && !defined(CC_USE_FENTRY) +#define HAVE_FUNCTION_GRAPH_FP_TEST +#endif + +#ifdef CONFIG_DYNAMIC_FTRACE +/* relocation of mcount call site is the same as the address */ +static inline unsigned long ftrace_call_adjust(unsigned long addr) +{ + return addr; +} + +struct dyn_arch_ftrace { +}; +#endif /* CONFIG_DYNAMIC_FTRACE */ + +unsigned long prepare_ftrace_return(unsigned long parent, + unsigned long self_addr, + unsigned long frame_pointer); + +#endif /* _ASM_SPARC64_FTRACE */ diff --git a/arch/sparc/include/asm/futex.h b/arch/sparc/include/asm/futex.h new file mode 100644 index 000000000..75a13a20c --- /dev/null +++ b/arch/sparc/include/asm/futex.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_FUTEX_H +#define ___ASM_SPARC_FUTEX_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/futex_32.h b/arch/sparc/include/asm/futex_32.h new file mode 100644 index 000000000..6a332a9f0 --- /dev/null +++ b/arch/sparc/include/asm/futex_32.h @@ -0,0 +1,6 @@ +#ifndef _ASM_FUTEX_H +#define _ASM_FUTEX_H + +#include + +#endif diff --git a/arch/sparc/include/asm/futex_64.h b/arch/sparc/include/asm/futex_64.h new file mode 100644 index 000000000..72de96731 --- /dev/null +++ b/arch/sparc/include/asm/futex_64.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_FUTEX_H +#define _SPARC64_FUTEX_H + +#include +#include +#include + +#define __futex_cas_op(insn, ret, oldval, uaddr, oparg) \ + __asm__ __volatile__( \ + "\n1: lduwa [%3] %%asi, %2\n" \ + " " insn "\n" \ + "2: casa [%3] %%asi, %2, %1\n" \ + " cmp %2, %1\n" \ + " bne,pn %%icc, 1b\n" \ + " mov 0, %0\n" \ + "3:\n" \ + " .section .fixup,#alloc,#execinstr\n" \ + " .align 4\n" \ + "4: sethi %%hi(3b), %0\n" \ + " jmpl %0 + %%lo(3b), %%g0\n" \ + " mov %5, %0\n" \ + " .previous\n" \ + " .section __ex_table,\"a\"\n" \ + " .align 4\n" \ + " .word 1b, 4b\n" \ + " .word 2b, 4b\n" \ + " .previous\n" \ + : "=&r" (ret), "=&r" (oldval), "=&r" (tem) \ + : "r" (uaddr), "r" (oparg), "i" (-EFAULT) \ + : "memory") + +static inline int arch_futex_atomic_op_inuser(int op, int oparg, int *oval, + u32 __user *uaddr) +{ + int oldval = 0, ret, tem; + + if (unlikely((((unsigned long) uaddr) & 0x3UL))) + return -EINVAL; + + switch (op) { + case FUTEX_OP_SET: + __futex_cas_op("mov\t%4, %1", ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ADD: + __futex_cas_op("add\t%2, %4, %1", ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_OR: + __futex_cas_op("or\t%2, %4, %1", ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_ANDN: + __futex_cas_op("andn\t%2, %4, %1", ret, oldval, uaddr, oparg); + break; + case FUTEX_OP_XOR: + __futex_cas_op("xor\t%2, %4, %1", ret, oldval, uaddr, oparg); + break; + default: + ret = -ENOSYS; + } + + if (!ret) + *oval = oldval; + + return ret; +} + +static inline int +futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr, + u32 oldval, u32 newval) +{ + int ret = 0; + + __asm__ __volatile__( + "\n1: casa [%4] %%asi, %3, %1\n" + "2:\n" + " .section .fixup,#alloc,#execinstr\n" + " .align 4\n" + "3: sethi %%hi(2b), %0\n" + " jmpl %0 + %%lo(2b), %%g0\n" + " mov %5, %0\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + " .align 4\n" + " .word 1b, 3b\n" + " .previous\n" + : "+r" (ret), "=r" (newval) + : "1" (newval), "r" (oldval), "r" (uaddr), "i" (-EFAULT) + : "memory"); + + *uval = newval; + return ret; +} + +#endif /* !(_SPARC64_FUTEX_H) */ diff --git a/arch/sparc/include/asm/hardirq.h b/arch/sparc/include/asm/hardirq.h new file mode 100644 index 000000000..a185f6647 --- /dev/null +++ b/arch/sparc/include/asm/hardirq.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_HARDIRQ_H +#define ___ASM_SPARC_HARDIRQ_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/hardirq_32.h b/arch/sparc/include/asm/hardirq_32.h new file mode 100644 index 000000000..9830d8731 --- /dev/null +++ b/arch/sparc/include/asm/hardirq_32.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* hardirq.h: 32-bit Sparc hard IRQ support. + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1998-2000 Anton Blanchard (anton@samba.org) + */ + +#ifndef __SPARC_HARDIRQ_H +#define __SPARC_HARDIRQ_H + +#include + +#endif /* __SPARC_HARDIRQ_H */ diff --git a/arch/sparc/include/asm/hardirq_64.h b/arch/sparc/include/asm/hardirq_64.h new file mode 100644 index 000000000..75b92bfe0 --- /dev/null +++ b/arch/sparc/include/asm/hardirq_64.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* hardirq.h: 64-bit Sparc hard IRQ support. + * + * Copyright (C) 1997, 1998, 2005 David S. Miller (davem@davemloft.net) + */ + +#ifndef __SPARC64_HARDIRQ_H +#define __SPARC64_HARDIRQ_H + +#include + +#define __ARCH_IRQ_STAT + +#define local_softirq_pending_ref \ + __cpu_data.__softirq_pending + +void ack_bad_irq(unsigned int irq); + +#endif /* !(__SPARC64_HARDIRQ_H) */ diff --git a/arch/sparc/include/asm/head.h b/arch/sparc/include/asm/head.h new file mode 100644 index 000000000..25299b701 --- /dev/null +++ b/arch/sparc/include/asm/head.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_HEAD_H +#define ___ASM_SPARC_HEAD_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/head_32.h b/arch/sparc/include/asm/head_32.h new file mode 100644 index 000000000..d2809c859 --- /dev/null +++ b/arch/sparc/include/asm/head_32.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_HEAD_H +#define __SPARC_HEAD_H + +#define KERNBASE 0xf0000000 /* First address the kernel will eventually be */ + +#define WRITE_PAUSE nop; nop; nop; /* Have to do this after %wim/%psr chg */ + +/* Here are some trap goodies */ + +/* Generic trap entry. */ +#define TRAP_ENTRY(type, label) \ + rd %psr, %l0; b label; rd %wim, %l3; nop; + +/* Data/text faults */ +#define SRMMU_TFAULT rd %psr, %l0; rd %wim, %l3; b srmmu_fault; mov 1, %l7; +#define SRMMU_DFAULT rd %psr, %l0; rd %wim, %l3; b srmmu_fault; mov 0, %l7; + +/* This is for traps we should NEVER get. */ +#define BAD_TRAP(num) \ + rd %psr, %l0; mov num, %l7; b bad_trap_handler; rd %wim, %l3; + +/* This is for traps when we want just skip the instruction which caused it */ +#define SKIP_TRAP(type, name) \ + jmpl %l2, %g0; rett %l2 + 4; nop; nop; + +/* Notice that for the system calls we pull a trick. We load up a + * different pointer to the system call vector table in %l7, but call + * the same generic system call low-level entry point. The trap table + * entry sequences are also HyperSparc pipeline friendly ;-) + */ + +/* Software trap for Linux system calls. */ +#define LINUX_SYSCALL_TRAP \ + sethi %hi(sys_call_table), %l7; \ + or %l7, %lo(sys_call_table), %l7; \ + b linux_sparc_syscall; \ + rd %psr, %l0; + +#define BREAKPOINT_TRAP \ + b breakpoint_trap; \ + rd %psr,%l0; \ + nop; \ + nop; + +#ifdef CONFIG_KGDB +#define KGDB_TRAP(num) \ + mov num, %l7; \ + b kgdb_trap_low; \ + rd %psr,%l0; \ + nop; +#else +#define KGDB_TRAP(num) \ + BAD_TRAP(num) +#endif + +/* The Get Condition Codes software trap for userland. */ +#define GETCC_TRAP \ + b getcc_trap_handler; rd %psr, %l0; nop; nop; + +/* The Set Condition Codes software trap for userland. */ +#define SETCC_TRAP \ + b setcc_trap_handler; rd %psr, %l0; nop; nop; + +/* The Get PSR software trap for userland. */ +#define GETPSR_TRAP \ + rd %psr, %i0; jmp %l2; rett %l2 + 4; nop; + +/* This is for hard interrupts from level 1-14, 15 is non-maskable (nmi) and + * gets handled with another macro. + */ +#define TRAP_ENTRY_INTERRUPT(int_level) \ + mov int_level, %l7; rd %psr, %l0; b real_irq_entry; rd %wim, %l3; + +/* Window overflows/underflows are special and we need to try to be as + * efficient as possible here.... + */ +#define WINDOW_SPILL \ + rd %psr, %l0; rd %wim, %l3; b spill_window_entry; andcc %l0, PSR_PS, %g0; + +#define WINDOW_FILL \ + rd %psr, %l0; rd %wim, %l3; b fill_window_entry; andcc %l0, PSR_PS, %g0; + +#endif /* __SPARC_HEAD_H */ diff --git a/arch/sparc/include/asm/head_64.h b/arch/sparc/include/asm/head_64.h new file mode 100644 index 000000000..69a2062d9 --- /dev/null +++ b/arch/sparc/include/asm/head_64.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_HEAD_H +#define _SPARC64_HEAD_H + +#include + + /* wrpr %g0, val, %gl */ +#define SET_GL(val) \ + .word 0xa1902000 | val + + /* rdpr %gl, %gN */ +#define GET_GL_GLOBAL(N) \ + .word 0x81540000 | (N << 25) + +#define KERNBASE 0x400000 + +#define PTREGS_OFF (STACK_BIAS + STACKFRAME_SZ) + +#define RTRAP_PSTATE (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_IE) +#define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) +#define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) + +#define __CHEETAH_ID 0x003e0014 +#define __JALAPENO_ID 0x003e0016 +#define __SERRANO_ID 0x003e0022 + +#define CHEETAH_MANUF 0x003e +#define CHEETAH_IMPL 0x0014 /* Ultra-III */ +#define CHEETAH_PLUS_IMPL 0x0015 /* Ultra-III+ */ +#define JALAPENO_IMPL 0x0016 /* Ultra-IIIi */ +#define JAGUAR_IMPL 0x0018 /* Ultra-IV */ +#define PANTHER_IMPL 0x0019 /* Ultra-IV+ */ +#define SERRANO_IMPL 0x0022 /* Ultra-IIIi+ */ + +#define BRANCH_IF_SUN4V(tmp1,label) \ + sethi %hi(is_sun4v), %tmp1; \ + lduw [%tmp1 + %lo(is_sun4v)], %tmp1; \ + brnz,pn %tmp1, label; \ + nop + +#define BRANCH_IF_CHEETAH_BASE(tmp1,tmp2,label) \ + rdpr %ver, %tmp1; \ + sethi %hi(__CHEETAH_ID), %tmp2; \ + srlx %tmp1, 32, %tmp1; \ + or %tmp2, %lo(__CHEETAH_ID), %tmp2;\ + cmp %tmp1, %tmp2; \ + be,pn %icc, label; \ + nop; + +#define BRANCH_IF_JALAPENO(tmp1,tmp2,label) \ + rdpr %ver, %tmp1; \ + sethi %hi(__JALAPENO_ID), %tmp2; \ + srlx %tmp1, 32, %tmp1; \ + or %tmp2, %lo(__JALAPENO_ID), %tmp2;\ + cmp %tmp1, %tmp2; \ + be,pn %icc, label; \ + nop; + +#define BRANCH_IF_CHEETAH_PLUS_OR_FOLLOWON(tmp1,tmp2,label) \ + rdpr %ver, %tmp1; \ + srlx %tmp1, (32 + 16), %tmp2; \ + cmp %tmp2, CHEETAH_MANUF; \ + bne,pt %xcc, 99f; \ + sllx %tmp1, 16, %tmp1; \ + srlx %tmp1, (32 + 16), %tmp2; \ + cmp %tmp2, CHEETAH_PLUS_IMPL; \ + bgeu,pt %xcc, label; \ +99: nop; + +#define BRANCH_IF_ANY_CHEETAH(tmp1,tmp2,label) \ + rdpr %ver, %tmp1; \ + srlx %tmp1, (32 + 16), %tmp2; \ + cmp %tmp2, CHEETAH_MANUF; \ + bne,pt %xcc, 99f; \ + sllx %tmp1, 16, %tmp1; \ + srlx %tmp1, (32 + 16), %tmp2; \ + cmp %tmp2, CHEETAH_IMPL; \ + bgeu,pt %xcc, label; \ +99: nop; + +#endif /* !(_SPARC64_HEAD_H) */ diff --git a/arch/sparc/include/asm/hibernate.h b/arch/sparc/include/asm/hibernate.h new file mode 100644 index 000000000..3bb0a96ec --- /dev/null +++ b/arch/sparc/include/asm/hibernate.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * hibernate.h: Hibernaton support specific for sparc64. + * + * Copyright (C) 2013 Kirill V Tkhai (tkhai@yandex.ru) + */ + +#ifndef ___SPARC_HIBERNATE_H +#define ___SPARC_HIBERNATE_H + +struct saved_context { + unsigned long fp; + unsigned long cwp; + unsigned long wstate; + + unsigned long tick; + unsigned long pstate; + + unsigned long g4; + unsigned long g5; + unsigned long g6; +}; + +#endif diff --git a/arch/sparc/include/asm/highmem.h b/arch/sparc/include/asm/highmem.h new file mode 100644 index 000000000..c7b2e2083 --- /dev/null +++ b/arch/sparc/include/asm/highmem.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * highmem.h: virtual kernel memory mappings for high memory + * + * Used in CONFIG_HIGHMEM systems for memory pages which + * are not addressable by direct kernel virtual addresses. + * + * Copyright (C) 1999 Gerhard Wichert, Siemens AG + * Gerhard.Wichert@pdb.siemens.de + * + * + * Redesigned the x86 32-bit VM architecture to deal with + * up to 16 Terrabyte physical memory. With current x86 CPUs + * we now support up to 64 Gigabytes physical RAM. + * + * Copyright (C) 1999 Ingo Molnar + */ + +#ifndef _ASM_HIGHMEM_H +#define _ASM_HIGHMEM_H + +#ifdef __KERNEL__ + +#include +#include +#include +#include + +/* declarations for highmem.c */ +extern unsigned long highstart_pfn, highend_pfn; + +#define kmap_prot __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE) +extern pte_t *pkmap_page_table; + +/* + * Right now we initialize only a single pte table. It can be extended + * easily, subsequent pte tables have to be allocated in one physical + * chunk of RAM. Currently the simplest way to do this is to align the + * pkmap region on a pagetable boundary (4MB). + */ +#define LAST_PKMAP 1024 +#define PKMAP_SIZE (LAST_PKMAP << PAGE_SHIFT) +#define PKMAP_BASE PMD_ALIGN(SRMMU_NOCACHE_VADDR + (SRMMU_MAX_NOCACHE_PAGES << PAGE_SHIFT)) + +#define LAST_PKMAP_MASK (LAST_PKMAP - 1) +#define PKMAP_NR(virt) ((virt - PKMAP_BASE) >> PAGE_SHIFT) +#define PKMAP_ADDR(nr) (PKMAP_BASE + ((nr) << PAGE_SHIFT)) + +#define PKMAP_END (PKMAP_ADDR(LAST_PKMAP)) + +#define flush_cache_kmaps() flush_cache_all() + +/* FIXME: Use __flush_*_one(vaddr) instead of flush_*_all() -- Anton */ +#define arch_kmap_local_pre_map(vaddr, pteval) flush_cache_all() +#define arch_kmap_local_pre_unmap(vaddr) flush_cache_all() +#define arch_kmap_local_post_map(vaddr, pteval) flush_tlb_all() +#define arch_kmap_local_post_unmap(vaddr) flush_tlb_all() + +#endif /* __KERNEL__ */ + +#endif /* _ASM_HIGHMEM_H */ diff --git a/arch/sparc/include/asm/hugetlb.h b/arch/sparc/include/asm/hugetlb.h new file mode 100644 index 000000000..d3bc16fbc --- /dev/null +++ b/arch/sparc/include/asm/hugetlb.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC64_HUGETLB_H +#define _ASM_SPARC64_HUGETLB_H + +#include + +#ifdef CONFIG_HUGETLB_PAGE +struct pud_huge_patch_entry { + unsigned int addr; + unsigned int insn; +}; +extern struct pud_huge_patch_entry __pud_huge_patch, __pud_huge_patch_end; +#endif + +#define __HAVE_ARCH_HUGE_SET_HUGE_PTE_AT +void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned long sz); +void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte); + +#define __HAVE_ARCH_HUGE_PTEP_GET_AND_CLEAR +pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz); + +#define __HAVE_ARCH_HUGE_PTEP_CLEAR_FLUSH +static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + return *ptep; +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_WRPROTECT +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pte_t old_pte = *ptep; + __set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte)); +} + +#define __HAVE_ARCH_HUGE_PTEP_SET_ACCESS_FLAGS +static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty) +{ + int changed = !pte_same(*ptep, pte); + if (changed) { + __set_huge_pte_at(vma->vm_mm, addr, ptep, pte); + flush_tlb_page(vma, addr); + } + return changed; +} + +#include + +#endif /* _ASM_SPARC64_HUGETLB_H */ diff --git a/arch/sparc/include/asm/hvtramp.h b/arch/sparc/include/asm/hvtramp.h new file mode 100644 index 000000000..8cf7a54fa --- /dev/null +++ b/arch/sparc/include/asm/hvtramp.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_HVTRAP_H +#define _SPARC64_HVTRAP_H + +#ifndef __ASSEMBLER__ + +#include + +struct hvtramp_mapping { + __u64 vaddr; + __u64 tte; +}; + +struct hvtramp_descr { + __u32 cpu; + __u32 num_mappings; + __u64 fault_info_va; + __u64 fault_info_pa; + __u64 thread_reg; + struct hvtramp_mapping maps[]; +}; + +void hv_cpu_startup(unsigned long hvdescr_pa); + +#endif + +#define HVTRAMP_DESCR_CPU 0x00 +#define HVTRAMP_DESCR_NUM_MAPPINGS 0x04 +#define HVTRAMP_DESCR_FAULT_INFO_VA 0x08 +#define HVTRAMP_DESCR_FAULT_INFO_PA 0x10 +#define HVTRAMP_DESCR_THREAD_REG 0x18 +#define HVTRAMP_DESCR_MAPS 0x20 + +#define HVTRAMP_MAPPING_VADDR 0x00 +#define HVTRAMP_MAPPING_TTE 0x08 +#define HVTRAMP_MAPPING_SIZE 0x10 + +#endif /* _SPARC64_HVTRAP_H */ diff --git a/arch/sparc/include/asm/hw_irq.h b/arch/sparc/include/asm/hw_irq.h new file mode 100644 index 000000000..8d30a7694 --- /dev/null +++ b/arch/sparc/include/asm/hw_irq.h @@ -0,0 +1,6 @@ +#ifndef __ASM_SPARC_HW_IRQ_H +#define __ASM_SPARC_HW_IRQ_H + +/* Dummy include. */ + +#endif diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h new file mode 100644 index 000000000..94ac56d43 --- /dev/null +++ b/arch/sparc/include/asm/hypervisor.h @@ -0,0 +1,3527 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_HYPERVISOR_H +#define _SPARC64_HYPERVISOR_H + +/* Sun4v hypervisor interfaces and defines. + * + * Hypervisor calls are made via traps to software traps number 0x80 + * and above. Registers %o0 to %o5 serve as argument, status, and + * return value registers. + * + * There are two kinds of these traps. First there are the normal + * "fast traps" which use software trap 0x80 and encode the function + * to invoke by number in register %o5. Argument and return value + * handling is as follows: + * + * ----------------------------------------------- + * | %o5 | function number | undefined | + * | %o0 | argument 0 | return status | + * | %o1 | argument 1 | return value 1 | + * | %o2 | argument 2 | return value 2 | + * | %o3 | argument 3 | return value 3 | + * | %o4 | argument 4 | return value 4 | + * ----------------------------------------------- + * + * The second type are "hyper-fast traps" which encode the function + * number in the software trap number itself. So these use trap + * numbers > 0x80. The register usage for hyper-fast traps is as + * follows: + * + * ----------------------------------------------- + * | %o0 | argument 0 | return status | + * | %o1 | argument 1 | return value 1 | + * | %o2 | argument 2 | return value 2 | + * | %o3 | argument 3 | return value 3 | + * | %o4 | argument 4 | return value 4 | + * ----------------------------------------------- + * + * Registers providing explicit arguments to the hypervisor calls + * are volatile across the call. Upon return their values are + * undefined unless explicitly specified as containing a particular + * return value by the specific call. The return status is always + * returned in register %o0, zero indicates a successful execution of + * the hypervisor call and other values indicate an error status as + * defined below. So, for example, if a hyper-fast trap takes + * arguments 0, 1, and 2, then %o0, %o1, and %o2 are volatile across + * the call and %o3, %o4, and %o5 would be preserved. + * + * If the hypervisor trap is invalid, or the fast trap function number + * is invalid, HV_EBADTRAP will be returned in %o0. Also, all 64-bits + * of the argument and return values are significant. + */ + +/* Trap numbers. */ +#define HV_FAST_TRAP 0x80 +#define HV_MMU_MAP_ADDR_TRAP 0x83 +#define HV_MMU_UNMAP_ADDR_TRAP 0x84 +#define HV_TTRACE_ADDENTRY_TRAP 0x85 +#define HV_CORE_TRAP 0xff + +/* Error codes. */ +#define HV_EOK 0 /* Successful return */ +#define HV_ENOCPU 1 /* Invalid CPU id */ +#define HV_ENORADDR 2 /* Invalid real address */ +#define HV_ENOINTR 3 /* Invalid interrupt id */ +#define HV_EBADPGSZ 4 /* Invalid pagesize encoding */ +#define HV_EBADTSB 5 /* Invalid TSB description */ +#define HV_EINVAL 6 /* Invalid argument */ +#define HV_EBADTRAP 7 /* Invalid function number */ +#define HV_EBADALIGN 8 /* Invalid address alignment */ +#define HV_EWOULDBLOCK 9 /* Cannot complete w/o blocking */ +#define HV_ENOACCESS 10 /* No access to resource */ +#define HV_EIO 11 /* I/O error */ +#define HV_ECPUERROR 12 /* CPU in error state */ +#define HV_ENOTSUPPORTED 13 /* Function not supported */ +#define HV_ENOMAP 14 /* No mapping found */ +#define HV_ETOOMANY 15 /* Too many items specified */ +#define HV_ECHANNEL 16 /* Invalid LDC channel */ +#define HV_EBUSY 17 /* Resource busy */ +#define HV_EUNAVAILABLE 23 /* Resource or operation not + * currently available, but may + * become available in the future + */ + +/* mach_exit() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MACH_EXIT + * ARG0: exit code + * ERRORS: This service does not return. + * + * Stop all CPUs in the virtual domain and place them into the stopped + * state. The 64-bit exit code may be passed to a service entity as + * the domain's exit status. On systems without a service entity, the + * domain will undergo a reset, and the boot firmware will be + * reloaded. + * + * This function will never return to the guest that invokes it. + * + * Note: By convention an exit code of zero denotes a successful exit by + * the guest code. A non-zero exit code denotes a guest specific + * error indication. + * + */ +#define HV_FAST_MACH_EXIT 0x00 + +#ifndef __ASSEMBLER__ +void sun4v_mach_exit(unsigned long exit_code); +#endif + +/* Domain services. */ + +/* mach_desc() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MACH_DESC + * ARG0: buffer + * ARG1: length + * RET0: status + * RET1: length + * ERRORS: HV_EBADALIGN Buffer is badly aligned + * HV_ENORADDR Buffer is to an illegal real address. + * HV_EINVAL Buffer length is too small for complete + * machine description. + * + * Copy the most current machine description into the buffer indicated + * by the real address in ARG0. The buffer provided must be 16 byte + * aligned. Upon success or HV_EINVAL, this service returns the + * actual size of the machine description in the RET1 return value. + * + * Note: A method of determining the appropriate buffer size for the + * machine description is to first call this service with a buffer + * length of 0 bytes. + */ +#define HV_FAST_MACH_DESC 0x01 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_mach_desc(unsigned long buffer_pa, + unsigned long buf_len, + unsigned long *real_buf_len); +#endif + +/* mach_sir() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MACH_SIR + * ERRORS: This service does not return. + * + * Perform a software initiated reset of the virtual machine domain. + * All CPUs are captured as soon as possible, all hardware devices are + * returned to the entry default state, and the domain is restarted at + * the SIR (trap type 0x04) real trap table (RTBA) entry point on one + * of the CPUs. The single CPU restarted is selected as determined by + * platform specific policy. Memory is preserved across this + * operation. + */ +#define HV_FAST_MACH_SIR 0x02 + +#ifndef __ASSEMBLER__ +void sun4v_mach_sir(void); +#endif + +/* mach_set_watchdog() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MACH_SET_WATCHDOG + * ARG0: timeout in milliseconds + * RET0: status + * RET1: time remaining in milliseconds + * + * A guest uses this API to set a watchdog timer. Once the gues has set + * the timer, it must call the timer service again either to disable or + * postpone the expiration. If the timer expires before being reset or + * disabled, then the hypervisor take a platform specific action leading + * to guest termination within a bounded time period. The platform action + * may include recovery actions such as reporting the expiration to a + * Service Processor, and/or automatically restarting the gues. + * + * The 'timeout' parameter is specified in milliseconds, however the + * implementated granularity is given by the 'watchdog-resolution' + * property in the 'platform' node of the guest's machine description. + * The largest allowed timeout value is specified by the + * 'watchdog-max-timeout' property of the 'platform' node. + * + * If the 'timeout' argument is not zero, the watchdog timer is set to + * expire after a minimum of 'timeout' milliseconds. + * + * If the 'timeout' argument is zero, the watchdog timer is disabled. + * + * If the 'timeout' value exceeds the value of the 'max-watchdog-timeout' + * property, the hypervisor leaves the watchdog timer state unchanged, + * and returns a status of EINVAL. + * + * The 'time remaining' return value is valid regardless of whether the + * return status is EOK or EINVAL. A non-zero return value indicates the + * number of milliseconds that were remaining until the timer was to expire. + * If less than one millisecond remains, the return value is '1'. If the + * watchdog timer was disabled at the time of the call, the return value is + * zero. + * + * If the hypervisor cannot support the exact timeout value requested, but + * can support a larger timeout value, the hypervisor may round the actual + * timeout to a value larger than the requested timeout, consequently the + * 'time remaining' return value may be larger than the previously requested + * timeout value. + * + * Any guest OS debugger should be aware that the watchdog service may be in + * use. Consequently, it is recommended that the watchdog service is + * disabled upon debugger entry (e.g. reaching a breakpoint), and then + * re-enabled upon returning to normal execution. The API has been designed + * with this in mind, and the 'time remaining' result of the disable call may + * be used directly as the timeout argument of the re-enable call. + */ +#define HV_FAST_MACH_SET_WATCHDOG 0x05 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_mach_set_watchdog(unsigned long timeout, + unsigned long *orig_timeout); +#endif + +/* CPU services. + * + * CPUs represent devices that can execute software threads. A single + * chip that contains multiple cores or strands is represented as + * multiple CPUs with unique CPU identifiers. CPUs are exported to + * OBP via the machine description (and to the OS via the OBP device + * tree). CPUs are always in one of three states: stopped, running, + * or error. + * + * A CPU ID is a pre-assigned 16-bit value that uniquely identifies a + * CPU within a logical domain. Operations that are to be performed + * on multiple CPUs specify them via a CPU list. A CPU list is an + * array in real memory, of which each 16-bit word is a CPU ID. CPU + * lists are passed through the API as two arguments. The first is + * the number of entries (16-bit words) in the CPU list, and the + * second is the (real address) pointer to the CPU ID list. + */ + +/* cpu_start() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_START + * ARG0: CPU ID + * ARG1: PC + * ARG2: RTBA + * ARG3: target ARG0 + * RET0: status + * ERRORS: ENOCPU Invalid CPU ID + * EINVAL Target CPU ID is not in the stopped state + * ENORADDR Invalid PC or RTBA real address + * EBADALIGN Unaligned PC or unaligned RTBA + * EWOULDBLOCK Starting resources are not available + * + * Start CPU with given CPU ID with PC in %pc and with a real trap + * base address value of RTBA. The indicated CPU must be in the + * stopped state. The supplied RTBA must be aligned on a 256 byte + * boundary. On successful completion, the specified CPU will be in + * the running state and will be supplied with "target ARG0" in %o0 + * and RTBA in %tba. + */ +#define HV_FAST_CPU_START 0x10 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_start(unsigned long cpuid, + unsigned long pc, + unsigned long rtba, + unsigned long arg0); +#endif + +/* cpu_stop() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_STOP + * ARG0: CPU ID + * RET0: status + * ERRORS: ENOCPU Invalid CPU ID + * EINVAL Target CPU ID is the current cpu + * EINVAL Target CPU ID is not in the running state + * EWOULDBLOCK Stopping resources are not available + * ENOTSUPPORTED Not supported on this platform + * + * The specified CPU is stopped. The indicated CPU must be in the + * running state. On completion, it will be in the stopped state. It + * is not legal to stop the current CPU. + * + * Note: As this service cannot be used to stop the current cpu, this service + * may not be used to stop the last running CPU in a domain. To stop + * and exit a running domain, a guest must use the mach_exit() service. + */ +#define HV_FAST_CPU_STOP 0x11 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_stop(unsigned long cpuid); +#endif + +/* cpu_yield() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_YIELD + * RET0: status + * ERRORS: No possible error. + * + * Suspend execution on the current CPU. Execution will resume when + * an interrupt (device, %stick_compare, or cross-call) is targeted to + * the CPU. On some CPUs, this API may be used by the hypervisor to + * save power by disabling hardware strands. + */ +#define HV_FAST_CPU_YIELD 0x12 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_yield(void); +#endif + +/* cpu_poke() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_POKE + * RET0: status + * ERRORS: ENOCPU cpuid refers to a CPU that does not exist + * EINVAL cpuid is current CPU + * + * Poke CPU cpuid. If the target CPU is currently suspended having + * invoked the cpu-yield service, that vCPU will be resumed. + * Poke interrupts may only be sent to valid, non-local CPUs. + * It is not legal to poke the current vCPU. + */ +#define HV_FAST_CPU_POKE 0x13 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_poke(unsigned long cpuid); +#endif + +/* cpu_qconf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_QCONF + * ARG0: queue + * ARG1: base real address + * ARG2: number of entries + * RET0: status + * ERRORS: ENORADDR Invalid base real address + * EINVAL Invalid queue or number of entries is less + * than 2 or too large. + * EBADALIGN Base real address is not correctly aligned + * for size. + * + * Configure the given queue to be placed at the given base real + * address, with the given number of entries. The number of entries + * must be a power of 2. The base real address must be aligned + * exactly to match the queue size. Each queue entry is 64 bytes + * long, so for example a 32 entry queue must be aligned on a 2048 + * byte real address boundary. + * + * The specified queue is unconfigured if the number of entries is given + * as zero. + * + * For the current version of this API service, the argument queue is defined + * as follows: + * + * queue description + * ----- ------------------------- + * 0x3c cpu mondo queue + * 0x3d device mondo queue + * 0x3e resumable error queue + * 0x3f non-resumable error queue + * + * Note: The maximum number of entries for each queue for a specific cpu may + * be determined from the machine description. + */ +#define HV_FAST_CPU_QCONF 0x14 +#define HV_CPU_QUEUE_CPU_MONDO 0x3c +#define HV_CPU_QUEUE_DEVICE_MONDO 0x3d +#define HV_CPU_QUEUE_RES_ERROR 0x3e +#define HV_CPU_QUEUE_NONRES_ERROR 0x3f + +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_qconf(unsigned long type, + unsigned long queue_paddr, + unsigned long num_queue_entries); +#endif + +/* cpu_qinfo() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_QINFO + * ARG0: queue + * RET0: status + * RET1: base real address + * RET1: number of entries + * ERRORS: EINVAL Invalid queue + * + * Return the configuration info for the given queue. The base real + * address and number of entries of the defined queue are returned. + * The queue argument values are the same as for cpu_qconf() above. + * + * If the specified queue is a valid queue number, but no queue has + * been defined, the number of entries will be set to zero and the + * base real address returned is undefined. + */ +#define HV_FAST_CPU_QINFO 0x15 + +/* cpu_mondo_send() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_MONDO_SEND + * ARG0-1: CPU list + * ARG2: data real address + * RET0: status + * ERRORS: EBADALIGN Mondo data is not 64-byte aligned or CPU list + * is not 2-byte aligned. + * ENORADDR Invalid data mondo address, or invalid cpu list + * address. + * ENOCPU Invalid cpu in CPU list + * EWOULDBLOCK Some or all of the listed CPUs did not receive + * the mondo + * ECPUERROR One or more of the listed CPUs are in error + * state, use HV_FAST_CPU_STATE to see which ones + * EINVAL CPU list includes caller's CPU ID + * + * Send a mondo interrupt to the CPUs in the given CPU list with the + * 64-bytes at the given data real address. The data must be 64-byte + * aligned. The mondo data will be delivered to the cpu_mondo queues + * of the recipient CPUs. + * + * In all cases, error or not, the CPUs in the CPU list to which the + * mondo has been successfully delivered will be indicated by having + * their entry in CPU list updated with the value 0xffff. + */ +#define HV_FAST_CPU_MONDO_SEND 0x42 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_cpu_mondo_send(unsigned long cpu_count, + unsigned long cpu_list_pa, + unsigned long mondo_block_pa); +#endif + +/* cpu_myid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_MYID + * RET0: status + * RET1: CPU ID + * ERRORS: No errors defined. + * + * Return the hypervisor ID handle for the current CPU. Use by a + * virtual CPU to discover its own identity. + */ +#define HV_FAST_CPU_MYID 0x16 + +/* cpu_state() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_STATE + * ARG0: CPU ID + * RET0: status + * RET1: state + * ERRORS: ENOCPU Invalid CPU ID + * + * Retrieve the current state of the CPU with the given CPU ID. + */ +#define HV_FAST_CPU_STATE 0x17 +#define HV_CPU_STATE_STOPPED 0x01 +#define HV_CPU_STATE_RUNNING 0x02 +#define HV_CPU_STATE_ERROR 0x03 + +#ifndef __ASSEMBLER__ +long sun4v_cpu_state(unsigned long cpuid); +#endif + +/* cpu_set_rtba() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_SET_RTBA + * ARG0: RTBA + * RET0: status + * RET1: previous RTBA + * ERRORS: ENORADDR Invalid RTBA real address + * EBADALIGN RTBA is incorrectly aligned for a trap table + * + * Set the real trap base address of the local cpu to the given RTBA. + * The supplied RTBA must be aligned on a 256 byte boundary. Upon + * success the previous value of the RTBA is returned in RET1. + * + * Note: This service does not affect %tba + */ +#define HV_FAST_CPU_SET_RTBA 0x18 + +/* cpu_set_rtba() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CPU_GET_RTBA + * RET0: status + * RET1: previous RTBA + * ERRORS: No possible error. + * + * Returns the current value of RTBA in RET1. + */ +#define HV_FAST_CPU_GET_RTBA 0x19 + +/* MMU services. + * + * Layout of a TSB description for mmu_tsb_ctx{,non}0() calls. + */ +#ifndef __ASSEMBLER__ +struct hv_tsb_descr { + unsigned short pgsz_idx; + unsigned short assoc; + unsigned int num_ttes; /* in TTEs */ + unsigned int ctx_idx; + unsigned int pgsz_mask; + unsigned long tsb_base; + unsigned long resv; +}; +#endif +#define HV_TSB_DESCR_PGSZ_IDX_OFFSET 0x00 +#define HV_TSB_DESCR_ASSOC_OFFSET 0x02 +#define HV_TSB_DESCR_NUM_TTES_OFFSET 0x04 +#define HV_TSB_DESCR_CTX_IDX_OFFSET 0x08 +#define HV_TSB_DESCR_PGSZ_MASK_OFFSET 0x0c +#define HV_TSB_DESCR_TSB_BASE_OFFSET 0x10 +#define HV_TSB_DESCR_RESV_OFFSET 0x18 + +/* Page size bitmask. */ +#define HV_PGSZ_MASK_8K (1 << 0) +#define HV_PGSZ_MASK_64K (1 << 1) +#define HV_PGSZ_MASK_512K (1 << 2) +#define HV_PGSZ_MASK_4MB (1 << 3) +#define HV_PGSZ_MASK_32MB (1 << 4) +#define HV_PGSZ_MASK_256MB (1 << 5) +#define HV_PGSZ_MASK_2GB (1 << 6) +#define HV_PGSZ_MASK_16GB (1 << 7) + +/* Page size index. The value given in the TSB descriptor must correspond + * to the smallest page size specified in the pgsz_mask page size bitmask. + */ +#define HV_PGSZ_IDX_8K 0 +#define HV_PGSZ_IDX_64K 1 +#define HV_PGSZ_IDX_512K 2 +#define HV_PGSZ_IDX_4MB 3 +#define HV_PGSZ_IDX_32MB 4 +#define HV_PGSZ_IDX_256MB 5 +#define HV_PGSZ_IDX_2GB 6 +#define HV_PGSZ_IDX_16GB 7 + +/* MMU fault status area. + * + * MMU related faults have their status and fault address information + * placed into a memory region made available by privileged code. Each + * virtual processor must make a mmu_fault_area_conf() call to tell the + * hypervisor where that processor's fault status should be stored. + * + * The fault status block is a multiple of 64-bytes and must be aligned + * on a 64-byte boundary. + */ +#ifndef __ASSEMBLER__ +struct hv_fault_status { + unsigned long i_fault_type; + unsigned long i_fault_addr; + unsigned long i_fault_ctx; + unsigned long i_reserved[5]; + unsigned long d_fault_type; + unsigned long d_fault_addr; + unsigned long d_fault_ctx; + unsigned long d_reserved[5]; +}; +#endif +#define HV_FAULT_I_TYPE_OFFSET 0x00 +#define HV_FAULT_I_ADDR_OFFSET 0x08 +#define HV_FAULT_I_CTX_OFFSET 0x10 +#define HV_FAULT_D_TYPE_OFFSET 0x40 +#define HV_FAULT_D_ADDR_OFFSET 0x48 +#define HV_FAULT_D_CTX_OFFSET 0x50 + +#define HV_FAULT_TYPE_FAST_MISS 1 +#define HV_FAULT_TYPE_FAST_PROT 2 +#define HV_FAULT_TYPE_MMU_MISS 3 +#define HV_FAULT_TYPE_INV_RA 4 +#define HV_FAULT_TYPE_PRIV_VIOL 5 +#define HV_FAULT_TYPE_PROT_VIOL 6 +#define HV_FAULT_TYPE_NFO 7 +#define HV_FAULT_TYPE_NFO_SEFF 8 +#define HV_FAULT_TYPE_INV_VA 9 +#define HV_FAULT_TYPE_INV_ASI 10 +#define HV_FAULT_TYPE_NC_ATOMIC 11 +#define HV_FAULT_TYPE_PRIV_ACT 12 +#define HV_FAULT_TYPE_RESV1 13 +#define HV_FAULT_TYPE_UNALIGNED 14 +#define HV_FAULT_TYPE_INV_PGSZ 15 +#define HV_FAULT_TYPE_MCD 17 +#define HV_FAULT_TYPE_MCD_DIS 18 +/* Values 16 --> -2 are reserved. */ +#define HV_FAULT_TYPE_MULTIPLE -1 + +/* Flags argument for mmu_{map,unmap}_addr(), mmu_demap_{page,context,all}(), + * and mmu_{map,unmap}_perm_addr(). + */ +#define HV_MMU_DMMU 0x01 +#define HV_MMU_IMMU 0x02 +#define HV_MMU_ALL (HV_MMU_DMMU | HV_MMU_IMMU) + +/* mmu_map_addr() + * TRAP: HV_MMU_MAP_ADDR_TRAP + * ARG0: virtual address + * ARG1: mmu context + * ARG2: TTE + * ARG3: flags (HV_MMU_{IMMU,DMMU}) + * ERRORS: EINVAL Invalid virtual address, mmu context, or flags + * EBADPGSZ Invalid page size value + * ENORADDR Invalid real address in TTE + * + * Create a non-permanent mapping using the given TTE, virtual + * address, and mmu context. The flags argument determines which + * (data, or instruction, or both) TLB the mapping gets loaded into. + * + * The behavior is undefined if the valid bit is clear in the TTE. + * + * Note: This API call is for privileged code to specify temporary translation + * mappings without the need to create and manage a TSB. + */ + +/* mmu_unmap_addr() + * TRAP: HV_MMU_UNMAP_ADDR_TRAP + * ARG0: virtual address + * ARG1: mmu context + * ARG2: flags (HV_MMU_{IMMU,DMMU}) + * ERRORS: EINVAL Invalid virtual address, mmu context, or flags + * + * Demaps the given virtual address in the given mmu context on this + * CPU. This function is intended to be used to demap pages mapped + * with mmu_map_addr. This service is equivalent to invoking + * mmu_demap_page() with only the current CPU in the CPU list. The + * flags argument determines which (data, or instruction, or both) TLB + * the mapping gets unmapped from. + * + * Attempting to perform an unmap operation for a previously defined + * permanent mapping will have undefined results. + */ + +/* mmu_tsb_ctx0() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_TSB_CTX0 + * ARG0: number of TSB descriptions + * ARG1: TSB descriptions pointer + * RET0: status + * ERRORS: ENORADDR Invalid TSB descriptions pointer or + * TSB base within a descriptor + * EBADALIGN TSB descriptions pointer is not aligned + * to an 8-byte boundary, or TSB base + * within a descriptor is not aligned for + * the given TSB size + * EBADPGSZ Invalid page size in a TSB descriptor + * EBADTSB Invalid associativity or size in a TSB + * descriptor + * EINVAL Invalid number of TSB descriptions, or + * invalid context index in a TSB + * descriptor, or index page size not + * equal to smallest page size in page + * size bitmask field. + * + * Configures the TSBs for the current CPU for virtual addresses with + * context zero. The TSB descriptions pointer is a pointer to an + * array of the given number of TSB descriptions. + * + * Note: The maximum number of TSBs available to a virtual CPU is given by the + * mmu-max-#tsbs property of the cpu's corresponding "cpu" node in the + * machine description. + */ +#define HV_FAST_MMU_TSB_CTX0 0x20 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_mmu_tsb_ctx0(unsigned long num_descriptions, + unsigned long tsb_desc_ra); +#endif + +/* mmu_tsb_ctxnon0() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_TSB_CTXNON0 + * ARG0: number of TSB descriptions + * ARG1: TSB descriptions pointer + * RET0: status + * ERRORS: Same as for mmu_tsb_ctx0() above. + * + * Configures the TSBs for the current CPU for virtual addresses with + * non-zero contexts. The TSB descriptions pointer is a pointer to an + * array of the given number of TSB descriptions. + * + * Note: A maximum of 16 TSBs may be specified in the TSB description list. + */ +#define HV_FAST_MMU_TSB_CTXNON0 0x21 + +/* mmu_demap_page() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_DEMAP_PAGE + * ARG0: reserved, must be zero + * ARG1: reserved, must be zero + * ARG2: virtual address + * ARG3: mmu context + * ARG4: flags (HV_MMU_{IMMU,DMMU}) + * RET0: status + * ERRORS: EINVAL Invalid virtual address, context, or + * flags value + * ENOTSUPPORTED ARG0 or ARG1 is non-zero + * + * Demaps any page mapping of the given virtual address in the given + * mmu context for the current virtual CPU. Any virtually tagged + * caches are guaranteed to be kept consistent. The flags argument + * determines which TLB (instruction, or data, or both) participate in + * the operation. + * + * ARG0 and ARG1 are both reserved and must be set to zero. + */ +#define HV_FAST_MMU_DEMAP_PAGE 0x22 + +/* mmu_demap_ctx() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_DEMAP_CTX + * ARG0: reserved, must be zero + * ARG1: reserved, must be zero + * ARG2: mmu context + * ARG3: flags (HV_MMU_{IMMU,DMMU}) + * RET0: status + * ERRORS: EINVAL Invalid context or flags value + * ENOTSUPPORTED ARG0 or ARG1 is non-zero + * + * Demaps all non-permanent virtual page mappings previously specified + * for the given context for the current virtual CPU. Any virtual + * tagged caches are guaranteed to be kept consistent. The flags + * argument determines which TLB (instruction, or data, or both) + * participate in the operation. + * + * ARG0 and ARG1 are both reserved and must be set to zero. + */ +#define HV_FAST_MMU_DEMAP_CTX 0x23 + +/* mmu_demap_all() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_DEMAP_ALL + * ARG0: reserved, must be zero + * ARG1: reserved, must be zero + * ARG2: flags (HV_MMU_{IMMU,DMMU}) + * RET0: status + * ERRORS: EINVAL Invalid flags value + * ENOTSUPPORTED ARG0 or ARG1 is non-zero + * + * Demaps all non-permanent virtual page mappings previously specified + * for the current virtual CPU. Any virtual tagged caches are + * guaranteed to be kept consistent. The flags argument determines + * which TLB (instruction, or data, or both) participate in the + * operation. + * + * ARG0 and ARG1 are both reserved and must be set to zero. + */ +#define HV_FAST_MMU_DEMAP_ALL 0x24 + +#ifndef __ASSEMBLER__ +void sun4v_mmu_demap_all(void); +#endif + +/* mmu_map_perm_addr() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_MAP_PERM_ADDR + * ARG0: virtual address + * ARG1: reserved, must be zero + * ARG2: TTE + * ARG3: flags (HV_MMU_{IMMU,DMMU}) + * RET0: status + * ERRORS: EINVAL Invalid virtual address or flags value + * EBADPGSZ Invalid page size value + * ENORADDR Invalid real address in TTE + * ETOOMANY Too many mappings (max of 8 reached) + * + * Create a permanent mapping using the given TTE and virtual address + * for context 0 on the calling virtual CPU. A maximum of 8 such + * permanent mappings may be specified by privileged code. Mappings + * may be removed with mmu_unmap_perm_addr(). + * + * The behavior is undefined if a TTE with the valid bit clear is given. + * + * Note: This call is used to specify address space mappings for which + * privileged code does not expect to receive misses. For example, + * this mechanism can be used to map kernel nucleus code and data. + */ +#define HV_FAST_MMU_MAP_PERM_ADDR 0x25 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_mmu_map_perm_addr(unsigned long vaddr, + unsigned long set_to_zero, + unsigned long tte, + unsigned long flags); +#endif + +/* mmu_fault_area_conf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_FAULT_AREA_CONF + * ARG0: real address + * RET0: status + * RET1: previous mmu fault area real address + * ERRORS: ENORADDR Invalid real address + * EBADALIGN Invalid alignment for fault area + * + * Configure the MMU fault status area for the calling CPU. A 64-byte + * aligned real address specifies where MMU fault status information + * is placed. The return value is the previously specified area, or 0 + * for the first invocation. Specifying a fault area at real address + * 0 is not allowed. + */ +#define HV_FAST_MMU_FAULT_AREA_CONF 0x26 + +/* mmu_enable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_ENABLE + * ARG0: enable flag + * ARG1: return target address + * RET0: status + * ERRORS: ENORADDR Invalid real address when disabling + * translation. + * EBADALIGN The return target address is not + * aligned to an instruction. + * EINVAL The enable flag request the current + * operating mode (e.g. disable if already + * disabled) + * + * Enable or disable virtual address translation for the calling CPU + * within the virtual machine domain. If the enable flag is zero, + * translation is disabled, any non-zero value will enable + * translation. + * + * When this function returns, the newly selected translation mode + * will be active. If the mmu is being enabled, then the return + * target address is a virtual address else it is a real address. + * + * Upon successful completion, control will be returned to the given + * return target address (ie. the cpu will jump to that address). On + * failure, the previous mmu mode remains and the trap simply returns + * as normal with the appropriate error code in RET0. + */ +#define HV_FAST_MMU_ENABLE 0x27 + +/* mmu_unmap_perm_addr() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_UNMAP_PERM_ADDR + * ARG0: virtual address + * ARG1: reserved, must be zero + * ARG2: flags (HV_MMU_{IMMU,DMMU}) + * RET0: status + * ERRORS: EINVAL Invalid virtual address or flags value + * ENOMAP Specified mapping was not found + * + * Demaps any permanent page mapping (established via + * mmu_map_perm_addr()) at the given virtual address for context 0 on + * the current virtual CPU. Any virtual tagged caches are guaranteed + * to be kept consistent. + */ +#define HV_FAST_MMU_UNMAP_PERM_ADDR 0x28 + +/* mmu_tsb_ctx0_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_TSB_CTX0_INFO + * ARG0: max TSBs + * ARG1: buffer pointer + * RET0: status + * RET1: number of TSBs + * ERRORS: EINVAL Supplied buffer is too small + * EBADALIGN The buffer pointer is badly aligned + * ENORADDR Invalid real address for buffer pointer + * + * Return the TSB configuration as previous defined by mmu_tsb_ctx0() + * into the provided buffer. The size of the buffer is given in ARG1 + * in terms of the number of TSB description entries. + * + * Upon return, RET1 always contains the number of TSB descriptions + * previously configured. If zero TSBs were configured, EOK is + * returned with RET1 containing 0. + */ +#define HV_FAST_MMU_TSB_CTX0_INFO 0x29 + +/* mmu_tsb_ctxnon0_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_TSB_CTXNON0_INFO + * ARG0: max TSBs + * ARG1: buffer pointer + * RET0: status + * RET1: number of TSBs + * ERRORS: EINVAL Supplied buffer is too small + * EBADALIGN The buffer pointer is badly aligned + * ENORADDR Invalid real address for buffer pointer + * + * Return the TSB configuration as previous defined by + * mmu_tsb_ctxnon0() into the provided buffer. The size of the buffer + * is given in ARG1 in terms of the number of TSB description entries. + * + * Upon return, RET1 always contains the number of TSB descriptions + * previously configured. If zero TSBs were configured, EOK is + * returned with RET1 containing 0. + */ +#define HV_FAST_MMU_TSB_CTXNON0_INFO 0x2a + +/* mmu_fault_area_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMU_FAULT_AREA_INFO + * RET0: status + * RET1: fault area real address + * ERRORS: No errors defined. + * + * Return the currently defined MMU fault status area for the current + * CPU. The real address of the fault status area is returned in + * RET1, or 0 is returned in RET1 if no fault status area is defined. + * + * Note: mmu_fault_area_conf() may be called with the return value (RET1) + * from this service if there is a need to save and restore the fault + * area for a cpu. + */ +#define HV_FAST_MMU_FAULT_AREA_INFO 0x2b + +/* Cache and Memory services. */ + +/* mem_scrub() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MEM_SCRUB + * ARG0: real address + * ARG1: length + * RET0: status + * RET1: length scrubbed + * ERRORS: ENORADDR Invalid real address + * EBADALIGN Start address or length are not correctly + * aligned + * EINVAL Length is zero + * + * Zero the memory contents in the range real address to real address + * plus length minus 1. Also, valid ECC will be generated for that + * memory address range. Scrubbing is started at the given real + * address, but may not scrub the entire given length. The actual + * length scrubbed will be returned in RET1. + * + * The real address and length must be aligned on an 8K boundary, or + * contain the start address and length from a sun4v error report. + * + * Note: There are two uses for this function. The first use is to block clear + * and initialize memory and the second is to scrub an u ncorrectable + * error reported via a resumable or non-resumable trap. The second + * use requires the arguments to be equal to the real address and length + * provided in a sun4v memory error report. + */ +#define HV_FAST_MEM_SCRUB 0x31 + +/* mem_sync() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MEM_SYNC + * ARG0: real address + * ARG1: length + * RET0: status + * RET1: length synced + * ERRORS: ENORADDR Invalid real address + * EBADALIGN Start address or length are not correctly + * aligned + * EINVAL Length is zero + * + * Force the next access within the real address to real address plus + * length minus 1 to be fetches from main system memory. Less than + * the given length may be synced, the actual amount synced is + * returned in RET1. The real address and length must be aligned on + * an 8K boundary. + */ +#define HV_FAST_MEM_SYNC 0x32 + +/* Coprocessor services + * + * M7 and later processors provide an on-chip coprocessor which + * accelerates database operations, and is known internally as + * DAX. + */ + +/* ccb_submit() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_CCB_SUBMIT + * ARG0: address of CCB array + * ARG1: size (in bytes) of CCB array being submitted + * ARG2: flags + * ARG3: reserved + * RET0: status (success or error code) + * RET1: size (in bytes) of CCB array that was accepted (might be less + * than arg1) + * RET2: status data + * if status == ENOMAP or ENOACCESS, identifies the VA in question + * if status == EUNAVAILBLE, unavailable code + * RET3: reserved + * + * ERRORS: EOK successful submission (check size) + * EWOULDBLOCK could not finish submissions, try again + * EBADALIGN array not 64B aligned or size not 64B multiple + * ENORADDR invalid RA for array or in CCB + * ENOMAP could not translate address (see status data) + * EINVAL invalid ccb or arguments + * ETOOMANY too many ccbs with all-or-nothing flag + * ENOACCESS guest has no access to submit ccbs or address + * in CCB does not have correct permissions (check + * status data) + * EUNAVAILABLE ccb operation could not be performed at this + * time (check status data) + * Status data codes: + * 0 - exact CCB could not be executed + * 1 - CCB opcode cannot be executed + * 2 - CCB version cannot be executed + * 3 - vcpu cannot execute CCBs + * 4 - no CCBs can be executed + */ + +#define HV_CCB_SUBMIT 0x34 +#ifndef __ASSEMBLER__ +unsigned long sun4v_ccb_submit(unsigned long ccb_buf, + unsigned long len, + unsigned long flags, + unsigned long reserved, + void *submitted_len, + void *status_data); +#endif + +/* flags (ARG2) */ +#define HV_CCB_QUERY_CMD BIT(1) +#define HV_CCB_ARG0_TYPE_REAL 0UL +#define HV_CCB_ARG0_TYPE_PRIMARY BIT(4) +#define HV_CCB_ARG0_TYPE_SECONDARY BIT(5) +#define HV_CCB_ARG0_TYPE_NUCLEUS GENMASK(5, 4) +#define HV_CCB_ARG0_PRIVILEGED BIT(6) +#define HV_CCB_ALL_OR_NOTHING BIT(7) +#define HV_CCB_QUEUE_INFO BIT(8) +#define HV_CCB_VA_REJECT 0UL +#define HV_CCB_VA_SECONDARY BIT(13) +#define HV_CCB_VA_NUCLEUS GENMASK(13, 12) +#define HV_CCB_VA_PRIVILEGED BIT(14) +#define HV_CCB_VA_READ_ADI_DISABLE BIT(15) /* DAX2 only */ + +/* ccb_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_CCB_INFO + * ARG0: real address of CCB completion area + * RET0: status (success or error code) + * RET1: info array + * - RET1[0]: CCB state + * - RET1[1]: dax unit + * - RET1[2]: queue number + * - RET1[3]: queue position + * + * ERRORS: EOK operation successful + * EBADALIGN address not 64B aligned + * ENORADDR RA in address not valid + * EINVAL CA not valid + * EWOULDBLOCK info not available for this CCB currently, try + * again + * ENOACCESS guest cannot use dax + */ + +#define HV_CCB_INFO 0x35 +#ifndef __ASSEMBLER__ +unsigned long sun4v_ccb_info(unsigned long ca, + void *info_arr); +#endif + +/* info array byte offsets (RET1) */ +#define CCB_INFO_OFFSET_CCB_STATE 0 +#define CCB_INFO_OFFSET_DAX_UNIT 2 +#define CCB_INFO_OFFSET_QUEUE_NUM 4 +#define CCB_INFO_OFFSET_QUEUE_POS 6 + +/* CCB state (RET1[0]) */ +#define HV_CCB_STATE_COMPLETED 0 +#define HV_CCB_STATE_ENQUEUED 1 +#define HV_CCB_STATE_INPROGRESS 2 +#define HV_CCB_STATE_NOTFOUND 3 + +/* ccb_kill() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_CCB_KILL + * ARG0: real address of CCB completion area + * RET0: status (success or error code) + * RET1: CCB kill status + * + * ERRORS: EOK operation successful + * EBADALIGN address not 64B aligned + * ENORADDR RA in address not valid + * EINVAL CA not valid + * EWOULDBLOCK kill not available for this CCB currently, try + * again + * ENOACCESS guest cannot use dax + */ + +#define HV_CCB_KILL 0x36 +#ifndef __ASSEMBLER__ +unsigned long sun4v_ccb_kill(unsigned long ca, + void *kill_status); +#endif + +/* CCB kill status (RET1) */ +#define HV_CCB_KILL_COMPLETED 0 +#define HV_CCB_KILL_DEQUEUED 1 +#define HV_CCB_KILL_KILLED 2 +#define HV_CCB_KILL_NOTFOUND 3 + +/* Time of day services. + * + * The hypervisor maintains the time of day on a per-domain basis. + * Changing the time of day in one domain does not affect the time of + * day on any other domain. + * + * Time is described by a single unsigned 64-bit word which is the + * number of seconds since the UNIX Epoch (00:00:00 UTC, January 1, + * 1970). + */ + +/* tod_get() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_TOD_GET + * RET0: status + * RET1: TOD + * ERRORS: EWOULDBLOCK TOD resource is temporarily unavailable + * ENOTSUPPORTED If TOD not supported on this platform + * + * Return the current time of day. May block if TOD access is + * temporarily not possible. + */ +#define HV_FAST_TOD_GET 0x50 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_tod_get(unsigned long *time); +#endif + +/* tod_set() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_TOD_SET + * ARG0: TOD + * RET0: status + * ERRORS: EWOULDBLOCK TOD resource is temporarily unavailable + * ENOTSUPPORTED If TOD not supported on this platform + * + * The current time of day is set to the value specified in ARG0. May + * block if TOD access is temporarily not possible. + */ +#define HV_FAST_TOD_SET 0x51 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_tod_set(unsigned long time); +#endif + +/* Console services */ + +/* con_getchar() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CONS_GETCHAR + * RET0: status + * RET1: character + * ERRORS: EWOULDBLOCK No character available. + * + * Returns a character from the console device. If no character is + * available then an EWOULDBLOCK error is returned. If a character is + * available, then the returned status is EOK and the character value + * is in RET1. + * + * A virtual BREAK is represented by the 64-bit value -1. + * + * A virtual HUP signal is represented by the 64-bit value -2. + */ +#define HV_FAST_CONS_GETCHAR 0x60 + +/* con_putchar() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CONS_PUTCHAR + * ARG0: character + * RET0: status + * ERRORS: EINVAL Illegal character + * EWOULDBLOCK Output buffer currently full, would block + * + * Send a character to the console device. Only character values + * between 0 and 255 may be used. Values outside this range are + * invalid except for the 64-bit value -1 which is used to send a + * virtual BREAK. + */ +#define HV_FAST_CONS_PUTCHAR 0x61 + +/* con_read() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CONS_READ + * ARG0: buffer real address + * ARG1: buffer size in bytes + * RET0: status + * RET1: bytes read or BREAK or HUP + * ERRORS: EWOULDBLOCK No character available. + * + * Reads characters into a buffer from the console device. If no + * character is available then an EWOULDBLOCK error is returned. + * If a character is available, then the returned status is EOK + * and the number of bytes read into the given buffer is provided + * in RET1. + * + * A virtual BREAK is represented by the 64-bit RET1 value -1. + * + * A virtual HUP signal is represented by the 64-bit RET1 value -2. + * + * If BREAK or HUP are indicated, no bytes were read into buffer. + */ +#define HV_FAST_CONS_READ 0x62 + +/* con_write() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_CONS_WRITE + * ARG0: buffer real address + * ARG1: buffer size in bytes + * RET0: status + * RET1: bytes written + * ERRORS: EWOULDBLOCK Output buffer currently full, would block + * + * Send a characters in buffer to the console device. Breaks must be + * sent using con_putchar(). + */ +#define HV_FAST_CONS_WRITE 0x63 + +#ifndef __ASSEMBLER__ +long sun4v_con_getchar(long *status); +long sun4v_con_putchar(long c); +long sun4v_con_read(unsigned long buffer, + unsigned long size, + unsigned long *bytes_read); +unsigned long sun4v_con_write(unsigned long buffer, + unsigned long size, + unsigned long *bytes_written); +#endif + +/* mach_set_soft_state() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MACH_SET_SOFT_STATE + * ARG0: software state + * ARG1: software state description pointer + * RET0: status + * ERRORS: EINVAL software state not valid or software state + * description is not NULL terminated + * ENORADDR software state description pointer is not a + * valid real address + * EBADALIGNED software state description is not correctly + * aligned + * + * This allows the guest to report its soft state to the hypervisor. There + * are two primary components to this state. The first part states whether + * the guest software is running or not. The second containts optional + * details specific to the software. + * + * The software state argument is defined below in HV_SOFT_STATE_*, and + * indicates whether the guest is operating normally or in a transitional + * state. + * + * The software state description argument is a real address of a data buffer + * of size 32-bytes aligned on a 32-byte boundary. It is treated as a NULL + * terminated 7-bit ASCII string of up to 31 characters not including the + * NULL termination. + */ +#define HV_FAST_MACH_SET_SOFT_STATE 0x70 +#define HV_SOFT_STATE_NORMAL 0x01 +#define HV_SOFT_STATE_TRANSITION 0x02 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_mach_set_soft_state(unsigned long soft_state, + unsigned long msg_string_ra); +#endif + +/* mach_get_soft_state() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MACH_GET_SOFT_STATE + * ARG0: software state description pointer + * RET0: status + * RET1: software state + * ERRORS: ENORADDR software state description pointer is not a + * valid real address + * EBADALIGNED software state description is not correctly + * aligned + * + * Retrieve the current value of the guest's software state. The rules + * for the software state pointer are the same as for mach_set_soft_state() + * above. + */ +#define HV_FAST_MACH_GET_SOFT_STATE 0x71 + +/* svc_send() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_SVC_SEND + * ARG0: service ID + * ARG1: buffer real address + * ARG2: buffer size + * RET0: STATUS + * RET1: sent_bytes + * + * Be careful, all output registers are clobbered by this operation, + * so for example it is not possible to save away a value in %o4 + * across the trap. + */ +#define HV_FAST_SVC_SEND 0x80 + +/* svc_recv() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_SVC_RECV + * ARG0: service ID + * ARG1: buffer real address + * ARG2: buffer size + * RET0: STATUS + * RET1: recv_bytes + * + * Be careful, all output registers are clobbered by this operation, + * so for example it is not possible to save away a value in %o4 + * across the trap. + */ +#define HV_FAST_SVC_RECV 0x81 + +/* svc_getstatus() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_SVC_GETSTATUS + * ARG0: service ID + * RET0: STATUS + * RET1: status bits + */ +#define HV_FAST_SVC_GETSTATUS 0x82 + +/* svc_setstatus() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_SVC_SETSTATUS + * ARG0: service ID + * ARG1: bits to set + * RET0: STATUS + */ +#define HV_FAST_SVC_SETSTATUS 0x83 + +/* svc_clrstatus() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_SVC_CLRSTATUS + * ARG0: service ID + * ARG1: bits to clear + * RET0: STATUS + */ +#define HV_FAST_SVC_CLRSTATUS 0x84 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_svc_send(unsigned long svc_id, + unsigned long buffer, + unsigned long buffer_size, + unsigned long *sent_bytes); +unsigned long sun4v_svc_recv(unsigned long svc_id, + unsigned long buffer, + unsigned long buffer_size, + unsigned long *recv_bytes); +unsigned long sun4v_svc_getstatus(unsigned long svc_id, + unsigned long *status_bits); +unsigned long sun4v_svc_setstatus(unsigned long svc_id, + unsigned long status_bits); +unsigned long sun4v_svc_clrstatus(unsigned long svc_id, + unsigned long status_bits); +#endif + +/* Trap trace services. + * + * The hypervisor provides a trap tracing capability for privileged + * code running on each virtual CPU. Privileged code provides a + * round-robin trap trace queue within which the hypervisor writes + * 64-byte entries detailing hyperprivileged traps taken n behalf of + * privileged code. This is provided as a debugging capability for + * privileged code. + * + * The trap trace control structure is 64-bytes long and placed at the + * start (offset 0) of the trap trace buffer, and is described as + * follows: + */ +#ifndef __ASSEMBLER__ +struct hv_trap_trace_control { + unsigned long head_offset; + unsigned long tail_offset; + unsigned long __reserved[0x30 / sizeof(unsigned long)]; +}; +#endif +#define HV_TRAP_TRACE_CTRL_HEAD_OFFSET 0x00 +#define HV_TRAP_TRACE_CTRL_TAIL_OFFSET 0x08 + +/* The head offset is the offset of the most recently completed entry + * in the trap-trace buffer. The tail offset is the offset of the + * next entry to be written. The control structure is owned and + * modified by the hypervisor. A guest may not modify the control + * structure contents. Attempts to do so will result in undefined + * behavior for the guest. + * + * Each trap trace buffer entry is laid out as follows: + */ +#ifndef __ASSEMBLER__ +struct hv_trap_trace_entry { + unsigned char type; /* Hypervisor or guest entry? */ + unsigned char hpstate; /* Hyper-privileged state */ + unsigned char tl; /* Trap level */ + unsigned char gl; /* Global register level */ + unsigned short tt; /* Trap type */ + unsigned short tag; /* Extended trap identifier */ + unsigned long tstate; /* Trap state */ + unsigned long tick; /* Tick */ + unsigned long tpc; /* Trap PC */ + unsigned long f1; /* Entry specific */ + unsigned long f2; /* Entry specific */ + unsigned long f3; /* Entry specific */ + unsigned long f4; /* Entry specific */ +}; +#endif +#define HV_TRAP_TRACE_ENTRY_TYPE 0x00 +#define HV_TRAP_TRACE_ENTRY_HPSTATE 0x01 +#define HV_TRAP_TRACE_ENTRY_TL 0x02 +#define HV_TRAP_TRACE_ENTRY_GL 0x03 +#define HV_TRAP_TRACE_ENTRY_TT 0x04 +#define HV_TRAP_TRACE_ENTRY_TAG 0x06 +#define HV_TRAP_TRACE_ENTRY_TSTATE 0x08 +#define HV_TRAP_TRACE_ENTRY_TICK 0x10 +#define HV_TRAP_TRACE_ENTRY_TPC 0x18 +#define HV_TRAP_TRACE_ENTRY_F1 0x20 +#define HV_TRAP_TRACE_ENTRY_F2 0x28 +#define HV_TRAP_TRACE_ENTRY_F3 0x30 +#define HV_TRAP_TRACE_ENTRY_F4 0x38 + +/* The type field is encoded as follows. */ +#define HV_TRAP_TYPE_UNDEF 0x00 /* Entry content undefined */ +#define HV_TRAP_TYPE_HV 0x01 /* Hypervisor trap entry */ +#define HV_TRAP_TYPE_GUEST 0xff /* Added via ttrace_addentry() */ + +/* ttrace_buf_conf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_TTRACE_BUF_CONF + * ARG0: real address + * ARG1: number of entries + * RET0: status + * RET1: number of entries + * ERRORS: ENORADDR Invalid real address + * EINVAL Size is too small + * EBADALIGN Real address not aligned on 64-byte boundary + * + * Requests hypervisor trap tracing and declares a virtual CPU's trap + * trace buffer to the hypervisor. The real address supplies the real + * base address of the trap trace queue and must be 64-byte aligned. + * Specifying a value of 0 for the number of entries disables trap + * tracing for the calling virtual CPU. The buffer allocated must be + * sized for a power of two number of 64-byte trap trace entries plus + * an initial 64-byte control structure. + * + * This may be invoked any number of times so that a virtual CPU may + * relocate a trap trace buffer or create "snapshots" of information. + * + * If the real address is illegal or badly aligned, then trap tracing + * is disabled and an error is returned. + * + * Upon failure with EINVAL, this service call returns in RET1 the + * minimum number of buffer entries required. Upon other failures + * RET1 is undefined. + */ +#define HV_FAST_TTRACE_BUF_CONF 0x90 + +/* ttrace_buf_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_TTRACE_BUF_INFO + * RET0: status + * RET1: real address + * RET2: size + * ERRORS: None defined. + * + * Returns the size and location of the previously declared trap-trace + * buffer. In the event that no buffer was previously defined, or the + * buffer is disabled, this call will return a size of zero bytes. + */ +#define HV_FAST_TTRACE_BUF_INFO 0x91 + +/* ttrace_enable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_TTRACE_ENABLE + * ARG0: enable + * RET0: status + * RET1: previous enable state + * ERRORS: EINVAL No trap trace buffer currently defined + * + * Enable or disable trap tracing, and return the previous enabled + * state in RET1. Future systems may define various flags for the + * enable argument (ARG0), for the moment a guest should pass + * "(uint64_t) -1" to enable, and "(uint64_t) 0" to disable all + * tracing - which will ensure future compatibility. + */ +#define HV_FAST_TTRACE_ENABLE 0x92 + +/* ttrace_freeze() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_TTRACE_FREEZE + * ARG0: freeze + * RET0: status + * RET1: previous freeze state + * ERRORS: EINVAL No trap trace buffer currently defined + * + * Freeze or unfreeze trap tracing, returning the previous freeze + * state in RET1. A guest should pass a non-zero value to freeze and + * a zero value to unfreeze all tracing. The returned previous state + * is 0 for not frozen and 1 for frozen. + */ +#define HV_FAST_TTRACE_FREEZE 0x93 + +/* ttrace_addentry() + * TRAP: HV_TTRACE_ADDENTRY_TRAP + * ARG0: tag (16-bits) + * ARG1: data word 0 + * ARG2: data word 1 + * ARG3: data word 2 + * ARG4: data word 3 + * RET0: status + * ERRORS: EINVAL No trap trace buffer currently defined + * + * Add an entry to the trap trace buffer. Upon return only ARG0/RET0 + * is modified - none of the other registers holding arguments are + * volatile across this hypervisor service. + */ + +/* Core dump services. + * + * Since the hypervisor viraulizes and thus obscures a lot of the + * physical machine layout and state, traditional OS crash dumps can + * be difficult to diagnose especially when the problem is a + * configuration error of some sort. + * + * The dump services provide an opaque buffer into which the + * hypervisor can place its internal state in order to assist in + * debugging such situations. The contents are opaque and extremely + * platform and hypervisor implementation specific. The guest, during + * a core dump, requests that the hypervisor update any information in + * the dump buffer in preparation to being dumped as part of the + * domain's memory image. + */ + +/* dump_buf_update() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_DUMP_BUF_UPDATE + * ARG0: real address + * ARG1: size + * RET0: status + * RET1: required size of dump buffer + * ERRORS: ENORADDR Invalid real address + * EBADALIGN Real address is not aligned on a 64-byte + * boundary + * EINVAL Size is non-zero but less than minimum size + * required + * ENOTSUPPORTED Operation not supported on current logical + * domain + * + * Declare a domain dump buffer to the hypervisor. The real address + * provided for the domain dump buffer must be 64-byte aligned. The + * size specifies the size of the dump buffer and may be larger than + * the minimum size specified in the machine description. The + * hypervisor will fill the dump buffer with opaque data. + * + * Note: A guest may elect to include dump buffer contents as part of a crash + * dump to assist with debugging. This function may be called any number + * of times so that a guest may relocate a dump buffer, or create + * "snapshots" of any dump-buffer information. Each call to + * dump_buf_update() atomically declares the new dump buffer to the + * hypervisor. + * + * A specified size of 0 unconfigures the dump buffer. If the real + * address is illegal or badly aligned, then any currently active dump + * buffer is disabled and an error is returned. + * + * In the event that the call fails with EINVAL, RET1 contains the + * minimum size requires by the hypervisor for a valid dump buffer. + */ +#define HV_FAST_DUMP_BUF_UPDATE 0x94 + +/* dump_buf_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_DUMP_BUF_INFO + * RET0: status + * RET1: real address of current dump buffer + * RET2: size of current dump buffer + * ERRORS: No errors defined. + * + * Return the currently configures dump buffer description. A + * returned size of 0 bytes indicates an undefined dump buffer. In + * this case the return address in RET1 is undefined. + */ +#define HV_FAST_DUMP_BUF_INFO 0x95 + +/* Device interrupt services. + * + * Device interrupts are allocated to system bus bridges by the hypervisor, + * and described to OBP in the machine description. OBP then describes + * these interrupts to the OS via properties in the device tree. + * + * Terminology: + * + * cpuid Unique opaque value which represents a target cpu. + * + * devhandle Device handle. It uniquely identifies a device, and + * consistes of the lower 28-bits of the hi-cell of the + * first entry of the device's "reg" property in the + * OBP device tree. + * + * devino Device interrupt number. Specifies the relative + * interrupt number within the device. The unique + * combination of devhandle and devino are used to + * identify a specific device interrupt. + * + * Note: The devino value is the same as the values in the + * "interrupts" property or "interrupt-map" property + * in the OBP device tree for that device. + * + * sysino System interrupt number. A 64-bit unsigned interger + * representing a unique interrupt within a virtual + * machine. + * + * intr_state A flag representing the interrupt state for a given + * sysino. The state values are defined below. + * + * intr_enabled A flag representing the 'enabled' state for a given + * sysino. The enable values are defined below. + */ + +#define HV_INTR_STATE_IDLE 0 /* Nothing pending */ +#define HV_INTR_STATE_RECEIVED 1 /* Interrupt received by hardware */ +#define HV_INTR_STATE_DELIVERED 2 /* Interrupt delivered to queue */ + +#define HV_INTR_DISABLED 0 /* sysino not enabled */ +#define HV_INTR_ENABLED 1 /* sysino enabled */ + +/* intr_devino_to_sysino() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_INTR_DEVINO2SYSINO + * ARG0: devhandle + * ARG1: devino + * RET0: status + * RET1: sysino + * ERRORS: EINVAL Invalid devhandle/devino + * + * Converts a device specific interrupt number of the given + * devhandle/devino into a system specific ino (sysino). + */ +#define HV_FAST_INTR_DEVINO2SYSINO 0xa0 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_devino_to_sysino(unsigned long devhandle, + unsigned long devino); +#endif + +/* intr_getenabled() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_INTR_GETENABLED + * ARG0: sysino + * RET0: status + * RET1: intr_enabled (HV_INTR_{DISABLED,ENABLED}) + * ERRORS: EINVAL Invalid sysino + * + * Returns interrupt enabled state in RET1 for the interrupt defined + * by the given sysino. + */ +#define HV_FAST_INTR_GETENABLED 0xa1 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_getenabled(unsigned long sysino); +#endif + +/* intr_setenabled() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_INTR_SETENABLED + * ARG0: sysino + * ARG1: intr_enabled (HV_INTR_{DISABLED,ENABLED}) + * RET0: status + * ERRORS: EINVAL Invalid sysino or intr_enabled value + * + * Set the 'enabled' state of the interrupt sysino. + */ +#define HV_FAST_INTR_SETENABLED 0xa2 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_setenabled(unsigned long sysino, + unsigned long intr_enabled); +#endif + +/* intr_getstate() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_INTR_GETSTATE + * ARG0: sysino + * RET0: status + * RET1: intr_state (HV_INTR_STATE_*) + * ERRORS: EINVAL Invalid sysino + * + * Returns current state of the interrupt defined by the given sysino. + */ +#define HV_FAST_INTR_GETSTATE 0xa3 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_getstate(unsigned long sysino); +#endif + +/* intr_setstate() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_INTR_SETSTATE + * ARG0: sysino + * ARG1: intr_state (HV_INTR_STATE_*) + * RET0: status + * ERRORS: EINVAL Invalid sysino or intr_state value + * + * Sets the current state of the interrupt described by the given sysino + * value. + * + * Note: Setting the state to HV_INTR_STATE_IDLE clears any pending + * interrupt for sysino. + */ +#define HV_FAST_INTR_SETSTATE 0xa4 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_setstate(unsigned long sysino, unsigned long intr_state); +#endif + +/* intr_gettarget() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_INTR_GETTARGET + * ARG0: sysino + * RET0: status + * RET1: cpuid + * ERRORS: EINVAL Invalid sysino + * + * Returns CPU that is the current target of the interrupt defined by + * the given sysino. The CPU value returned is undefined if the target + * has not been set via intr_settarget(). + */ +#define HV_FAST_INTR_GETTARGET 0xa5 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_gettarget(unsigned long sysino); +#endif + +/* intr_settarget() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_INTR_SETTARGET + * ARG0: sysino + * ARG1: cpuid + * RET0: status + * ERRORS: EINVAL Invalid sysino + * ENOCPU Invalid cpuid + * + * Set the target CPU for the interrupt defined by the given sysino. + */ +#define HV_FAST_INTR_SETTARGET 0xa6 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_intr_settarget(unsigned long sysino, unsigned long cpuid); +#endif + +/* vintr_get_cookie() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_VINTR_GET_COOKIE + * ARG0: device handle + * ARG1: device ino + * RET0: status + * RET1: cookie + */ +#define HV_FAST_VINTR_GET_COOKIE 0xa7 + +/* vintr_set_cookie() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_VINTR_SET_COOKIE + * ARG0: device handle + * ARG1: device ino + * ARG2: cookie + * RET0: status + */ +#define HV_FAST_VINTR_SET_COOKIE 0xa8 + +/* vintr_get_valid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_VINTR_GET_VALID + * ARG0: device handle + * ARG1: device ino + * RET0: status + * RET1: valid state + */ +#define HV_FAST_VINTR_GET_VALID 0xa9 + +/* vintr_set_valid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_VINTR_SET_VALID + * ARG0: device handle + * ARG1: device ino + * ARG2: valid state + * RET0: status + */ +#define HV_FAST_VINTR_SET_VALID 0xaa + +/* vintr_get_state() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_VINTR_GET_STATE + * ARG0: device handle + * ARG1: device ino + * RET0: status + * RET1: state + */ +#define HV_FAST_VINTR_GET_STATE 0xab + +/* vintr_set_state() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_VINTR_SET_STATE + * ARG0: device handle + * ARG1: device ino + * ARG2: state + * RET0: status + */ +#define HV_FAST_VINTR_SET_STATE 0xac + +/* vintr_get_target() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_VINTR_GET_TARGET + * ARG0: device handle + * ARG1: device ino + * RET0: status + * RET1: cpuid + */ +#define HV_FAST_VINTR_GET_TARGET 0xad + +/* vintr_set_target() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_VINTR_SET_TARGET + * ARG0: device handle + * ARG1: device ino + * ARG2: cpuid + * RET0: status + */ +#define HV_FAST_VINTR_SET_TARGET 0xae + +#ifndef __ASSEMBLER__ +unsigned long sun4v_vintr_get_cookie(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *cookie); +unsigned long sun4v_vintr_set_cookie(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long cookie); +unsigned long sun4v_vintr_get_valid(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *valid); +unsigned long sun4v_vintr_set_valid(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long valid); +unsigned long sun4v_vintr_get_state(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *state); +unsigned long sun4v_vintr_set_state(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long state); +unsigned long sun4v_vintr_get_target(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long *cpuid); +unsigned long sun4v_vintr_set_target(unsigned long dev_handle, + unsigned long dev_ino, + unsigned long cpuid); +#endif + +/* PCI IO services. + * + * See the terminology descriptions in the device interrupt services + * section above as those apply here too. Here are terminology + * definitions specific to these PCI IO services: + * + * tsbnum TSB number. Indentifies which io-tsb is used. + * For this version of the specification, tsbnum + * must be zero. + * + * tsbindex TSB index. Identifies which entry in the TSB + * is used. The first entry is zero. + * + * tsbid A 64-bit aligned data structure which contains + * a tsbnum and a tsbindex. Bits 63:32 contain the + * tsbnum and bits 31:00 contain the tsbindex. + * + * Use the HV_PCI_TSBID() macro to construct such + * values. + * + * io_attributes IO attributes for IOMMU mappings. One of more + * of the attritbute bits are stores in a 64-bit + * value. The values are defined below. + * + * r_addr 64-bit real address + * + * pci_device PCI device address. A PCI device address identifies + * a specific device on a specific PCI bus segment. + * A PCI device address ia a 32-bit unsigned integer + * with the following format: + * + * 00000000.bbbbbbbb.dddddfff.00000000 + * + * Use the HV_PCI_DEVICE_BUILD() macro to construct + * such values. + * + * pci_config_offset + * PCI configureation space offset. For conventional + * PCI a value between 0 and 255. For extended + * configuration space, a value between 0 and 4095. + * + * Note: For PCI configuration space accesses, the offset + * must be aligned to the access size. + * + * error_flag A return value which specifies if the action succeeded + * or failed. 0 means no error, non-0 means some error + * occurred while performing the service. + * + * io_sync_direction + * Direction definition for pci_dma_sync(), defined + * below in HV_PCI_SYNC_*. + * + * io_page_list A list of io_page_addresses, an io_page_address is + * a real address. + * + * io_page_list_p A pointer to an io_page_list. + * + * "size based byte swap" - Some functions do size based byte swapping + * which allows sw to access pointers and + * counters in native form when the processor + * operates in a different endianness than the + * IO bus. Size-based byte swapping converts a + * multi-byte field between big-endian and + * little-endian format. + */ + +#define HV_PCI_MAP_ATTR_READ 0x01 +#define HV_PCI_MAP_ATTR_WRITE 0x02 +#define HV_PCI_MAP_ATTR_RELAXED_ORDER 0x04 + +#define HV_PCI_DEVICE_BUILD(b,d,f) \ + ((((b) & 0xff) << 16) | \ + (((d) & 0x1f) << 11) | \ + (((f) & 0x07) << 8)) + +#define HV_PCI_TSBID(__tsb_num, __tsb_index) \ + ((((u64)(__tsb_num)) << 32UL) | ((u64)(__tsb_index))) + +#define HV_PCI_SYNC_FOR_DEVICE 0x01 +#define HV_PCI_SYNC_FOR_CPU 0x02 + +/* pci_iommu_map() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOMMU_MAP + * ARG0: devhandle + * ARG1: tsbid + * ARG2: #ttes + * ARG3: io_attributes + * ARG4: io_page_list_p + * RET0: status + * RET1: #ttes mapped + * ERRORS: EINVAL Invalid devhandle/tsbnum/tsbindex/io_attributes + * EBADALIGN Improperly aligned real address + * ENORADDR Invalid real address + * + * Create IOMMU mappings in the sun4v device defined by the given + * devhandle. The mappings are created in the TSB defined by the + * tsbnum component of the given tsbid. The first mapping is created + * in the TSB i ndex defined by the tsbindex component of the given tsbid. + * The call creates up to #ttes mappings, the first one at tsbnum, tsbindex, + * the second at tsbnum, tsbindex + 1, etc. + * + * All mappings are created with the attributes defined by the io_attributes + * argument. The page mapping addresses are described in the io_page_list + * defined by the given io_page_list_p, which is a pointer to the io_page_list. + * The first entry in the io_page_list is the address for the first iotte, the + * 2nd for the 2nd iotte, and so on. + * + * Each io_page_address in the io_page_list must be appropriately aligned. + * #ttes must be greater than zero. For this version of the spec, the tsbnum + * component of the given tsbid must be zero. + * + * Returns the actual number of mappings creates, which may be less than + * or equal to the argument #ttes. If the function returns a value which + * is less than the #ttes, the caller may continus to call the function with + * an updated tsbid, #ttes, io_page_list_p arguments until all pages are + * mapped. + * + * Note: This function does not imply an iotte cache flush. The guest must + * demap an entry before re-mapping it. + */ +#define HV_FAST_PCI_IOMMU_MAP 0xb0 + +/* pci_iommu_demap() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOMMU_DEMAP + * ARG0: devhandle + * ARG1: tsbid + * ARG2: #ttes + * RET0: status + * RET1: #ttes demapped + * ERRORS: EINVAL Invalid devhandle/tsbnum/tsbindex + * + * Demap and flush IOMMU mappings in the device defined by the given + * devhandle. Demaps up to #ttes entries in the TSB defined by the tsbnum + * component of the given tsbid, starting at the TSB index defined by the + * tsbindex component of the given tsbid. + * + * For this version of the spec, the tsbnum of the given tsbid must be zero. + * #ttes must be greater than zero. + * + * Returns the actual number of ttes demapped, which may be less than or equal + * to the argument #ttes. If #ttes demapped is less than #ttes, the caller + * may continue to call this function with updated tsbid and #ttes arguments + * until all pages are demapped. + * + * Note: Entries do not have to be mapped to be demapped. A demap of an + * unmapped page will flush the entry from the tte cache. + */ +#define HV_FAST_PCI_IOMMU_DEMAP 0xb1 + +/* pci_iommu_getmap() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOMMU_GETMAP + * ARG0: devhandle + * ARG1: tsbid + * RET0: status + * RET1: io_attributes + * RET2: real address + * ERRORS: EINVAL Invalid devhandle/tsbnum/tsbindex + * ENOMAP Mapping is not valid, no translation exists + * + * Read and return the mapping in the device described by the given devhandle + * and tsbid. If successful, the io_attributes shall be returned in RET1 + * and the page address of the mapping shall be returned in RET2. + * + * For this version of the spec, the tsbnum component of the given tsbid + * must be zero. + */ +#define HV_FAST_PCI_IOMMU_GETMAP 0xb2 + +/* pci_iommu_getbypass() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOMMU_GETBYPASS + * ARG0: devhandle + * ARG1: real address + * ARG2: io_attributes + * RET0: status + * RET1: io_addr + * ERRORS: EINVAL Invalid devhandle/io_attributes + * ENORADDR Invalid real address + * ENOTSUPPORTED Function not supported in this implementation. + * + * Create a "special" mapping in the device described by the given devhandle, + * for the given real address and attributes. Return the IO address in RET1 + * if successful. + */ +#define HV_FAST_PCI_IOMMU_GETBYPASS 0xb3 + +/* pci_config_get() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_CONFIG_GET + * ARG0: devhandle + * ARG1: pci_device + * ARG2: pci_config_offset + * ARG3: size + * RET0: status + * RET1: error_flag + * RET2: data + * ERRORS: EINVAL Invalid devhandle/pci_device/offset/size + * EBADALIGN pci_config_offset not size aligned + * ENOACCESS Access to this offset is not permitted + * + * Read PCI configuration space for the adapter described by the given + * devhandle. Read size (1, 2, or 4) bytes of data from the given + * pci_device, at pci_config_offset from the beginning of the device's + * configuration space. If there was no error, RET1 is set to zero and + * RET2 is set to the data read. Insignificant bits in RET2 are not + * guaranteed to have any specific value and therefore must be ignored. + * + * The data returned in RET2 is size based byte swapped. + * + * If an error occurs during the read, set RET1 to a non-zero value. The + * given pci_config_offset must be 'size' aligned. + */ +#define HV_FAST_PCI_CONFIG_GET 0xb4 + +/* pci_config_put() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_CONFIG_PUT + * ARG0: devhandle + * ARG1: pci_device + * ARG2: pci_config_offset + * ARG3: size + * ARG4: data + * RET0: status + * RET1: error_flag + * ERRORS: EINVAL Invalid devhandle/pci_device/offset/size + * EBADALIGN pci_config_offset not size aligned + * ENOACCESS Access to this offset is not permitted + * + * Write PCI configuration space for the adapter described by the given + * devhandle. Write size (1, 2, or 4) bytes of data in a single operation, + * at pci_config_offset from the beginning of the device's configuration + * space. The data argument contains the data to be written to configuration + * space. Prior to writing, the data is size based byte swapped. + * + * If an error occurs during the write access, do not generate an error + * report, do set RET1 to a non-zero value. Otherwise RET1 is zero. + * The given pci_config_offset must be 'size' aligned. + * + * This function is permitted to read from offset zero in the configuration + * space described by the given pci_device if necessary to ensure that the + * write access to config space completes. + */ +#define HV_FAST_PCI_CONFIG_PUT 0xb5 + +/* pci_peek() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PEEK + * ARG0: devhandle + * ARG1: real address + * ARG2: size + * RET0: status + * RET1: error_flag + * RET2: data + * ERRORS: EINVAL Invalid devhandle or size + * EBADALIGN Improperly aligned real address + * ENORADDR Bad real address + * ENOACCESS Guest access prohibited + * + * Attempt to read the IO address given by the given devhandle, real address, + * and size. Size must be 1, 2, 4, or 8. The read is performed as a single + * access operation using the given size. If an error occurs when reading + * from the given location, do not generate an error report, but return a + * non-zero value in RET1. If the read was successful, return zero in RET1 + * and return the actual data read in RET2. The data returned is size based + * byte swapped. + * + * Non-significant bits in RET2 are not guaranteed to have any specific value + * and therefore must be ignored. If RET1 is returned as non-zero, the data + * value is not guaranteed to have any specific value and should be ignored. + * + * The caller must have permission to read from the given devhandle, real + * address, which must be an IO address. The argument real address must be a + * size aligned address. + * + * The hypervisor implementation of this function must block access to any + * IO address that the guest does not have explicit permission to access. + */ +#define HV_FAST_PCI_PEEK 0xb6 + +/* pci_poke() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_POKE + * ARG0: devhandle + * ARG1: real address + * ARG2: size + * ARG3: data + * ARG4: pci_device + * RET0: status + * RET1: error_flag + * ERRORS: EINVAL Invalid devhandle, size, or pci_device + * EBADALIGN Improperly aligned real address + * ENORADDR Bad real address + * ENOACCESS Guest access prohibited + * ENOTSUPPORTED Function is not supported by implementation + * + * Attempt to write data to the IO address given by the given devhandle, + * real address, and size. Size must be 1, 2, 4, or 8. The write is + * performed as a single access operation using the given size. Prior to + * writing the data is size based swapped. + * + * If an error occurs when writing to the given location, do not generate an + * error report, but return a non-zero value in RET1. If the write was + * successful, return zero in RET1. + * + * pci_device describes the configuration address of the device being + * written to. The implementation may safely read from offset 0 with + * the configuration space of the device described by devhandle and + * pci_device in order to guarantee that the write portion of the operation + * completes + * + * Any error that occurs due to the read shall be reported using the normal + * error reporting mechanisms .. the read error is not suppressed. + * + * The caller must have permission to write to the given devhandle, real + * address, which must be an IO address. The argument real address must be a + * size aligned address. The caller must have permission to read from + * the given devhandle, pci_device cofiguration space offset 0. + * + * The hypervisor implementation of this function must block access to any + * IO address that the guest does not have explicit permission to access. + */ +#define HV_FAST_PCI_POKE 0xb7 + +/* pci_dma_sync() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_DMA_SYNC + * ARG0: devhandle + * ARG1: real address + * ARG2: size + * ARG3: io_sync_direction + * RET0: status + * RET1: #synced + * ERRORS: EINVAL Invalid devhandle or io_sync_direction + * ENORADDR Bad real address + * + * Synchronize a memory region described by the given real address and size, + * for the device defined by the given devhandle using the direction(s) + * defined by the given io_sync_direction. The argument size is the size of + * the memory region in bytes. + * + * Return the actual number of bytes synchronized in the return value #synced, + * which may be less than or equal to the argument size. If the return + * value #synced is less than size, the caller must continue to call this + * function with updated real address and size arguments until the entire + * memory region is synchronized. + */ +#define HV_FAST_PCI_DMA_SYNC 0xb8 + +/* PCI MSI services. */ + +#define HV_MSITYPE_MSI32 0x00 +#define HV_MSITYPE_MSI64 0x01 + +#define HV_MSIQSTATE_IDLE 0x00 +#define HV_MSIQSTATE_ERROR 0x01 + +#define HV_MSIQ_INVALID 0x00 +#define HV_MSIQ_VALID 0x01 + +#define HV_MSISTATE_IDLE 0x00 +#define HV_MSISTATE_DELIVERED 0x01 + +#define HV_MSIVALID_INVALID 0x00 +#define HV_MSIVALID_VALID 0x01 + +#define HV_PCIE_MSGTYPE_PME_MSG 0x18 +#define HV_PCIE_MSGTYPE_PME_ACK_MSG 0x1b +#define HV_PCIE_MSGTYPE_CORR_MSG 0x30 +#define HV_PCIE_MSGTYPE_NONFATAL_MSG 0x31 +#define HV_PCIE_MSGTYPE_FATAL_MSG 0x33 + +#define HV_MSG_INVALID 0x00 +#define HV_MSG_VALID 0x01 + +/* pci_msiq_conf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSIQ_CONF + * ARG0: devhandle + * ARG1: msiqid + * ARG2: real address + * ARG3: number of entries + * RET0: status + * ERRORS: EINVAL Invalid devhandle, msiqid or nentries + * EBADALIGN Improperly aligned real address + * ENORADDR Bad real address + * + * Configure the MSI queue given by the devhandle and msiqid arguments, + * and to be placed at the given real address and be of the given + * number of entries. The real address must be aligned exactly to match + * the queue size. Each queue entry is 64-bytes long, so f.e. a 32 entry + * queue must be aligned on a 2048 byte real address boundary. The MSI-EQ + * Head and Tail are initialized so that the MSI-EQ is 'empty'. + * + * Implementation Note: Certain implementations have fixed sized queues. In + * that case, number of entries must contain the correct + * value. + */ +#define HV_FAST_PCI_MSIQ_CONF 0xc0 + +/* pci_msiq_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSIQ_INFO + * ARG0: devhandle + * ARG1: msiqid + * RET0: status + * RET1: real address + * RET2: number of entries + * ERRORS: EINVAL Invalid devhandle or msiqid + * + * Return the configuration information for the MSI queue described + * by the given devhandle and msiqid. The base address of the queue + * is returned in ARG1 and the number of entries is returned in ARG2. + * If the queue is unconfigured, the real address is undefined and the + * number of entries will be returned as zero. + */ +#define HV_FAST_PCI_MSIQ_INFO 0xc1 + +/* pci_msiq_getvalid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSIQ_GETVALID + * ARG0: devhandle + * ARG1: msiqid + * RET0: status + * RET1: msiqvalid (HV_MSIQ_VALID or HV_MSIQ_INVALID) + * ERRORS: EINVAL Invalid devhandle or msiqid + * + * Get the valid state of the MSI-EQ described by the given devhandle and + * msiqid. + */ +#define HV_FAST_PCI_MSIQ_GETVALID 0xc2 + +/* pci_msiq_setvalid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSIQ_SETVALID + * ARG0: devhandle + * ARG1: msiqid + * ARG2: msiqvalid (HV_MSIQ_VALID or HV_MSIQ_INVALID) + * RET0: status + * ERRORS: EINVAL Invalid devhandle or msiqid or msiqvalid + * value or MSI EQ is uninitialized + * + * Set the valid state of the MSI-EQ described by the given devhandle and + * msiqid to the given msiqvalid. + */ +#define HV_FAST_PCI_MSIQ_SETVALID 0xc3 + +/* pci_msiq_getstate() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSIQ_GETSTATE + * ARG0: devhandle + * ARG1: msiqid + * RET0: status + * RET1: msiqstate (HV_MSIQSTATE_IDLE or HV_MSIQSTATE_ERROR) + * ERRORS: EINVAL Invalid devhandle or msiqid + * + * Get the state of the MSI-EQ described by the given devhandle and + * msiqid. + */ +#define HV_FAST_PCI_MSIQ_GETSTATE 0xc4 + +/* pci_msiq_getvalid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSIQ_GETVALID + * ARG0: devhandle + * ARG1: msiqid + * ARG2: msiqstate (HV_MSIQSTATE_IDLE or HV_MSIQSTATE_ERROR) + * RET0: status + * ERRORS: EINVAL Invalid devhandle or msiqid or msiqstate + * value or MSI EQ is uninitialized + * + * Set the state of the MSI-EQ described by the given devhandle and + * msiqid to the given msiqvalid. + */ +#define HV_FAST_PCI_MSIQ_SETSTATE 0xc5 + +/* pci_msiq_gethead() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSIQ_GETHEAD + * ARG0: devhandle + * ARG1: msiqid + * RET0: status + * RET1: msiqhead + * ERRORS: EINVAL Invalid devhandle or msiqid + * + * Get the current MSI EQ queue head for the MSI-EQ described by the + * given devhandle and msiqid. + */ +#define HV_FAST_PCI_MSIQ_GETHEAD 0xc6 + +/* pci_msiq_sethead() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSIQ_SETHEAD + * ARG0: devhandle + * ARG1: msiqid + * ARG2: msiqhead + * RET0: status + * ERRORS: EINVAL Invalid devhandle or msiqid or msiqhead, + * or MSI EQ is uninitialized + * + * Set the current MSI EQ queue head for the MSI-EQ described by the + * given devhandle and msiqid. + */ +#define HV_FAST_PCI_MSIQ_SETHEAD 0xc7 + +/* pci_msiq_gettail() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSIQ_GETTAIL + * ARG0: devhandle + * ARG1: msiqid + * RET0: status + * RET1: msiqtail + * ERRORS: EINVAL Invalid devhandle or msiqid + * + * Get the current MSI EQ queue tail for the MSI-EQ described by the + * given devhandle and msiqid. + */ +#define HV_FAST_PCI_MSIQ_GETTAIL 0xc8 + +/* pci_msi_getvalid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSI_GETVALID + * ARG0: devhandle + * ARG1: msinum + * RET0: status + * RET1: msivalidstate + * ERRORS: EINVAL Invalid devhandle or msinum + * + * Get the current valid/enabled state for the MSI defined by the + * given devhandle and msinum. + */ +#define HV_FAST_PCI_MSI_GETVALID 0xc9 + +/* pci_msi_setvalid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSI_SETVALID + * ARG0: devhandle + * ARG1: msinum + * ARG2: msivalidstate + * RET0: status + * ERRORS: EINVAL Invalid devhandle or msinum or msivalidstate + * + * Set the current valid/enabled state for the MSI defined by the + * given devhandle and msinum. + */ +#define HV_FAST_PCI_MSI_SETVALID 0xca + +/* pci_msi_getmsiq() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSI_GETMSIQ + * ARG0: devhandle + * ARG1: msinum + * RET0: status + * RET1: msiqid + * ERRORS: EINVAL Invalid devhandle or msinum or MSI is unbound + * + * Get the MSI EQ that the MSI defined by the given devhandle and + * msinum is bound to. + */ +#define HV_FAST_PCI_MSI_GETMSIQ 0xcb + +/* pci_msi_setmsiq() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSI_SETMSIQ + * ARG0: devhandle + * ARG1: msinum + * ARG2: msitype + * ARG3: msiqid + * RET0: status + * ERRORS: EINVAL Invalid devhandle or msinum or msiqid + * + * Set the MSI EQ that the MSI defined by the given devhandle and + * msinum is bound to. + */ +#define HV_FAST_PCI_MSI_SETMSIQ 0xcc + +/* pci_msi_getstate() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSI_GETSTATE + * ARG0: devhandle + * ARG1: msinum + * RET0: status + * RET1: msistate + * ERRORS: EINVAL Invalid devhandle or msinum + * + * Get the state of the MSI defined by the given devhandle and msinum. + * If not initialized, return HV_MSISTATE_IDLE. + */ +#define HV_FAST_PCI_MSI_GETSTATE 0xcd + +/* pci_msi_setstate() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSI_SETSTATE + * ARG0: devhandle + * ARG1: msinum + * ARG2: msistate + * RET0: status + * ERRORS: EINVAL Invalid devhandle or msinum or msistate + * + * Set the state of the MSI defined by the given devhandle and msinum. + */ +#define HV_FAST_PCI_MSI_SETSTATE 0xce + +/* pci_msg_getmsiq() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSG_GETMSIQ + * ARG0: devhandle + * ARG1: msgtype + * RET0: status + * RET1: msiqid + * ERRORS: EINVAL Invalid devhandle or msgtype + * + * Get the MSI EQ of the MSG defined by the given devhandle and msgtype. + */ +#define HV_FAST_PCI_MSG_GETMSIQ 0xd0 + +/* pci_msg_setmsiq() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSG_SETMSIQ + * ARG0: devhandle + * ARG1: msgtype + * ARG2: msiqid + * RET0: status + * ERRORS: EINVAL Invalid devhandle, msgtype, or msiqid + * + * Set the MSI EQ of the MSG defined by the given devhandle and msgtype. + */ +#define HV_FAST_PCI_MSG_SETMSIQ 0xd1 + +/* pci_msg_getvalid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSG_GETVALID + * ARG0: devhandle + * ARG1: msgtype + * RET0: status + * RET1: msgvalidstate + * ERRORS: EINVAL Invalid devhandle or msgtype + * + * Get the valid/enabled state of the MSG defined by the given + * devhandle and msgtype. + */ +#define HV_FAST_PCI_MSG_GETVALID 0xd2 + +/* pci_msg_setvalid() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_MSG_SETVALID + * ARG0: devhandle + * ARG1: msgtype + * ARG2: msgvalidstate + * RET0: status + * ERRORS: EINVAL Invalid devhandle or msgtype or msgvalidstate + * + * Set the valid/enabled state of the MSG defined by the given + * devhandle and msgtype. + */ +#define HV_FAST_PCI_MSG_SETVALID 0xd3 + +/* PCI IOMMU v2 definitions and services + * + * While the PCI IO definitions above is valid IOMMU v2 adds new PCI IO + * definitions and services. + * + * CTE Clump Table Entry. First level table entry in the ATU. + * + * pci_device_list + * A 32-bit aligned list of pci_devices. + * + * pci_device_listp + * real address of a pci_device_list. 32-bit aligned. + * + * iotte IOMMU translation table entry. + * + * iotte_attributes + * IO Attributes for IOMMU v2 mappings. In addition to + * read, write IOMMU v2 supports relax ordering + * + * io_page_list A 64-bit aligned list of real addresses. Each real + * address in an io_page_list must be properly aligned + * to the pagesize of the given IOTSB. + * + * io_page_list_p Real address of an io_page_list, 64-bit aligned. + * + * IOTSB IO Translation Storage Buffer. An aligned table of + * IOTTEs. Each IOTSB has a pagesize, table size, and + * virtual address associated with it that must match + * a pagesize and table size supported by the un-derlying + * hardware implementation. The alignment requirements + * for an IOTSB depend on the pagesize used for that IOTSB. + * Each IOTTE in an IOTSB maps one pagesize-sized page. + * The size of the IOTSB dictates how large of a virtual + * address space the IOTSB is capable of mapping. + * + * iotsb_handle An opaque identifier for an IOTSB. A devhandle plus + * iotsb_handle represents a binding of an IOTSB to a + * PCI root complex. + * + * iotsb_index Zero-based IOTTE number within an IOTSB. + */ + +/* The index_count argument consists of two fields: + * bits 63:48 #iottes and bits 47:0 iotsb_index + */ +#define HV_PCI_IOTSB_INDEX_COUNT(__iottes, __iotsb_index) \ + (((u64)(__iottes) << 48UL) | ((u64)(__iotsb_index))) + +/* pci_iotsb_conf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_CONF + * ARG0: devhandle + * ARG1: r_addr + * ARG2: size + * ARG3: pagesize + * ARG4: iova + * RET0: status + * RET1: iotsb_handle + * ERRORS: EINVAL Invalid devhandle, size, iova, or pagesize + * EBADALIGN r_addr is not properly aligned + * ENORADDR r_addr is not a valid real address + * ETOOMANY No further IOTSBs may be configured + * EBUSY Duplicate devhandle, raddir, iova combination + * + * Create an IOTSB suitable for the PCI root complex identified by devhandle, + * for the DMA virtual address defined by the argument iova. + * + * r_addr is the properly aligned base address of the IOTSB and size is the + * IOTSB (table) size in bytes.The IOTSB is required to be zeroed prior to + * being configured. If it contains any values other than zeros then the + * behavior is undefined. + * + * pagesize is the size of each page in the IOTSB. Note that the combination of + * size (table size) and pagesize must be valid. + * + * virt is the DMA virtual address this IOTSB will map. + * + * If successful, the opaque 64-bit handle iotsb_handle is returned in ret1. + * Once configured, privileged access to the IOTSB memory is prohibited and + * creates undefined behavior. The only permitted access is indirect via these + * services. + */ +#define HV_FAST_PCI_IOTSB_CONF 0x190 + +/* pci_iotsb_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_INFO + * ARG0: devhandle + * ARG1: iotsb_handle + * RET0: status + * RET1: r_addr + * RET2: size + * RET3: pagesize + * RET4: iova + * RET5: #bound + * ERRORS: EINVAL Invalid devhandle or iotsb_handle + * + * This service returns configuration information about an IOTSB previously + * created with pci_iotsb_conf. + * + * iotsb_handle value 0 may be used with this service to inquire about the + * legacy IOTSB that may or may not exist. If the service succeeds, the return + * values describe the legacy IOTSB and I/O virtual addresses mapped by that + * table. However, the table base address r_addr may contain the value -1 which + * indicates a memory range that cannot be accessed or be reclaimed. + * + * The return value #bound contains the number of PCI devices that iotsb_handle + * is currently bound to. + */ +#define HV_FAST_PCI_IOTSB_INFO 0x191 + +/* pci_iotsb_unconf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_UNCONF + * ARG0: devhandle + * ARG1: iotsb_handle + * RET0: status + * ERRORS: EINVAL Invalid devhandle or iotsb_handle + * EBUSY The IOTSB is bound and may not be unconfigured + * + * This service unconfigures the IOTSB identified by the devhandle and + * iotsb_handle arguments, previously created with pci_iotsb_conf. + * The IOTSB must not be currently bound to any device or the service will fail + * + * If the call succeeds, iotsb_handle is no longer valid. + */ +#define HV_FAST_PCI_IOTSB_UNCONF 0x192 + +/* pci_iotsb_bind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_BIND + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: pci_device + * RET0: status + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, or pci_device + * EBUSY A PCI function is already bound to an IOTSB at the same + * address range as specified by devhandle, iotsb_handle. + * + * This service binds the PCI function specified by the argument pci_device to + * the IOTSB specified by the arguments devhandle and iotsb_handle. + * + * The PCI device function is bound to the specified IOTSB with the IOVA range + * specified when the IOTSB was configured via pci_iotsb_conf. If the function + * is already bound then it is unbound first. + */ +#define HV_FAST_PCI_IOTSB_BIND 0x193 + +/* pci_iotsb_unbind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_UNBIND + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: pci_device + * RET0: status + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, or pci_device + * ENOMAP The PCI function was not bound to the specified IOTSB + * + * This service unbinds the PCI device specified by the argument pci_device + * from the IOTSB identified * by the arguments devhandle and iotsb_handle. + * + * If the PCI device is not bound to the specified IOTSB then this service will + * fail with status ENOMAP + */ +#define HV_FAST_PCI_IOTSB_UNBIND 0x194 + +/* pci_iotsb_get_binding() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_GET_BINDING + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iova + * RET0: status + * RET1: iotsb_handle + * ERRORS: EINVAL Invalid devhandle, pci_device, or iova + * ENOMAP The PCI function is not bound to an IOTSB at iova + * + * This service returns the IOTSB binding, iotsb_handle, for a given pci_device + * and DMA virtual address, iova. + * + * iova must be the base address of a DMA virtual address range as defined by + * the iommu-address-ranges property in the root complex device node defined + * by the argument devhandle. + */ +#define HV_FAST_PCI_IOTSB_GET_BINDING 0x195 + +/* pci_iotsb_map() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_MAP + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: index_count + * ARG3: iotte_attributes + * ARG4: io_page_list_p + * RET0: status + * RET1: #mapped + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, #iottes, + * iotsb_index or iotte_attributes + * EBADALIGN Improperly aligned io_page_list_p or I/O page + * address in the I/O page list. + * ENORADDR Invalid io_page_list_p or I/O page address in + * the I/O page list. + * + * This service creates and flushes mappings in the IOTSB defined by the + * arguments devhandle, iotsb. + * + * The index_count argument consists of two fields. Bits 63:48 contain #iotte + * and bits 47:0 contain iotsb_index + * + * The first mapping is created in the IOTSB index specified by iotsb_index. + * Subsequent mappings are created at iotsb_index+1 and so on. + * + * The attributes of each mapping are defined by the argument iotte_attributes. + * + * The io_page_list_p specifies the real address of the 64-bit-aligned list of + * #iottes I/O page addresses. Each page address must be a properly aligned + * real address of a page to be mapped in the IOTSB. The first entry in the I/O + * page list contains the real address of the first page, the 2nd entry for the + * 2nd page, and so on. + * + * #iottes must be greater than zero. + * + * The return value #mapped is the actual number of mappings created, which may + * be less than or equal to the argument #iottes. If the function returns + * successfully with a #mapped value less than the requested #iottes then the + * caller should continue to invoke the service with updated iotsb_index, + * #iottes, and io_page_list_p arguments until all pages are mapped. + * + * This service must not be used to demap a mapping. In other words, all + * mappings must be valid and have one or both of the RW attribute bits set. + * + * Note: + * It is implementation-defined whether I/O page real address validity checking + * is done at time mappings are established or deferred until they are + * accessed. + */ +#define HV_FAST_PCI_IOTSB_MAP 0x196 + +/* pci_iotsb_map_one() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_MAP_ONE + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iotsb_index + * ARG3: iotte_attributes + * ARG4: r_addr + * RET0: status + * ERRORS: EINVAL Invalid devhandle,iotsb_handle, iotsb_index + * or iotte_attributes + * EBADALIGN Improperly aligned r_addr + * ENORADDR Invalid r_addr + * + * This service creates and flushes a single mapping in the IOTSB defined by the + * arguments devhandle, iotsb. + * + * The mapping for the page at r_addr is created at the IOTSB index specified by + * iotsb_index with the attributes iotte_attributes. + * + * This service must not be used to demap a mapping. In other words, the mapping + * must be valid and have one or both of the RW attribute bits set. + * + * Note: + * It is implementation-defined whether I/O page real address validity checking + * is done at time mappings are established or deferred until they are + * accessed. + */ +#define HV_FAST_PCI_IOTSB_MAP_ONE 0x197 + +/* pci_iotsb_demap() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_DEMAP + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iotsb_index + * ARG3: #iottes + * RET0: status + * RET1: #unmapped + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, iotsb_index or #iottes + * + * This service unmaps and flushes up to #iottes mappings starting at index + * iotsb_index from the IOTSB defined by the arguments devhandle, iotsb. + * + * #iottes must be greater than zero. + * + * The actual number of IOTTEs unmapped is returned in #unmapped and may be less + * than or equal to the requested number of IOTTEs, #iottes. + * + * If #unmapped is less than #iottes, the caller should continue to invoke this + * service with updated iotsb_index and #iottes arguments until all pages are + * demapped. + */ +#define HV_FAST_PCI_IOTSB_DEMAP 0x198 + +/* pci_iotsb_getmap() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_GETMAP + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iotsb_index + * RET0: status + * RET1: r_addr + * RET2: iotte_attributes + * ERRORS: EINVAL Invalid devhandle, iotsb_handle, or iotsb_index + * ENOMAP No mapping was found + * + * This service returns the mapping specified by index iotsb_index from the + * IOTSB defined by the arguments devhandle, iotsb. + * + * Upon success, the real address of the mapping shall be returned in + * r_addr and thethe IOTTE mapping attributes shall be returned in + * iotte_attributes. + * + * The return value iotte_attributes may not include optional features used in + * the call to create the mapping. + */ +#define HV_FAST_PCI_IOTSB_GETMAP 0x199 + +/* pci_iotsb_sync_mappings() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOTSB_SYNC_MAPPINGS + * ARG0: devhandle + * ARG1: iotsb_handle + * ARG2: iotsb_index + * ARG3: #iottes + * RET0: status + * RET1: #synced + * ERROS: EINVAL Invalid devhandle, iotsb_handle, iotsb_index, or #iottes + * + * This service synchronizes #iottes mappings starting at index iotsb_index in + * the IOTSB defined by the arguments devhandle, iotsb. + * + * #iottes must be greater than zero. + * + * The actual number of IOTTEs synchronized is returned in #synced, which may + * be less than or equal to the requested number, #iottes. + * + * Upon a successful return, #synced is less than #iottes, the caller should + * continue to invoke this service with updated iotsb_index and #iottes + * arguments until all pages are synchronized. + */ +#define HV_FAST_PCI_IOTSB_SYNC_MAPPINGS 0x19a + +/* Logical Domain Channel services. */ + +#define LDC_CHANNEL_DOWN 0 +#define LDC_CHANNEL_UP 1 +#define LDC_CHANNEL_RESETTING 2 + +/* ldc_tx_qconf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_TX_QCONF + * ARG0: channel ID + * ARG1: real address base of queue + * ARG2: num entries in queue + * RET0: status + * + * Configure transmit queue for the LDC endpoint specified by the + * given channel ID, to be placed at the given real address, and + * be of the given num entries. Num entries must be a power of two. + * The real address base of the queue must be aligned on the queue + * size. Each queue entry is 64-bytes, so for example, a 32 entry + * queue must be aligned on a 2048 byte real address boundary. + * + * Upon configuration of a valid transmit queue the head and tail + * pointers are set to a hypervisor specific identical value indicating + * that the queue initially is empty. + * + * The endpoint's transmit queue is un-configured if num entries is zero. + * + * The maximum number of entries for each queue for a specific cpu may be + * determined from the machine description. A transmit queue may be + * specified even in the event that the LDC is down (peer endpoint has no + * receive queue specified). Transmission will begin as soon as the peer + * endpoint defines a receive queue. + * + * It is recommended that a guest wait for a transmit queue to empty prior + * to reconfiguring it, or un-configuring it. Re or un-configuring of a + * non-empty transmit queue behaves exactly as defined above, however it + * is undefined as to how many of the pending entries in the original queue + * will be delivered prior to the re-configuration taking effect. + * Furthermore, as the queue configuration causes a reset of the head and + * tail pointers there is no way for a guest to determine how many entries + * have been sent after the configuration operation. + */ +#define HV_FAST_LDC_TX_QCONF 0xe0 + +/* ldc_tx_qinfo() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_TX_QINFO + * ARG0: channel ID + * RET0: status + * RET1: real address base of queue + * RET2: num entries in queue + * + * Return the configuration info for the transmit queue of LDC endpoint + * defined by the given channel ID. The real address is the currently + * defined real address base of the defined queue, and num entries is the + * size of the queue in terms of number of entries. + * + * If the specified channel ID is a valid endpoint number, but no transmit + * queue has been defined this service will return success, but with num + * entries set to zero and the real address will have an undefined value. + */ +#define HV_FAST_LDC_TX_QINFO 0xe1 + +/* ldc_tx_get_state() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_TX_GET_STATE + * ARG0: channel ID + * RET0: status + * RET1: head offset + * RET2: tail offset + * RET3: channel state + * + * Return the transmit state, and the head and tail queue pointers, for + * the transmit queue of the LDC endpoint defined by the given channel ID. + * The head and tail values are the byte offset of the head and tail + * positions of the transmit queue for the specified endpoint. + */ +#define HV_FAST_LDC_TX_GET_STATE 0xe2 + +/* ldc_tx_set_qtail() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_TX_SET_QTAIL + * ARG0: channel ID + * ARG1: tail offset + * RET0: status + * + * Update the tail pointer for the transmit queue associated with the LDC + * endpoint defined by the given channel ID. The tail offset specified + * must be aligned on a 64 byte boundary, and calculated so as to increase + * the number of pending entries on the transmit queue. Any attempt to + * decrease the number of pending transmit queue entires is considered + * an invalid tail offset and will result in an EINVAL error. + * + * Since the tail of the transmit queue may not be moved backwards, the + * transmit queue may be flushed by configuring a new transmit queue, + * whereupon the hypervisor will configure the initial transmit head and + * tail pointers to be equal. + */ +#define HV_FAST_LDC_TX_SET_QTAIL 0xe3 + +/* ldc_rx_qconf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_RX_QCONF + * ARG0: channel ID + * ARG1: real address base of queue + * ARG2: num entries in queue + * RET0: status + * + * Configure receive queue for the LDC endpoint specified by the + * given channel ID, to be placed at the given real address, and + * be of the given num entries. Num entries must be a power of two. + * The real address base of the queue must be aligned on the queue + * size. Each queue entry is 64-bytes, so for example, a 32 entry + * queue must be aligned on a 2048 byte real address boundary. + * + * The endpoint's transmit queue is un-configured if num entries is zero. + * + * If a valid receive queue is specified for a local endpoint the LDC is + * in the up state for the purpose of transmission to this endpoint. + * + * The maximum number of entries for each queue for a specific cpu may be + * determined from the machine description. + * + * As receive queue configuration causes a reset of the queue's head and + * tail pointers there is no way for a gues to determine how many entries + * have been received between a preceding ldc_get_rx_state() API call + * and the completion of the configuration operation. It should be noted + * that datagram delivery is not guaranteed via domain channels anyway, + * and therefore any higher protocol should be resilient to datagram + * loss if necessary. However, to overcome this specific race potential + * it is recommended, for example, that a higher level protocol be employed + * to ensure either retransmission, or ensure that no datagrams are pending + * on the peer endpoint's transmit queue prior to the configuration process. + */ +#define HV_FAST_LDC_RX_QCONF 0xe4 + +/* ldc_rx_qinfo() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_RX_QINFO + * ARG0: channel ID + * RET0: status + * RET1: real address base of queue + * RET2: num entries in queue + * + * Return the configuration info for the receive queue of LDC endpoint + * defined by the given channel ID. The real address is the currently + * defined real address base of the defined queue, and num entries is the + * size of the queue in terms of number of entries. + * + * If the specified channel ID is a valid endpoint number, but no receive + * queue has been defined this service will return success, but with num + * entries set to zero and the real address will have an undefined value. + */ +#define HV_FAST_LDC_RX_QINFO 0xe5 + +/* ldc_rx_get_state() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_RX_GET_STATE + * ARG0: channel ID + * RET0: status + * RET1: head offset + * RET2: tail offset + * RET3: channel state + * + * Return the receive state, and the head and tail queue pointers, for + * the receive queue of the LDC endpoint defined by the given channel ID. + * The head and tail values are the byte offset of the head and tail + * positions of the receive queue for the specified endpoint. + */ +#define HV_FAST_LDC_RX_GET_STATE 0xe6 + +/* ldc_rx_set_qhead() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_RX_SET_QHEAD + * ARG0: channel ID + * ARG1: head offset + * RET0: status + * + * Update the head pointer for the receive queue associated with the LDC + * endpoint defined by the given channel ID. The head offset specified + * must be aligned on a 64 byte boundary, and calculated so as to decrease + * the number of pending entries on the receive queue. Any attempt to + * increase the number of pending receive queue entires is considered + * an invalid head offset and will result in an EINVAL error. + * + * The receive queue may be flushed by setting the head offset equal + * to the current tail offset. + */ +#define HV_FAST_LDC_RX_SET_QHEAD 0xe7 + +/* LDC Map Table Entry. Each slot is defined by a translation table + * entry, as specified by the LDC_MTE_* bits below, and a 64-bit + * hypervisor invalidation cookie. + */ +#define LDC_MTE_PADDR 0x0fffffffffffe000 /* pa[55:13] */ +#define LDC_MTE_COPY_W 0x0000000000000400 /* copy write access */ +#define LDC_MTE_COPY_R 0x0000000000000200 /* copy read access */ +#define LDC_MTE_IOMMU_W 0x0000000000000100 /* IOMMU write access */ +#define LDC_MTE_IOMMU_R 0x0000000000000080 /* IOMMU read access */ +#define LDC_MTE_EXEC 0x0000000000000040 /* execute */ +#define LDC_MTE_WRITE 0x0000000000000020 /* read */ +#define LDC_MTE_READ 0x0000000000000010 /* write */ +#define LDC_MTE_SZALL 0x000000000000000f /* page size bits */ +#define LDC_MTE_SZ16GB 0x0000000000000007 /* 16GB page */ +#define LDC_MTE_SZ2GB 0x0000000000000006 /* 2GB page */ +#define LDC_MTE_SZ256MB 0x0000000000000005 /* 256MB page */ +#define LDC_MTE_SZ32MB 0x0000000000000004 /* 32MB page */ +#define LDC_MTE_SZ4MB 0x0000000000000003 /* 4MB page */ +#define LDC_MTE_SZ512K 0x0000000000000002 /* 512K page */ +#define LDC_MTE_SZ64K 0x0000000000000001 /* 64K page */ +#define LDC_MTE_SZ8K 0x0000000000000000 /* 8K page */ + +#ifndef __ASSEMBLER__ +struct ldc_mtable_entry { + unsigned long mte; + unsigned long cookie; +}; +#endif + +/* ldc_set_map_table() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_SET_MAP_TABLE + * ARG0: channel ID + * ARG1: table real address + * ARG2: num entries + * RET0: status + * + * Register the MTE table at the given table real address, with the + * specified num entries, for the LDC indicated by the given channel + * ID. + */ +#define HV_FAST_LDC_SET_MAP_TABLE 0xea + +/* ldc_get_map_table() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_GET_MAP_TABLE + * ARG0: channel ID + * RET0: status + * RET1: table real address + * RET2: num entries + * + * Return the configuration of the current mapping table registered + * for the given channel ID. + */ +#define HV_FAST_LDC_GET_MAP_TABLE 0xeb + +#define LDC_COPY_IN 0 +#define LDC_COPY_OUT 1 + +/* ldc_copy() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_COPY + * ARG0: channel ID + * ARG1: LDC_COPY_* direction code + * ARG2: target real address + * ARG3: local real address + * ARG4: length in bytes + * RET0: status + * RET1: actual length in bytes + */ +#define HV_FAST_LDC_COPY 0xec + +#define LDC_MEM_READ 1 +#define LDC_MEM_WRITE 2 +#define LDC_MEM_EXEC 4 + +/* ldc_mapin() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_MAPIN + * ARG0: channel ID + * ARG1: cookie + * RET0: status + * RET1: real address + * RET2: LDC_MEM_* permissions + */ +#define HV_FAST_LDC_MAPIN 0xed + +/* ldc_unmap() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_UNMAP + * ARG0: real address + * RET0: status + */ +#define HV_FAST_LDC_UNMAP 0xee + +/* ldc_revoke() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_LDC_REVOKE + * ARG0: channel ID + * ARG1: cookie + * ARG2: ldc_mtable_entry cookie + * RET0: status + */ +#define HV_FAST_LDC_REVOKE 0xef + +#ifndef __ASSEMBLER__ +unsigned long sun4v_ldc_tx_qconf(unsigned long channel, + unsigned long ra, + unsigned long num_entries); +unsigned long sun4v_ldc_tx_qinfo(unsigned long channel, + unsigned long *ra, + unsigned long *num_entries); +unsigned long sun4v_ldc_tx_get_state(unsigned long channel, + unsigned long *head_off, + unsigned long *tail_off, + unsigned long *chan_state); +unsigned long sun4v_ldc_tx_set_qtail(unsigned long channel, + unsigned long tail_off); +unsigned long sun4v_ldc_rx_qconf(unsigned long channel, + unsigned long ra, + unsigned long num_entries); +unsigned long sun4v_ldc_rx_qinfo(unsigned long channel, + unsigned long *ra, + unsigned long *num_entries); +unsigned long sun4v_ldc_rx_get_state(unsigned long channel, + unsigned long *head_off, + unsigned long *tail_off, + unsigned long *chan_state); +unsigned long sun4v_ldc_rx_set_qhead(unsigned long channel, + unsigned long head_off); +unsigned long sun4v_ldc_set_map_table(unsigned long channel, + unsigned long ra, + unsigned long num_entries); +unsigned long sun4v_ldc_get_map_table(unsigned long channel, + unsigned long *ra, + unsigned long *num_entries); +unsigned long sun4v_ldc_copy(unsigned long channel, + unsigned long dir_code, + unsigned long tgt_raddr, + unsigned long lcl_raddr, + unsigned long len, + unsigned long *actual_len); +unsigned long sun4v_ldc_mapin(unsigned long channel, + unsigned long cookie, + unsigned long *ra, + unsigned long *perm); +unsigned long sun4v_ldc_unmap(unsigned long ra); +unsigned long sun4v_ldc_revoke(unsigned long channel, + unsigned long cookie, + unsigned long mte_cookie); +#endif + +/* Performance counter services. */ + +#define HV_PERF_JBUS_PERF_CTRL_REG 0x00 +#define HV_PERF_JBUS_PERF_CNT_REG 0x01 +#define HV_PERF_DRAM_PERF_CTRL_REG_0 0x02 +#define HV_PERF_DRAM_PERF_CNT_REG_0 0x03 +#define HV_PERF_DRAM_PERF_CTRL_REG_1 0x04 +#define HV_PERF_DRAM_PERF_CNT_REG_1 0x05 +#define HV_PERF_DRAM_PERF_CTRL_REG_2 0x06 +#define HV_PERF_DRAM_PERF_CNT_REG_2 0x07 +#define HV_PERF_DRAM_PERF_CTRL_REG_3 0x08 +#define HV_PERF_DRAM_PERF_CNT_REG_3 0x09 + +/* get_perfreg() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_GET_PERFREG + * ARG0: performance reg number + * RET0: status + * RET1: performance reg value + * ERRORS: EINVAL Invalid performance register number + * ENOACCESS No access allowed to performance counters + * + * Read the value of the given DRAM/JBUS performance counter/control register. + */ +#define HV_FAST_GET_PERFREG 0x100 + +/* set_perfreg() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_SET_PERFREG + * ARG0: performance reg number + * ARG1: performance reg value + * RET0: status + * ERRORS: EINVAL Invalid performance register number + * ENOACCESS No access allowed to performance counters + * + * Write the given performance reg value to the given DRAM/JBUS + * performance counter/control register. + */ +#define HV_FAST_SET_PERFREG 0x101 + +#define HV_N2_PERF_SPARC_CTL 0x0 +#define HV_N2_PERF_DRAM_CTL0 0x1 +#define HV_N2_PERF_DRAM_CNT0 0x2 +#define HV_N2_PERF_DRAM_CTL1 0x3 +#define HV_N2_PERF_DRAM_CNT1 0x4 +#define HV_N2_PERF_DRAM_CTL2 0x5 +#define HV_N2_PERF_DRAM_CNT2 0x6 +#define HV_N2_PERF_DRAM_CTL3 0x7 +#define HV_N2_PERF_DRAM_CNT3 0x8 + +#define HV_FAST_N2_GET_PERFREG 0x104 +#define HV_FAST_N2_SET_PERFREG 0x105 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_niagara_getperf(unsigned long reg, + unsigned long *val); +unsigned long sun4v_niagara_setperf(unsigned long reg, + unsigned long val); +unsigned long sun4v_niagara2_getperf(unsigned long reg, + unsigned long *val); +unsigned long sun4v_niagara2_setperf(unsigned long reg, + unsigned long val); +#endif + +/* MMU statistics services. + * + * The hypervisor maintains MMU statistics and privileged code provides + * a buffer where these statistics can be collected. It is continually + * updated once configured. The layout is as follows: + */ +#ifndef __ASSEMBLER__ +struct hv_mmu_statistics { + unsigned long immu_tsb_hits_ctx0_8k_tte; + unsigned long immu_tsb_ticks_ctx0_8k_tte; + unsigned long immu_tsb_hits_ctx0_64k_tte; + unsigned long immu_tsb_ticks_ctx0_64k_tte; + unsigned long __reserved1[2]; + unsigned long immu_tsb_hits_ctx0_4mb_tte; + unsigned long immu_tsb_ticks_ctx0_4mb_tte; + unsigned long __reserved2[2]; + unsigned long immu_tsb_hits_ctx0_256mb_tte; + unsigned long immu_tsb_ticks_ctx0_256mb_tte; + unsigned long __reserved3[4]; + unsigned long immu_tsb_hits_ctxnon0_8k_tte; + unsigned long immu_tsb_ticks_ctxnon0_8k_tte; + unsigned long immu_tsb_hits_ctxnon0_64k_tte; + unsigned long immu_tsb_ticks_ctxnon0_64k_tte; + unsigned long __reserved4[2]; + unsigned long immu_tsb_hits_ctxnon0_4mb_tte; + unsigned long immu_tsb_ticks_ctxnon0_4mb_tte; + unsigned long __reserved5[2]; + unsigned long immu_tsb_hits_ctxnon0_256mb_tte; + unsigned long immu_tsb_ticks_ctxnon0_256mb_tte; + unsigned long __reserved6[4]; + unsigned long dmmu_tsb_hits_ctx0_8k_tte; + unsigned long dmmu_tsb_ticks_ctx0_8k_tte; + unsigned long dmmu_tsb_hits_ctx0_64k_tte; + unsigned long dmmu_tsb_ticks_ctx0_64k_tte; + unsigned long __reserved7[2]; + unsigned long dmmu_tsb_hits_ctx0_4mb_tte; + unsigned long dmmu_tsb_ticks_ctx0_4mb_tte; + unsigned long __reserved8[2]; + unsigned long dmmu_tsb_hits_ctx0_256mb_tte; + unsigned long dmmu_tsb_ticks_ctx0_256mb_tte; + unsigned long __reserved9[4]; + unsigned long dmmu_tsb_hits_ctxnon0_8k_tte; + unsigned long dmmu_tsb_ticks_ctxnon0_8k_tte; + unsigned long dmmu_tsb_hits_ctxnon0_64k_tte; + unsigned long dmmu_tsb_ticks_ctxnon0_64k_tte; + unsigned long __reserved10[2]; + unsigned long dmmu_tsb_hits_ctxnon0_4mb_tte; + unsigned long dmmu_tsb_ticks_ctxnon0_4mb_tte; + unsigned long __reserved11[2]; + unsigned long dmmu_tsb_hits_ctxnon0_256mb_tte; + unsigned long dmmu_tsb_ticks_ctxnon0_256mb_tte; + unsigned long __reserved12[4]; +}; +#endif + +/* mmustat_conf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMUSTAT_CONF + * ARG0: real address + * RET0: status + * RET1: real address + * ERRORS: ENORADDR Invalid real address + * EBADALIGN Real address not aligned on 64-byte boundary + * EBADTRAP API not supported on this processor + * + * Enable MMU statistic gathering using the buffer at the given real + * address on the current virtual CPU. The new buffer real address + * is given in ARG1, and the previously specified buffer real address + * is returned in RET1, or is returned as zero for the first invocation. + * + * If the passed in real address argument is zero, this will disable + * MMU statistic collection on the current virtual CPU. If an error is + * returned then no statistics are collected. + * + * The buffer contents should be initialized to all zeros before being + * given to the hypervisor or else the statistics will be meaningless. + */ +#define HV_FAST_MMUSTAT_CONF 0x102 + +/* mmustat_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_MMUSTAT_INFO + * RET0: status + * RET1: real address + * ERRORS: EBADTRAP API not supported on this processor + * + * Return the current state and real address of the currently configured + * MMU statistics buffer on the current virtual CPU. + */ +#define HV_FAST_MMUSTAT_INFO 0x103 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_mmustat_conf(unsigned long ra, unsigned long *orig_ra); +unsigned long sun4v_mmustat_info(unsigned long *ra); +#endif + +/* NCS crypto services */ + +/* ncs_request() sub-function numbers */ +#define HV_NCS_QCONF 0x01 +#define HV_NCS_QTAIL_UPDATE 0x02 + +#ifndef __ASSEMBLER__ +struct hv_ncs_queue_entry { + /* MAU Control Register */ + unsigned long mau_control; +#define MAU_CONTROL_INV_PARITY 0x0000000000002000 +#define MAU_CONTROL_STRAND 0x0000000000001800 +#define MAU_CONTROL_BUSY 0x0000000000000400 +#define MAU_CONTROL_INT 0x0000000000000200 +#define MAU_CONTROL_OP 0x00000000000001c0 +#define MAU_CONTROL_OP_SHIFT 6 +#define MAU_OP_LOAD_MA_MEMORY 0x0 +#define MAU_OP_STORE_MA_MEMORY 0x1 +#define MAU_OP_MODULAR_MULT 0x2 +#define MAU_OP_MODULAR_REDUCE 0x3 +#define MAU_OP_MODULAR_EXP_LOOP 0x4 +#define MAU_CONTROL_LEN 0x000000000000003f +#define MAU_CONTROL_LEN_SHIFT 0 + + /* Real address of bytes to load or store bytes + * into/out-of the MAU. + */ + unsigned long mau_mpa; + + /* Modular Arithmetic MA Offset Register. */ + unsigned long mau_ma; + + /* Modular Arithmetic N Prime Register. */ + unsigned long mau_np; +}; + +struct hv_ncs_qconf_arg { + unsigned long mid; /* MAU ID, 1 per core on Niagara */ + unsigned long base; /* Real address base of queue */ + unsigned long end; /* Real address end of queue */ + unsigned long num_ents; /* Number of entries in queue */ +}; + +struct hv_ncs_qtail_update_arg { + unsigned long mid; /* MAU ID, 1 per core on Niagara */ + unsigned long tail; /* New tail index to use */ + unsigned long syncflag; /* only SYNCFLAG_SYNC is implemented */ +#define HV_NCS_SYNCFLAG_SYNC 0x00 +#define HV_NCS_SYNCFLAG_ASYNC 0x01 +}; +#endif + +/* ncs_request() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_NCS_REQUEST + * ARG0: NCS sub-function + * ARG1: sub-function argument real address + * ARG2: size in bytes of sub-function argument + * RET0: status + * + * The MAU chip of the Niagara processor is not directly accessible + * to privileged code, instead it is programmed indirectly via this + * hypervisor API. + * + * The interfaces defines a queue of MAU operations to perform. + * Privileged code registers a queue with the hypervisor by invoking + * this HVAPI with the HV_NCS_QCONF sub-function, which defines the + * base, end, and number of entries of the queue. Each queue entry + * contains a MAU register struct block. + * + * The privileged code then proceeds to add entries to the queue and + * then invoke the HV_NCS_QTAIL_UPDATE sub-function. Since only + * synchronous operations are supported by the current hypervisor, + * HV_NCS_QTAIL_UPDATE will run all the pending queue entries to + * completion and return HV_EOK, or return an error code. + * + * The real address of the sub-function argument must be aligned on at + * least an 8-byte boundary. + * + * The tail argument of HV_NCS_QTAIL_UPDATE is an index, not a byte + * offset, into the queue and must be less than or equal the 'num_ents' + * argument given in the HV_NCS_QCONF call. + */ +#define HV_FAST_NCS_REQUEST 0x110 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_ncs_request(unsigned long request, + unsigned long arg_ra, + unsigned long arg_size); +#endif + +#define HV_FAST_FIRE_GET_PERFREG 0x120 +#define HV_FAST_FIRE_SET_PERFREG 0x121 + +#define HV_FAST_REBOOT_DATA_SET 0x172 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_reboot_data_set(unsigned long ra, + unsigned long len); +#endif + +#define HV_FAST_VT_GET_PERFREG 0x184 +#define HV_FAST_VT_SET_PERFREG 0x185 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_vt_get_perfreg(unsigned long reg_num, + unsigned long *reg_val); +unsigned long sun4v_vt_set_perfreg(unsigned long reg_num, + unsigned long reg_val); +#endif + +#define HV_FAST_T5_GET_PERFREG 0x1a8 +#define HV_FAST_T5_SET_PERFREG 0x1a9 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_t5_get_perfreg(unsigned long reg_num, + unsigned long *reg_val); +unsigned long sun4v_t5_set_perfreg(unsigned long reg_num, + unsigned long reg_val); +#endif + + +#define HV_FAST_M7_GET_PERFREG 0x43 +#define HV_FAST_M7_SET_PERFREG 0x44 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_m7_get_perfreg(unsigned long reg_num, + unsigned long *reg_val); +unsigned long sun4v_m7_set_perfreg(unsigned long reg_num, + unsigned long reg_val); +#endif + +/* Function numbers for HV_CORE_TRAP. */ +#define HV_CORE_SET_VER 0x00 +#define HV_CORE_PUTCHAR 0x01 +#define HV_CORE_EXIT 0x02 +#define HV_CORE_GET_VER 0x03 + +/* Hypervisor API groups for use with HV_CORE_SET_VER and + * HV_CORE_GET_VER. + */ +#define HV_GRP_SUN4V 0x0000 +#define HV_GRP_CORE 0x0001 +#define HV_GRP_INTR 0x0002 +#define HV_GRP_SOFT_STATE 0x0003 +#define HV_GRP_TM 0x0080 +#define HV_GRP_PCI 0x0100 +#define HV_GRP_LDOM 0x0101 +#define HV_GRP_SVC_CHAN 0x0102 +#define HV_GRP_NCS 0x0103 +#define HV_GRP_RNG 0x0104 +#define HV_GRP_PBOOT 0x0105 +#define HV_GRP_TPM 0x0107 +#define HV_GRP_SDIO 0x0108 +#define HV_GRP_SDIO_ERR 0x0109 +#define HV_GRP_REBOOT_DATA 0x0110 +#define HV_GRP_ATU 0x0111 +#define HV_GRP_DAX 0x0113 +#define HV_GRP_M7_PERF 0x0114 +#define HV_GRP_NIAG_PERF 0x0200 +#define HV_GRP_FIRE_PERF 0x0201 +#define HV_GRP_N2_CPU 0x0202 +#define HV_GRP_NIU 0x0204 +#define HV_GRP_VF_CPU 0x0205 +#define HV_GRP_KT_CPU 0x0209 +#define HV_GRP_VT_CPU 0x020c +#define HV_GRP_T5_CPU 0x0211 +#define HV_GRP_DIAG 0x0300 + +#ifndef __ASSEMBLER__ +unsigned long sun4v_get_version(unsigned long group, + unsigned long *major, + unsigned long *minor); +unsigned long sun4v_set_version(unsigned long group, + unsigned long major, + unsigned long minor, + unsigned long *actual_minor); + +int sun4v_hvapi_register(unsigned long group, unsigned long major, + unsigned long *minor); +void sun4v_hvapi_unregister(unsigned long group); +int sun4v_hvapi_get(unsigned long group, + unsigned long *major, + unsigned long *minor); +void sun4v_hvapi_init(void); +#endif + +#endif /* !(_SPARC64_HYPERVISOR_H) */ diff --git a/arch/sparc/include/asm/idprom.h b/arch/sparc/include/asm/idprom.h new file mode 100644 index 000000000..4c372199c --- /dev/null +++ b/arch/sparc/include/asm/idprom.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * idprom.h: Macros and defines for idprom routines + * + * Copyright (C) 1995,1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_IDPROM_H +#define _SPARC_IDPROM_H + +#include + +struct idprom { + u8 id_format; /* Format identifier (always 0x01) */ + u8 id_machtype; /* Machine type */ + u8 id_ethaddr[6]; /* Hardware ethernet address */ + s32 id_date; /* Date of manufacture */ + u32 id_sernum:24; /* Unique serial number */ + u8 id_cksum; /* Checksum - xor of the data bytes */ + u8 reserved[16]; +}; + +extern struct idprom *idprom; +void idprom_init(void); + +#endif /* !(_SPARC_IDPROM_H) */ diff --git a/arch/sparc/include/asm/intr_queue.h b/arch/sparc/include/asm/intr_queue.h new file mode 100644 index 000000000..d61be2a3a --- /dev/null +++ b/arch/sparc/include/asm/intr_queue.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_INTR_QUEUE_H +#define _SPARC64_INTR_QUEUE_H + +/* Sun4v interrupt queue registers, accessed via ASI_QUEUE. */ + +#define INTRQ_CPU_MONDO_HEAD 0x3c0 /* CPU mondo head */ +#define INTRQ_CPU_MONDO_TAIL 0x3c8 /* CPU mondo tail */ +#define INTRQ_DEVICE_MONDO_HEAD 0x3d0 /* Device mondo head */ +#define INTRQ_DEVICE_MONDO_TAIL 0x3d8 /* Device mondo tail */ +#define INTRQ_RESUM_MONDO_HEAD 0x3e0 /* Resumable error mondo head */ +#define INTRQ_RESUM_MONDO_TAIL 0x3e8 /* Resumable error mondo tail */ +#define INTRQ_NONRESUM_MONDO_HEAD 0x3f0 /* Non-resumable error mondo head */ +#define INTRQ_NONRESUM_MONDO_TAIL 0x3f8 /* Non-resumable error mondo head */ + +#endif /* !(_SPARC64_INTR_QUEUE_H) */ diff --git a/arch/sparc/include/asm/io-unit.h b/arch/sparc/include/asm/io-unit.h new file mode 100644 index 000000000..8c38f5b9f --- /dev/null +++ b/arch/sparc/include/asm/io-unit.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* io-unit.h: Definitions for the sun4d IO-UNIT. + * + * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ +#ifndef _SPARC_IO_UNIT_H +#define _SPARC_IO_UNIT_H + +#include +#include +#include + +/* The io-unit handles all virtual to physical address translations + * that occur between the SBUS and physical memory. Access by + * the cpu to IO registers and similar go over the xdbus so are + * translated by the on chip SRMMU. The io-unit and the srmmu do + * not need to have the same translations at all, in fact most + * of the time the translations they handle are a disjunct set. + * Basically the io-unit handles all dvma sbus activity. + */ + +/* AIEEE, unlike the nice sun4m, these monsters have + fixed DMA range 64M */ + +#define IOUNIT_DMA_BASE 0xfc000000 /* TOP - 64M */ +#define IOUNIT_DMA_SIZE 0x04000000 /* 64M */ +/* We use last 1M for sparc_dvma_malloc */ +#define IOUNIT_DVMA_SIZE 0x00100000 /* 1M */ + +/* The format of an iopte in the external page tables */ +#define IOUPTE_PAGE 0xffffff00 /* Physical page number (PA[35:12]) */ +#define IOUPTE_CACHE 0x00000080 /* Cached (in Viking/MXCC) */ +/* XXX Jakub, find out how to program SBUS streaming cache on XDBUS/sun4d. + * XXX Actually, all you should need to do is find out where the registers + * XXX are and copy over the sparc64 implementation I wrote. There may be + * XXX some horrible hwbugs though, so be careful. -DaveM + */ +#define IOUPTE_STREAM 0x00000040 /* Translation can use streaming cache */ +#define IOUPTE_INTRA 0x00000008 /* SBUS direct slot->slot transfer */ +#define IOUPTE_WRITE 0x00000004 /* Writeable */ +#define IOUPTE_VALID 0x00000002 /* IOPTE is valid */ +#define IOUPTE_PARITY 0x00000001 /* Parity is checked during DVMA */ + +struct iounit_struct { + unsigned long bmap[(IOUNIT_DMA_SIZE >> (PAGE_SHIFT + 3)) / sizeof(unsigned long)]; + spinlock_t lock; + iopte_t __iomem *page_table; + unsigned long rotor[3]; + unsigned long limit[4]; +}; + +#define IOUNIT_BMAP1_START 0x00000000 +#define IOUNIT_BMAP1_END (IOUNIT_DMA_SIZE >> (PAGE_SHIFT + 1)) +#define IOUNIT_BMAP2_START IOUNIT_BMAP1_END +#define IOUNIT_BMAP2_END IOUNIT_BMAP2_START + (IOUNIT_DMA_SIZE >> (PAGE_SHIFT + 2)) +#define IOUNIT_BMAPM_START IOUNIT_BMAP2_END +#define IOUNIT_BMAPM_END ((IOUNIT_DMA_SIZE - IOUNIT_DVMA_SIZE) >> PAGE_SHIFT) + +#endif /* !(_SPARC_IO_UNIT_H) */ diff --git a/arch/sparc/include/asm/io.h b/arch/sparc/include/asm/io.h new file mode 100644 index 000000000..2dad9be9e --- /dev/null +++ b/arch/sparc/include/asm/io.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_IO_H +#define ___ASM_SPARC_IO_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif + +/* + * Defines used for both SPARC32 and SPARC64 + */ + +/* Big endian versions of memory read/write routines */ +#define readb_be(__addr) __raw_readb(__addr) +#define readw_be(__addr) __raw_readw(__addr) +#define readl_be(__addr) __raw_readl(__addr) +#define writeb_be(__b, __addr) __raw_writeb(__b, __addr) +#define writel_be(__w, __addr) __raw_writel(__w, __addr) +#define writew_be(__l, __addr) __raw_writew(__l, __addr) + +#include + +#endif diff --git a/arch/sparc/include/asm/io_32.h b/arch/sparc/include/asm/io_32.h new file mode 100644 index 000000000..549f0a722 --- /dev/null +++ b/arch/sparc/include/asm/io_32.h @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_IO_H +#define __SPARC_IO_H + +#include +#include /* struct resource */ + +#define IO_SPACE_LIMIT 0xffffffff + +#define memset_io(d,c,sz) _memset_io(d,c,sz) +#define memcpy_fromio(d,s,sz) _memcpy_fromio(d,s,sz) +#define memcpy_toio(d,s,sz) _memcpy_toio(d,s,sz) + +/* + * Bus number may be embedded in the higher bits of the physical address. + * This is why we have no bus number argument to ioremap(). + */ +void __iomem *ioremap(phys_addr_t offset, size_t size); +void iounmap(volatile void __iomem *addr); + +#include + +static inline void _memset_io(volatile void __iomem *dst, + int c, __kernel_size_t n) +{ + volatile void __iomem *d = dst; + + while (n--) { + writeb(c, d); + d++; + } +} + +static inline void _memcpy_fromio(void *dst, const volatile void __iomem *src, + __kernel_size_t n) +{ + char *d = dst; + + while (n--) { + char tmp = readb(src); + *d++ = tmp; + src++; + } +} + +static inline void _memcpy_toio(volatile void __iomem *dst, const void *src, + __kernel_size_t n) +{ + const char *s = src; + volatile void __iomem *d = dst; + + while (n--) { + char tmp = *s++; + writeb(tmp, d); + d++; + } +} + +/* + * SBus accessors. + * + * SBus has only one, memory mapped, I/O space. + * We do not need to flip bytes for SBus of course. + */ +static inline u8 sbus_readb(const volatile void __iomem *addr) +{ + return *(__force volatile u8 *)addr; +} + +static inline u16 sbus_readw(const volatile void __iomem *addr) +{ + return *(__force volatile u16 *)addr; +} + +static inline u32 sbus_readl(const volatile void __iomem *addr) +{ + return *(__force volatile u32 *)addr; +} + +static inline void sbus_writeb(u8 b, volatile void __iomem *addr) +{ + *(__force volatile u8 *)addr = b; +} + +static inline void sbus_writew(u16 w, volatile void __iomem *addr) +{ + *(__force volatile u16 *)addr = w; +} + +static inline void sbus_writel(u32 l, volatile void __iomem *addr) +{ + *(__force volatile u32 *)addr = l; +} + +static inline void sbus_memset_io(volatile void __iomem *__dst, int c, + __kernel_size_t n) +{ + while(n--) { + sbus_writeb(c, __dst); + __dst++; + } +} + +static inline void sbus_memcpy_fromio(void *dst, + const volatile void __iomem *src, + __kernel_size_t n) +{ + char *d = dst; + + while (n--) { + char tmp = sbus_readb(src); + *d++ = tmp; + src++; + } +} + +static inline void sbus_memcpy_toio(volatile void __iomem *dst, + const void *src, + __kernel_size_t n) +{ + const char *s = src; + volatile void __iomem *d = dst; + + while (n--) { + char tmp = *s++; + sbus_writeb(tmp, d); + d++; + } +} + +/* Create a virtual mapping cookie for an IO port range */ +void __iomem *ioport_map(unsigned long port, unsigned int nr); +void ioport_unmap(void __iomem *); + +/* Create a virtual mapping cookie for a PCI BAR (memory or IO) */ +struct pci_dev; +void pci_iounmap(struct pci_dev *dev, void __iomem *); + +static inline int sbus_can_dma_64bit(void) +{ + return 0; /* actually, sparc_cpu_model==sun4d */ +} +static inline int sbus_can_burst64(void) +{ + return 0; /* actually, sparc_cpu_model==sun4d */ +} +struct device; +void sbus_set_sbus64(struct device *, int); + +#define __ARCH_HAS_NO_PAGE_ZERO_MAPPED 1 + + +#endif /* !(__SPARC_IO_H) */ diff --git a/arch/sparc/include/asm/io_64.h b/arch/sparc/include/asm/io_64.h new file mode 100644 index 000000000..d8ed29662 --- /dev/null +++ b/arch/sparc/include/asm/io_64.h @@ -0,0 +1,474 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC64_IO_H +#define __SPARC64_IO_H + +#include +#include +#include + +#include /* IO address mapping routines need this */ +#include +#include +#define pci_iomap pci_iomap + +/* BIO layer definitions. */ +extern unsigned long kern_base, kern_size; + +/* __raw_{read,write}{b,w,l,q} uses direct access. + * Access the memory as big endian bypassing the cache + * by using ASI_PHYS_BYPASS_EC_E + */ +#define __raw_readb __raw_readb +static inline u8 __raw_readb(const volatile void __iomem *addr) +{ + u8 ret; + + __asm__ __volatile__("lduba\t[%1] %2, %0\t/* pci_raw_readb */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + + return ret; +} + +#define __raw_readw __raw_readw +static inline u16 __raw_readw(const volatile void __iomem *addr) +{ + u16 ret; + + __asm__ __volatile__("lduha\t[%1] %2, %0\t/* pci_raw_readw */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + + return ret; +} + +#define __raw_readl __raw_readl +static inline u32 __raw_readl(const volatile void __iomem *addr) +{ + u32 ret; + + __asm__ __volatile__("lduwa\t[%1] %2, %0\t/* pci_raw_readl */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + + return ret; +} + +#define __raw_readq __raw_readq +static inline u64 __raw_readq(const volatile void __iomem *addr) +{ + u64 ret; + + __asm__ __volatile__("ldxa\t[%1] %2, %0\t/* pci_raw_readq */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + + return ret; +} + +#define __raw_writeb __raw_writeb +static inline void __raw_writeb(u8 b, const volatile void __iomem *addr) +{ + __asm__ __volatile__("stba\t%r0, [%1] %2\t/* pci_raw_writeb */" + : /* no outputs */ + : "Jr" (b), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); +} + +#define __raw_writew __raw_writew +static inline void __raw_writew(u16 w, const volatile void __iomem *addr) +{ + __asm__ __volatile__("stha\t%r0, [%1] %2\t/* pci_raw_writew */" + : /* no outputs */ + : "Jr" (w), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); +} + +#define __raw_writel __raw_writel +static inline void __raw_writel(u32 l, const volatile void __iomem *addr) +{ + __asm__ __volatile__("stwa\t%r0, [%1] %2\t/* pci_raw_writel */" + : /* no outputs */ + : "Jr" (l), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); +} + +#define __raw_writeq __raw_writeq +static inline void __raw_writeq(u64 q, const volatile void __iomem *addr) +{ + __asm__ __volatile__("stxa\t%r0, [%1] %2\t/* pci_raw_writeq */" + : /* no outputs */ + : "Jr" (q), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); +} + +/* Memory functions, same as I/O accesses on Ultra. + * Access memory as little endian bypassing + * the cache by using ASI_PHYS_BYPASS_EC_E_L + */ +#define readb readb +#define readb_relaxed readb +static inline u8 readb(const volatile void __iomem *addr) +{ u8 ret; + + __asm__ __volatile__("lduba\t[%1] %2, %0\t/* pci_readb */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + return ret; +} + +#define readw readw +#define readw_relaxed readw +static inline u16 readw(const volatile void __iomem *addr) +{ u16 ret; + + __asm__ __volatile__("lduha\t[%1] %2, %0\t/* pci_readw */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + + return ret; +} + +#define readl readl +#define readl_relaxed readl +static inline u32 readl(const volatile void __iomem *addr) +{ u32 ret; + + __asm__ __volatile__("lduwa\t[%1] %2, %0\t/* pci_readl */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + + return ret; +} + +#define readq readq +#define readq_relaxed readq +static inline u64 readq(const volatile void __iomem *addr) +{ u64 ret; + + __asm__ __volatile__("ldxa\t[%1] %2, %0\t/* pci_readq */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + + return ret; +} + +#define writeb writeb +#define writeb_relaxed writeb +static inline void writeb(u8 b, volatile void __iomem *addr) +{ + __asm__ __volatile__("stba\t%r0, [%1] %2\t/* pci_writeb */" + : /* no outputs */ + : "Jr" (b), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); +} + +#define writew writew +#define writew_relaxed writew +static inline void writew(u16 w, volatile void __iomem *addr) +{ + __asm__ __volatile__("stha\t%r0, [%1] %2\t/* pci_writew */" + : /* no outputs */ + : "Jr" (w), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); +} + +#define writel writel +#define writel_relaxed writel +static inline void writel(u32 l, volatile void __iomem *addr) +{ + __asm__ __volatile__("stwa\t%r0, [%1] %2\t/* pci_writel */" + : /* no outputs */ + : "Jr" (l), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); +} + +#define writeq writeq +#define writeq_relaxed writeq +static inline void writeq(u64 q, volatile void __iomem *addr) +{ + __asm__ __volatile__("stxa\t%r0, [%1] %2\t/* pci_writeq */" + : /* no outputs */ + : "Jr" (q), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); +} + +#define inb inb +static inline u8 inb(unsigned long addr) +{ + return readb((volatile void __iomem *)addr); +} + +#define inw inw +static inline u16 inw(unsigned long addr) +{ + return readw((volatile void __iomem *)addr); +} + +#define inl inl +static inline u32 inl(unsigned long addr) +{ + return readl((volatile void __iomem *)addr); +} + +#define outb outb +static inline void outb(u8 b, unsigned long addr) +{ + writeb(b, (volatile void __iomem *)addr); +} + +#define outw outw +static inline void outw(u16 w, unsigned long addr) +{ + writew(w, (volatile void __iomem *)addr); +} + +#define outl outl +static inline void outl(u32 l, unsigned long addr) +{ + writel(l, (volatile void __iomem *)addr); +} + + +#define inb_p(__addr) inb(__addr) +#define outb_p(__b, __addr) outb(__b, __addr) +#define inw_p(__addr) inw(__addr) +#define outw_p(__w, __addr) outw(__w, __addr) +#define inl_p(__addr) inl(__addr) +#define outl_p(__l, __addr) outl(__l, __addr) + +void outsb(unsigned long, const void *, unsigned long); +void outsw(unsigned long, const void *, unsigned long); +void outsl(unsigned long, const void *, unsigned long); +#define outsb outsb +#define outsw outsw +#define outsl outsl +void insb(unsigned long, void *, unsigned long); +void insw(unsigned long, void *, unsigned long); +void insl(unsigned long, void *, unsigned long); +#define insb insb +#define insw insw +#define insl insl + +static inline void readsb(const volatile void __iomem *port, void *buf, unsigned long count) +{ + insb((unsigned long __force)port, buf, count); +} +#define readsb readsb + +static inline void readsw(const volatile void __iomem *port, void *buf, unsigned long count) +{ + insw((unsigned long __force)port, buf, count); +} +#define readsw readsw + +static inline void readsl(const volatile void __iomem *port, void *buf, unsigned long count) +{ + insl((unsigned long __force)port, buf, count); +} +#define readsl readsl + +static inline void writesb(void __iomem *port, const void *buf, unsigned long count) +{ + outsb((unsigned long __force)port, buf, count); +} +#define writesb writesb + +static inline void writesw(void __iomem *port, const void *buf, unsigned long count) +{ + outsw((unsigned long __force)port, buf, count); +} +#define writesw writesw + +static inline void writesl(void __iomem *port, const void *buf, unsigned long count) +{ + outsl((unsigned long __force)port, buf, count); +} +#define writesl writesl + +#define ioread8_rep(p,d,l) readsb(p,d,l) +#define ioread16_rep(p,d,l) readsw(p,d,l) +#define ioread32_rep(p,d,l) readsl(p,d,l) +#define iowrite8_rep(p,d,l) writesb(p,d,l) +#define iowrite16_rep(p,d,l) writesw(p,d,l) +#define iowrite32_rep(p,d,l) writesl(p,d,l) + +/* Valid I/O Space regions are anywhere, because each PCI bus supported + * can live in an arbitrary area of the physical address range. + */ +#define IO_SPACE_LIMIT 0xffffffffffffffffUL + +/* Now, SBUS variants, only difference from PCI is that we do + * not use little-endian ASIs. + */ +static inline u8 sbus_readb(const volatile void __iomem *addr) +{ + return __raw_readb(addr); +} + +static inline u16 sbus_readw(const volatile void __iomem *addr) +{ + return __raw_readw(addr); +} + +static inline u32 sbus_readl(const volatile void __iomem *addr) +{ + return __raw_readl(addr); +} + +static inline u64 sbus_readq(const volatile void __iomem *addr) +{ + return __raw_readq(addr); +} + +static inline void sbus_writeb(u8 b, volatile void __iomem *addr) +{ + __raw_writeb(b, addr); +} + +static inline void sbus_writew(u16 w, volatile void __iomem *addr) +{ + __raw_writew(w, addr); +} + +static inline void sbus_writel(u32 l, volatile void __iomem *addr) +{ + __raw_writel(l, addr); +} + +static inline void sbus_writeq(u64 q, volatile void __iomem *addr) +{ + __raw_writeq(q, addr); +} + +static inline void sbus_memset_io(volatile void __iomem *dst, int c, __kernel_size_t n) +{ + while(n--) { + sbus_writeb(c, dst); + dst++; + } +} + +static inline void memset_io(volatile void __iomem *dst, int c, __kernel_size_t n) +{ + volatile void __iomem *d = dst; + + while (n--) { + writeb(c, d); + d++; + } +} +#define memset_io memset_io + +static inline void sbus_memcpy_fromio(void *dst, const volatile void __iomem *src, + __kernel_size_t n) +{ + char *d = dst; + + while (n--) { + char tmp = sbus_readb(src); + *d++ = tmp; + src++; + } +} + + +static inline void memcpy_fromio(void *dst, const volatile void __iomem *src, + __kernel_size_t n) +{ + char *d = dst; + + while (n--) { + char tmp = readb(src); + *d++ = tmp; + src++; + } +} +#define memcpy_fromio memcpy_fromio + +static inline void sbus_memcpy_toio(volatile void __iomem *dst, const void *src, + __kernel_size_t n) +{ + const char *s = src; + volatile void __iomem *d = dst; + + while (n--) { + char tmp = *s++; + sbus_writeb(tmp, d); + d++; + } +} + +static inline void memcpy_toio(volatile void __iomem *dst, const void *src, + __kernel_size_t n) +{ + const char *s = src; + volatile void __iomem *d = dst; + + while (n--) { + char tmp = *s++; + writeb(tmp, d); + d++; + } +} +#define memcpy_toio memcpy_toio + +#ifdef __KERNEL__ + +/* On sparc64 we have the whole physical IO address space accessible + * using physically addressed loads and stores, so this does nothing. + */ +static inline void __iomem *ioremap(unsigned long offset, unsigned long size) +{ + return (void __iomem *)offset; +} + +#define ioremap_wc(X,Y) ioremap((X),(Y)) +#define ioremap_wt(X,Y) ioremap((X),(Y)) +static inline void __iomem *ioremap_np(unsigned long offset, unsigned long size) +{ + return NULL; + +} +#define ioremap_np ioremap_np + +static inline void iounmap(volatile void __iomem *addr) +{ +} + +#define ioread8 readb +#define ioread16 readw +#define ioread16be __raw_readw +#define ioread32 readl +#define ioread32be __raw_readl +#define iowrite8 writeb +#define iowrite16 writew +#define iowrite16be __raw_writew +#define iowrite32 writel +#define iowrite32be __raw_writel + +/* Create a virtual mapping cookie for an IO port range */ +void __iomem *ioport_map(unsigned long port, unsigned int nr); +void ioport_unmap(void __iomem *); +#define ioport_map ioport_map +#define ioport_unmap ioport_unmap + +/* Create a virtual mapping cookie for a PCI BAR (memory or IO) */ +struct pci_dev; +void pci_iounmap(struct pci_dev *dev, void __iomem *); +#define pci_iounmap pci_iounmap + +static inline int sbus_can_dma_64bit(void) +{ + return 1; +} +static inline int sbus_can_burst64(void) +{ + return 1; +} +struct device; +void sbus_set_sbus64(struct device *, int); + +#endif + +#endif /* !(__SPARC64_IO_H) */ diff --git a/arch/sparc/include/asm/ioctls.h b/arch/sparc/include/asm/ioctls.h new file mode 100644 index 000000000..c92ac780c --- /dev/null +++ b/arch/sparc/include/asm/ioctls.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC_IOCTLS_H +#define _ASM_SPARC_IOCTLS_H + +#include + +#define TIOCGETC __TIOCGETC +#define TIOCGETP __TIOCGETP +#define TIOCGLTC __TIOCGLTC +#define TIOCSLTC __TIOCSLTC +#define TIOCSETP __TIOCSETP +#define TIOCSETN __TIOCSETN +#define TIOCSETC __TIOCSETC +#endif /* !(_ASM_SPARC_IOCTLS_H) */ diff --git a/arch/sparc/include/asm/iommu-common.h b/arch/sparc/include/asm/iommu-common.h new file mode 100644 index 000000000..802c90c79 --- /dev/null +++ b/arch/sparc/include/asm/iommu-common.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_IOMMU_COMMON_H +#define _LINUX_IOMMU_COMMON_H + +#include +#include +#include + +#define IOMMU_POOL_HASHBITS 4 +#define IOMMU_NR_POOLS (1 << IOMMU_POOL_HASHBITS) +#define IOMMU_ERROR_CODE (~(unsigned long) 0) + +struct iommu_pool { + unsigned long start; + unsigned long end; + unsigned long hint; + spinlock_t lock; +}; + +struct iommu_map_table { + unsigned long table_map_base; + unsigned long table_shift; + unsigned long nr_pools; + void (*lazy_flush)(struct iommu_map_table *); + unsigned long poolsize; + struct iommu_pool pools[IOMMU_NR_POOLS]; + u32 flags; +#define IOMMU_HAS_LARGE_POOL 0x00000001 +#define IOMMU_NO_SPAN_BOUND 0x00000002 +#define IOMMU_NEED_FLUSH 0x00000004 + struct iommu_pool large_pool; + unsigned long *map; +}; + +extern void iommu_tbl_pool_init(struct iommu_map_table *iommu, + unsigned long num_entries, + u32 table_shift, + void (*lazy_flush)(struct iommu_map_table *), + bool large_pool, u32 npools, + bool skip_span_boundary_check); + +extern unsigned long iommu_tbl_range_alloc(struct device *dev, + struct iommu_map_table *iommu, + unsigned long npages, + unsigned long *handle, + unsigned long mask, + unsigned int align_order); + +extern void iommu_tbl_range_free(struct iommu_map_table *iommu, + u64 dma_addr, unsigned long npages, + unsigned long entry); + +#endif diff --git a/arch/sparc/include/asm/iommu.h b/arch/sparc/include/asm/iommu.h new file mode 100644 index 000000000..37935cb34 --- /dev/null +++ b/arch/sparc/include/asm/iommu.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_IOMMU_H +#define ___ASM_SPARC_IOMMU_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/iommu_32.h b/arch/sparc/include/asm/iommu_32.h new file mode 100644 index 000000000..af51cd5ea --- /dev/null +++ b/arch/sparc/include/asm/iommu_32.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* iommu.h: Definitions for the sun4m IOMMU. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ +#ifndef _SPARC_IOMMU_H +#define _SPARC_IOMMU_H + +#include +#include + +/* The iommu handles all virtual to physical address translations + * that occur between the SBUS and physical memory. Access by + * the cpu to IO registers and similar go over the mbus so are + * translated by the on chip SRMMU. The iommu and the srmmu do + * not need to have the same translations at all, in fact most + * of the time the translations they handle are a disjunct set. + * Basically the iommu handles all dvma sbus activity. + */ + +/* The IOMMU registers occupy three pages in IO space. */ +struct iommu_regs { + /* First page */ + volatile unsigned long control; /* IOMMU control */ + volatile unsigned long base; /* Physical base of iopte page table */ + volatile unsigned long _unused1[3]; + volatile unsigned long tlbflush; /* write only */ + volatile unsigned long pageflush; /* write only */ + volatile unsigned long _unused2[1017]; + /* Second page */ + volatile unsigned long afsr; /* Async-fault status register */ + volatile unsigned long afar; /* Async-fault physical address */ + volatile unsigned long _unused3[2]; + volatile unsigned long sbuscfg0; /* SBUS configuration registers, per-slot */ + volatile unsigned long sbuscfg1; + volatile unsigned long sbuscfg2; + volatile unsigned long sbuscfg3; + volatile unsigned long mfsr; /* Memory-fault status register */ + volatile unsigned long mfar; /* Memory-fault physical address */ + volatile unsigned long _unused4[1014]; + /* Third page */ + volatile unsigned long mid; /* IOMMU module-id */ +}; + +#define IOMMU_CTRL_IMPL 0xf0000000 /* Implementation */ +#define IOMMU_CTRL_VERS 0x0f000000 /* Version */ +#define IOMMU_CTRL_RNGE 0x0000001c /* Mapping RANGE */ +#define IOMMU_RNGE_16MB 0x00000000 /* 0xff000000 -> 0xffffffff */ +#define IOMMU_RNGE_32MB 0x00000004 /* 0xfe000000 -> 0xffffffff */ +#define IOMMU_RNGE_64MB 0x00000008 /* 0xfc000000 -> 0xffffffff */ +#define IOMMU_RNGE_128MB 0x0000000c /* 0xf8000000 -> 0xffffffff */ +#define IOMMU_RNGE_256MB 0x00000010 /* 0xf0000000 -> 0xffffffff */ +#define IOMMU_RNGE_512MB 0x00000014 /* 0xe0000000 -> 0xffffffff */ +#define IOMMU_RNGE_1GB 0x00000018 /* 0xc0000000 -> 0xffffffff */ +#define IOMMU_RNGE_2GB 0x0000001c /* 0x80000000 -> 0xffffffff */ +#define IOMMU_CTRL_ENAB 0x00000001 /* IOMMU Enable */ + +#define IOMMU_AFSR_ERR 0x80000000 /* LE, TO, or BE asserted */ +#define IOMMU_AFSR_LE 0x40000000 /* SBUS reports error after transaction */ +#define IOMMU_AFSR_TO 0x20000000 /* Write access took more than 12.8 us. */ +#define IOMMU_AFSR_BE 0x10000000 /* Write access received error acknowledge */ +#define IOMMU_AFSR_SIZE 0x0e000000 /* Size of transaction causing error */ +#define IOMMU_AFSR_S 0x01000000 /* Sparc was in supervisor mode */ +#define IOMMU_AFSR_RESV 0x00f00000 /* Reserver, forced to 0x8 by hardware */ +#define IOMMU_AFSR_ME 0x00080000 /* Multiple errors occurred */ +#define IOMMU_AFSR_RD 0x00040000 /* A read operation was in progress */ +#define IOMMU_AFSR_FAV 0x00020000 /* IOMMU afar has valid contents */ + +#define IOMMU_SBCFG_SAB30 0x00010000 /* Phys-address bit 30 when bypass enabled */ +#define IOMMU_SBCFG_BA16 0x00000004 /* Slave supports 16 byte bursts */ +#define IOMMU_SBCFG_BA8 0x00000002 /* Slave supports 8 byte bursts */ +#define IOMMU_SBCFG_BYPASS 0x00000001 /* Bypass IOMMU, treat all addresses + produced by this device as pure + physical. */ + +#define IOMMU_MFSR_ERR 0x80000000 /* One or more of PERR1 or PERR0 */ +#define IOMMU_MFSR_S 0x01000000 /* Sparc was in supervisor mode */ +#define IOMMU_MFSR_CPU 0x00800000 /* CPU transaction caused parity error */ +#define IOMMU_MFSR_ME 0x00080000 /* Multiple parity errors occurred */ +#define IOMMU_MFSR_PERR 0x00006000 /* high bit indicates parity error occurred + on the even word of the access, low bit + indicated odd word caused the parity error */ +#define IOMMU_MFSR_BM 0x00001000 /* Error occurred while in boot mode */ +#define IOMMU_MFSR_C 0x00000800 /* Address causing error was marked cacheable */ +#define IOMMU_MFSR_RTYP 0x000000f0 /* Memory request transaction type */ + +#define IOMMU_MID_SBAE 0x001f0000 /* SBus arbitration enable */ +#define IOMMU_MID_SE 0x00100000 /* Enables SCSI/ETHERNET arbitration */ +#define IOMMU_MID_SB3 0x00080000 /* Enable SBUS device 3 arbitration */ +#define IOMMU_MID_SB2 0x00040000 /* Enable SBUS device 2 arbitration */ +#define IOMMU_MID_SB1 0x00020000 /* Enable SBUS device 1 arbitration */ +#define IOMMU_MID_SB0 0x00010000 /* Enable SBUS device 0 arbitration */ +#define IOMMU_MID_MID 0x0000000f /* Module-id, hardcoded to 0x8 */ + +/* The format of an iopte in the page tables */ +#define IOPTE_PAGE 0x07ffff00 /* Physical page number (PA[30:12]) */ +#define IOPTE_CACHE 0x00000080 /* Cached (in vme IOCACHE or Viking/MXCC) */ +#define IOPTE_WRITE 0x00000004 /* Writeable */ +#define IOPTE_VALID 0x00000002 /* IOPTE is valid */ +#define IOPTE_WAZ 0x00000001 /* Write as zeros */ + +struct iommu_struct { + struct iommu_regs __iomem *regs; + iopte_t *page_table; + /* For convenience */ + unsigned long start; /* First managed virtual address */ + unsigned long end; /* Last managed virtual address */ + + struct bit_map usemap; +}; + +static inline void iommu_invalidate(struct iommu_regs __iomem *regs) +{ + sbus_writel(0, ®s->tlbflush); +} + +static inline void iommu_invalidate_page(struct iommu_regs __iomem *regs, unsigned long ba) +{ + sbus_writel(ba & PAGE_MASK, ®s->pageflush); +} + +#endif /* !(_SPARC_IOMMU_H) */ diff --git a/arch/sparc/include/asm/iommu_64.h b/arch/sparc/include/asm/iommu_64.h new file mode 100644 index 000000000..0ef6dedf7 --- /dev/null +++ b/arch/sparc/include/asm/iommu_64.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* iommu.h: Definitions for the sun5 IOMMU. + * + * Copyright (C) 1996, 1999, 2007 David S. Miller (davem@davemloft.net) + */ +#ifndef _SPARC64_IOMMU_H +#define _SPARC64_IOMMU_H + +/* The format of an iopte in the page tables. */ +#define IOPTE_VALID 0x8000000000000000UL +#define IOPTE_64K 0x2000000000000000UL +#define IOPTE_STBUF 0x1000000000000000UL +#define IOPTE_INTRA 0x0800000000000000UL +#define IOPTE_CONTEXT 0x07ff800000000000UL +#define IOPTE_PAGE 0x00007fffffffe000UL +#define IOPTE_CACHE 0x0000000000000010UL +#define IOPTE_WRITE 0x0000000000000002UL + +#define IOMMU_NUM_CTXS 4096 +#include + +struct iommu_arena { + unsigned long *map; + unsigned int hint; + unsigned int limit; +}; + +#define ATU_64_SPACE_SIZE 0x800000000 /* 32G */ + +/* Data structures for SPARC ATU architecture */ +struct atu_iotsb { + void *table; /* IOTSB table base virtual addr*/ + u64 ra; /* IOTSB table real addr */ + u64 dvma_size; /* ranges[3].size or OS slected 32G size */ + u64 dvma_base; /* ranges[3].base */ + u64 table_size; /* IOTSB table size */ + u64 page_size; /* IO PAGE size for IOTSB */ + u32 iotsb_num; /* tsbnum is same as iotsb_handle */ +}; + +struct atu_ranges { + u64 base; + u64 size; +}; + +struct atu { + struct atu_ranges *ranges; + struct atu_iotsb *iotsb; + struct iommu_map_table tbl; + u64 base; + u64 size; + u64 dma_addr_mask; +}; + +struct iommu { + struct iommu_map_table tbl; + struct atu *atu; + spinlock_t lock; + u32 dma_addr_mask; + iopte_t *page_table; + unsigned long iommu_control; + unsigned long iommu_tsbbase; + unsigned long iommu_flush; + unsigned long iommu_flushinv; + unsigned long iommu_tags; + unsigned long iommu_ctxflush; + unsigned long write_complete_reg; + unsigned long dummy_page; + unsigned long dummy_page_pa; + unsigned long ctx_lowest_free; + DECLARE_BITMAP(ctx_bitmap, IOMMU_NUM_CTXS); +}; + +struct strbuf { + int strbuf_enabled; + unsigned long strbuf_control; + unsigned long strbuf_pflush; + unsigned long strbuf_fsync; + unsigned long strbuf_err_stat; + unsigned long strbuf_tag_diag; + unsigned long strbuf_line_diag; + unsigned long strbuf_ctxflush; + unsigned long strbuf_ctxmatch_base; + unsigned long strbuf_flushflag_pa; + volatile unsigned long *strbuf_flushflag; + volatile unsigned long __flushflag_buf[(64+(64-1)) / sizeof(long)]; +}; + +int iommu_table_init(struct iommu *iommu, int tsbsize, + u32 dma_offset, u32 dma_addr_mask, + int numa_node); + +#endif /* !(_SPARC64_IOMMU_H) */ diff --git a/arch/sparc/include/asm/irq.h b/arch/sparc/include/asm/irq.h new file mode 100644 index 000000000..a2efc274d --- /dev/null +++ b/arch/sparc/include/asm/irq.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_IRQ_H +#define ___ASM_SPARC_IRQ_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/irq_32.h b/arch/sparc/include/asm/irq_32.h new file mode 100644 index 000000000..6ee48321c --- /dev/null +++ b/arch/sparc/include/asm/irq_32.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* irq.h: IRQ registers on the Sparc. + * + * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) + */ + +#ifndef _SPARC_IRQ_H +#define _SPARC_IRQ_H + +/* Allocated number of logical irq numbers. + * sun4d boxes (ss2000e) should be OK with ~32. + * Be on the safe side and make room for 64 + */ +#define NR_IRQS 64 + +#include + +#define irq_canonicalize(irq) (irq) + +void __init sun4d_init_sbi_irq(void); + +#define NO_IRQ 0xffffffff + +#endif diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h new file mode 100644 index 000000000..8c4c0c87f --- /dev/null +++ b/arch/sparc/include/asm/irq_64.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* irq.h: IRQ registers on the 64-bit Sparc. + * + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) + * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#ifndef _SPARC64_IRQ_H +#define _SPARC64_IRQ_H + +#include +#include +#include +#include +#include +#include + +/* IMAP/ICLR register defines */ +#define IMAP_VALID 0x80000000UL /* IRQ Enabled */ +#define IMAP_TID_UPA 0x7c000000UL /* UPA TargetID */ +#define IMAP_TID_JBUS 0x7c000000UL /* JBUS TargetID */ +#define IMAP_TID_SHIFT 26 +#define IMAP_AID_SAFARI 0x7c000000UL /* Safari AgentID */ +#define IMAP_AID_SHIFT 26 +#define IMAP_NID_SAFARI 0x03e00000UL /* Safari NodeID */ +#define IMAP_NID_SHIFT 21 +#define IMAP_IGN 0x000007c0UL /* IRQ Group Number */ +#define IMAP_INO 0x0000003fUL /* IRQ Number */ +#define IMAP_INR 0x000007ffUL /* Full interrupt number*/ + +#define ICLR_IDLE 0x00000000UL /* Idle state */ +#define ICLR_TRANSMIT 0x00000001UL /* Transmit state */ +#define ICLR_PENDING 0x00000003UL /* Pending state */ + +/* The largest number of unique interrupt sources we support. + * If this needs to ever be larger than 255, you need to change + * the type of ino_bucket->irq as appropriate. + * + * ino_bucket->irq allocation is made during {sun4v_,}build_irq(). + */ +#define NR_IRQS (2048) + +void irq_install_pre_handler(int irq, + void (*func)(unsigned int, void *, void *), + void *arg1, void *arg2); +#define irq_canonicalize(irq) (irq) +unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap); +unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino); +unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino); +unsigned int sun4v_build_msi(u32 devhandle, unsigned int *irq_p, + unsigned int msi_devino_start, + unsigned int msi_devino_end); +void sun4v_destroy_msi(unsigned int irq); +unsigned int sun4u_build_msi(u32 portid, unsigned int *irq_p, + unsigned int msi_devino_start, + unsigned int msi_devino_end, + unsigned long imap_base, + unsigned long iclr_base); +void sun4u_destroy_msi(unsigned int irq); + +unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino); +void irq_free(unsigned int irq); + +void fixup_irqs(void); + +static inline void set_softint(unsigned long bits) +{ + __asm__ __volatile__("wr %0, 0x0, %%set_softint" + : /* No outputs */ + : "r" (bits)); +} + +static inline void clear_softint(unsigned long bits) +{ + __asm__ __volatile__("wr %0, 0x0, %%clear_softint" + : /* No outputs */ + : "r" (bits)); +} + +static inline unsigned long get_softint(void) +{ + unsigned long retval; + + __asm__ __volatile__("rd %%softint, %0" + : "=r" (retval)); + return retval; +} + +void arch_trigger_cpumask_backtrace(const struct cpumask *mask, + int exclude_cpu); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace + +extern void *hardirq_stack[NR_CPUS]; +extern void *softirq_stack[NR_CPUS]; + +#define NO_IRQ 0xffffffff + +#endif diff --git a/arch/sparc/include/asm/irqflags.h b/arch/sparc/include/asm/irqflags.h new file mode 100644 index 000000000..d1bdf6c39 --- /dev/null +++ b/arch/sparc/include/asm/irqflags.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_IRQFLAGS_H +#define ___ASM_SPARC_IRQFLAGS_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/irqflags_32.h b/arch/sparc/include/asm/irqflags_32.h new file mode 100644 index 000000000..f5f20774f --- /dev/null +++ b/arch/sparc/include/asm/irqflags_32.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/asm/irqflags.h + * + * IRQ flags handling + * + * This file gets included from lowlevel asm headers too, to provide + * wrapped versions of the local_irq_*() APIs, based on the + * arch_local_irq_*() functions from the lowlevel headers. + */ +#ifndef _ASM_IRQFLAGS_H +#define _ASM_IRQFLAGS_H + +#ifndef __ASSEMBLER__ + +#include +#include + +void arch_local_irq_restore(unsigned long); +unsigned long arch_local_irq_save(void); +void arch_local_irq_enable(void); + +static inline notrace unsigned long arch_local_save_flags(void) +{ + unsigned long flags; + + asm volatile("rd %%psr, %0" : "=r" (flags)); + return flags; +} + +static inline notrace void arch_local_irq_disable(void) +{ + arch_local_irq_save(); +} + +static inline notrace bool arch_irqs_disabled_flags(unsigned long flags) +{ + return (flags & PSR_PIL) != 0; +} + +static inline notrace bool arch_irqs_disabled(void) +{ + return arch_irqs_disabled_flags(arch_local_save_flags()); +} + +#endif /* (__ASSEMBLER__) */ + +#endif /* !(_ASM_IRQFLAGS_H) */ diff --git a/arch/sparc/include/asm/irqflags_64.h b/arch/sparc/include/asm/irqflags_64.h new file mode 100644 index 000000000..0071566c2 --- /dev/null +++ b/arch/sparc/include/asm/irqflags_64.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/asm/irqflags.h + * + * IRQ flags handling + * + * This file gets included from lowlevel asm headers too, to provide + * wrapped versions of the local_irq_*() APIs, based on the + * arch_local_irq_*() functions from the lowlevel headers. + */ +#ifndef _ASM_IRQFLAGS_H +#define _ASM_IRQFLAGS_H + +#include + +#ifndef __ASSEMBLER__ + +static inline notrace unsigned long arch_local_save_flags(void) +{ + unsigned long flags; + + __asm__ __volatile__( + "rdpr %%pil, %0" + : "=r" (flags) + ); + + return flags; +} + +static inline notrace void arch_local_irq_restore(unsigned long flags) +{ + __asm__ __volatile__( + "wrpr %0, %%pil" + : /* no output */ + : "r" (flags) + : "memory" + ); +} + +static inline notrace void arch_local_irq_disable(void) +{ + __asm__ __volatile__( + "wrpr %0, %%pil" + : /* no outputs */ + : "i" (PIL_NORMAL_MAX) + : "memory" + ); +} + +static inline notrace void arch_local_irq_enable(void) +{ + __asm__ __volatile__( + "wrpr 0, %%pil" + : /* no outputs */ + : /* no inputs */ + : "memory" + ); +} + +static inline notrace int arch_irqs_disabled_flags(unsigned long flags) +{ + return (flags > 0); +} + +static inline notrace int arch_irqs_disabled(void) +{ + return arch_irqs_disabled_flags(arch_local_save_flags()); +} + +static inline notrace unsigned long arch_local_irq_save(void) +{ + unsigned long flags, tmp; + + /* Disable interrupts to PIL_NORMAL_MAX unless we already + * are using PIL_NMI, in which case PIL_NMI is retained. + * + * The only values we ever program into the %pil are 0, + * PIL_NORMAL_MAX and PIL_NMI. + * + * Since PIL_NMI is the largest %pil value and all bits are + * set in it (0xf), it doesn't matter what PIL_NORMAL_MAX + * actually is. + */ + __asm__ __volatile__( + "rdpr %%pil, %0\n\t" + "or %0, %2, %1\n\t" + "wrpr %1, 0x0, %%pil" + : "=r" (flags), "=r" (tmp) + : "i" (PIL_NORMAL_MAX) + : "memory" + ); + + return flags; +} + +#endif /* (__ASSEMBLER__) */ + +#endif /* !(_ASM_IRQFLAGS_H) */ diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h new file mode 100644 index 000000000..f49d1e610 --- /dev/null +++ b/arch/sparc/include/asm/jump_label.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC_JUMP_LABEL_H +#define _ASM_SPARC_JUMP_LABEL_H + +#ifndef __ASSEMBLER__ + +#include + +#define JUMP_LABEL_NOP_SIZE 4 + +static __always_inline bool arch_static_branch(struct static_key *key, bool branch) +{ + asm goto("1:\n\t" + "nop\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) +{ + asm goto("1:\n\t" + "b %l[l_yes]\n\t" + "nop\n\t" + ".pushsection __jump_table, \"aw\"\n\t" + ".align 4\n\t" + ".word 1b, %l[l_yes], %c0\n\t" + ".popsection \n\t" + : : "i" (&((char *)key)[branch]) : : l_yes); + + return false; +l_yes: + return true; +} + +typedef u32 jump_label_t; + +struct jump_entry { + jump_label_t code; + jump_label_t target; + jump_label_t key; +}; + +#endif /* __ASSEMBLER__ */ +#endif diff --git a/arch/sparc/include/asm/kdebug.h b/arch/sparc/include/asm/kdebug.h new file mode 100644 index 000000000..830053f91 --- /dev/null +++ b/arch/sparc/include/asm/kdebug.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_KDEBUG_H +#define ___ASM_SPARC_KDEBUG_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/kdebug_32.h b/arch/sparc/include/asm/kdebug_32.h new file mode 100644 index 000000000..7627701a0 --- /dev/null +++ b/arch/sparc/include/asm/kdebug_32.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * kdebug.h: Defines and definitions for debugging the Linux kernel + * under various kernel debuggers. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ +#ifndef _SPARC_KDEBUG_H +#define _SPARC_KDEBUG_H + +#include +#include + +/* Breakpoints are enter through trap table entry 126. So in sparc assembly + * if you want to drop into the debugger you do: + * + * t DEBUG_BP_TRAP + */ + +#define DEBUG_BP_TRAP 126 + +#ifndef __ASSEMBLER__ +/* The debug vector is passed in %o1 at boot time. It is a pointer to + * a structure in the debuggers address space. Here is its format. + */ + +typedef unsigned int (*debugger_funct)(void); + +struct kernel_debug { + /* First the entry point into the debugger. You jump here + * to give control over to the debugger. + */ + unsigned long kdebug_entry; + unsigned long kdebug_trapme; /* Figure out later... */ + /* The following is the number of pages that the debugger has + * taken from to total pool. + */ + unsigned long *kdebug_stolen_pages; + /* Ok, after you remap yourself and/or change the trap table + * from what you were left with at boot time you have to call + * this synchronization function so the debugger can check out + * what you have done. + */ + debugger_funct teach_debugger; +}; /* I think that is it... */ + +extern struct kernel_debug *linux_dbvec; + +/* Use this macro in C-code to enter the debugger. */ +static inline void sp_enter_debugger(void) +{ + __asm__ __volatile__("jmpl %0, %%o7\n\t" + "nop\n\t" : : + "r" (linux_dbvec) : "o7", "memory"); +} + +#define SP_ENTER_DEBUGGER do { \ + if((linux_dbvec!=0) && ((*(short *)linux_dbvec)!=-1)) \ + sp_enter_debugger(); \ + } while(0) + +enum die_val { + DIE_UNUSED, + DIE_OOPS, +}; + +#endif /* !(__ASSEMBLER__) */ + +/* Some nice offset defines for assembler code. */ +#define KDEBUG_ENTRY_OFF 0x0 +#define KDEBUG_DUNNO_OFF 0x4 +#define KDEBUG_DUNNO2_OFF 0x8 +#define KDEBUG_TEACH_OFF 0xc + +#endif /* !(_SPARC_KDEBUG_H) */ diff --git a/arch/sparc/include/asm/kdebug_64.h b/arch/sparc/include/asm/kdebug_64.h new file mode 100644 index 000000000..89428bd4f --- /dev/null +++ b/arch/sparc/include/asm/kdebug_64.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_KDEBUG_H +#define _SPARC64_KDEBUG_H + +struct pt_regs; + +void bad_trap(struct pt_regs *, long); + +/* Grossly misnamed. */ +enum die_val { + DIE_OOPS = 1, + DIE_DEBUG, /* ta 0x70 */ + DIE_DEBUG_2, /* ta 0x71 */ + DIE_BPT, /* ta 0x73 */ + DIE_SSTEP, /* ta 0x74 */ + DIE_DIE, + DIE_TRAP, + DIE_TRAP_TL1, + DIE_CALL, + DIE_NMI, + DIE_NMIWATCHDOG, +}; + +#endif diff --git a/arch/sparc/include/asm/kgdb.h b/arch/sparc/include/asm/kgdb.h new file mode 100644 index 000000000..deabe0279 --- /dev/null +++ b/arch/sparc/include/asm/kgdb.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_KGDB_H +#define _SPARC_KGDB_H + +#ifdef CONFIG_SPARC32 +#define BUFMAX 2048 +#else +#define BUFMAX 4096 +#endif + +enum regnames { + GDB_G0, GDB_G1, GDB_G2, GDB_G3, GDB_G4, GDB_G5, GDB_G6, GDB_G7, + GDB_O0, GDB_O1, GDB_O2, GDB_O3, GDB_O4, GDB_O5, GDB_SP, GDB_O7, + GDB_L0, GDB_L1, GDB_L2, GDB_L3, GDB_L4, GDB_L5, GDB_L6, GDB_L7, + GDB_I0, GDB_I1, GDB_I2, GDB_I3, GDB_I4, GDB_I5, GDB_FP, GDB_I7, + GDB_F0, + GDB_F31 = GDB_F0 + 31, +#ifdef CONFIG_SPARC32 + GDB_Y, GDB_PSR, GDB_WIM, GDB_TBR, GDB_PC, GDB_NPC, + GDB_FSR, GDB_CSR, +#else + GDB_F32 = GDB_F0 + 32, + GDB_F62 = GDB_F32 + 15, + GDB_PC, GDB_NPC, GDB_STATE, GDB_FSR, GDB_FPRS, GDB_Y, +#endif +}; + +#ifdef CONFIG_SPARC32 +#define NUMREGBYTES ((GDB_CSR + 1) * 4) +#else +#define NUMREGBYTES ((GDB_Y + 1) * 8) +#endif + +struct pt_regs; +asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs); + +void arch_kgdb_breakpoint(void); + +#define BREAK_INSTR_SIZE 4 +#define CACHE_FLUSH_IS_SAFE 1 + +#endif /* _SPARC_KGDB_H */ diff --git a/arch/sparc/include/asm/kprobes.h b/arch/sparc/include/asm/kprobes.h new file mode 100644 index 000000000..aec742cd8 --- /dev/null +++ b/arch/sparc/include/asm/kprobes.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_KPROBES_H +#define _SPARC64_KPROBES_H + +#include + +#define BREAKPOINT_INSTRUCTION 0x91d02070 /* ta 0x70 */ +#define BREAKPOINT_INSTRUCTION_2 0x91d02071 /* ta 0x71 */ + +#ifdef CONFIG_KPROBES +#include +#include + +typedef u32 kprobe_opcode_t; + +#define MAX_INSN_SIZE 2 + +#define kretprobe_blacklist_size 0 + +#define arch_remove_kprobe(p) do {} while (0) + +#define flush_insn_slot(p) \ +do { flushi(&(p)->ainsn.insn[0]); \ + flushi(&(p)->ainsn.insn[1]); \ +} while (0) + +void __kretprobe_trampoline(void); + +/* Architecture specific copy of original instruction*/ +struct arch_specific_insn { + /* copy of the original instruction */ + kprobe_opcode_t insn[MAX_INSN_SIZE]; +}; + +struct prev_kprobe { + struct kprobe *kp; + unsigned long status; + unsigned long orig_tnpc; + unsigned long orig_tstate_pil; +}; + +/* per-cpu kprobe control block */ +struct kprobe_ctlblk { + unsigned long kprobe_status; + unsigned long kprobe_orig_tnpc; + unsigned long kprobe_orig_tstate_pil; + struct prev_kprobe prev_kprobe; +}; + +int kprobe_fault_handler(struct pt_regs *regs, int trapnr); +asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, + struct pt_regs *regs); + +#endif /* CONFIG_KPROBES */ +#endif /* _SPARC64_KPROBES_H */ diff --git a/arch/sparc/include/asm/ldc.h b/arch/sparc/include/asm/ldc.h new file mode 100644 index 000000000..4294738d4 --- /dev/null +++ b/arch/sparc/include/asm/ldc.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_LDC_H +#define _SPARC64_LDC_H + +#include + +extern int ldom_domaining_enabled; +void ldom_set_var(const char *var, const char *value); +void ldom_reboot(const char *boot_command); +void ldom_power_off(void); + +/* The event handler will be evoked when link state changes + * or data becomes available on the receive side. + * + * For non-RAW links, if the LDC_EVENT_RESET event arrives the + * driver should reset all of its internal state and reinvoke + * ldc_connect() to try and bring the link up again. + * + * For RAW links, ldc_connect() is not used. Instead the driver + * just waits for the LDC_EVENT_UP event. + */ +struct ldc_channel_config { + void (*event)(void *arg, int event); + + u32 mtu; + unsigned int rx_irq; + unsigned int tx_irq; + u8 mode; +#define LDC_MODE_RAW 0x00 +#define LDC_MODE_UNRELIABLE 0x01 +#define LDC_MODE_RESERVED 0x02 +#define LDC_MODE_STREAM 0x03 + + u8 debug; +#define LDC_DEBUG_HS 0x01 +#define LDC_DEBUG_STATE 0x02 +#define LDC_DEBUG_RX 0x04 +#define LDC_DEBUG_TX 0x08 +#define LDC_DEBUG_DATA 0x10 +}; + +#define LDC_EVENT_RESET 0x01 +#define LDC_EVENT_UP 0x02 +#define LDC_EVENT_DATA_READY 0x04 + +#define LDC_STATE_INVALID 0x00 +#define LDC_STATE_INIT 0x01 +#define LDC_STATE_BOUND 0x02 +#define LDC_STATE_READY 0x03 +#define LDC_STATE_CONNECTED 0x04 + +#define LDC_PACKET_SIZE 64 + +struct ldc_channel; + +/* Allocate state for a channel. */ +struct ldc_channel *ldc_alloc(unsigned long id, + const struct ldc_channel_config *cfgp, + void *event_arg, + const char *name); + +/* Shut down and free state for a channel. */ +void ldc_free(struct ldc_channel *lp); + +/* Register TX and RX queues of the link with the hypervisor. */ +int ldc_bind(struct ldc_channel *lp); +void ldc_unbind(struct ldc_channel *lp); + +/* For non-RAW protocols we need to complete a handshake before + * communication can proceed. ldc_connect() does that, if the + * handshake completes successfully, an LDC_EVENT_UP event will + * be sent up to the driver. + */ +int ldc_connect(struct ldc_channel *lp); +int ldc_disconnect(struct ldc_channel *lp); + +int ldc_state(struct ldc_channel *lp); +void ldc_set_state(struct ldc_channel *lp, u8 state); +int ldc_mode(struct ldc_channel *lp); +void __ldc_print(struct ldc_channel *lp, const char *caller); +int ldc_rx_reset(struct ldc_channel *lp); + +#define ldc_print(chan) __ldc_print(chan, __func__) + +/* Read and write operations. Only valid when the link is up. */ +int ldc_write(struct ldc_channel *lp, const void *buf, + unsigned int size); +int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size); + +#define LDC_MAP_SHADOW 0x01 +#define LDC_MAP_DIRECT 0x02 +#define LDC_MAP_IO 0x04 +#define LDC_MAP_R 0x08 +#define LDC_MAP_W 0x10 +#define LDC_MAP_X 0x20 +#define LDC_MAP_RW (LDC_MAP_R | LDC_MAP_W) +#define LDC_MAP_RWX (LDC_MAP_R | LDC_MAP_W | LDC_MAP_X) +#define LDC_MAP_ALL 0x03f + +struct ldc_trans_cookie { + u64 cookie_addr; + u64 cookie_size; +}; + +struct scatterlist; +int ldc_map_sg(struct ldc_channel *lp, + struct scatterlist *sg, int num_sg, + struct ldc_trans_cookie *cookies, int ncookies, + unsigned int map_perm); + +int ldc_map_single(struct ldc_channel *lp, + void *buf, unsigned int len, + struct ldc_trans_cookie *cookies, int ncookies, + unsigned int map_perm); + +void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, + int ncookies); + +int ldc_copy(struct ldc_channel *lp, int copy_dir, + void *buf, unsigned int len, unsigned long offset, + struct ldc_trans_cookie *cookies, int ncookies); + +static inline int ldc_get_dring_entry(struct ldc_channel *lp, + void *buf, unsigned int len, + unsigned long offset, + struct ldc_trans_cookie *cookies, + int ncookies) +{ + return ldc_copy(lp, LDC_COPY_IN, buf, len, offset, cookies, ncookies); +} + +static inline int ldc_put_dring_entry(struct ldc_channel *lp, + void *buf, unsigned int len, + unsigned long offset, + struct ldc_trans_cookie *cookies, + int ncookies) +{ + return ldc_copy(lp, LDC_COPY_OUT, buf, len, offset, cookies, ncookies); +} + +void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, + struct ldc_trans_cookie *cookies, + int *ncookies, unsigned int map_perm); + +void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, + unsigned int len, + struct ldc_trans_cookie *cookies, int ncookies); + +#endif /* _SPARC64_LDC_H */ diff --git a/arch/sparc/include/asm/leon.h b/arch/sparc/include/asm/leon.h new file mode 100644 index 000000000..053a24b67 --- /dev/null +++ b/arch/sparc/include/asm/leon.h @@ -0,0 +1,266 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2004 Konrad Eisele (eiselekd@web.de,konrad@gaisler.com) Gaisler Research + * Copyright (C) 2004 Stefan Holst (mail@s-holst.de) Uni-Stuttgart + * Copyright (C) 2009 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB + * Copyright (C) 2009 Konrad Eisele (konrad@gaisler.com) Aeroflex Gaisler AB + */ + +#ifndef LEON_H_INCLUDE +#define LEON_H_INCLUDE + +/* mmu register access, ASI_LEON_MMUREGS */ +#define LEON_CNR_CTRL 0x000 +#define LEON_CNR_CTXP 0x100 +#define LEON_CNR_CTX 0x200 +#define LEON_CNR_F 0x300 +#define LEON_CNR_FADDR 0x400 + +#define LEON_CNR_CTX_NCTX 256 /*number of MMU ctx */ + +#define LEON_CNR_CTRL_TLBDIS 0x80000000 + +#define LEON_MMUTLB_ENT_MAX 64 + +/* + * diagnostic access from mmutlb.vhd: + * 0: pte address + * 4: pte + * 8: additional flags + */ +#define LEON_DIAGF_LVL 0x3 +#define LEON_DIAGF_WR 0x8 +#define LEON_DIAGF_WR_SHIFT 3 +#define LEON_DIAGF_HIT 0x10 +#define LEON_DIAGF_HIT_SHIFT 4 +#define LEON_DIAGF_CTX 0x1fe0 +#define LEON_DIAGF_CTX_SHIFT 5 +#define LEON_DIAGF_VALID 0x2000 +#define LEON_DIAGF_VALID_SHIFT 13 + +/* irq masks */ +#define LEON_HARD_INT(x) (1 << (x)) /* irq 0-15 */ +#define LEON_IRQMASK_R 0x0000fffe /* bit 15- 1 of lregs.irqmask */ +#define LEON_IRQPRIO_R 0xfffe0000 /* bit 31-17 of lregs.irqmask */ + +#define LEON_MCFG2_SRAMDIS 0x00002000 +#define LEON_MCFG2_SDRAMEN 0x00004000 +#define LEON_MCFG2_SRAMBANKSZ 0x00001e00 /* [12-9] */ +#define LEON_MCFG2_SRAMBANKSZ_SHIFT 9 +#define LEON_MCFG2_SDRAMBANKSZ 0x03800000 /* [25-23] */ +#define LEON_MCFG2_SDRAMBANKSZ_SHIFT 23 + +#define LEON_TCNT0_MASK 0x7fffff + + +#define ASI_LEON3_SYSCTRL 0x02 +#define ASI_LEON3_SYSCTRL_ICFG 0x08 +#define ASI_LEON3_SYSCTRL_DCFG 0x0c +#define ASI_LEON3_SYSCTRL_CFG_SNOOPING (1 << 27) +#define ASI_LEON3_SYSCTRL_CFG_SSIZE(c) (1 << ((c >> 20) & 0xf)) + +#ifndef __ASSEMBLER__ + +/* do a physical address bypass write, i.e. for 0x80000000 */ +static inline void leon_store_reg(unsigned long paddr, unsigned long value) +{ + __asm__ __volatile__("sta %0, [%1] %2\n\t" : : "r"(value), "r"(paddr), + "i"(ASI_LEON_BYPASS) : "memory"); +} + +/* do a physical address bypass load, i.e. for 0x80000000 */ +static inline unsigned long leon_load_reg(unsigned long paddr) +{ + unsigned long retval; + __asm__ __volatile__("lda [%1] %2, %0\n\t" : + "=r"(retval) : "r"(paddr), "i"(ASI_LEON_BYPASS)); + return retval; +} + +/* macro access for leon_load_reg() and leon_store_reg() */ +#define LEON3_BYPASS_LOAD_PA(x) (leon_load_reg((unsigned long)(x))) +#define LEON3_BYPASS_STORE_PA(x, v) (leon_store_reg((unsigned long)(x), (unsigned long)(v))) +#define LEON_BYPASS_LOAD_PA(x) leon_load_reg((unsigned long)(x)) +#define LEON_BYPASS_STORE_PA(x, v) leon_store_reg((unsigned long)(x), (unsigned long)(v)) + +void leon_switch_mm(void); +void leon_init_IRQ(void); + +static inline unsigned long sparc_leon3_get_dcachecfg(void) +{ + unsigned int retval; + __asm__ __volatile__("lda [%1] %2, %0\n\t" : + "=r"(retval) : + "r"(ASI_LEON3_SYSCTRL_DCFG), + "i"(ASI_LEON3_SYSCTRL)); + return retval; +} + +/* enable snooping */ +static inline void sparc_leon3_enable_snooping(void) +{ + __asm__ __volatile__ ("lda [%%g0] 2, %%l1\n\t" + "set 0x800000, %%l2\n\t" + "or %%l2, %%l1, %%l2\n\t" + "sta %%l2, [%%g0] 2\n\t" : : : "l1", "l2"); +}; + +static inline int sparc_leon3_snooping_enabled(void) +{ + u32 cctrl; + __asm__ __volatile__("lda [%%g0] 2, %0\n\t" : "=r"(cctrl)); + return ((cctrl >> 23) & 1) && ((cctrl >> 17) & 1); +}; + +static inline void sparc_leon3_disable_cache(void) +{ + __asm__ __volatile__ ("lda [%%g0] 2, %%l1\n\t" + "set 0x00000f, %%l2\n\t" + "andn %%l2, %%l1, %%l2\n\t" + "sta %%l2, [%%g0] 2\n\t" : : : "l1", "l2"); +}; + +static inline unsigned long sparc_leon3_asr17(void) +{ + u32 asr17; + __asm__ __volatile__ ("rd %%asr17, %0\n\t" : "=r"(asr17)); + return asr17; +}; + +static inline int sparc_leon3_cpuid(void) +{ + return sparc_leon3_asr17() >> 28; +} + +#endif /*!__ASSEMBLER__*/ + +#ifdef CONFIG_SMP +# define LEON3_IRQ_IPI_DEFAULT 13 +# define LEON3_IRQ_TICKER (leon3_gptimer_irq) +# define LEON3_IRQ_CROSS_CALL 15 +#endif + +#if defined(PAGE_SIZE_LEON_8K) +#define LEON_PAGE_SIZE_LEON 1 +#elif defined(PAGE_SIZE_LEON_16K) +#define LEON_PAGE_SIZE_LEON 2) +#else +#define LEON_PAGE_SIZE_LEON 0 +#endif + +#if LEON_PAGE_SIZE_LEON == 0 +/* [ 8, 6, 6 ] + 12 */ +#define LEON_PGD_SH 24 +#define LEON_PGD_M 0xff +#define LEON_PMD_SH 18 +#define LEON_PMD_SH_V (LEON_PGD_SH-2) +#define LEON_PMD_M 0x3f +#define LEON_PTE_SH 12 +#define LEON_PTE_M 0x3f +#elif LEON_PAGE_SIZE_LEON == 1 +/* [ 7, 6, 6 ] + 13 */ +#define LEON_PGD_SH 25 +#define LEON_PGD_M 0x7f +#define LEON_PMD_SH 19 +#define LEON_PMD_SH_V (LEON_PGD_SH-1) +#define LEON_PMD_M 0x3f +#define LEON_PTE_SH 13 +#define LEON_PTE_M 0x3f +#elif LEON_PAGE_SIZE_LEON == 2 +/* [ 6, 6, 6 ] + 14 */ +#define LEON_PGD_SH 26 +#define LEON_PGD_M 0x3f +#define LEON_PMD_SH 20 +#define LEON_PMD_SH_V (LEON_PGD_SH-0) +#define LEON_PMD_M 0x3f +#define LEON_PTE_SH 14 +#define LEON_PTE_M 0x3f +#elif LEON_PAGE_SIZE_LEON == 3 +/* [ 4, 7, 6 ] + 15 */ +#define LEON_PGD_SH 28 +#define LEON_PGD_M 0x0f +#define LEON_PMD_SH 21 +#define LEON_PMD_SH_V (LEON_PGD_SH-0) +#define LEON_PMD_M 0x7f +#define LEON_PTE_SH 15 +#define LEON_PTE_M 0x3f +#else +#error cannot determine LEON_PAGE_SIZE_LEON +#endif + +#define LEON3_XCCR_SETS_MASK 0x07000000UL +#define LEON3_XCCR_SSIZE_MASK 0x00f00000UL + +#define LEON2_CCR_DSETS_MASK 0x03000000UL +#define LEON2_CFG_SSIZE_MASK 0x00007000UL + +#ifndef __ASSEMBLER__ +struct vm_area_struct; + +unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr); +void leon_flush_icache_all(void); +void leon_flush_dcache_all(void); +void leon_flush_cache_all(void); +void leon_flush_tlb_all(void); +extern int leon_flush_during_switch; +int leon_flush_needed(void); +void leon_flush_pcache_all(struct vm_area_struct *vma, unsigned long page); + +/* struct that hold LEON3 cache configuration registers */ +struct leon3_cacheregs { + unsigned long ccr; /* 0x00 - Cache Control Register */ + unsigned long iccr; /* 0x08 - Instruction Cache Configuration Register */ + unsigned long dccr; /* 0x0c - Data Cache Configuration Register */ +}; + +#include +#include + +struct device_node; +struct task_struct; +unsigned int leon_build_device_irq(unsigned int real_irq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack); +void leon_update_virq_handling(unsigned int virq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack); +void leon_init_timers(void); +void leon_node_init(struct device_node *dp, struct device_node ***nextp); +void init_leon(void); +void poke_leonsparc(void); +void leon3_getCacheRegs(struct leon3_cacheregs *regs); +extern int leon3_ticker_irq; + +#ifdef CONFIG_SMP +int leon_smp_nrcpus(void); +void leon_clear_profile_irq(int cpu); +void leon_smp_done(void); +void leon_boot_cpus(void); +int leon_boot_one_cpu(int i, struct task_struct *); +void leon_init_smp(void); +void leon_enable_irq_cpu(unsigned int irq_nr, unsigned int cpu); +irqreturn_t leon_percpu_timer_interrupt(int irq, void *unused); + +extern unsigned int smpleon_ipi[]; +extern unsigned int linux_trap_ipi15_leon[]; +extern int leon_ipi_irq; + +#endif /* CONFIG_SMP */ + +#endif /* __ASSEMBLER__ */ + +/* macros used in leon_mm.c */ +#define PFN(x) ((x) >> PAGE_SHIFT) +#define _pfn_valid(pfn) ((pfn < last_valid_pfn) && (pfn >= PFN(phys_base))) +#define _SRMMU_PTE_PMASK_LEON 0xffffffff + +/* + * On LEON PCI Memory space is mapped 1:1 with physical address space. + * + * I/O space is located at low 64Kbytes in PCI I/O space. The I/O addresses + * are converted into CPU addresses to virtual addresses that are mapped with + * MMU to the PCI Host PCI I/O space window which are translated to the low + * 64Kbytes by the Host controller. + */ + +#endif diff --git a/arch/sparc/include/asm/leon_amba.h b/arch/sparc/include/asm/leon_amba.h new file mode 100644 index 000000000..2ff5714d7 --- /dev/null +++ b/arch/sparc/include/asm/leon_amba.h @@ -0,0 +1,267 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* +*Copyright (C) 2004 Konrad Eisele (eiselekd@web.de,konrad@gaisler.com), Gaisler Research +*Copyright (C) 2004 Stefan Holst (mail@s-holst.de), Uni-Stuttgart +*Copyright (C) 2009 Daniel Hellstrom (daniel@gaisler.com),Konrad Eisele (konrad@gaisler.com) Aeroflex Gaisler AB +*/ + +#ifndef LEON_AMBA_H_INCLUDE +#define LEON_AMBA_H_INCLUDE + +#ifndef __ASSEMBLER__ + +struct amba_prom_registers { + unsigned int phys_addr; /* The physical address of this register */ + unsigned int reg_size; /* How many bytes does this register take up? */ +}; + +#endif + +/* + * The following defines the bits in the LEON UART Status Registers. + */ + +#define LEON_REG_UART_STATUS_DR 0x00000001 /* Data Ready */ +#define LEON_REG_UART_STATUS_TSE 0x00000002 /* TX Send Register Empty */ +#define LEON_REG_UART_STATUS_THE 0x00000004 /* TX Hold Register Empty */ +#define LEON_REG_UART_STATUS_BR 0x00000008 /* Break Error */ +#define LEON_REG_UART_STATUS_OE 0x00000010 /* RX Overrun Error */ +#define LEON_REG_UART_STATUS_PE 0x00000020 /* RX Parity Error */ +#define LEON_REG_UART_STATUS_FE 0x00000040 /* RX Framing Error */ +#define LEON_REG_UART_STATUS_ERR 0x00000078 /* Error Mask */ + +/* + * The following defines the bits in the LEON UART Ctrl Registers. + */ + +#define LEON_REG_UART_CTRL_RE 0x00000001 /* Receiver enable */ +#define LEON_REG_UART_CTRL_TE 0x00000002 /* Transmitter enable */ +#define LEON_REG_UART_CTRL_RI 0x00000004 /* Receiver interrupt enable */ +#define LEON_REG_UART_CTRL_TI 0x00000008 /* Transmitter irq */ +#define LEON_REG_UART_CTRL_PS 0x00000010 /* Parity select */ +#define LEON_REG_UART_CTRL_PE 0x00000020 /* Parity enable */ +#define LEON_REG_UART_CTRL_FL 0x00000040 /* Flow control enable */ +#define LEON_REG_UART_CTRL_LB 0x00000080 /* Loop Back enable */ + +#define LEON3_GPTIMER_EN 1 +#define LEON3_GPTIMER_RL 2 +#define LEON3_GPTIMER_LD 4 +#define LEON3_GPTIMER_IRQEN 8 +#define LEON3_GPTIMER_SEPIRQ 8 +#define LEON3_GPTIMER_TIMERS 0x7 + +#define LEON23_REG_TIMER_CONTROL_EN 0x00000001 /* 1 = enable counting */ +/* 0 = hold scalar and counter */ +#define LEON23_REG_TIMER_CONTROL_RL 0x00000002 /* 1 = reload at 0 */ + /* 0 = stop at 0 */ +#define LEON23_REG_TIMER_CONTROL_LD 0x00000004 /* 1 = load counter */ + /* 0 = no function */ +#define LEON23_REG_TIMER_CONTROL_IQ 0x00000008 /* 1 = irq enable */ + /* 0 = no function */ + +/* + * The following defines the bits in the LEON PS/2 Status Registers. + */ + +#define LEON_REG_PS2_STATUS_DR 0x00000001 /* Data Ready */ +#define LEON_REG_PS2_STATUS_PE 0x00000002 /* Parity error */ +#define LEON_REG_PS2_STATUS_FE 0x00000004 /* Framing error */ +#define LEON_REG_PS2_STATUS_KI 0x00000008 /* Keyboard inhibit */ +#define LEON_REG_PS2_STATUS_RF 0x00000010 /* RX buffer full */ +#define LEON_REG_PS2_STATUS_TF 0x00000020 /* TX buffer full */ + +/* + * The following defines the bits in the LEON PS/2 Ctrl Registers. + */ + +#define LEON_REG_PS2_CTRL_RE 0x00000001 /* Receiver enable */ +#define LEON_REG_PS2_CTRL_TE 0x00000002 /* Transmitter enable */ +#define LEON_REG_PS2_CTRL_RI 0x00000004 /* Keyboard receive irq */ +#define LEON_REG_PS2_CTRL_TI 0x00000008 /* Keyboard transmit irq */ + +#define LEON3_IRQMPSTATUS_CPUNR 28 +#define LEON3_IRQMPSTATUS_BROADCAST 27 + +#define GPTIMER_CONFIG_IRQNT(a) (((a) >> 3) & 0x1f) +#define GPTIMER_CONFIG_ISSEP(a) ((a) & (1 << 8)) +#define GPTIMER_CONFIG_NTIMERS(a) ((a) & (0x7)) +#define LEON3_GPTIMER_CTRL_PENDING 0x10 +#define LEON3_GPTIMER_CONFIG_NRTIMERS(c) ((c)->config & 0x7) +#define LEON3_GPTIMER_CTRL_ISPENDING(r) (((r)&LEON3_GPTIMER_CTRL_PENDING) ? 1 : 0) + +#ifndef __ASSEMBLER__ + +struct leon3_irqctrl_regs_map { + u32 ilevel; + u32 ipend; + u32 iforce; + u32 iclear; + u32 mpstatus; + u32 mpbroadcast; + u32 notused02; + u32 notused03; + u32 ampctrl; + u32 icsel[2]; + u32 notused13; + u32 notused20; + u32 notused21; + u32 notused22; + u32 notused23; + u32 mask[16]; + u32 force[16]; + /* Extended IRQ registers */ + u32 intid[16]; /* 0xc0 */ + u32 unused[(0x1000-0x100)/4]; +}; + +struct leon3_apbuart_regs_map { + u32 data; + u32 status; + u32 ctrl; + u32 scaler; +}; + +struct leon3_gptimerelem_regs_map { + u32 val; + u32 rld; + u32 ctrl; + u32 unused; +}; + +struct leon3_gptimer_regs_map { + u32 scalar; + u32 scalar_reload; + u32 config; + u32 unused; + struct leon3_gptimerelem_regs_map e[8]; +}; + +/* + * Types and structure used for AMBA Plug & Play bus scanning + */ + +#define AMBA_MAXAPB_DEVS 64 +#define AMBA_MAXAPB_DEVS_PERBUS 16 + +struct amba_device_table { + int devnr; /* number of devices on AHB or APB bus */ + unsigned int *addr[16]; /* addresses to the devices configuration tables */ + unsigned int allocbits[1]; /* 0=unallocated, 1=allocated driver */ +}; + +struct amba_apbslv_device_table { + int devnr; /* number of devices on AHB or APB bus */ + unsigned int *addr[AMBA_MAXAPB_DEVS]; /* addresses to the devices configuration tables */ + unsigned int apbmst[AMBA_MAXAPB_DEVS]; /* apb master if a entry is a apb slave */ + unsigned int apbmstidx[AMBA_MAXAPB_DEVS]; /* apb master idx if a entry is a apb slave */ + unsigned int allocbits[4]; /* 0=unallocated, 1=allocated driver */ +}; + +struct amba_confarea_type { + struct amba_confarea_type *next;/* next bus in chain */ + struct amba_device_table ahbmst; + struct amba_device_table ahbslv; + struct amba_apbslv_device_table apbslv; + unsigned int apbmst; +}; + +/* collect apb slaves */ +struct amba_apb_device { + unsigned int start, irq, bus_id; + struct amba_confarea_type *bus; +}; + +/* collect ahb slaves */ +struct amba_ahb_device { + unsigned int start[4], irq, bus_id; + struct amba_confarea_type *bus; +}; + +struct device_node; +void _amba_init(struct device_node *dp, struct device_node ***nextp); + +extern unsigned long amba_system_id; +extern struct leon3_irqctrl_regs_map *leon3_irqctrl_regs; +extern struct leon3_gptimer_regs_map *leon3_gptimer_regs; +extern struct amba_apb_device leon_percpu_timer_dev[16]; +extern int leondebug_irq_disable; +extern int leon_debug_irqout; +extern unsigned long leon3_gptimer_irq; +extern unsigned int sparc_leon_eirq; + +#endif /* __ASSEMBLER__ */ + +#define LEON3_IO_AREA 0xfff00000 +#define LEON3_CONF_AREA 0xff000 +#define LEON3_AHB_SLAVE_CONF_AREA (1 << 11) + +#define LEON3_AHB_CONF_WORDS 8 +#define LEON3_APB_CONF_WORDS 2 +#define LEON3_AHB_MASTERS 16 +#define LEON3_AHB_SLAVES 16 +#define LEON3_APB_SLAVES 16 +#define LEON3_APBUARTS 8 + +/* Vendor codes */ +#define VENDOR_GAISLER 1 +#define VENDOR_PENDER 2 +#define VENDOR_ESA 4 +#define VENDOR_OPENCORES 8 + +/* Gaisler Research device id's */ +#define GAISLER_LEON3 0x003 +#define GAISLER_LEON3DSU 0x004 +#define GAISLER_ETHAHB 0x005 +#define GAISLER_APBMST 0x006 +#define GAISLER_AHBUART 0x007 +#define GAISLER_SRCTRL 0x008 +#define GAISLER_SDCTRL 0x009 +#define GAISLER_APBUART 0x00C +#define GAISLER_IRQMP 0x00D +#define GAISLER_AHBRAM 0x00E +#define GAISLER_GPTIMER 0x011 +#define GAISLER_PCITRG 0x012 +#define GAISLER_PCISBRG 0x013 +#define GAISLER_PCIFBRG 0x014 +#define GAISLER_PCITRACE 0x015 +#define GAISLER_PCIDMA 0x016 +#define GAISLER_AHBTRACE 0x017 +#define GAISLER_ETHDSU 0x018 +#define GAISLER_PIOPORT 0x01A +#define GAISLER_GRGPIO 0x01A +#define GAISLER_AHBJTAG 0x01c +#define GAISLER_ETHMAC 0x01D +#define GAISLER_AHB2AHB 0x020 +#define GAISLER_USBDC 0x021 +#define GAISLER_ATACTRL 0x024 +#define GAISLER_DDRSPA 0x025 +#define GAISLER_USBEHC 0x026 +#define GAISLER_USBUHC 0x027 +#define GAISLER_I2CMST 0x028 +#define GAISLER_SPICTRL 0x02D +#define GAISLER_DDR2SPA 0x02E +#define GAISLER_SPIMCTRL 0x045 +#define GAISLER_LEON4 0x048 +#define GAISLER_LEON4DSU 0x049 +#define GAISLER_AHBSTAT 0x052 +#define GAISLER_FTMCTRL 0x054 +#define GAISLER_KBD 0x060 +#define GAISLER_VGA 0x061 +#define GAISLER_SVGA 0x063 +#define GAISLER_GRSYSMON 0x066 +#define GAISLER_GRACECTRL 0x067 + +#define GAISLER_L2TIME 0xffd /* internal device: leon2 timer */ +#define GAISLER_L2C 0xffe /* internal device: leon2compat */ +#define GAISLER_PLUGPLAY 0xfff /* internal device: plug & play configarea */ + +/* Chip IDs */ +#define AEROFLEX_UT699 0x0699 +#define LEON4_NEXTREME1 0x0102 +#define GAISLER_GR712RC 0x0712 + +#define amba_vendor(x) (((x) >> 24) & 0xff) + +#define amba_device(x) (((x) >> 12) & 0xfff) + +#endif diff --git a/arch/sparc/include/asm/leon_pci.h b/arch/sparc/include/asm/leon_pci.h new file mode 100644 index 000000000..484a56e81 --- /dev/null +++ b/arch/sparc/include/asm/leon_pci.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * asm/leon_pci.h + * + * Copyright (C) 2011 Aeroflex Gaisler AB, Daniel Hellstrom + */ + +#ifndef _ASM_LEON_PCI_H_ +#define _ASM_LEON_PCI_H_ + +/* PCI related definitions */ +struct leon_pci_info { + struct pci_ops *ops; + struct resource io_space; + struct resource mem_space; + struct resource busn; + int (*map_irq)(const struct pci_dev *dev, u8 slot, u8 pin); +}; + +void leon_pci_init(struct platform_device *ofdev, + struct leon_pci_info *info); + +#endif /* _ASM_LEON_PCI_H_ */ diff --git a/arch/sparc/include/asm/lsu.h b/arch/sparc/include/asm/lsu.h new file mode 100644 index 000000000..154ae76c0 --- /dev/null +++ b/arch/sparc/include/asm/lsu.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_LSU_H +#define _SPARC64_LSU_H + +#include + +/* LSU Control Register */ +#define LSU_CONTROL_PM _AC(0x000001fe00000000,UL) /* Phys-watchpoint byte mask*/ +#define LSU_CONTROL_VM _AC(0x00000001fe000000,UL) /* Virt-watchpoint byte mask*/ +#define LSU_CONTROL_PR _AC(0x0000000001000000,UL) /* Phys-rd watchpoint enable*/ +#define LSU_CONTROL_PW _AC(0x0000000000800000,UL) /* Phys-wr watchpoint enable*/ +#define LSU_CONTROL_VR _AC(0x0000000000400000,UL) /* Virt-rd watchpoint enable*/ +#define LSU_CONTROL_VW _AC(0x0000000000200000,UL) /* Virt-wr watchpoint enable*/ +#define LSU_CONTROL_FM _AC(0x00000000000ffff0,UL) /* Parity mask enables. */ +#define LSU_CONTROL_DM _AC(0x0000000000000008,UL) /* Data MMU enable. */ +#define LSU_CONTROL_IM _AC(0x0000000000000004,UL) /* Instruction MMU enable. */ +#define LSU_CONTROL_DC _AC(0x0000000000000002,UL) /* Data cache enable. */ +#define LSU_CONTROL_IC _AC(0x0000000000000001,UL) /* Instruction cache enable.*/ + +#endif /* !(_SPARC64_LSU_H) */ diff --git a/arch/sparc/include/asm/machines.h b/arch/sparc/include/asm/machines.h new file mode 100644 index 000000000..9f78f70c6 --- /dev/null +++ b/arch/sparc/include/asm/machines.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * machines.h: Defines for taking apart the machine type value in the + * idprom and determining the kind of machine we are on. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ +#ifndef _SPARC_MACHINES_H +#define _SPARC_MACHINES_H + +struct Sun_Machine_Models { + char *name; + unsigned char id_machtype; +}; + +/* The machine type in the idprom area looks like this: + * + * --------------- + * | ARCH | MACH | + * --------------- + * 7 4 3 0 + * + * The ARCH field determines the architecture line (sun4m, etc). + * The MACH field determines the machine make within that architecture. + */ + +#define SM_ARCH_MASK 0xf0 +#define M_LEON 0x30 +#define SM_SUN4M 0x70 +#define SM_SUN4M_OBP 0x80 + +#define SM_TYP_MASK 0x0f + +/* Leon machines */ +#define M_LEON3_SOC 0x02 /* Leon3 SoC */ + +/* Sun4m machines, these predate the OpenBoot. These values only mean + * something if the value in the ARCH field is SM_SUN4M, if it is + * SM_SUN4M_OBP then you have the following situation: + * 1) You either have a sun4d, a sun4e, or a recently made sun4m. + * 2) You have to consult OpenBoot to determine which machine this is. + */ +#define SM_4M_SS60 0x01 /* Sun4m SparcSystem 600 */ +#define SM_4M_SS50 0x02 /* Sun4m SparcStation 10 */ +#define SM_4M_SS40 0x03 /* Sun4m SparcStation 5 */ + +/* Sun4d machines -- N/A */ +/* Sun4e machines -- N/A */ +/* Sun4u machines -- N/A */ + +#endif /* !(_SPARC_MACHINES_H) */ diff --git a/arch/sparc/include/asm/mbus.h b/arch/sparc/include/asm/mbus.h new file mode 100644 index 000000000..8b6dbe701 --- /dev/null +++ b/arch/sparc/include/asm/mbus.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * mbus.h: Various defines for MBUS modules. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_MBUS_H +#define _SPARC_MBUS_H + +#include /* HyperSparc stuff */ +#include /* Ugh, bug city... */ + +enum mbus_module { + HyperSparc = 0, + Swift_ok = 4, + Swift_bad_c = 5, + Swift_lots_o_bugs = 6, + Tsunami = 7, + Viking_12 = 8, + Viking_2x = 9, + Viking_30 = 10, + Viking_35 = 11, + Viking_new = 12, + TurboSparc = 13, + SRMMU_INVAL_MOD = 14, +}; + +extern enum mbus_module srmmu_modtype; +extern unsigned int viking_rev, swift_rev, cypress_rev; + +/* HW Mbus module bugs we have to deal with */ +#define HWBUG_COPYBACK_BROKEN 0x00000001 +#define HWBUG_ASIFLUSH_BROKEN 0x00000002 +#define HWBUG_VACFLUSH_BITROT 0x00000004 +#define HWBUG_KERN_ACCBROKEN 0x00000008 +#define HWBUG_KERN_CBITBROKEN 0x00000010 +#define HWBUG_MODIFIED_BITROT 0x00000020 +#define HWBUG_PC_BADFAULT_ADDR 0x00000040 +#define HWBUG_SUPERSCALAR_BAD 0x00000080 +#define HWBUG_PACINIT_BITROT 0x00000100 + +/* First the module type values. To find out which you have, just load + * the mmu control register from ASI_M_MMUREG alternate address space and + * shift the value right 28 bits. + */ +/* IMPL field means the company which produced the chip. */ +#define MBUS_VIKING 0x4 /* bleech, Texas Instruments Module */ +#define MBUS_LSI 0x3 /* LSI Logics */ +#define MBUS_ROSS 0x1 /* Ross is nice */ +#define MBUS_FMI 0x0 /* Fujitsu Microelectronics/Swift */ + +/* Ross Module versions */ +#define ROSS_604_REV_CDE 0x0 /* revisions c, d, and e */ +#define ROSS_604_REV_F 0x1 /* revision f */ +#define ROSS_605 0xf /* revision a, a.1, and a.2 */ +#define ROSS_605_REV_B 0xe /* revision b */ + +/* TI Viking Module versions */ +#define VIKING_REV_12 0x1 /* Version 1.2 or SPARCclassic's CPU */ +#define VIKING_REV_2 0x2 /* Version 2.1, 2.2, 2.3, and 2.4 */ +#define VIKING_REV_30 0x3 /* Version 3.0 */ +#define VIKING_REV_35 0x4 /* Version 3.5 */ + +/* LSI Logics. */ +#define LSI_L64815 0x0 + +/* Fujitsu */ +#define FMI_AURORA 0x4 /* MB8690x, a Swift module... */ +#define FMI_TURBO 0x5 /* MB86907, a TurboSparc module... */ + +/* For multiprocessor support we need to be able to obtain the CPU id and + * the MBUS Module id. + */ + +/* The CPU ID is encoded in the trap base register, 20 bits to the left of + * bit zero, with 2 bits being significant. + */ +#define TBR_ID_SHIFT 20 + +static inline int get_cpuid(void) +{ + register int retval; + __asm__ __volatile__("rd %%tbr, %0\n\t" + "srl %0, %1, %0\n\t" : + "=r" (retval) : + "i" (TBR_ID_SHIFT)); + return (retval & 3); +} + +static inline int get_modid(void) +{ + return (get_cpuid() | 0x8); +} + + +#endif /* !(_SPARC_MBUS_H) */ diff --git a/arch/sparc/include/asm/mc146818rtc.h b/arch/sparc/include/asm/mc146818rtc.h new file mode 100644 index 000000000..07faf754e --- /dev/null +++ b/arch/sparc/include/asm/mc146818rtc.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_MC146818RTC_H +#define ___ASM_SPARC_MC146818RTC_H + +#include + +extern spinlock_t rtc_lock; + +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/mc146818rtc_32.h b/arch/sparc/include/asm/mc146818rtc_32.h new file mode 100644 index 000000000..d8fd75da8 --- /dev/null +++ b/arch/sparc/include/asm/mc146818rtc_32.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Machine dependent access functions for RTC registers. + */ +#ifndef __ASM_SPARC_MC146818RTC_H +#define __ASM_SPARC_MC146818RTC_H + +#include + +#ifndef RTC_PORT +#define RTC_PORT(x) (0x70 + (x)) +#define RTC_ALWAYS_BCD 1 /* RTC operates in binary mode */ +#endif + +/* + * The yet supported machines all access the RTC index register via + * an ISA port access but the way to access the date register differs ... + */ +#define CMOS_READ(addr) ({ \ +outb_p((addr),RTC_PORT(0)); \ +inb_p(RTC_PORT(1)); \ +}) +#define CMOS_WRITE(val, addr) ({ \ +outb_p((addr),RTC_PORT(0)); \ +outb_p((val),RTC_PORT(1)); \ +}) + +#define RTC_IRQ 8 + +#endif /* __ASM_SPARC_MC146818RTC_H */ diff --git a/arch/sparc/include/asm/mc146818rtc_64.h b/arch/sparc/include/asm/mc146818rtc_64.h new file mode 100644 index 000000000..b1708a7e5 --- /dev/null +++ b/arch/sparc/include/asm/mc146818rtc_64.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Machine dependent access functions for RTC registers. + */ +#ifndef __ASM_SPARC64_MC146818RTC_H +#define __ASM_SPARC64_MC146818RTC_H + +#include + +#ifndef RTC_PORT +extern unsigned long cmos_regs; +#define RTC_PORT(x) (cmos_regs + (x)) +#define RTC_ALWAYS_BCD 0 +#endif + +/* + * The yet supported machines all access the RTC index register via + * an ISA port access but the way to access the date register differs ... + */ +#define CMOS_READ(addr) ({ \ +outb_p((addr),RTC_PORT(0)); \ +inb_p(RTC_PORT(1)); \ +}) +#define CMOS_WRITE(val, addr) ({ \ +outb_p((addr),RTC_PORT(0)); \ +outb_p((val),RTC_PORT(1)); \ +}) + +#endif /* __ASM_SPARC64_MC146818RTC_H */ diff --git a/arch/sparc/include/asm/mdesc.h b/arch/sparc/include/asm/mdesc.h new file mode 100644 index 000000000..ec31a06d0 --- /dev/null +++ b/arch/sparc/include/asm/mdesc.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_MDESC_H +#define _SPARC64_MDESC_H + +#include +#include +#include + +struct mdesc_handle; + +/* Machine description operations are to be surrounded by grab and + * release calls. The mdesc_handle returned from the grab is + * the first argument to all of the operational calls that work + * on mdescs. + */ +struct mdesc_handle *mdesc_grab(void); +void mdesc_release(struct mdesc_handle *); + +#define MDESC_NODE_NULL (~(u64)0) +#define MDESC_MAX_STR_LEN 256 + +u64 mdesc_node_by_name(struct mdesc_handle *handle, + u64 from_node, const char *name); +#define mdesc_for_each_node_by_name(__hdl, __node, __name) \ + for (__node = mdesc_node_by_name(__hdl, MDESC_NODE_NULL, __name); \ + (__node) != MDESC_NODE_NULL; \ + __node = mdesc_node_by_name(__hdl, __node, __name)) + +/* Access to property values returned from mdesc_get_property() are + * only valid inside of a mdesc_grab()/mdesc_release() sequence. + * Once mdesc_release() is called, the memory backed up by these + * pointers may reference freed up memory. + * + * Therefore callers must make copies of any property values + * they need. + * + * These same rules apply to mdesc_node_name(). + */ +const void *mdesc_get_property(struct mdesc_handle *handle, + u64 node, const char *name, int *lenp); +const char *mdesc_node_name(struct mdesc_handle *hp, u64 node); + +/* MD arc iteration, the standard sequence is: + * + * unsigned long arc; + * mdesc_for_each_arc(arc, handle, node, MDESC_ARC_TYPE_{FWD,BACK}) { + * unsigned long target = mdesc_arc_target(handle, arc); + * ... + * } + */ + +#define MDESC_ARC_TYPE_FWD "fwd" +#define MDESC_ARC_TYPE_BACK "back" + +u64 mdesc_next_arc(struct mdesc_handle *handle, u64 from, + const char *arc_type); +#define mdesc_for_each_arc(__arc, __hdl, __node, __type) \ + for (__arc = mdesc_next_arc(__hdl, __node, __type); \ + (__arc) != MDESC_NODE_NULL; \ + __arc = mdesc_next_arc(__hdl, __arc, __type)) + +u64 mdesc_arc_target(struct mdesc_handle *hp, u64 arc); + +void mdesc_update(void); + +struct mdesc_notifier_client { + void (*add)(struct mdesc_handle *handle, u64 node, + const char *node_name); + void (*remove)(struct mdesc_handle *handle, u64 node, + const char *node_name); + const char *node_name; + struct mdesc_notifier_client *next; +}; + +void mdesc_register_notifier(struct mdesc_notifier_client *client); + +union md_node_info { + struct vdev_port { + u64 id; /* id */ + u64 parent_cfg_hdl; /* parent config handle */ + const char *name; /* name (property) */ + } vdev_port; + struct ds_port { + u64 id; /* id */ + } ds_port; +}; + +u64 mdesc_get_node(struct mdesc_handle *hp, const char *node_name, + union md_node_info *node_info); +int mdesc_get_node_info(struct mdesc_handle *hp, u64 node, + const char *node_name, union md_node_info *node_info); + +void mdesc_fill_in_cpu_data(cpumask_t *mask); +void mdesc_populate_present_mask(cpumask_t *mask); +void mdesc_get_page_sizes(cpumask_t *mask, unsigned long *pgsz_mask); + +void sun4v_mdesc_init(void); + +#endif diff --git a/arch/sparc/include/asm/memctrl.h b/arch/sparc/include/asm/memctrl.h new file mode 100644 index 000000000..6790ed608 --- /dev/null +++ b/arch/sparc/include/asm/memctrl.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_MEMCTRL_H +#define _SPARC_MEMCTRL_H + +typedef int (*dimm_printer_t)(int synd_code, unsigned long paddr, char *buf, int buflen); + +int register_dimm_printer(dimm_printer_t func); +void unregister_dimm_printer(dimm_printer_t func); + +#endif /* _SPARC_MEMCTRL_H */ diff --git a/arch/sparc/include/asm/mman.h b/arch/sparc/include/asm/mman.h new file mode 100644 index 000000000..a8bae8ad2 --- /dev/null +++ b/arch/sparc/include/asm/mman.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_MMAN_H__ +#define __SPARC_MMAN_H__ + +#include + +#ifndef __ASSEMBLER__ +#define arch_mmap_check(addr,len,flags) sparc_mmap_check(addr,len) +int sparc_mmap_check(unsigned long addr, unsigned long len); + +#ifdef CONFIG_SPARC64 +#include + +static inline void ipi_set_tstate_mcde(void *arg) +{ + struct mm_struct *mm = arg; + + /* Set TSTATE_MCDE for the task using address map that ADI has been + * enabled on if the task is running. If not, it will be set + * automatically at the next context switch + */ + if (current->mm == mm) { + struct pt_regs *regs; + + regs = task_pt_regs(current); + regs->tstate |= TSTATE_MCDE; + } +} + +#define arch_calc_vm_prot_bits(prot, pkey) sparc_calc_vm_prot_bits(prot) +static inline vm_flags_t sparc_calc_vm_prot_bits(unsigned long prot) +{ + if (adi_capable() && (prot & PROT_ADI)) { + struct pt_regs *regs; + + if (!current->mm->context.adi) { + regs = task_pt_regs(current); + regs->tstate |= TSTATE_MCDE; + current->mm->context.adi = true; + on_each_cpu_mask(mm_cpumask(current->mm), + ipi_set_tstate_mcde, current->mm, 0); + } + return VM_SPARC_ADI; + } else { + return 0; + } +} + +#define arch_validate_prot(prot, addr) sparc_validate_prot(prot, addr) +static inline int sparc_validate_prot(unsigned long prot, unsigned long addr) +{ + if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC | PROT_SEM | PROT_ADI)) + return 0; + return 1; +} + +#define arch_validate_flags(vm_flags) arch_validate_flags(vm_flags) +/* arch_validate_flags() - Ensure combination of flags is valid for a + * VMA. + */ +static inline bool arch_validate_flags(vm_flags_t vm_flags) +{ + /* If ADI is being enabled on this VMA, check for ADI + * capability on the platform and ensure VMA is suitable + * for ADI + */ + if (vm_flags & VM_SPARC_ADI) { + if (!adi_capable()) + return false; + + /* ADI can not be enabled on PFN mapped pages */ + if (vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) + return false; + + /* Mergeable pages can become unmergeable + * if ADI is enabled on them even if they + * have identical data on them. This can be + * because ADI enabled pages with identical + * data may still not have identical ADI + * tags on them. Disallow ADI on mergeable + * pages. + */ + if (vm_flags & VM_MERGEABLE) + return false; + } + return true; +} +#endif /* CONFIG_SPARC64 */ + +#endif /* __ASSEMBLER__ */ +#endif /* __SPARC_MMAN_H__ */ diff --git a/arch/sparc/include/asm/mmu.h b/arch/sparc/include/asm/mmu.h new file mode 100644 index 000000000..286457e58 --- /dev/null +++ b/arch/sparc/include/asm/mmu.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_MMU_H +#define ___ASM_SPARC_MMU_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/mmu_32.h b/arch/sparc/include/asm/mmu_32.h new file mode 100644 index 000000000..d5b220ca7 --- /dev/null +++ b/arch/sparc/include/asm/mmu_32.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __MMU_H +#define __MMU_H + +/* Default "unsigned long" context */ +typedef unsigned long mm_context_t; + +/* mm/srmmu.c */ +extern ctxd_t *srmmu_ctx_table_phys; + +#endif diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h new file mode 100644 index 000000000..4eeb938f3 --- /dev/null +++ b/arch/sparc/include/asm/mmu_64.h @@ -0,0 +1,129 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __MMU_H +#define __MMU_H + +#include +#include +#include + +#define CTX_NR_BITS 13 + +#define TAG_CONTEXT_BITS ((_AC(1,UL) << CTX_NR_BITS) - _AC(1,UL)) + +/* UltraSPARC-III+ and later have a feature whereby you can + * select what page size the various Data-TLB instances in the + * chip. In order to gracefully support this, we put the version + * field in a spot outside of the areas of the context register + * where this parameter is specified. + */ +#define CTX_VERSION_SHIFT 22 +#define CTX_VERSION_MASK ((~0UL) << CTX_VERSION_SHIFT) + +#define CTX_PGSZ_8KB _AC(0x0,UL) +#define CTX_PGSZ_64KB _AC(0x1,UL) +#define CTX_PGSZ_512KB _AC(0x2,UL) +#define CTX_PGSZ_4MB _AC(0x3,UL) +#define CTX_PGSZ_BITS _AC(0x7,UL) +#define CTX_PGSZ0_NUC_SHIFT 61 +#define CTX_PGSZ1_NUC_SHIFT 58 +#define CTX_PGSZ0_SHIFT 16 +#define CTX_PGSZ1_SHIFT 19 +#define CTX_PGSZ_MASK ((CTX_PGSZ_BITS << CTX_PGSZ0_SHIFT) | \ + (CTX_PGSZ_BITS << CTX_PGSZ1_SHIFT)) + +#define CTX_PGSZ_BASE CTX_PGSZ_8KB +#define CTX_PGSZ_HUGE CTX_PGSZ_4MB +#define CTX_PGSZ_KERN CTX_PGSZ_4MB + +/* Thus, when running on UltraSPARC-III+ and later, we use the following + * PRIMARY_CONTEXT register values for the kernel context. + */ +#define CTX_CHEETAH_PLUS_NUC \ + ((CTX_PGSZ_KERN << CTX_PGSZ0_NUC_SHIFT) | \ + (CTX_PGSZ_BASE << CTX_PGSZ1_NUC_SHIFT)) + +#define CTX_CHEETAH_PLUS_CTX0 \ + ((CTX_PGSZ_KERN << CTX_PGSZ0_SHIFT) | \ + (CTX_PGSZ_BASE << CTX_PGSZ1_SHIFT)) + +/* If you want "the TLB context number" use CTX_NR_MASK. If you + * want "the bits I program into the context registers" use + * CTX_HW_MASK. + */ +#define CTX_NR_MASK TAG_CONTEXT_BITS +#define CTX_HW_MASK (CTX_NR_MASK | CTX_PGSZ_MASK) + +#define CTX_FIRST_VERSION BIT(CTX_VERSION_SHIFT) +#define CTX_VALID(__ctx) \ + (!(((__ctx.sparc64_ctx_val) ^ tlb_context_cache) & CTX_VERSION_MASK)) +#define CTX_HWBITS(__ctx) ((__ctx.sparc64_ctx_val) & CTX_HW_MASK) +#define CTX_NRBITS(__ctx) ((__ctx.sparc64_ctx_val) & CTX_NR_MASK) + +#ifndef __ASSEMBLER__ + +#define TSB_ENTRY_ALIGNMENT 16 + +struct tsb { + unsigned long tag; + unsigned long pte; +} __attribute__((aligned(TSB_ENTRY_ALIGNMENT))); + +void __tsb_insert(unsigned long ent, unsigned long tag, unsigned long pte); +void tsb_flush(unsigned long ent, unsigned long tag); +void tsb_init(struct tsb *tsb, unsigned long size); + +struct tsb_config { + struct tsb *tsb; + unsigned long tsb_rss_limit; + unsigned long tsb_nentries; + unsigned long tsb_reg_val; + unsigned long tsb_map_vaddr; + unsigned long tsb_map_pte; +}; + +#define MM_TSB_BASE 0 + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +#define MM_TSB_HUGE 1 +#define MM_NUM_TSBS 2 +#else +#define MM_NUM_TSBS 1 +#endif + +/* ADI tags are stored when a page is swapped out and the storage for + * tags is allocated dynamically. There is a tag storage descriptor + * associated with each set of tag storage pages. Tag storage descriptors + * are allocated dynamically. Since kernel will allocate a full page for + * each tag storage descriptor, we can store up to + * PAGE_SIZE/sizeof(tag storage descriptor) descriptors on that page. + */ +typedef struct { + unsigned long start; /* Start address for this tag storage */ + unsigned long end; /* Last address for tag storage */ + unsigned char *tags; /* Where the tags are */ + unsigned long tag_users; /* number of references to descriptor */ +} tag_storage_desc_t; + +typedef struct { + spinlock_t lock; + unsigned long sparc64_ctx_val; + unsigned long hugetlb_pte_count; + unsigned long thp_pte_count; + struct tsb_config tsb_block[MM_NUM_TSBS]; + struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; + void *vdso; + bool adi; + tag_storage_desc_t *tag_store; + spinlock_t tag_lock; +} mm_context_t; + +#endif /* !__ASSEMBLER__ */ + +#define TSB_CONFIG_TSB 0x00 +#define TSB_CONFIG_RSS_LIMIT 0x08 +#define TSB_CONFIG_NENTRIES 0x10 +#define TSB_CONFIG_REG_VAL 0x18 +#define TSB_CONFIG_MAP_VADDR 0x20 +#define TSB_CONFIG_MAP_PTE 0x28 + +#endif /* __MMU_H */ diff --git a/arch/sparc/include/asm/mmu_context.h b/arch/sparc/include/asm/mmu_context.h new file mode 100644 index 000000000..5e1ea3851 --- /dev/null +++ b/arch/sparc/include/asm/mmu_context.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_MMU_CONTEXT_H +#define ___ASM_SPARC_MMU_CONTEXT_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/mmu_context_32.h b/arch/sparc/include/asm/mmu_context_32.h new file mode 100644 index 000000000..d9ff73f77 --- /dev/null +++ b/arch/sparc/include/asm/mmu_context_32.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_MMU_CONTEXT_H +#define __SPARC_MMU_CONTEXT_H + +#ifndef __ASSEMBLER__ + +#include + +/* Initialize a new mmu context. This is invoked when a new + * address space instance (unique or shared) is instantiated. + */ +#define init_new_context init_new_context +int init_new_context(struct task_struct *tsk, struct mm_struct *mm); + +/* Destroy a dead context. This occurs when mmput drops the + * mm_users count to zero, the mmaps have been released, and + * all the page tables have been flushed. Our job is to destroy + * any remaining processor-specific state. + */ +#define destroy_context destroy_context +void destroy_context(struct mm_struct *mm); + +/* Switch the current MM context. */ +void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, + struct task_struct *tsk); + +/* Activate a new MM instance for the current task. */ +#define activate_mm(active_mm, mm) switch_mm((active_mm), (mm), NULL) + +#include + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(__SPARC_MMU_CONTEXT_H) */ diff --git a/arch/sparc/include/asm/mmu_context_64.h b/arch/sparc/include/asm/mmu_context_64.h new file mode 100644 index 000000000..78bbacc14 --- /dev/null +++ b/arch/sparc/include/asm/mmu_context_64.h @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC64_MMU_CONTEXT_H +#define __SPARC64_MMU_CONTEXT_H + +/* Derived heavily from Linus's Alpha/AXP ASN code... */ + +#ifndef __ASSEMBLER__ + +#include +#include +#include +#include + +#include +#include +#include +#include + +extern spinlock_t ctx_alloc_lock; +extern unsigned long tlb_context_cache; +extern unsigned long mmu_context_bmap[]; + +DECLARE_PER_CPU(struct mm_struct *, per_cpu_secondary_mm); +void get_new_mmu_context(struct mm_struct *mm); + +#define init_new_context init_new_context +int init_new_context(struct task_struct *tsk, struct mm_struct *mm); +#define destroy_context destroy_context +void destroy_context(struct mm_struct *mm); + +void __tsb_context_switch(unsigned long pgd_pa, + struct tsb_config *tsb_base, + struct tsb_config *tsb_huge, + unsigned long tsb_descr_pa, + unsigned long secondary_ctx); + +static inline void tsb_context_switch_ctx(struct mm_struct *mm, + unsigned long ctx) +{ + __tsb_context_switch(__pa(mm->pgd), + &mm->context.tsb_block[MM_TSB_BASE], +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + (mm->context.tsb_block[MM_TSB_HUGE].tsb ? + &mm->context.tsb_block[MM_TSB_HUGE] : + NULL) +#else + NULL +#endif + , __pa(&mm->context.tsb_descr[MM_TSB_BASE]), + ctx); +} + +#define tsb_context_switch(X) tsb_context_switch_ctx(X, 0) + +void tsb_grow(struct mm_struct *mm, + unsigned long tsb_index, + unsigned long mm_rss); +#ifdef CONFIG_SMP +void smp_tsb_sync(struct mm_struct *mm); +#else +#define smp_tsb_sync(__mm) do { } while (0) +#endif + +/* Set MMU context in the actual hardware. */ +#define load_secondary_context(__mm) \ + __asm__ __volatile__( \ + "\n661: stxa %0, [%1] %2\n" \ + " .section .sun4v_1insn_patch, \"ax\"\n" \ + " .word 661b\n" \ + " stxa %0, [%1] %3\n" \ + " .previous\n" \ + " flush %%g6\n" \ + : /* No outputs */ \ + : "r" (CTX_HWBITS((__mm)->context)), \ + "r" (SECONDARY_CONTEXT), "i" (ASI_DMMU), "i" (ASI_MMU)) + +void __flush_tlb_mm(unsigned long, unsigned long); + +/* Switch the current MM context. */ +static inline void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk) +{ + unsigned long ctx_valid, flags; + int cpu = smp_processor_id(); + + per_cpu(per_cpu_secondary_mm, cpu) = mm; + if (unlikely(mm == &init_mm)) + return; + + spin_lock_irqsave(&mm->context.lock, flags); + ctx_valid = CTX_VALID(mm->context); + if (!ctx_valid) + get_new_mmu_context(mm); + + /* We have to be extremely careful here or else we will miss + * a TSB grow if we switch back and forth between a kernel + * thread and an address space which has its TSB size increased + * on another processor. + * + * It is possible to play some games in order to optimize the + * switch, but the safest thing to do is to unconditionally + * perform the secondary context load and the TSB context switch. + * + * For reference the bad case is, for address space "A": + * + * CPU 0 CPU 1 + * run address space A + * set cpu0's bits in cpu_vm_mask + * switch to kernel thread, borrow + * address space A via entry_lazy_tlb + * run address space A + * set cpu1's bit in cpu_vm_mask + * flush_tlb_pending() + * reset cpu_vm_mask to just cpu1 + * TSB grow + * run address space A + * context was valid, so skip + * TSB context switch + * + * At that point cpu0 continues to use a stale TSB, the one from + * before the TSB grow performed on cpu1. cpu1 did not cross-call + * cpu0 to update its TSB because at that point the cpu_vm_mask + * only had cpu1 set in it. + */ + tsb_context_switch_ctx(mm, CTX_HWBITS(mm->context)); + + /* Any time a processor runs a context on an address space + * for the first time, we must flush that context out of the + * local TLB. + */ + if (!ctx_valid || !cpumask_test_cpu(cpu, mm_cpumask(mm))) { + cpumask_set_cpu(cpu, mm_cpumask(mm)); + __flush_tlb_mm(CTX_HWBITS(mm->context), + SECONDARY_CONTEXT); + } + spin_unlock_irqrestore(&mm->context.lock, flags); +} + +#define activate_mm(active_mm, mm) switch_mm(active_mm, mm, NULL) + +#define __HAVE_ARCH_START_CONTEXT_SWITCH +static inline void arch_start_context_switch(struct task_struct *prev) +{ + /* Save the current state of MCDPER register for the process + * we are switching from + */ + if (adi_capable()) { + register unsigned long tmp_mcdper; + + __asm__ __volatile__( + ".word 0x83438000\n\t" /* rd %mcdper, %g1 */ + "mov %%g1, %0\n\t" + : "=r" (tmp_mcdper) + : + : "g1"); + if (tmp_mcdper) + set_tsk_thread_flag(prev, TIF_MCDPER); + else + clear_tsk_thread_flag(prev, TIF_MCDPER); + } +} + +#define finish_arch_post_lock_switch finish_arch_post_lock_switch +static inline void finish_arch_post_lock_switch(void) +{ + /* Restore the state of MCDPER register for the new process + * just switched to. + */ + if (adi_capable()) { + register unsigned long tmp_mcdper; + + tmp_mcdper = test_thread_flag(TIF_MCDPER); + __asm__ __volatile__( + "mov %0, %%g1\n\t" + ".word 0x9d800001\n\t" /* wr %g0, %g1, %mcdper" */ + ".word 0xaf902001\n\t" /* wrpr %g0, 1, %pmcdper */ + : + : "ir" (tmp_mcdper) + : "g1"); + if (current && current->mm && current->mm->context.adi) { + struct pt_regs *regs; + + regs = task_pt_regs(current); + regs->tstate |= TSTATE_MCDE; + } + } +} + +#define mm_untag_mask mm_untag_mask +static inline unsigned long mm_untag_mask(struct mm_struct *mm) +{ + return -1UL >> adi_nbits(); +} + +#include + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(__SPARC64_MMU_CONTEXT_H) */ diff --git a/arch/sparc/include/asm/mmzone.h b/arch/sparc/include/asm/mmzone.h new file mode 100644 index 000000000..74eb2c71d --- /dev/null +++ b/arch/sparc/include/asm/mmzone.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_MMZONE_H +#define _SPARC64_MMZONE_H + +#ifdef CONFIG_NUMA + +#include + +extern int numa_cpu_lookup_table[]; +extern cpumask_t numa_cpumask_lookup_table[]; + +#endif /* CONFIG_NUMA */ + +#endif /* _SPARC64_MMZONE_H */ diff --git a/arch/sparc/include/asm/mxcc.h b/arch/sparc/include/asm/mxcc.h new file mode 100644 index 000000000..bd6339dcf --- /dev/null +++ b/arch/sparc/include/asm/mxcc.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * mxcc.h: Definitions of the Viking MXCC registers + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_MXCC_H +#define _SPARC_MXCC_H + +/* These registers are accessed through ASI 0x2. */ +#define MXCC_DATSTREAM 0x1C00000 /* Data stream register */ +#define MXCC_SRCSTREAM 0x1C00100 /* Source stream register */ +#define MXCC_DESSTREAM 0x1C00200 /* Destination stream register */ +#define MXCC_RMCOUNT 0x1C00300 /* Count of references and misses */ +#define MXCC_STEST 0x1C00804 /* Internal self-test */ +#define MXCC_CREG 0x1C00A04 /* Control register */ +#define MXCC_SREG 0x1C00B00 /* Status register */ +#define MXCC_RREG 0x1C00C04 /* Reset register */ +#define MXCC_EREG 0x1C00E00 /* Error code register */ +#define MXCC_PREG 0x1C00F04 /* Address port register */ + +/* Some MXCC constants. */ +#define MXCC_STREAM_SIZE 0x20 /* Size in bytes of one stream r/w */ + +/* The MXCC Control Register: + * + * ---------------------------------------------------------------------- + * | | RRC | RSV |PRE|MCE|PARE|ECE|RSV| + * ---------------------------------------------------------------------- + * 31 10 9 8-6 5 4 3 2 1-0 + * + * RRC: Controls what you read from MXCC_RMCOUNT reg. + * 0=Misses 1=References + * PRE: Prefetch enable + * MCE: Multiple Command Enable + * PARE: Parity enable + * ECE: External cache enable + */ + +#define MXCC_CTL_RRC 0x00000200 +#define MXCC_CTL_PRE 0x00000020 +#define MXCC_CTL_MCE 0x00000010 +#define MXCC_CTL_PARE 0x00000008 +#define MXCC_CTL_ECE 0x00000004 + +/* The MXCC Error Register: + * + * -------------------------------------------------------- + * |ME| RSV|CE|PEW|PEE|ASE|EIV| MOPC|ECODE|PRIV|RSV|HPADDR| + * -------------------------------------------------------- + * 31 30 29 28 27 26 25 24-15 14-7 6 5-3 2-0 + * + * ME: Multiple Errors have occurred + * CE: Cache consistency Error + * PEW: Parity Error during a Write operation + * PEE: Parity Error involving the External cache + * ASE: ASynchronous Error + * EIV: This register is toast + * MOPC: MXCC Operation Code for instance causing error + * ECODE: The Error CODE + * PRIV: A privileged mode error? 0=no 1=yes + * HPADDR: High PhysicalADDRess bits (35-32) + */ + +#define MXCC_ERR_ME 0x80000000 +#define MXCC_ERR_CE 0x20000000 +#define MXCC_ERR_PEW 0x10000000 +#define MXCC_ERR_PEE 0x08000000 +#define MXCC_ERR_ASE 0x04000000 +#define MXCC_ERR_EIV 0x02000000 +#define MXCC_ERR_MOPC 0x01FF8000 +#define MXCC_ERR_ECODE 0x00007F80 +#define MXCC_ERR_PRIV 0x00000040 +#define MXCC_ERR_HPADDR 0x0000000f + +/* The MXCC Port register: + * + * ----------------------------------------------------- + * | | MID | | + * ----------------------------------------------------- + * 31 21 20-18 17 0 + * + * MID: The moduleID of the cpu your read this from. + */ + +#ifndef __ASSEMBLER__ + +static inline void mxcc_set_stream_src(unsigned long *paddr) +{ + unsigned long data0 = paddr[0]; + unsigned long data1 = paddr[1]; + + __asm__ __volatile__ ("or %%g0, %0, %%g2\n\t" + "or %%g0, %1, %%g3\n\t" + "stda %%g2, [%2] %3\n\t" : : + "r" (data0), "r" (data1), + "r" (MXCC_SRCSTREAM), + "i" (ASI_M_MXCC) : "g2", "g3"); +} + +static inline void mxcc_set_stream_dst(unsigned long *paddr) +{ + unsigned long data0 = paddr[0]; + unsigned long data1 = paddr[1]; + + __asm__ __volatile__ ("or %%g0, %0, %%g2\n\t" + "or %%g0, %1, %%g3\n\t" + "stda %%g2, [%2] %3\n\t" : : + "r" (data0), "r" (data1), + "r" (MXCC_DESSTREAM), + "i" (ASI_M_MXCC) : "g2", "g3"); +} + +static inline unsigned long mxcc_get_creg(void) +{ + unsigned long mxcc_control; + + __asm__ __volatile__("set 0xffffffff, %%g2\n\t" + "set 0xffffffff, %%g3\n\t" + "stda %%g2, [%1] %2\n\t" + "lda [%3] %2, %0\n\t" : + "=r" (mxcc_control) : + "r" (MXCC_EREG), "i" (ASI_M_MXCC), + "r" (MXCC_CREG) : "g2", "g3"); + return mxcc_control; +} + +static inline void mxcc_set_creg(unsigned long mxcc_control) +{ + __asm__ __volatile__("sta %0, [%1] %2\n\t" : : + "r" (mxcc_control), "r" (MXCC_CREG), + "i" (ASI_M_MXCC)); +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* !(_SPARC_MXCC_H) */ diff --git a/arch/sparc/include/asm/nmi.h b/arch/sparc/include/asm/nmi.h new file mode 100644 index 000000000..920dc23f4 --- /dev/null +++ b/arch/sparc/include/asm/nmi.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __NMI_H +#define __NMI_H + +int __init nmi_init(void); +void perfctr_irq(int irq, struct pt_regs *regs); +void nmi_adjust_hz(unsigned int new_hz); + +extern atomic_t nmi_active; + +void start_nmi_watchdog(void *unused); +void stop_nmi_watchdog(void *unused); + +#endif /* __NMI_H */ diff --git a/arch/sparc/include/asm/ns87303.h b/arch/sparc/include/asm/ns87303.h new file mode 100644 index 000000000..5401894a6 --- /dev/null +++ b/arch/sparc/include/asm/ns87303.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* ns87303.h: Configuration Register Description for the + * National Semiconductor PC87303 (SuperIO). + * + * Copyright (C) 1997 Eddie C. Dost (ecd@skynet.be) + */ + +#ifndef _SPARC_NS87303_H +#define _SPARC_NS87303_H 1 + +/* + * Control Register Index Values + */ +#define FER 0x00 +#define FAR 0x01 +#define PTR 0x02 +#define FCR 0x03 +#define PCR 0x04 +#define KRR 0x05 +#define PMC 0x06 +#define TUP 0x07 +#define SID 0x08 +#define ASC 0x09 +#define CS0CF0 0x0a +#define CS0CF1 0x0b +#define CS1CF0 0x0c +#define CS1CF1 0x0d + +/* Function Enable Register (FER) bits */ +#define FER_EDM 0x10 /* Encoded Drive and Motor pin information */ + +/* Function Address Register (FAR) bits */ +#define FAR_LPT_MASK 0x03 +#define FAR_LPTB 0x00 +#define FAR_LPTA 0x01 +#define FAR_LPTC 0x02 + +/* Power and Test Register (PTR) bits */ +#define PTR_LPTB_IRQ7 0x08 +#define PTR_LEVEL_IRQ 0x80 /* When not ECP/EPP: Use level IRQ */ +#define PTR_LPT_REG_DIR 0x80 /* When ECP/EPP: LPT CTR controls direction */ + /* of the parallel port */ + +/* Function Control Register (FCR) bits */ +#define FCR_LDE 0x10 /* Logical Drive Exchange */ +#define FCR_ZWS_ENA 0x20 /* Enable short host read/write in ECP/EPP */ + +/* Printer Control Register (PCR) bits */ +#define PCR_EPP_ENABLE 0x01 +#define PCR_EPP_IEEE 0x02 /* Enable EPP Version 1.9 (IEEE 1284) */ +#define PCR_ECP_ENABLE 0x04 +#define PCR_ECP_CLK_ENA 0x08 /* If 0 ECP Clock is stopped on Power down */ +#define PCR_IRQ_POLAR 0x20 /* If 0 IRQ is level high or negative pulse, */ + /* if 1 polarity is inverted */ +#define PCR_IRQ_ODRAIN 0x40 /* If 1, IRQ is open drain */ + +/* Tape UARTs and Parallel Port Config Register (TUP) bits */ +#define TUP_EPP_TIMO 0x02 /* Enable EPP timeout IRQ */ + +/* Advanced SuperIO Config Register (ASC) bits */ +#define ASC_LPT_IRQ7 0x01 /* Always use IRQ7 for LPT */ +#define ASC_DRV2_SEL 0x02 /* Logical Drive Exchange controlled by TDR */ + +#define FER_RESERVED 0x00 +#define FAR_RESERVED 0x00 +#define PTR_RESERVED 0x73 +#define FCR_RESERVED 0xc4 +#define PCR_RESERVED 0x10 +#define KRR_RESERVED 0x00 +#define PMC_RESERVED 0x98 +#define TUP_RESERVED 0xfb +#define SIP_RESERVED 0x00 +#define ASC_RESERVED 0x18 +#define CS0CF0_RESERVED 0x00 +#define CS0CF1_RESERVED 0x08 +#define CS1CF0_RESERVED 0x00 +#define CS1CF1_RESERVED 0x08 + +#ifdef __KERNEL__ + +#include + +#include + +extern spinlock_t ns87303_lock; + +static inline int ns87303_modify(unsigned long port, unsigned int index, + unsigned char clr, unsigned char set) +{ + static unsigned char reserved[] = { + FER_RESERVED, FAR_RESERVED, PTR_RESERVED, FCR_RESERVED, + PCR_RESERVED, KRR_RESERVED, PMC_RESERVED, TUP_RESERVED, + SIP_RESERVED, ASC_RESERVED, CS0CF0_RESERVED, CS0CF1_RESERVED, + CS1CF0_RESERVED, CS1CF1_RESERVED + }; + unsigned long flags; + unsigned char value; + + if (index > 0x0d) + return -EINVAL; + + spin_lock_irqsave(&ns87303_lock, flags); + + outb(index, port); + value = inb(port + 1); + value &= ~(reserved[index] | clr); + value |= set; + outb(value, port + 1); + outb(value, port + 1); + + spin_unlock_irqrestore(&ns87303_lock, flags); + + return 0; +} + +#endif /* __KERNEL__ */ + +#endif /* !(_SPARC_NS87303_H) */ diff --git a/arch/sparc/include/asm/obio.h b/arch/sparc/include/asm/obio.h new file mode 100644 index 000000000..f1ad7f7bc --- /dev/null +++ b/arch/sparc/include/asm/obio.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * obio.h: Some useful locations in 0xFXXXXXXXX PA obio space on sun4d. + * + * Copyright (C) 1997 Jakub Jelinek + */ + +#ifndef _SPARC_OBIO_H +#define _SPARC_OBIO_H + +#include + +/* This weird monster likes to use the very upper parts of + 36bit PA for these things :) */ + +/* CSR space (for each XDBUS) + * ------------------------------------------------------------------------ + * | 0xFE | DEVID | | XDBUS ID | | + * ------------------------------------------------------------------------ + * 35 28 27 20 19 10 9 8 7 0 + */ + +#define CSR_BASE_ADDR 0xe0000000 +#define CSR_CPU_SHIFT (32 - 4 - 5) +#define CSR_XDBUS_SHIFT 8 + +#define CSR_BASE(cpu) (((CSR_BASE_ADDR >> CSR_CPU_SHIFT) + cpu) << CSR_CPU_SHIFT) + +/* ECSR space (not for each XDBUS) + * ------------------------------------------------------------------------ + * | 0xF | DEVID[7:1] | | + * ------------------------------------------------------------------------ + * 35 32 31 25 24 0 + */ + +#define ECSR_BASE_ADDR 0x00000000 +#define ECSR_CPU_SHIFT (32 - 5) +#define ECSR_DEV_SHIFT (32 - 8) + +#define ECSR_BASE(cpu) ((cpu) << ECSR_CPU_SHIFT) +#define ECSR_DEV_BASE(devid) ((devid) << ECSR_DEV_SHIFT) + +/* Bus Watcher */ +#define BW_LOCAL_BASE 0xfff00000 + +#define BW_CID 0x00000000 +#define BW_DBUS_CTRL 0x00000008 +#define BW_DBUS_DATA 0x00000010 +#define BW_CTRL 0x00001000 +#define BW_INTR_TABLE 0x00001040 +#define BW_INTR_TABLE_CLEAR 0x00001080 +#define BW_PRESCALER 0x000010c0 +#define BW_PTIMER_LIMIT 0x00002000 +#define BW_PTIMER_COUNTER2 0x00002004 +#define BW_PTIMER_NDLIMIT 0x00002008 +#define BW_PTIMER_CTRL 0x0000200c +#define BW_PTIMER_COUNTER 0x00002010 +#define BW_TIMER_LIMIT 0x00003000 +#define BW_TIMER_COUNTER2 0x00003004 +#define BW_TIMER_NDLIMIT 0x00003008 +#define BW_TIMER_CTRL 0x0000300c +#define BW_TIMER_COUNTER 0x00003010 + +/* BW Control */ +#define BW_CTRL_USER_TIMER 0x00000004 /* Is User Timer Free run enabled */ + +/* Boot Bus */ +#define BB_LOCAL_BASE 0xf0000000 + +#define BB_STAT1 0x00100000 +#define BB_STAT2 0x00120000 +#define BB_STAT3 0x00140000 +#define BB_LEDS 0x002e0000 + +/* Bits in BB_STAT2 */ +#define BB_STAT2_AC_INTR 0x04 /* Aiee! 5ms and power is gone... */ +#define BB_STAT2_TMP_INTR 0x10 /* My Penguins are burning. Are you able to smell it? */ +#define BB_STAT2_FAN_INTR 0x20 /* My fan refuses to work */ +#define BB_STAT2_PWR_INTR 0x40 /* On SC2000, one of the two ACs died. Ok, we go on... */ +#define BB_STAT2_MASK (BB_STAT2_AC_INTR|BB_STAT2_TMP_INTR|BB_STAT2_FAN_INTR|BB_STAT2_PWR_INTR) + +/* Cache Controller */ +#define CC_BASE 0x1F00000 +#define CC_DATSTREAM 0x1F00000 /* Data stream register */ +#define CC_DATSIZE 0x1F0003F /* Size */ +#define CC_SRCSTREAM 0x1F00100 /* Source stream register */ +#define CC_DESSTREAM 0x1F00200 /* Destination stream register */ +#define CC_RMCOUNT 0x1F00300 /* Count of references and misses */ +#define CC_IPEN 0x1F00406 /* Pending Interrupts */ +#define CC_IMSK 0x1F00506 /* Interrupt Mask */ +#define CC_ICLR 0x1F00606 /* Clear pending Interrupts */ +#define CC_IGEN 0x1F00704 /* Generate Interrupt register */ +#define CC_STEST 0x1F00804 /* Internal self-test */ +#define CC_CREG 0x1F00A04 /* Control register */ +#define CC_SREG 0x1F00B00 /* Status register */ +#define CC_RREG 0x1F00C04 /* Reset register */ +#define CC_EREG 0x1F00E00 /* Error code register */ +#define CC_CID 0x1F00F04 /* Component ID */ + +#ifndef __ASSEMBLER__ + +static inline int bw_get_intr_mask(int sbus_level) +{ + int mask; + + __asm__ __volatile__ ("lduha [%1] %2, %0" : + "=r" (mask) : + "r" (BW_LOCAL_BASE + BW_INTR_TABLE + (sbus_level << 3)), + "i" (ASI_M_CTL)); + return mask; +} + +static inline void bw_clear_intr_mask(int sbus_level, int mask) +{ + __asm__ __volatile__ ("stha %0, [%1] %2" : : + "r" (mask), + "r" (BW_LOCAL_BASE + BW_INTR_TABLE_CLEAR + (sbus_level << 3)), + "i" (ASI_M_CTL)); +} + +static inline unsigned int bw_get_prof_limit(int cpu) +{ + unsigned int limit; + + __asm__ __volatile__ ("lda [%1] %2, %0" : + "=r" (limit) : + "r" (CSR_BASE(cpu) + BW_PTIMER_LIMIT), + "i" (ASI_M_CTL)); + return limit; +} + +static inline void bw_set_prof_limit(int cpu, unsigned int limit) +{ + __asm__ __volatile__ ("sta %0, [%1] %2" : : + "r" (limit), + "r" (CSR_BASE(cpu) + BW_PTIMER_LIMIT), + "i" (ASI_M_CTL)); +} + +static inline unsigned int bw_get_ctrl(int cpu) +{ + unsigned int ctrl; + + __asm__ __volatile__ ("lda [%1] %2, %0" : + "=r" (ctrl) : + "r" (CSR_BASE(cpu) + BW_CTRL), + "i" (ASI_M_CTL)); + return ctrl; +} + +static inline void bw_set_ctrl(int cpu, unsigned int ctrl) +{ + __asm__ __volatile__ ("sta %0, [%1] %2" : : + "r" (ctrl), + "r" (CSR_BASE(cpu) + BW_CTRL), + "i" (ASI_M_CTL)); +} + +static inline unsigned int cc_get_ipen(void) +{ + unsigned int pending; + + __asm__ __volatile__ ("lduha [%1] %2, %0" : + "=r" (pending) : + "r" (CC_IPEN), + "i" (ASI_M_MXCC)); + return pending; +} + +static inline void cc_set_iclr(unsigned int clear) +{ + __asm__ __volatile__ ("stha %0, [%1] %2" : : + "r" (clear), + "r" (CC_ICLR), + "i" (ASI_M_MXCC)); +} + +static inline unsigned int cc_get_imsk(void) +{ + unsigned int mask; + + __asm__ __volatile__ ("lduha [%1] %2, %0" : + "=r" (mask) : + "r" (CC_IMSK), + "i" (ASI_M_MXCC)); + return mask; +} + +static inline void cc_set_imsk(unsigned int mask) +{ + __asm__ __volatile__ ("stha %0, [%1] %2" : : + "r" (mask), + "r" (CC_IMSK), + "i" (ASI_M_MXCC)); +} + +static inline unsigned int cc_get_imsk_other(int cpuid) +{ + unsigned int mask; + + __asm__ __volatile__ ("lduha [%1] %2, %0" : + "=r" (mask) : + "r" (ECSR_BASE(cpuid) | CC_IMSK), + "i" (ASI_M_CTL)); + return mask; +} + +static inline void cc_set_imsk_other(int cpuid, unsigned int mask) +{ + __asm__ __volatile__ ("stha %0, [%1] %2" : : + "r" (mask), + "r" (ECSR_BASE(cpuid) | CC_IMSK), + "i" (ASI_M_CTL)); +} + +static inline void cc_set_igen(unsigned int gen) +{ + __asm__ __volatile__ ("sta %0, [%1] %2" : : + "r" (gen), + "r" (CC_IGEN), + "i" (ASI_M_MXCC)); +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* !(_SPARC_OBIO_H) */ diff --git a/arch/sparc/include/asm/opcodes.h b/arch/sparc/include/asm/opcodes.h new file mode 100644 index 000000000..ebfda6eb4 --- /dev/null +++ b/arch/sparc/include/asm/opcodes.h @@ -0,0 +1,100 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_ASM_OPCODES_H +#define _SPARC_ASM_OPCODES_H + +#define SPARC_CR_OPCODE_PRIORITY 300 + +#define F3F(x,y,z) (((x)<<30)|((y)<<19)|((z)<<5)) + +#define FPD_ENCODE(x) (((x) >> 5) | ((x) & ~(0x20))) + +#define RS1(x) (FPD_ENCODE(x) << 14) +#define RS2(x) (FPD_ENCODE(x) << 0) +#define RS3(x) (FPD_ENCODE(x) << 9) +#define RD(x) (FPD_ENCODE(x) << 25) +#define IMM5_0(x) ((x) << 0) +#define IMM5_9(x) ((x) << 9) + +#define CRC32C(a,b,c) \ + .word (F3F(2,0x36,0x147)|RS1(a)|RS2(b)|RD(c)); + +#define MD5 \ + .word 0x81b02800; +#define SHA1 \ + .word 0x81b02820; +#define SHA256 \ + .word 0x81b02840; +#define SHA512 \ + .word 0x81b02860; + +#define AES_EROUND01(a,b,c,d) \ + .word (F3F(2, 0x19, 0)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_EROUND23(a,b,c,d) \ + .word (F3F(2, 0x19, 1)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_DROUND01(a,b,c,d) \ + .word (F3F(2, 0x19, 2)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_DROUND23(a,b,c,d) \ + .word (F3F(2, 0x19, 3)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_EROUND01_L(a,b,c,d) \ + .word (F3F(2, 0x19, 4)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_EROUND23_L(a,b,c,d) \ + .word (F3F(2, 0x19, 5)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_DROUND01_L(a,b,c,d) \ + .word (F3F(2, 0x19, 6)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_DROUND23_L(a,b,c,d) \ + .word (F3F(2, 0x19, 7)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define AES_KEXPAND1(a,b,c,d) \ + .word (F3F(2, 0x19, 8)|RS1(a)|RS2(b)|IMM5_9(c)|RD(d)); +#define AES_KEXPAND0(a,b,c) \ + .word (F3F(2, 0x36, 0x130)|RS1(a)|RS2(b)|RD(c)); +#define AES_KEXPAND2(a,b,c) \ + .word (F3F(2, 0x36, 0x131)|RS1(a)|RS2(b)|RD(c)); + +#define DES_IP(a,b) \ + .word (F3F(2, 0x36, 0x134)|RS1(a)|RD(b)); +#define DES_IIP(a,b) \ + .word (F3F(2, 0x36, 0x135)|RS1(a)|RD(b)); +#define DES_KEXPAND(a,b,c) \ + .word (F3F(2, 0x36, 0x136)|RS1(a)|IMM5_0(b)|RD(c)); +#define DES_ROUND(a,b,c,d) \ + .word (F3F(2, 0x19, 0x009)|RS1(a)|RS2(b)|RS3(c)|RD(d)); + +#define CAMELLIA_F(a,b,c,d) \ + .word (F3F(2, 0x19, 0x00c)|RS1(a)|RS2(b)|RS3(c)|RD(d)); +#define CAMELLIA_FL(a,b,c) \ + .word (F3F(2, 0x36, 0x13c)|RS1(a)|RS2(b)|RD(c)); +#define CAMELLIA_FLI(a,b,c) \ + .word (F3F(2, 0x36, 0x13d)|RS1(a)|RS2(b)|RD(c)); + +#define MOVDTOX_F0_O4 \ + .word 0x99b02200 +#define MOVDTOX_F2_O5 \ + .word 0x9bb02202 +#define MOVXTOD_G1_F60 \ + .word 0xbbb02301 +#define MOVXTOD_G1_F62 \ + .word 0xbfb02301 +#define MOVXTOD_G3_F4 \ + .word 0x89b02303; +#define MOVXTOD_G7_F6 \ + .word 0x8db02307; +#define MOVXTOD_G3_F0 \ + .word 0x81b02303; +#define MOVXTOD_G7_F2 \ + .word 0x85b02307; +#define MOVXTOD_O0_F0 \ + .word 0x81b02308; +#define MOVXTOD_O5_F0 \ + .word 0x81b0230d; +#define MOVXTOD_O5_F2 \ + .word 0x85b0230d; +#define MOVXTOD_O5_F4 \ + .word 0x89b0230d; +#define MOVXTOD_O5_F6 \ + .word 0x8db0230d; +#define MOVXTOD_G3_F60 \ + .word 0xbbb02303; +#define MOVXTOD_G7_F62 \ + .word 0xbfb02307; + +#endif /* _SPARC_ASM_OPCODES_H */ diff --git a/arch/sparc/include/asm/openprom.h b/arch/sparc/include/asm/openprom.h new file mode 100644 index 000000000..ce68000df --- /dev/null +++ b/arch/sparc/include/asm/openprom.h @@ -0,0 +1,280 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_OPENPROM_H +#define __SPARC_OPENPROM_H + +/* openprom.h: Prom structures and defines for access to the OPENBOOT + * prom routines and data areas. + * + * Copyright (C) 1995,1996 David S. Miller (davem@caip.rutgers.edu) + */ + +/* Empirical constants... */ +#define LINUX_OPPROM_MAGIC 0x10010407 + +#ifndef __ASSEMBLER__ +#include + +/* V0 prom device operations. */ +struct linux_dev_v0_funcs { + int (*v0_devopen)(char *device_str); + int (*v0_devclose)(int dev_desc); + int (*v0_rdblkdev)(int dev_desc, int num_blks, int blk_st, char *buf); + int (*v0_wrblkdev)(int dev_desc, int num_blks, int blk_st, char *buf); + int (*v0_wrnetdev)(int dev_desc, int num_bytes, char *buf); + int (*v0_rdnetdev)(int dev_desc, int num_bytes, char *buf); + int (*v0_rdchardev)(int dev_desc, int num_bytes, int dummy, char *buf); + int (*v0_wrchardev)(int dev_desc, int num_bytes, int dummy, char *buf); + int (*v0_seekdev)(int dev_desc, long logical_offst, int from); +}; + +/* V2 and later prom device operations. */ +struct linux_dev_v2_funcs { + phandle (*v2_inst2pkg)(int d); /* Convert ihandle to phandle */ + char * (*v2_dumb_mem_alloc)(char *va, unsigned int sz); + void (*v2_dumb_mem_free)(char *va, unsigned int sz); + + /* To map devices into virtual I/O space. */ + char * (*v2_dumb_mmap)(char *virta, int which_io, unsigned int paddr, unsigned int sz); + void (*v2_dumb_munmap)(char *virta, unsigned int size); + + int (*v2_dev_open)(char *devpath); + void (*v2_dev_close)(int d); + int (*v2_dev_read)(int d, char *buf, int nbytes); + int (*v2_dev_write)(int d, const char *buf, int nbytes); + int (*v2_dev_seek)(int d, int hi, int lo); + + /* Never issued (multistage load support) */ + void (*v2_wheee2)(void); + void (*v2_wheee3)(void); +}; + +struct linux_mlist_v0 { + struct linux_mlist_v0 *theres_more; + unsigned int start_adr; + unsigned int num_bytes; +}; + +struct linux_mem_v0 { + struct linux_mlist_v0 **v0_totphys; + struct linux_mlist_v0 **v0_prommap; + struct linux_mlist_v0 **v0_available; /* What we can use */ +}; + +/* Arguments sent to the kernel from the boot prompt. */ +struct linux_arguments_v0 { + char *argv[8]; + char args[100]; + char boot_dev[2]; + int boot_dev_ctrl; + int boot_dev_unit; + int dev_partition; + char *kernel_file_name; + void *aieee1; /* XXX */ +}; + +/* V2 and up boot things. */ +struct linux_bootargs_v2 { + char **bootpath; + char **bootargs; + int *fd_stdin; + int *fd_stdout; +}; + +/* The top level PROM vector. */ +struct linux_romvec { + /* Version numbers. */ + unsigned int pv_magic_cookie; + unsigned int pv_romvers; + unsigned int pv_plugin_revision; + unsigned int pv_printrev; + + /* Version 0 memory descriptors. */ + struct linux_mem_v0 pv_v0mem; + + /* Node operations. */ + struct linux_nodeops *pv_nodeops; + + char **pv_bootstr; + struct linux_dev_v0_funcs pv_v0devops; + + char *pv_stdin; + char *pv_stdout; +#define PROMDEV_KBD 0 /* input from keyboard */ +#define PROMDEV_SCREEN 0 /* output to screen */ +#define PROMDEV_TTYA 1 /* in/out to ttya */ +#define PROMDEV_TTYB 2 /* in/out to ttyb */ + + /* Blocking getchar/putchar. NOT REENTRANT! (grr) */ + int (*pv_getchar)(void); + void (*pv_putchar)(int ch); + + /* Non-blocking variants. */ + int (*pv_nbgetchar)(void); + int (*pv_nbputchar)(int ch); + + void (*pv_putstr)(char *str, int len); + + /* Miscellany. */ + void (*pv_reboot)(char *bootstr); + void (*pv_printf)(__const__ char *fmt, ...); + void (*pv_abort)(void); + __volatile__ int *pv_ticks; + void (*pv_halt)(void); + void (**pv_synchook)(void); + + /* Evaluate a forth string, not different proto for V0 and V2->up. */ + union { + void (*v0_eval)(int len, char *str); + void (*v2_eval)(char *str); + } pv_fortheval; + + struct linux_arguments_v0 **pv_v0bootargs; + + /* Get ether address. */ + unsigned int (*pv_enaddr)(int d, char *enaddr); + + struct linux_bootargs_v2 pv_v2bootargs; + struct linux_dev_v2_funcs pv_v2devops; + + int filler[15]; + + /* This one is sun4c/sun4 only. */ + void (*pv_setctxt)(int ctxt, char *va, int pmeg); + + /* Prom version 3 Multiprocessor routines. This stuff is crazy. + * No joke. Calling these when there is only one cpu probably + * crashes the machine, have to test this. :-) + */ + + /* v3_cpustart() will start the cpu 'whichcpu' in mmu-context + * 'thiscontext' executing at address 'prog_counter' + */ + int (*v3_cpustart)(unsigned int whichcpu, int ctxtbl_ptr, + int thiscontext, char *prog_counter); + + /* v3_cpustop() will cause cpu 'whichcpu' to stop executing + * until a resume cpu call is made. + */ + int (*v3_cpustop)(unsigned int whichcpu); + + /* v3_cpuidle() will idle cpu 'whichcpu' until a stop or + * resume cpu call is made. + */ + int (*v3_cpuidle)(unsigned int whichcpu); + + /* v3_cpuresume() will resume processor 'whichcpu' executing + * starting with whatever 'pc' and 'npc' were left at the + * last 'idle' or 'stop' call. + */ + int (*v3_cpuresume)(unsigned int whichcpu); +}; + +/* Routines for traversing the prom device tree. */ +struct linux_nodeops { + phandle (*no_nextnode)(phandle node); + phandle (*no_child)(phandle node); + int (*no_proplen)(phandle node, const char *name); + int (*no_getprop)(phandle node, const char *name, char *val); + int (*no_setprop)(phandle node, const char *name, char *val, int len); + char * (*no_nextprop)(phandle node, char *name); +}; + +/* More fun PROM structures for device probing. */ +#if defined(__sparc__) && defined(__arch64__) +#define PROMREG_MAX 24 +#define PROMVADDR_MAX 16 +#define PROMINTR_MAX 32 +#else +#define PROMREG_MAX 16 +#define PROMVADDR_MAX 16 +#define PROMINTR_MAX 15 +#endif + +struct linux_prom_registers { + unsigned int which_io; /* hi part of physical address */ + unsigned int phys_addr; /* The physical address of this register */ + unsigned int reg_size; /* How many bytes does this register take up? */ +}; + +struct linux_prom64_registers { + unsigned long phys_addr; + unsigned long reg_size; +}; + +struct linux_prom_irqs { + int pri; /* IRQ priority */ + int vector; /* This is foobar, what does it do? */ +}; + +/* Element of the "ranges" vector */ +struct linux_prom_ranges { + unsigned int ot_child_space; + unsigned int ot_child_base; /* Bus feels this */ + unsigned int ot_parent_space; + unsigned int ot_parent_base; /* CPU looks from here */ + unsigned int or_size; +}; + +/* + * Ranges and reg properties are a bit different for PCI. + */ +#if defined(__sparc__) && defined(__arch64__) +struct linux_prom_pci_registers { + unsigned int phys_hi; + unsigned int phys_mid; + unsigned int phys_lo; + + unsigned int size_hi; + unsigned int size_lo; +}; +#else +struct linux_prom_pci_registers { + /* + * We don't know what information this field contain. + * We guess, PCI device function is in bits 15:8 + * So, ... + */ + unsigned int which_io; /* Let it be which_io */ + + unsigned int phys_hi; + unsigned int phys_lo; + + unsigned int size_hi; + unsigned int size_lo; +}; + +#endif + +struct linux_prom_pci_ranges { + unsigned int child_phys_hi; /* Only certain bits are encoded here. */ + unsigned int child_phys_mid; + unsigned int child_phys_lo; + + unsigned int parent_phys_hi; + unsigned int parent_phys_lo; + + unsigned int size_hi; + unsigned int size_lo; +}; + +struct linux_prom_pci_intmap { + unsigned int phys_hi; + unsigned int phys_mid; + unsigned int phys_lo; + + unsigned int interrupt; + + int cnode; + unsigned int cinterrupt; +}; + +struct linux_prom_pci_intmask { + unsigned int phys_hi; + unsigned int phys_mid; + unsigned int phys_lo; + unsigned int interrupt; +}; + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(__SPARC_OPENPROM_H) */ diff --git a/arch/sparc/include/asm/oplib.h b/arch/sparc/include/asm/oplib.h new file mode 100644 index 000000000..df49a7700 --- /dev/null +++ b/arch/sparc/include/asm/oplib.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_OPLIB_H +#define ___ASM_SPARC_OPLIB_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/oplib_32.h b/arch/sparc/include/asm/oplib_32.h new file mode 100644 index 000000000..d1cf3a27a --- /dev/null +++ b/arch/sparc/include/asm/oplib_32.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * oplib.h: Describes the interface and available routines in the + * Linux Prom library. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef __SPARC_OPLIB_H +#define __SPARC_OPLIB_H + +#include +#include +#include + +/* The master romvec pointer... */ +extern struct linux_romvec *romvec; + +/* Enumeration to describe the prom major version we have detected. */ +enum prom_major_version { + PROM_V0, /* Original sun4c V0 prom */ + PROM_V2, /* sun4c and early sun4m V2 prom */ + PROM_V3, /* sun4m and later, up to sun4d/sun4e machines V3 */ + PROM_P1275, /* IEEE compliant ISA based Sun PROM, only sun4u */ +}; + +extern enum prom_major_version prom_vers; +/* Revision, and firmware revision. */ +extern unsigned int prom_rev, prom_prev; + +/* Root node of the prom device tree, this stays constant after + * initialization is complete. + */ +extern phandle prom_root_node; + +/* Pointer to prom structure containing the device tree traversal + * and usage utility functions. Only prom-lib should use these, + * users use the interface defined by the library only! + */ +extern struct linux_nodeops *prom_nodeops; + +/* The functions... */ + +/* You must call prom_init() before using any of the library services, + * preferably as early as possible. Pass it the romvec pointer. + */ +void prom_init(struct linux_romvec *rom_ptr); + +/* Boot argument acquisition, returns the boot command line string. */ +char *prom_getbootargs(void); + +/* Miscellaneous routines, don't really fit in any category per se. */ + +/* Reboot the machine with the command line passed. */ +void prom_reboot(char *boot_command); + +/* Evaluate the forth string passed. */ +void prom_feval(char *forth_string); + +/* Enter the prom, with possibility of continuation with the 'go' + * command in newer proms. + */ +void prom_cmdline(void); + +/* Enter the prom, with no chance of continuation for the stand-alone + * which calls this. + */ +void __noreturn prom_halt(void); + +/* Set the PROM 'sync' callback function to the passed function pointer. + * When the user gives the 'sync' command at the prom prompt while the + * kernel is still active, the prom will call this routine. + * + * XXX The arguments are different on V0 vs. V2->higher proms, grrr! XXX + */ +typedef void (*sync_func_t)(void); +void prom_setsync(sync_func_t func_ptr); + +/* Acquire the IDPROM of the root node in the prom device tree. This + * gets passed a buffer where you would like it stuffed. The return value + * is the format type of this idprom or 0xff on error. + */ +unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); + +/* Get the prom major version. */ +int prom_version(void); + +/* Get the prom plugin revision. */ +int prom_getrev(void); + +/* Get the prom firmware revision. */ +int prom_getprev(void); + +/* Write a buffer of characters to the console. */ +void prom_console_write_buf(const char *buf, int len); + +/* Prom's internal routines, don't use in kernel/boot code. */ +__printf(1, 2) void prom_printf(const char *fmt, ...); +void prom_write(const char *buf, unsigned int len); + +/* Multiprocessor operations... */ + +/* Start the CPU with the given device tree node, context table, and context + * at the passed program counter. + */ +int prom_startcpu(int cpunode, struct linux_prom_registers *context_table, + int context, char *program_counter); + +/* Initialize the memory lists based upon the prom version. */ +void prom_meminit(void); + +/* PROM device tree traversal functions... */ + +/* Get the child node of the given node, or zero if no child exists. */ +phandle prom_getchild(phandle parent_node); + +/* Get the next sibling node of the given node, or zero if no further + * siblings exist. + */ +phandle prom_getsibling(phandle node); + +/* Get the length, at the passed node, of the given property type. + * Returns -1 on error (ie. no such property at this node). + */ +int prom_getproplen(phandle thisnode, const char *property); + +/* Fetch the requested property using the given buffer. Returns + * the number of bytes the prom put into your buffer or -1 on error. + */ +int __must_check prom_getproperty(phandle thisnode, const char *property, + char *prop_buffer, int propbuf_size); + +/* Acquire an integer property. */ +int prom_getint(phandle node, char *property); + +/* Acquire an integer property, with a default value. */ +int prom_getintdefault(phandle node, char *property, int defval); + +/* Acquire a boolean property, 0=FALSE 1=TRUE. */ +int prom_getbool(phandle node, char *prop); + +/* Acquire a string property, null string on error. */ +void prom_getstring(phandle node, char *prop, char *buf, int bufsize); + +/* Search all siblings starting at the passed node for "name" matching + * the given string. Returns the node on success, zero on failure. + */ +phandle prom_searchsiblings(phandle node_start, char *name); + +/* Returns the next property after the passed property for the given + * node. Returns null string on failure. + */ +char *prom_nextprop(phandle node, char *prev_property, char *buffer); + +/* Returns phandle of the path specified */ +phandle prom_finddevice(char *name); + +/* Set the indicated property at the given node with the passed value. + * Returns the number of bytes of your value that the prom took. + */ +int prom_setprop(phandle node, const char *prop_name, char *prop_value, + int value_size); + +phandle prom_inst2pkg(int); + +/* Dorking with Bus ranges... */ + +/* Apply promlib probes OBIO ranges to registers. */ +void prom_apply_obio_ranges(struct linux_prom_registers *obioregs, int nregs); + +/* Apply ranges of any prom node (and optionally parent node as well) to registers. */ +void prom_apply_generic_ranges(phandle node, phandle parent, + struct linux_prom_registers *sbusregs, int nregs); + +void prom_ranges_init(void); + +/* CPU probing helpers. */ +int cpu_find_by_instance(int instance, phandle *prom_node, int *mid); +int cpu_find_by_mid(int mid, phandle *prom_node); +int cpu_get_hwmid(phandle prom_node); + +extern spinlock_t prom_lock; + +#endif /* !(__SPARC_OPLIB_H) */ diff --git a/arch/sparc/include/asm/oplib_64.h b/arch/sparc/include/asm/oplib_64.h new file mode 100644 index 000000000..1b86d02a8 --- /dev/null +++ b/arch/sparc/include/asm/oplib_64.h @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* oplib.h: Describes the interface and available routines in the + * Linux Prom library. + * + * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#ifndef __SPARC64_OPLIB_H +#define __SPARC64_OPLIB_H + +#include + +/* OBP version string. */ +extern char prom_version[]; + +/* Root node of the prom device tree, this stays constant after + * initialization is complete. + */ +extern phandle prom_root_node; + +/* PROM stdout */ +extern int prom_stdout; + +/* /chosen node of the prom device tree, this stays constant after + * initialization is complete. + */ +extern phandle prom_chosen_node; + +/* Helper values and strings in arch/sparc64/kernel/head.S */ +extern const char prom_peer_name[]; +extern const char prom_compatible_name[]; +extern const char prom_root_compatible[]; +extern const char prom_cpu_compatible[]; +extern const char prom_finddev_name[]; +extern const char prom_chosen_path[]; +extern const char prom_cpu_path[]; +extern const char prom_getprop_name[]; +extern const char prom_mmu_name[]; +extern const char prom_callmethod_name[]; +extern const char prom_translate_name[]; +extern const char prom_map_name[]; +extern const char prom_unmap_name[]; +extern int prom_mmu_ihandle_cache; +extern unsigned int prom_boot_mapped_pc; +extern unsigned int prom_boot_mapping_mode; +extern unsigned long prom_boot_mapping_phys_high, prom_boot_mapping_phys_low; + +struct linux_mlist_p1275 { + struct linux_mlist_p1275 *theres_more; + unsigned long start_adr; + unsigned long num_bytes; +}; + +struct linux_mem_p1275 { + struct linux_mlist_p1275 **p1275_totphys; + struct linux_mlist_p1275 **p1275_prommap; + struct linux_mlist_p1275 **p1275_available; /* What we can use */ +}; + +/* The functions... */ + +/* You must call prom_init() before using any of the library services, + * preferably as early as possible. Pass it the romvec pointer. + */ +void prom_init(void *cif_handler); +void prom_init_report(void); + +/* Boot argument acquisition, returns the boot command line string. */ +char *prom_getbootargs(void); + +/* Miscellaneous routines, don't really fit in any category per se. */ + +/* Reboot the machine with the command line passed. */ +void prom_reboot(const char *boot_command); + +/* Evaluate the forth string passed. */ +void prom_feval(const char *forth_string); + +/* Enter the prom, with possibility of continuation with the 'go' + * command in newer proms. + */ +void prom_cmdline(void); + +/* Enter the prom, with no chance of continuation for the stand-alone + * which calls this. + */ +void prom_halt(void) __attribute__ ((noreturn)); + +/* Halt and power-off the machine. */ +void prom_halt_power_off(void) __attribute__ ((noreturn)); + +/* Acquire the IDPROM of the root node in the prom device tree. This + * gets passed a buffer where you would like it stuffed. The return value + * is the format type of this idprom or 0xff on error. + */ +unsigned char prom_get_idprom(char *idp_buffer, int idpbuf_size); + +/* Write a buffer of characters to the console. */ +void prom_console_write_buf(const char *buf, int len); + +/* Prom's internal routines, don't use in kernel/boot code. */ +__printf(1, 2) void prom_printf(const char *fmt, ...); +void prom_write(const char *buf, unsigned int len); + +/* Multiprocessor operations... */ +#ifdef CONFIG_SMP +/* Start the CPU with the given device tree node at the passed program + * counter with the given arg passed in via register %o0. + */ +void prom_startcpu(int cpunode, unsigned long pc, unsigned long arg); + +/* Start the CPU with the given cpu ID at the passed program + * counter with the given arg passed in via register %o0. + */ +void prom_startcpu_cpuid(int cpuid, unsigned long pc, unsigned long arg); + +/* Stop the CPU with the given cpu ID. */ +void prom_stopcpu_cpuid(int cpuid); + +/* Stop the current CPU. */ +void prom_stopself(void); + +/* Idle the current CPU. */ +void prom_idleself(void); + +/* Resume the CPU with the passed device tree node. */ +void prom_resumecpu(int cpunode); +#endif + +/* Power management interfaces. */ + +/* Put the current CPU to sleep. */ +void prom_sleepself(void); + +/* Put the entire system to sleep. */ +int prom_sleepsystem(void); + +/* Initiate a wakeup event. */ +int prom_wakeupsystem(void); + +/* MMU and memory related OBP interfaces. */ + +/* Get unique string identifying SIMM at given physical address. */ +int prom_getunumber(int syndrome_code, + unsigned long phys_addr, + char *buf, int buflen); + +/* Retain physical memory to the caller across soft resets. */ +int prom_retain(const char *name, unsigned long size, + unsigned long align, unsigned long *paddr); + +/* Load explicit I/D TLB entries into the calling processor. */ +long prom_itlb_load(unsigned long index, + unsigned long tte_data, + unsigned long vaddr); + +long prom_dtlb_load(unsigned long index, + unsigned long tte_data, + unsigned long vaddr); + +/* Map/Unmap client program address ranges. First the format of + * the mapping mode argument. + */ +#define PROM_MAP_WRITE 0x0001 /* Writable */ +#define PROM_MAP_READ 0x0002 /* Readable - sw */ +#define PROM_MAP_EXEC 0x0004 /* Executable - sw */ +#define PROM_MAP_LOCKED 0x0010 /* Locked, use i/dtlb load calls for this instead */ +#define PROM_MAP_CACHED 0x0020 /* Cacheable in both L1 and L2 caches */ +#define PROM_MAP_SE 0x0040 /* Side-Effects */ +#define PROM_MAP_GLOB 0x0080 /* Global */ +#define PROM_MAP_IE 0x0100 /* Invert-Endianness */ +#define PROM_MAP_DEFAULT (PROM_MAP_WRITE | PROM_MAP_READ | PROM_MAP_EXEC | PROM_MAP_CACHED) + +int prom_map(int mode, unsigned long size, + unsigned long vaddr, unsigned long paddr); +void prom_unmap(unsigned long size, unsigned long vaddr); + + +/* PROM device tree traversal functions... */ + +/* Get the child node of the given node, or zero if no child exists. */ +phandle prom_getchild(phandle parent_node); + +/* Get the next sibling node of the given node, or zero if no further + * siblings exist. + */ +phandle prom_getsibling(phandle node); + +/* Get the length, at the passed node, of the given property type. + * Returns -1 on error (ie. no such property at this node). + */ +int prom_getproplen(phandle thisnode, const char *property); + +/* Fetch the requested property using the given buffer. Returns + * the number of bytes the prom put into your buffer or -1 on error. + */ +int prom_getproperty(phandle thisnode, const char *property, + char *prop_buffer, int propbuf_size); + +/* Acquire an integer property. */ +int prom_getint(phandle node, const char *property); + +/* Acquire an integer property, with a default value. */ +int prom_getintdefault(phandle node, const char *property, int defval); + +/* Acquire a boolean property, 0=FALSE 1=TRUE. */ +int prom_getbool(phandle node, const char *prop); + +/* Acquire a string property, null string on error. */ +void prom_getstring(phandle node, const char *prop, char *buf, + int bufsize); + +/* Does the passed node have the given "name"? YES=1 NO=0 */ +int prom_nodematch(phandle thisnode, const char *name); + +/* Search all siblings starting at the passed node for "name" matching + * the given string. Returns the node on success, zero on failure. + */ +phandle prom_searchsiblings(phandle node_start, const char *name); + +/* Return the first property type, as a string, for the given node. + * Returns a null string on error. Buffer should be at least 32B long. + */ +char *prom_firstprop(phandle node, char *buffer); + +/* Returns the next property after the passed property for the given + * node. Returns null string on failure. Buffer should be at least 32B long. + */ +char *prom_nextprop(phandle node, const char *prev_property, char *buf); + +/* Returns 1 if the specified node has given property. */ +int prom_node_has_property(phandle node, const char *property); + +/* Returns phandle of the path specified */ +phandle prom_finddevice(const char *name); + +/* Set the indicated property at the given node with the passed value. + * Returns the number of bytes of your value that the prom took. + */ +int prom_setprop(phandle node, const char *prop_name, char *prop_value, + int value_size); + +phandle prom_inst2pkg(int); +void prom_sun4v_guest_soft_state(void); + +int prom_ihandle2path(int handle, char *buffer, int bufsize); + +/* Client interface level routines. */ +void prom_cif_init(void *cif_handler); +void p1275_cmd_direct(unsigned long *); + +#endif /* !(__SPARC64_OPLIB_H) */ diff --git a/arch/sparc/include/asm/page.h b/arch/sparc/include/asm/page.h new file mode 100644 index 000000000..1a00cc0a1 --- /dev/null +++ b/arch/sparc/include/asm/page.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_PAGE_H +#define ___ASM_SPARC_PAGE_H + +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/page_32.h b/arch/sparc/include/asm/page_32.h new file mode 100644 index 000000000..9f0b54f70 --- /dev/null +++ b/arch/sparc/include/asm/page_32.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * page.h: Various defines and such for MMU operations on the Sparc for + * the Linux kernel. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_PAGE_H +#define _SPARC_PAGE_H + +#include + +#include + +#ifndef __ASSEMBLER__ + +#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) +#define copy_page(to,from) memcpy((void *)(to), (void *)(from), PAGE_SIZE) +#define clear_user_page(addr, vaddr, page) \ + do { clear_page(addr); \ + sparc_flush_page_to_ram(page); \ + (void)(vaddr); \ + } while (0) +#define copy_user_page(to, from, vaddr, page) \ + do { copy_page(to, from); \ + sparc_flush_page_to_ram(page); \ + (void)(vaddr); \ + } while (0) + +/* The following structure is used to hold the physical + * memory configuration of the machine. This is filled in + * prom_meminit() and is later used by mem_init() to set up + * mem_map[]. We statically allocate SPARC_PHYS_BANKS+1 of + * these structs, this is arbitrary. The entry after the + * last valid one has num_bytes==0. + */ +struct sparc_phys_banks { + unsigned long base_addr; + unsigned long num_bytes; +}; + +#define SPARC_PHYS_BANKS 32 + +extern struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1]; + +/* passing structs on the Sparc slow us down tremendously... */ + +/* #define STRICT_MM_TYPECHECKS */ + +#ifdef STRICT_MM_TYPECHECKS +/* + * These are used to make use of C type-checking.. + */ +typedef struct { unsigned long pte; } pte_t; +typedef struct { unsigned long iopte; } iopte_t; +typedef struct { unsigned long pmd; } pmd_t; +typedef struct { unsigned long pgd; } pgd_t; +typedef struct { unsigned long ctxd; } ctxd_t; +typedef struct { unsigned long pgprot; } pgprot_t; +typedef struct { unsigned long iopgprot; } iopgprot_t; + +#define pte_val(x) ((x).pte) +#define iopte_val(x) ((x).iopte) +#define pmd_val(x) ((x).pmd) +#define pgd_val(x) ((x).pgd) +#define ctxd_val(x) ((x).ctxd) +#define pgprot_val(x) ((x).pgprot) +#define iopgprot_val(x) ((x).iopgprot) + +#define __pte(x) ((pte_t) { (x) } ) +#define __pmd(x) ((pmd_t) { { (x) }, }) +#define __iopte(x) ((iopte_t) { (x) } ) +#define __pgd(x) ((pgd_t) { (x) } ) +#define __ctxd(x) ((ctxd_t) { (x) } ) +#define __pgprot(x) ((pgprot_t) { (x) } ) +#define __iopgprot(x) ((iopgprot_t) { (x) } ) + +#else +/* + * .. while these make it easier on the compiler + */ +typedef unsigned long pte_t; +typedef unsigned long iopte_t; +typedef unsigned long pmd_t; +typedef unsigned long pgd_t; +typedef unsigned long ctxd_t; +typedef unsigned long pgprot_t; +typedef unsigned long iopgprot_t; + +#define pte_val(x) (x) +#define iopte_val(x) (x) +#define pmd_val(x) (x) +#define pgd_val(x) (x) +#define ctxd_val(x) (x) +#define pgprot_val(x) (x) +#define iopgprot_val(x) (x) + +#define __pte(x) (x) +#define __pmd(x) (x) +#define __iopte(x) (x) +#define __pgd(x) (x) +#define __ctxd(x) (x) +#define __pgprot(x) (x) +#define __iopgprot(x) (x) + +#endif + +typedef pte_t *pgtable_t; + +#define TASK_UNMAPPED_BASE 0x50000000 + +#else /* !(__ASSEMBLER__) */ + +#define __pgprot(x) (x) + +#endif /* !(__ASSEMBLER__) */ + +#define PAGE_OFFSET 0xf0000000 +#ifndef __ASSEMBLER__ +extern unsigned long phys_base; +extern unsigned long pfn_base; +#endif +#define __pa(x) ((unsigned long)(x) - PAGE_OFFSET + phys_base) +#define __va(x) ((void *)((unsigned long) (x) - phys_base + PAGE_OFFSET)) + +#define virt_to_phys __pa +#define phys_to_virt __va + +#define ARCH_PFN_OFFSET (pfn_base) +#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT) + +#define virt_addr_valid(kaddr) ((((unsigned long)(kaddr)-PAGE_OFFSET)>>PAGE_SHIFT) < max_mapnr) + +#include +#include + +#endif /* _SPARC_PAGE_H */ diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h new file mode 100644 index 000000000..fd4dc85fb --- /dev/null +++ b/arch/sparc/include/asm/page_64.h @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_PAGE_H +#define _SPARC64_PAGE_H + +#include + +#include + +/* Flushing for D-cache alias handling is only needed if + * the page size is smaller than 16K. + */ +#if PAGE_SHIFT < 14 +#define DCACHE_ALIASING_POSSIBLE +#endif + +#define HPAGE_SHIFT 23 +#define REAL_HPAGE_SHIFT 22 +#define HPAGE_16GB_SHIFT 34 +#define HPAGE_2GB_SHIFT 31 +#define HPAGE_256MB_SHIFT 28 +#define HPAGE_64K_SHIFT 16 +#define REAL_HPAGE_SIZE (_AC(1,UL) << REAL_HPAGE_SHIFT) + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +#define HPAGE_SIZE (_AC(1,UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1UL)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) +#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA +#define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT)) +#define HUGE_MAX_HSTATE 5 +#endif + +#ifndef __ASSEMBLER__ + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +struct pt_regs; +void hugetlb_setup(struct pt_regs *regs); +#endif + +#define WANT_PAGE_VIRTUAL + +void _clear_page(void *page); +#define clear_page(X) _clear_page((void *)(X)) +struct page; +void clear_user_page(void *addr, unsigned long vaddr, struct page *page); +#define clear_user_page clear_user_page +#define copy_page(X,Y) memcpy((void *)(X), (void *)(Y), PAGE_SIZE) +void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topage); +#define __HAVE_ARCH_COPY_USER_HIGHPAGE +struct vm_area_struct; +void copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma); +#define __HAVE_ARCH_COPY_HIGHPAGE +void copy_highpage(struct page *to, struct page *from); + +/* Unlike sparc32, sparc64's parameter passing API is more + * sane in that structures which as small enough are passed + * in registers instead of on the stack. Thus, setting + * STRICT_MM_TYPECHECKS does not generate worse code so + * let's enable it to get the type checking. + */ + +#define STRICT_MM_TYPECHECKS + +#ifdef STRICT_MM_TYPECHECKS +/* These are used to make use of C type-checking.. */ +typedef struct { unsigned long pte; } pte_t; +typedef struct { unsigned long iopte; } iopte_t; +typedef struct { unsigned long pmd; } pmd_t; +typedef struct { unsigned long pud; } pud_t; +typedef struct { unsigned long pgd; } pgd_t; +typedef struct { unsigned long pgprot; } pgprot_t; + +#define pte_val(x) ((x).pte) +#define iopte_val(x) ((x).iopte) +#define pmd_val(x) ((x).pmd) +#define pud_val(x) ((x).pud) +#define pgd_val(x) ((x).pgd) +#define pgprot_val(x) ((x).pgprot) + +#define __pte(x) ((pte_t) { (x) } ) +#define __iopte(x) ((iopte_t) { (x) } ) +#define __pmd(x) ((pmd_t) { (x) } ) +#define __pud(x) ((pud_t) { (x) } ) +#define __pgd(x) ((pgd_t) { (x) } ) +#define __pgprot(x) ((pgprot_t) { (x) } ) + +#else +/* .. while these make it easier on the compiler */ +typedef unsigned long pte_t; +typedef unsigned long iopte_t; +typedef unsigned long pmd_t; +typedef unsigned long pud_t; +typedef unsigned long pgd_t; +typedef unsigned long pgprot_t; + +#define pte_val(x) (x) +#define iopte_val(x) (x) +#define pmd_val(x) (x) +#define pud_val(x) (x) +#define pgd_val(x) (x) +#define pgprot_val(x) (x) + +#define __pte(x) (x) +#define __iopte(x) (x) +#define __pmd(x) (x) +#define __pud(x) (x) +#define __pgd(x) (x) +#define __pgprot(x) (x) + +#endif /* (STRICT_MM_TYPECHECKS) */ + +typedef pte_t *pgtable_t; + +extern unsigned long sparc64_va_hole_top; +extern unsigned long sparc64_va_hole_bottom; + +/* The next two defines specify the actual exclusion region we + * enforce, wherein we use a 4GB red zone on each side of the VA hole. + */ +#define VA_EXCLUDE_START (sparc64_va_hole_bottom - (1UL << 32UL)) +#define VA_EXCLUDE_END (sparc64_va_hole_top + (1UL << 32UL)) + +#define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ + _AC(0x0000000070000000,UL) : \ + VA_EXCLUDE_END) + +#include + +extern unsigned long PAGE_OFFSET; + +#endif /* !(__ASSEMBLER__) */ + +/* The maximum number of physical memory address bits we support. The + * largest value we can support is whatever "KPGD_SHIFT + KPTE_BITS" + * evaluates to. + */ +#define MAX_PHYS_ADDRESS_BITS 53 + +#define ILOG2_4MB 22 +#define ILOG2_256MB 28 + +#ifndef __ASSEMBLER__ + +#define __pa(x) ((unsigned long)(x) - PAGE_OFFSET) +#define __va(x) ((void *)((unsigned long) (x) + PAGE_OFFSET)) + +#define pfn_to_kaddr(pfn) __va((pfn) << PAGE_SHIFT) + +#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr)>>PAGE_SHIFT) + +#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) + +#define virt_to_phys __pa +#define phys_to_virt __va + +#endif /* !(__ASSEMBLER__) */ + +#include + +#endif /* _SPARC64_PAGE_H */ diff --git a/arch/sparc/include/asm/parport.h b/arch/sparc/include/asm/parport.h new file mode 100644 index 000000000..e2eed8f97 --- /dev/null +++ b/arch/sparc/include/asm/parport.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_PARPORT_H +#define ___ASM_SPARC_PARPORT_H + +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif + diff --git a/arch/sparc/include/asm/parport_64.h b/arch/sparc/include/asm/parport_64.h new file mode 100644 index 000000000..78f14d662 --- /dev/null +++ b/arch/sparc/include/asm/parport_64.h @@ -0,0 +1,255 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* parport.h: sparc64 specific parport initialization and dma. + * + * Copyright (C) 1999 Eddie C. Dost (ecd@skynet.be) + */ + +#ifndef _ASM_SPARC64_PARPORT_H +#define _ASM_SPARC64_PARPORT_H 1 + +#include +#include +#include + +#include +#include +#include + +#define PARPORT_PC_MAX_PORTS PARPORT_MAX + +/* + * While sparc64 doesn't have an ISA DMA API, we provide something that looks + * close enough to make parport_pc happy + */ +#define HAS_DMA + +#ifdef CONFIG_PARPORT_PC_FIFO +static DEFINE_SPINLOCK(dma_spin_lock); + +#define claim_dma_lock() \ +({ unsigned long flags; \ + spin_lock_irqsave(&dma_spin_lock, flags); \ + flags; \ +}) + +#define release_dma_lock(__flags) \ + spin_unlock_irqrestore(&dma_spin_lock, __flags); +#endif + +static struct sparc_ebus_info { + struct ebus_dma_info info; + unsigned int addr; + unsigned int count; + int lock; + + struct parport *port; +} sparc_ebus_dmas[PARPORT_PC_MAX_PORTS]; + +static DECLARE_BITMAP(dma_slot_map, PARPORT_PC_MAX_PORTS); + +static inline int request_dma(unsigned int dmanr, const char *device_id) +{ + if (dmanr >= PARPORT_PC_MAX_PORTS) + return -EINVAL; + if (xchg(&sparc_ebus_dmas[dmanr].lock, 1) != 0) + return -EBUSY; + return 0; +} + +static inline void free_dma(unsigned int dmanr) +{ + if (dmanr >= PARPORT_PC_MAX_PORTS) { + printk(KERN_WARNING "Trying to free DMA%d\n", dmanr); + return; + } + if (xchg(&sparc_ebus_dmas[dmanr].lock, 0) == 0) { + printk(KERN_WARNING "Trying to free free DMA%d\n", dmanr); + return; + } +} + +static inline void enable_dma(unsigned int dmanr) +{ + ebus_dma_enable(&sparc_ebus_dmas[dmanr].info, 1); + + if (ebus_dma_request(&sparc_ebus_dmas[dmanr].info, + sparc_ebus_dmas[dmanr].addr, + sparc_ebus_dmas[dmanr].count)) + BUG(); +} + +static inline void disable_dma(unsigned int dmanr) +{ + ebus_dma_enable(&sparc_ebus_dmas[dmanr].info, 0); +} + +static inline void clear_dma_ff(unsigned int dmanr) +{ + /* nothing */ +} + +static inline void set_dma_mode(unsigned int dmanr, char mode) +{ + ebus_dma_prepare(&sparc_ebus_dmas[dmanr].info, (mode != DMA_MODE_WRITE)); +} + +static inline void set_dma_addr(unsigned int dmanr, unsigned int addr) +{ + sparc_ebus_dmas[dmanr].addr = addr; +} + +static inline void set_dma_count(unsigned int dmanr, unsigned int count) +{ + sparc_ebus_dmas[dmanr].count = count; +} + +static inline unsigned int get_dma_residue(unsigned int dmanr) +{ + return ebus_dma_residue(&sparc_ebus_dmas[dmanr].info); +} + +static int ecpp_probe(struct platform_device *op) +{ + unsigned long base = op->resource[0].start; + unsigned long config = op->resource[1].start; + unsigned long d_base = op->resource[2].start; + unsigned long d_len; + struct device_node *parent; + struct parport *p; + int slot, err; + + parent = op->dev.of_node->parent; + if (of_node_name_eq(parent, "dma")) { + p = parport_pc_probe_port(base, base + 0x400, + op->archdata.irqs[0], PARPORT_DMA_NOFIFO, + op->dev.parent->parent, 0); + if (!p) + return -ENOMEM; + dev_set_drvdata(&op->dev, p); + return 0; + } + + for (slot = 0; slot < PARPORT_PC_MAX_PORTS; slot++) { + if (!test_and_set_bit(slot, dma_slot_map)) + break; + } + err = -ENODEV; + if (slot >= PARPORT_PC_MAX_PORTS) + goto out_err; + + spin_lock_init(&sparc_ebus_dmas[slot].info.lock); + + d_len = (op->resource[2].end - d_base) + 1UL; + sparc_ebus_dmas[slot].info.regs = + of_ioremap(&op->resource[2], 0, d_len, "ECPP DMA"); + + if (!sparc_ebus_dmas[slot].info.regs) + goto out_clear_map; + + sparc_ebus_dmas[slot].info.flags = 0; + sparc_ebus_dmas[slot].info.callback = NULL; + sparc_ebus_dmas[slot].info.client_cookie = NULL; + sparc_ebus_dmas[slot].info.irq = 0xdeadbeef; + strscpy(sparc_ebus_dmas[slot].info.name, "parport"); + if (ebus_dma_register(&sparc_ebus_dmas[slot].info)) + goto out_unmap_regs; + + ebus_dma_irq_enable(&sparc_ebus_dmas[slot].info, 1); + + /* Configure IRQ to Push Pull, Level Low */ + /* Enable ECP, set bit 2 of the CTR first */ + outb(0x04, base + 0x02); + ns87303_modify(config, PCR, + PCR_EPP_ENABLE | + PCR_IRQ_ODRAIN, + PCR_ECP_ENABLE | + PCR_ECP_CLK_ENA | + PCR_IRQ_POLAR); + + /* CTR bit 5 controls direction of port */ + ns87303_modify(config, PTR, + 0, PTR_LPT_REG_DIR); + + p = parport_pc_probe_port(base, base + 0x400, + op->archdata.irqs[0], + slot, + op->dev.parent, + 0); + err = -ENOMEM; + if (!p) + goto out_disable_irq; + + dev_set_drvdata(&op->dev, p); + + return 0; + +out_disable_irq: + ebus_dma_irq_enable(&sparc_ebus_dmas[slot].info, 0); + ebus_dma_unregister(&sparc_ebus_dmas[slot].info); + +out_unmap_regs: + of_iounmap(&op->resource[2], sparc_ebus_dmas[slot].info.regs, d_len); + +out_clear_map: + clear_bit(slot, dma_slot_map); + +out_err: + return err; +} + +static void ecpp_remove(struct platform_device *op) +{ + struct parport *p = dev_get_drvdata(&op->dev); + int slot = p->dma; + + parport_pc_unregister_port(p); + + if (slot != PARPORT_DMA_NOFIFO) { + unsigned long d_base = op->resource[2].start; + unsigned long d_len; + + d_len = (op->resource[2].end - d_base) + 1UL; + + ebus_dma_irq_enable(&sparc_ebus_dmas[slot].info, 0); + ebus_dma_unregister(&sparc_ebus_dmas[slot].info); + of_iounmap(&op->resource[2], + sparc_ebus_dmas[slot].info.regs, + d_len); + clear_bit(slot, dma_slot_map); + } +} + +static const struct of_device_id ecpp_match[] = { + { + .name = "ecpp", + }, + { + .name = "parallel", + .compatible = "ecpp", + }, + { + .name = "parallel", + .compatible = "ns87317-ecpp", + }, + { + .name = "parallel", + .compatible = "pnpALI,1533,3", + }, + {}, +}; + +static struct platform_driver ecpp_driver = { + .driver = { + .name = "ecpp", + .of_match_table = ecpp_match, + }, + .probe = ecpp_probe, + .remove = ecpp_remove, +}; + +static int parport_pc_find_nonpci_ports(int autoirq, int autodma) +{ + return platform_driver_register(&ecpp_driver); +} + +#endif /* !(_ASM_SPARC64_PARPORT_H */ diff --git a/arch/sparc/include/asm/pbm.h b/arch/sparc/include/asm/pbm.h new file mode 100644 index 000000000..0c8626172 --- /dev/null +++ b/arch/sparc/include/asm/pbm.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * + * pbm.h: PCI bus module pseudo driver software state + * Adopted from sparc64 by V. Roganov and G. Raiko + * + * Original header: + * pbm.h: U2P PCI bus module pseudo driver software state. + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + * + * To put things into perspective, consider sparc64 with a few PCI controllers. + * Each type would have an own structure, with instances related one to one. + * We have only pcic on sparc, but we want to be compatible with sparc64 pbm.h. + * All three represent different abstractions. + * pci_bus - Linux PCI subsystem view of a PCI bus (including bridged buses) + * pbm - Arch-specific view of a PCI bus (sparc or sparc64) + * pcic - Chip-specific information for PCIC. + */ + +#ifndef __SPARC_PBM_H +#define __SPARC_PBM_H + +#include +#include +#include + +struct linux_pbm_info { + int prom_node; + char prom_name[64]; + /* struct linux_prom_pci_ranges pbm_ranges[PROMREG_MAX]; */ + /* int num_pbm_ranges; */ + + /* Now things for the actual PCI bus probes. */ + unsigned int pci_first_busno; /* Can it be nonzero? */ + struct pci_bus *pci_bus; /* Was inline, MJ allocs now */ +}; + +/* PCI devices which are not bridges have this placed in their pci_dev + * sysdata member. This makes OBP aware PCI device drivers easier to + * code. + */ +struct pcidev_cookie { + struct linux_pbm_info *pbm; + struct device_node *prom_node; +}; + +#endif /* !(__SPARC_PBM_H) */ diff --git a/arch/sparc/include/asm/pci.h b/arch/sparc/include/asm/pci.h new file mode 100644 index 000000000..1419aa84d --- /dev/null +++ b/arch/sparc/include/asm/pci.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_PCI_H +#define ___ASM_SPARC_PCI_H + + +/* Can be used to override the logic in pci_scan_bus for skipping + * already-configured bus numbers - to be used for buggy BIOSes + * or architectures with incomplete PCI setup by the loader. + */ +#define pcibios_assign_all_busses() 0 + +#define PCIBIOS_MIN_IO 0UL +#define PCIBIOS_MIN_MEM 0UL + +#define PCI_IRQ_NONE 0xffffffff + + +#ifdef CONFIG_SPARC64 + +/* PCI IOMMU mapping bypass support. */ + +/* PCI 64-bit addressing works for all slots on all controller + * types on sparc64. However, it requires that the device + * can drive enough of the 64 bits. + */ +#define PCI64_REQUIRED_MASK (~(u64)0) +#define PCI64_ADDR_BASE 0xfffc000000000000UL + +/* Return the index of the PCI controller for device PDEV. */ +int pci_domain_nr(struct pci_bus *bus); +static inline int pci_proc_domain(struct pci_bus *bus) +{ + return 1; +} + +/* Platform support for /proc/bus/pci/X/Y mmap()s. */ +#define HAVE_PCI_MMAP +#define arch_can_pci_mmap_io() 1 +#define HAVE_ARCH_PCI_GET_UNMAPPED_AREA +#define ARCH_GENERIC_PCI_MMAP_RESOURCE +#define get_pci_unmapped_area get_fb_unmapped_area +#endif /* CONFIG_SPARC64 */ + +#endif /* ___ASM_SPARC_PCI_H */ diff --git a/arch/sparc/include/asm/pcic.h b/arch/sparc/include/asm/pcic.h new file mode 100644 index 000000000..fb5ed6a59 --- /dev/null +++ b/arch/sparc/include/asm/pcic.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * pcic.h: JavaEngine 1 specific PCI definitions. + * + * Copyright (C) 1998 V. Roganov and G. Raiko + */ + +#ifndef __SPARC_PCIC_H +#define __SPARC_PCIC_H + +#ifndef __ASSEMBLER__ + +#include +#include +#include +#include +#include + +struct linux_pcic { + void __iomem *pcic_regs; + unsigned long pcic_io; + void __iomem *pcic_config_space_addr; + void __iomem *pcic_config_space_data; + struct resource pcic_res_regs; + struct resource pcic_res_io; + struct resource pcic_res_cfg_addr; + struct resource pcic_res_cfg_data; + struct linux_pbm_info pbm; + struct pcic_ca2irq *pcic_imap; + int pcic_imdim; +}; + +#ifdef CONFIG_PCIC_PCI +int pcic_present(void); +int pcic_probe(void); +void pci_time_init(void); +void sun4m_pci_init_IRQ(void); +#else +static inline int pcic_present(void) { return 0; } +static inline int pcic_probe(void) { return 0; } +static inline void pci_time_init(void) {} +static inline void sun4m_pci_init_IRQ(void) {} +#endif +#endif + +/* Size of PCI I/O space which we relocate. */ +#define PCI_SPACE_SIZE 0x1000000 /* 16 MB */ + +/* PCIC Register Set. */ +#define PCI_DIAGNOSTIC_0 0x40 /* 32 bits */ +#define PCI_SIZE_0 0x44 /* 32 bits */ +#define PCI_SIZE_1 0x48 /* 32 bits */ +#define PCI_SIZE_2 0x4c /* 32 bits */ +#define PCI_SIZE_3 0x50 /* 32 bits */ +#define PCI_SIZE_4 0x54 /* 32 bits */ +#define PCI_SIZE_5 0x58 /* 32 bits */ +#define PCI_PIO_CONTROL 0x60 /* 8 bits */ +#define PCI_DVMA_CONTROL 0x62 /* 8 bits */ +#define PCI_DVMA_CONTROL_INACTIVITY_REQ (1<<0) +#define PCI_DVMA_CONTROL_IOTLB_ENABLE (1<<0) +#define PCI_DVMA_CONTROL_IOTLB_DISABLE 0 +#define PCI_DVMA_CONTROL_INACTIVITY_ACK (1<<4) +#define PCI_INTERRUPT_CONTROL 0x63 /* 8 bits */ +#define PCI_CPU_INTERRUPT_PENDING 0x64 /* 32 bits */ +#define PCI_DIAGNOSTIC_1 0x68 /* 16 bits */ +#define PCI_SOFTWARE_INT_CLEAR 0x6a /* 16 bits */ +#define PCI_SOFTWARE_INT_SET 0x6e /* 16 bits */ +#define PCI_SYS_INT_PENDING 0x70 /* 32 bits */ +#define PCI_SYS_INT_PENDING_PIO 0x40000000 +#define PCI_SYS_INT_PENDING_DMA 0x20000000 +#define PCI_SYS_INT_PENDING_PCI 0x10000000 +#define PCI_SYS_INT_PENDING_APSR 0x08000000 +#define PCI_SYS_INT_TARGET_MASK 0x74 /* 32 bits */ +#define PCI_SYS_INT_TARGET_MASK_CLEAR 0x78 /* 32 bits */ +#define PCI_SYS_INT_TARGET_MASK_SET 0x7c /* 32 bits */ +#define PCI_SYS_INT_PENDING_CLEAR 0x83 /* 8 bits */ +#define PCI_SYS_INT_PENDING_CLEAR_ALL 0x80 +#define PCI_SYS_INT_PENDING_CLEAR_PIO 0x40 +#define PCI_SYS_INT_PENDING_CLEAR_DMA 0x20 +#define PCI_SYS_INT_PENDING_CLEAR_PCI 0x10 +#define PCI_IOTLB_CONTROL 0x84 /* 8 bits */ +#define PCI_INT_SELECT_LO 0x88 /* 16 bits */ +#define PCI_ARBITRATION_SELECT 0x8a /* 16 bits */ +#define PCI_INT_SELECT_HI 0x8c /* 16 bits */ +#define PCI_HW_INT_OUTPUT 0x8e /* 16 bits */ +#define PCI_IOTLB_RAM_INPUT 0x90 /* 32 bits */ +#define PCI_IOTLB_CAM_INPUT 0x94 /* 32 bits */ +#define PCI_IOTLB_RAM_OUTPUT 0x98 /* 32 bits */ +#define PCI_IOTLB_CAM_OUTPUT 0x9c /* 32 bits */ +#define PCI_SMBAR0 0xa0 /* 8 bits */ +#define PCI_MSIZE0 0xa1 /* 8 bits */ +#define PCI_PMBAR0 0xa2 /* 8 bits */ +#define PCI_SMBAR1 0xa4 /* 8 bits */ +#define PCI_MSIZE1 0xa5 /* 8 bits */ +#define PCI_PMBAR1 0xa6 /* 8 bits */ +#define PCI_SIBAR 0xa8 /* 8 bits */ +#define PCI_SIBAR_ADDRESS_MASK 0xf +#define PCI_ISIZE 0xa9 /* 8 bits */ +#define PCI_ISIZE_16M 0xf +#define PCI_ISIZE_32M 0xe +#define PCI_ISIZE_64M 0xc +#define PCI_ISIZE_128M 0x8 +#define PCI_ISIZE_256M 0x0 +#define PCI_PIBAR 0xaa /* 8 bits */ +#define PCI_CPU_COUNTER_LIMIT_HI 0xac /* 32 bits */ +#define PCI_CPU_COUNTER_LIMIT_LO 0xb0 /* 32 bits */ +#define PCI_CPU_COUNTER_LIMIT 0xb4 /* 32 bits */ +#define PCI_SYS_LIMIT 0xb8 /* 32 bits */ +#define PCI_SYS_COUNTER 0xbc /* 32 bits */ +#define PCI_SYS_COUNTER_OVERFLOW (1<<31) /* Limit reached */ +#define PCI_SYS_LIMIT_PSEUDO 0xc0 /* 32 bits */ +#define PCI_USER_TIMER_CONTROL 0xc4 /* 8 bits */ +#define PCI_USER_TIMER_CONFIG 0xc5 /* 8 bits */ +#define PCI_COUNTER_IRQ 0xc6 /* 8 bits */ +#define PCI_COUNTER_IRQ_SET(sys_irq, cpu_irq) ((((sys_irq) & 0xf) << 4) | \ + ((cpu_irq) & 0xf)) +#define PCI_COUNTER_IRQ_SYS(v) (((v) >> 4) & 0xf) +#define PCI_COUNTER_IRQ_CPU(v) ((v) & 0xf) +#define PCI_PIO_ERROR_COMMAND 0xc7 /* 8 bits */ +#define PCI_PIO_ERROR_ADDRESS 0xc8 /* 32 bits */ +#define PCI_IOTLB_ERROR_ADDRESS 0xcc /* 32 bits */ +#define PCI_SYS_STATUS 0xd0 /* 8 bits */ +#define PCI_SYS_STATUS_RESET_ENABLE (1<<0) +#define PCI_SYS_STATUS_RESET (1<<1) +#define PCI_SYS_STATUS_WATCHDOG_RESET (1<<4) +#define PCI_SYS_STATUS_PCI_RESET (1<<5) +#define PCI_SYS_STATUS_PCI_RESET_ENABLE (1<<6) +#define PCI_SYS_STATUS_PCI_SATTELITE_MODE (1<<7) + +#endif /* !(__SPARC_PCIC_H) */ diff --git a/arch/sparc/include/asm/pcr.h b/arch/sparc/include/asm/pcr.h new file mode 100644 index 000000000..da834ffbe --- /dev/null +++ b/arch/sparc/include/asm/pcr.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PCR_H +#define __PCR_H + +struct pcr_ops { + u64 (*read_pcr)(unsigned long); + void (*write_pcr)(unsigned long, u64); + u64 (*read_pic)(unsigned long); + void (*write_pic)(unsigned long, u64); + u64 (*nmi_picl_value)(unsigned int nmi_hz); + u64 pcr_nmi_enable; + u64 pcr_nmi_disable; +}; +extern const struct pcr_ops *pcr_ops; + +void deferred_pcr_work_irq(int irq, struct pt_regs *regs); +void schedule_deferred_pcr_work(void); + +#define PCR_PIC_PRIV 0x00000001 /* PIC access is privileged */ +#define PCR_STRACE 0x00000002 /* Trace supervisor events */ +#define PCR_UTRACE 0x00000004 /* Trace user events */ +#define PCR_N2_HTRACE 0x00000008 /* Trace hypervisor events */ +#define PCR_N2_TOE_OV0 0x00000010 /* Trap if PIC 0 overflows */ +#define PCR_N2_TOE_OV1 0x00000020 /* Trap if PIC 1 overflows */ +#define PCR_N2_MASK0 0x00003fc0 +#define PCR_N2_MASK0_SHIFT 6 +#define PCR_N2_SL0 0x0003c000 +#define PCR_N2_SL0_SHIFT 14 +#define PCR_N2_OV0 0x00040000 +#define PCR_N2_MASK1 0x07f80000 +#define PCR_N2_MASK1_SHIFT 19 +#define PCR_N2_SL1 0x78000000 +#define PCR_N2_SL1_SHIFT 27 +#define PCR_N2_OV1 0x80000000 + +#define PCR_N4_OV 0x00000001 /* PIC overflow */ +#define PCR_N4_TOE 0x00000002 /* Trap On Event */ +#define PCR_N4_UTRACE 0x00000004 /* Trace user events */ +#define PCR_N4_STRACE 0x00000008 /* Trace supervisor events */ +#define PCR_N4_HTRACE 0x00000010 /* Trace hypervisor events */ +#define PCR_N4_MASK 0x000007e0 /* Event mask */ +#define PCR_N4_MASK_SHIFT 5 +#define PCR_N4_SL 0x0000f800 /* Event Select */ +#define PCR_N4_SL_SHIFT 11 +#define PCR_N4_PICNPT 0x00010000 /* PIC non-privileged trap */ +#define PCR_N4_PICNHT 0x00020000 /* PIC non-hypervisor trap */ +#define PCR_N4_NTC 0x00040000 /* Next-To-Commit wrap */ + +int pcr_arch_init(void); + +#endif /* __PCR_H */ diff --git a/arch/sparc/include/asm/percpu.h b/arch/sparc/include/asm/percpu.h new file mode 100644 index 000000000..0cd704ef5 --- /dev/null +++ b/arch/sparc/include/asm/percpu.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_PERCPU_H +#define ___ASM_SPARC_PERCPU_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/percpu_32.h b/arch/sparc/include/asm/percpu_32.h new file mode 100644 index 000000000..ee6c7c193 --- /dev/null +++ b/arch/sparc/include/asm/percpu_32.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ARCH_SPARC_PERCPU__ +#define __ARCH_SPARC_PERCPU__ + +#include + +#endif /* __ARCH_SPARC_PERCPU__ */ diff --git a/arch/sparc/include/asm/percpu_64.h b/arch/sparc/include/asm/percpu_64.h new file mode 100644 index 000000000..a8786a4b9 --- /dev/null +++ b/arch/sparc/include/asm/percpu_64.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ARCH_SPARC64_PERCPU__ +#define __ARCH_SPARC64_PERCPU__ + +#include + +#ifndef BUILD_VDSO +register unsigned long __local_per_cpu_offset asm("g5"); +#endif + +#ifdef CONFIG_SMP + +#include + +#define __per_cpu_offset(__cpu) \ + (trap_block[(__cpu)].__per_cpu_base) +#define per_cpu_offset(x) (__per_cpu_offset(x)) + +#define __my_cpu_offset __local_per_cpu_offset + +#else /* ! SMP */ + +#endif /* SMP */ + +#include + +#endif /* __ARCH_SPARC64_PERCPU__ */ diff --git a/arch/sparc/include/asm/perf_event.h b/arch/sparc/include/asm/perf_event.h new file mode 100644 index 000000000..c2aec0c7f --- /dev/null +++ b/arch/sparc/include/asm/perf_event.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SPARC_PERF_EVENT_H +#define __ASM_SPARC_PERF_EVENT_H + +#ifdef CONFIG_PERF_EVENTS +#include + +#define perf_arch_fetch_caller_regs(regs, ip) \ +do { \ + unsigned long _pstate, _asi, _pil, _i7, _fp; \ + __asm__ __volatile__("rdpr %%pstate, %0\n\t" \ + "rd %%asi, %1\n\t" \ + "rdpr %%pil, %2\n\t" \ + "mov %%i7, %3\n\t" \ + "mov %%i6, %4\n\t" \ + : "=r" (_pstate), \ + "=r" (_asi), \ + "=r" (_pil), \ + "=r" (_i7), \ + "=r" (_fp)); \ + (regs)->tstate = (_pstate << 8) | \ + (_asi << 24) | (_pil << 20); \ + (regs)->tpc = (ip); \ + (regs)->tnpc = (regs)->tpc + 4; \ + (regs)->u_regs[UREG_I6] = _fp; \ + (regs)->u_regs[UREG_I7] = _i7; \ +} while (0) +#endif + +#endif diff --git a/arch/sparc/include/asm/pgalloc.h b/arch/sparc/include/asm/pgalloc.h new file mode 100644 index 000000000..9ea0b3758 --- /dev/null +++ b/arch/sparc/include/asm/pgalloc.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_PGALLOC_H +#define ___ASM_SPARC_PGALLOC_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h new file mode 100644 index 000000000..4f73e87b2 --- /dev/null +++ b/arch/sparc/include/asm/pgalloc_32.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_PGALLOC_H +#define _SPARC_PGALLOC_H + +#include +#include +#include + +#include +#include +#include + +struct page; + +void *srmmu_get_nocache(int size, int align); +void srmmu_free_nocache(void *addr, int size); + +extern struct resource sparc_iomap; + +pgd_t *get_pgd_fast(void); +static inline void free_pgd_fast(pgd_t *pgd) +{ + srmmu_free_nocache(pgd, SRMMU_PGD_TABLE_SIZE); +} + +#define pgd_free(mm, pgd) free_pgd_fast(pgd) +#define pgd_alloc(mm) get_pgd_fast() + +static inline void pud_set(pud_t * pudp, pmd_t * pmdp) +{ + unsigned long pa = __nocache_pa(pmdp); + + set_pte((pte_t *)pudp, __pte((SRMMU_ET_PTD | (pa >> 4)))); +} + +#define pud_populate(MM, PGD, PMD) pud_set(PGD, PMD) + +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, + unsigned long address) +{ + return srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, + SRMMU_PMD_TABLE_SIZE); +} + +static inline void free_pmd_fast(pmd_t * pmd) +{ + srmmu_free_nocache(pmd, SRMMU_PMD_TABLE_SIZE); +} + +#define pmd_free(mm, pmd) free_pmd_fast(pmd) +#define __pmd_free_tlb(tlb, pmd, addr) pmd_free((tlb)->mm, pmd) + +#define pmd_populate(mm, pmd, pte) pmd_set(pmd, pte) + +void pmd_set(pmd_t *pmdp, pte_t *ptep); +#define pmd_populate_kernel pmd_populate + +pgtable_t pte_alloc_one(struct mm_struct *mm); + +static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) +{ + return srmmu_get_nocache(SRMMU_PTE_TABLE_SIZE, + SRMMU_PTE_TABLE_SIZE); +} + + +static inline void free_pte_fast(pte_t *pte) +{ + srmmu_free_nocache(pte, SRMMU_PTE_TABLE_SIZE); +} + +#define pte_free_kernel(mm, pte) free_pte_fast(pte) + +void pte_free(struct mm_struct * mm, pgtable_t pte); +#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) + +#endif /* _SPARC_PGALLOC_H */ diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h new file mode 100644 index 000000000..caa7632be --- /dev/null +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -0,0 +1,118 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_PGALLOC_H +#define _SPARC64_PGALLOC_H + +#include +#include +#include +#include + +#include +#include +#include +#include + +/* Page table allocation/freeing. */ + +extern struct kmem_cache *pgtable_cache; + +static inline void __p4d_populate(p4d_t *p4d, pud_t *pud) +{ + p4d_set(p4d, pud); +} + +#define p4d_populate(MM, P4D, PUD) __p4d_populate(P4D, PUD) + +static inline pgd_t *pgd_alloc(struct mm_struct *mm) +{ + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); +} + +static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) +{ + kmem_cache_free(pgtable_cache, pgd); +} + +static inline void __pud_populate(pud_t *pud, pmd_t *pmd) +{ + pud_set(pud, pmd); +} + +#define pud_populate(MM, PUD, PMD) __pud_populate(PUD, PMD) + +static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); +} + +static inline void pud_free(struct mm_struct *mm, pud_t *pud) +{ + kmem_cache_free(pgtable_cache, pud); +} + +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); +} + +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +{ + kmem_cache_free(pgtable_cache, pmd); +} + +pte_t *pte_alloc_one_kernel(struct mm_struct *mm); +pgtable_t pte_alloc_one(struct mm_struct *mm); +void pte_free_kernel(struct mm_struct *mm, pte_t *pte); +void pte_free(struct mm_struct *mm, pgtable_t ptepage); + +/* arch use pte_free_defer() implementation in arch/sparc/mm/init_64.c */ +#define pte_free_defer pte_free_defer +void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable); + +#define pmd_populate_kernel(MM, PMD, PTE) pmd_set(MM, PMD, PTE) +#define pmd_populate(MM, PMD, PTE) pmd_set(MM, PMD, PTE) + +void pgtable_free(void *table, bool is_page); + +#ifdef CONFIG_SMP + +struct mmu_gather; +void tlb_remove_table(struct mmu_gather *, void *); + +static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) +{ + unsigned long pgf = (unsigned long)table; + if (is_page) + pgf |= 0x1UL; + tlb_remove_table(tlb, (void *)pgf); +} + +static inline void __tlb_remove_table(void *_table) +{ + void *table = (void *)((unsigned long)_table & ~0x1UL); + bool is_page = false; + + if ((unsigned long)_table & 0x1UL) + is_page = true; + pgtable_free(table, is_page); +} +#else /* CONFIG_SMP */ +static inline void pgtable_free_tlb(struct mmu_gather *tlb, void *table, bool is_page) +{ + pgtable_free(table, is_page); +} +#endif /* !CONFIG_SMP */ + +static inline void __pte_free_tlb(struct mmu_gather *tlb, pte_t *pte, + unsigned long address) +{ + pgtable_free_tlb(tlb, pte, true); +} + +#define __pmd_free_tlb(tlb, pmd, addr) \ + pgtable_free_tlb(tlb, pmd, false) + +#define __pud_free_tlb(tlb, pud, addr) \ + pgtable_free_tlb(tlb, pud, false) + +#endif /* _SPARC64_PGALLOC_H */ diff --git a/arch/sparc/include/asm/pgtable.h b/arch/sparc/include/asm/pgtable.h new file mode 100644 index 000000000..e48891170 --- /dev/null +++ b/arch/sparc/include/asm/pgtable.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_PGTABLE_H +#define ___ASM_SPARC_PGTABLE_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h new file mode 100644 index 000000000..f89b12506 --- /dev/null +++ b/arch/sparc/include/asm/pgtable_32.h @@ -0,0 +1,426 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_PGTABLE_H +#define _SPARC_PGTABLE_H + +/* asm/pgtable.h: Defines and functions used to work + * with Sparc page tables. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include + +#define PMD_SHIFT 18 +#define PMD_SIZE (1UL << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE-1)) +#define PMD_ALIGN(__addr) (((__addr) + ~PMD_MASK) & PMD_MASK) + +#define PGDIR_SHIFT 24 +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE-1)) +#define PGDIR_ALIGN(__addr) (((__addr) + ~PGDIR_MASK) & PGDIR_MASK) + +#ifndef __ASSEMBLER__ +#include + +#include +#include +#include +#include +#include +#include +#include + + +struct vm_area_struct; +struct page; + +void load_mmu(void); +unsigned long calc_highpages(void); +unsigned long __init bootmem_init(unsigned long *pages_avail); + +#define pte_ERROR(e) __builtin_trap() +#define pmd_ERROR(e) __builtin_trap() +#define pgd_ERROR(e) __builtin_trap() + +#define PTRS_PER_PTE 64 +#define PTRS_PER_PMD 64 +#define PTRS_PER_PGD 256 +#define USER_PTRS_PER_PGD PAGE_OFFSET / PGDIR_SIZE +#define PTE_SIZE (PTRS_PER_PTE*4) + +#define PAGE_NONE SRMMU_PAGE_NONE +#define PAGE_SHARED SRMMU_PAGE_SHARED +#define PAGE_COPY SRMMU_PAGE_COPY +#define PAGE_READONLY SRMMU_PAGE_RDONLY +#define PAGE_KERNEL SRMMU_PAGE_KERNEL + +/* Top-level page directory - dummy used by init-mm. + * srmmu.c will assign the real one (which is dynamically sized) */ +#define swapper_pg_dir NULL + +void paging_init(void); + +extern unsigned long ptr_in_current_pgd; + +/* First physical page can be anywhere, the following is needed so that + * va-->pa and vice versa conversions work properly without performance + * hit for all __pa()/__va() operations. + */ +extern unsigned long phys_base; +extern unsigned long pfn_base; + +/* + * In general all page table modifications should use the V8 atomic + * swap instruction. This insures the mmu and the cpu are in sync + * with respect to ref/mod bits in the page tables. + */ +static inline unsigned long srmmu_swap(unsigned long *addr, unsigned long value) +{ + __asm__ __volatile__("swap [%2], %0" : + "=&r" (value) : "0" (value), "r" (addr) : "memory"); + return value; +} + +/* Certain architectures need to do special things when pte's + * within a page table are directly modified. Thus, the following + * hook is made available. + */ + +static inline void set_pte(pte_t *ptep, pte_t pteval) +{ + srmmu_swap((unsigned long *)ptep, pte_val(pteval)); +} + +static inline int srmmu_device_memory(unsigned long x) +{ + return ((x & 0xF0000000) != 0); +} + +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + return (pmd_val(pmd) & SRMMU_PTD_PMASK) >> (PAGE_SHIFT-4); +} + +static inline struct page *pmd_page(pmd_t pmd) +{ + if (srmmu_device_memory(pmd_val(pmd))) + BUG(); + return pfn_to_page(pmd_pfn(pmd)); +} + +static inline unsigned long __pmd_page(pmd_t pmd) +{ + unsigned long v; + + if (srmmu_device_memory(pmd_val(pmd))) + BUG(); + + v = pmd_val(pmd) & SRMMU_PTD_PMASK; + return (unsigned long)__nocache_va(v << 4); +} + +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + unsigned long v = pmd_val(pmd) & SRMMU_PTD_PMASK; + return (unsigned long)__nocache_va(v << 4); +} + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + if (srmmu_device_memory(pud_val(pud))) { + return (pmd_t *)~0; + } else { + unsigned long v = pud_val(pud) & SRMMU_PTD_PMASK; + return (pmd_t *)__nocache_va(v << 4); + } +} + +static inline int pte_present(pte_t pte) +{ + return ((pte_val(pte) & SRMMU_ET_MASK) == SRMMU_ET_PTE); +} + +static inline int pte_none(pte_t pte) +{ + return !pte_val(pte); +} + +static inline void __pte_clear(pte_t *ptep) +{ + set_pte(ptep, __pte(0)); +} + +static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +{ + __pte_clear(ptep); +} + +static inline int pmd_bad(pmd_t pmd) +{ + return (pmd_val(pmd) & SRMMU_ET_MASK) != SRMMU_ET_PTD; +} + +static inline int pmd_present(pmd_t pmd) +{ + return ((pmd_val(pmd) & SRMMU_ET_MASK) == SRMMU_ET_PTD); +} + +static inline int pmd_none(pmd_t pmd) +{ + return !pmd_val(pmd); +} + +static inline void pmd_clear(pmd_t *pmdp) +{ + set_pte((pte_t *)&pmd_val(*pmdp), __pte(0)); +} + +static inline int pud_none(pud_t pud) +{ + return !(pud_val(pud) & 0xFFFFFFF); +} + +static inline int pud_bad(pud_t pud) +{ + return (pud_val(pud) & SRMMU_ET_MASK) != SRMMU_ET_PTD; +} + +static inline int pud_present(pud_t pud) +{ + return ((pud_val(pud) & SRMMU_ET_MASK) == SRMMU_ET_PTD); +} + +static inline void pud_clear(pud_t *pudp) +{ + set_pte((pte_t *)pudp, __pte(0)); +} + +/* + * The following only work if pte_present() is true. + * Undefined behaviour if not.. + */ +static inline int pte_write(pte_t pte) +{ + return pte_val(pte) & SRMMU_WRITE; +} + +static inline int pte_dirty(pte_t pte) +{ + return pte_val(pte) & SRMMU_DIRTY; +} + +static inline int pte_young(pte_t pte) +{ + return pte_val(pte) & SRMMU_REF; +} + +static inline pte_t pte_wrprotect(pte_t pte) +{ + return __pte(pte_val(pte) & ~SRMMU_WRITE); +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + return __pte(pte_val(pte) & ~SRMMU_DIRTY); +} + +static inline pte_t pte_mkold(pte_t pte) +{ + return __pte(pte_val(pte) & ~SRMMU_REF); +} + +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + return __pte(pte_val(pte) | SRMMU_WRITE); +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + return __pte(pte_val(pte) | SRMMU_DIRTY); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + return __pte(pte_val(pte) | SRMMU_REF); +} + +#define PFN_PTE_SHIFT (PAGE_SHIFT - 4) + +static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) +{ + return __pte((pfn << PFN_PTE_SHIFT) | pgprot_val(pgprot)); +} + +static inline unsigned long pte_pfn(pte_t pte) +{ + if (srmmu_device_memory(pte_val(pte))) { + /* Just return something that will cause + * pfn_valid() to return false. This makes + * copy_one_pte() to just directly copy to + * PTE over. + */ + return ~0UL; + } + return (pte_val(pte) & SRMMU_PTE_PMASK) >> PFN_PTE_SHIFT; +} + +#define pte_page(pte) pfn_to_page(pte_pfn(pte)) + +static inline pte_t mk_pte_phys(unsigned long page, pgprot_t pgprot) +{ + return __pte(((page) >> 4) | pgprot_val(pgprot)); +} + +static inline pte_t mk_pte_io(unsigned long page, pgprot_t pgprot, int space) +{ + return __pte(((page) >> 4) | (space << 28) | pgprot_val(pgprot)); +} + +#define pgprot_noncached pgprot_noncached +static inline pgprot_t pgprot_noncached(pgprot_t prot) +{ + pgprot_val(prot) &= ~pgprot_val(__pgprot(SRMMU_CACHE)); + return prot; +} + +static pte_t pte_modify(pte_t pte, pgprot_t newprot) __attribute_const__; +static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) +{ + return __pte((pte_val(pte) & SRMMU_CHG_MASK) | + pgprot_val(newprot)); +} + +/* only used by the huge vmap code, should never be called */ +#define pud_page(pud) NULL + +struct seq_file; +void mmu_info(struct seq_file *m); + +/* Fault handler stuff... */ +#define FAULT_CODE_PROT 0x1 +#define FAULT_CODE_WRITE 0x2 +#define FAULT_CODE_USER 0x4 + +#define update_mmu_cache(vma, address, ptep) do { } while (0) +#define update_mmu_cache_range(vmf, vma, address, ptep, nr) do { } while (0) + +void srmmu_mapiorange(unsigned int bus, unsigned long xpa, + unsigned long xva, unsigned int len); +void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len); + +/* + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTEs: + * + * 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * <-------------- offset ---------------> < type -> E 0 0 0 0 0 0 + */ +static inline unsigned long __swp_type(swp_entry_t entry) +{ + return (entry.val >> SRMMU_SWP_TYPE_SHIFT) & SRMMU_SWP_TYPE_MASK; +} + +static inline unsigned long __swp_offset(swp_entry_t entry) +{ + return (entry.val >> SRMMU_SWP_OFF_SHIFT) & SRMMU_SWP_OFF_MASK; +} + +static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset) +{ + return (swp_entry_t) { + (type & SRMMU_SWP_TYPE_MASK) << SRMMU_SWP_TYPE_SHIFT + | (offset & SRMMU_SWP_OFF_MASK) << SRMMU_SWP_OFF_SHIFT }; +} + +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) +#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) + +static inline bool pte_swp_exclusive(pte_t pte) +{ + return pte_val(pte) & SRMMU_SWP_EXCLUSIVE; +} + +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + return __pte(pte_val(pte) | SRMMU_SWP_EXCLUSIVE); +} + +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return __pte(pte_val(pte) & ~SRMMU_SWP_EXCLUSIVE); +} + +static inline unsigned long +__get_phys (unsigned long addr) +{ + switch (sparc_cpu_model){ + case sun4m: + case sun4d: + return ((srmmu_get_pte (addr) & 0xffffff00) << 4); + default: + return 0; + } +} + +static inline int +__get_iospace (unsigned long addr) +{ + switch (sparc_cpu_model){ + case sun4m: + case sun4d: + return (srmmu_get_pte (addr) >> 28); + default: + return -1; + } +} + +/* + * For sparc32&64, the pfn in io_remap_pfn_range() carries in + * its high 4 bits. These macros/functions put it there or get it from there. + */ +#define MK_IOSPACE_PFN(space, pfn) (pfn | (space << (BITS_PER_LONG - 4))) +#define GET_IOSPACE(pfn) (pfn >> (BITS_PER_LONG - 4)) +#define GET_PFN(pfn) (pfn & 0x0fffffffUL) + +static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, + unsigned long size) +{ + unsigned long long offset, space, phys_base; + + offset = ((unsigned long long) GET_PFN(pfn)) << PAGE_SHIFT; + space = GET_IOSPACE(pfn); + phys_base = offset | (space << 32ULL); + + return phys_base >> PAGE_SHIFT; +} +#define io_remap_pfn_range_pfn io_remap_pfn_range_pfn + +#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS +#define ptep_set_access_flags(__vma, __address, __ptep, __entry, __dirty) \ +({ \ + int __changed = !pte_same(*(__ptep), __entry); \ + if (__changed) { \ + set_pte(__ptep, __entry); \ + flush_tlb_page(__vma, __address); \ + } \ + __changed; \ +}) + +#endif /* !(__ASSEMBLER__) */ + +#define VMALLOC_START _AC(0xfe600000,UL) +#define VMALLOC_END _AC(0xffc00000,UL) +#define MODULES_VADDR VMALLOC_START +#define MODULES_END VMALLOC_END + +/* We provide our own get_unmapped_area to cope with VA holes for userland */ +#define HAVE_ARCH_UNMAPPED_AREA + +#define pmd_pgtable(pmd) ((pgtable_t)__pmd_page(pmd)) + +#endif /* !(_SPARC_PGTABLE_H) */ diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h new file mode 100644 index 000000000..0837ebbc5 --- /dev/null +++ b/arch/sparc/include/asm/pgtable_64.h @@ -0,0 +1,1171 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * pgtable.h: SpitFire page table operations. + * + * Copyright 1996,1997 David S. Miller (davem@caip.rutgers.edu) + * Copyright 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#ifndef _SPARC64_PGTABLE_H +#define _SPARC64_PGTABLE_H + +/* This file contains the functions and defines necessary to modify and use + * the SpitFire page tables. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* The kernel image occupies 0x4000000 to 0x6000000 (4MB --> 96MB). + * The page copy blockops can use 0x6000000 to 0x8000000. + * The 8K TSB is mapped in the 0x8000000 to 0x8400000 range. + * The 4M TSB is mapped in the 0x8400000 to 0x8800000 range. + * The PROM resides in an area spanning 0xf0000000 to 0x100000000. + * The vmalloc area spans 0x100000000 to 0x200000000. + * Since modules need to be in the lowest 32-bits of the address space, + * we place them right before the OBP area from 0x10000000 to 0xf0000000. + * There is a single static kernel PMD which maps from 0x0 to address + * 0x400000000. + */ +#define TLBTEMP_BASE _AC(0x0000000006000000,UL) +#define TSBMAP_8K_BASE _AC(0x0000000008000000,UL) +#define TSBMAP_4M_BASE _AC(0x0000000008400000,UL) +#define MODULES_VADDR _AC(0x0000000010000000,UL) +#define MODULES_LEN _AC(0x00000000e0000000,UL) +#define MODULES_END _AC(0x00000000f0000000,UL) +#define LOW_OBP_ADDRESS _AC(0x00000000f0000000,UL) +#define HI_OBP_ADDRESS _AC(0x0000000100000000,UL) +#define VMALLOC_START _AC(0x0000000100000000,UL) +#define VMEMMAP_BASE VMALLOC_END + +/* PMD_SHIFT determines the size of the area a second-level page + * table can map + */ +#define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT-3)) +#define PMD_SIZE (_AC(1,UL) << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE-1)) +#define PMD_BITS (PAGE_SHIFT - 3) + +/* PUD_SHIFT determines the size of the area a third-level page + * table can map + */ +#define PUD_SHIFT (PMD_SHIFT + PMD_BITS) +#define PUD_SIZE (_AC(1,UL) << PUD_SHIFT) +#define PUD_MASK (~(PUD_SIZE-1)) +#define PUD_BITS (PAGE_SHIFT - 3) + +/* PGDIR_SHIFT determines what a fourth-level page table entry can map */ +#define PGDIR_SHIFT (PUD_SHIFT + PUD_BITS) +#define PGDIR_SIZE (_AC(1,UL) << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE-1)) +#define PGDIR_BITS (PAGE_SHIFT - 3) + +#if (MAX_PHYS_ADDRESS_BITS > PGDIR_SHIFT + PGDIR_BITS) +#error MAX_PHYS_ADDRESS_BITS exceeds what kernel page tables can support +#endif + +#if (PGDIR_SHIFT + PGDIR_BITS) != 53 +#error Page table parameters do not cover virtual address space properly. +#endif + +#if (PMD_SHIFT != HPAGE_SHIFT) +#error PMD_SHIFT must equal HPAGE_SHIFT for transparent huge pages. +#endif + +#ifndef __ASSEMBLER__ + +extern unsigned long VMALLOC_END; + +#define vmemmap ((struct page *)VMEMMAP_BASE) + +#include +#include + +bool kern_addr_valid(unsigned long addr); + +/* Entries per page directory level. */ +#define PTRS_PER_PTE (1UL << (PAGE_SHIFT-3)) +#define PTRS_PER_PMD (1UL << PMD_BITS) +#define PTRS_PER_PUD (1UL << PUD_BITS) +#define PTRS_PER_PGD (1UL << PGDIR_BITS) + +#define pmd_ERROR(e) \ + pr_err("%s:%d: bad pmd %p(%016lx) seen at (%pS)\n", \ + __FILE__, __LINE__, &(e), pmd_val(e), __builtin_return_address(0)) +#define pud_ERROR(e) \ + pr_err("%s:%d: bad pud %p(%016lx) seen at (%pS)\n", \ + __FILE__, __LINE__, &(e), pud_val(e), __builtin_return_address(0)) +#define pgd_ERROR(e) \ + pr_err("%s:%d: bad pgd %p(%016lx) seen at (%pS)\n", \ + __FILE__, __LINE__, &(e), pgd_val(e), __builtin_return_address(0)) + +#endif /* !(__ASSEMBLER__) */ + +/* PTE bits which are the same in SUN4U and SUN4V format. */ +#define _PAGE_VALID _AC(0x8000000000000000,UL) /* Valid TTE */ +#define _PAGE_R _AC(0x8000000000000000,UL) /* Keep ref bit uptodate*/ +#define _PAGE_SPECIAL _AC(0x0200000000000000,UL) /* Special page */ +#define _PAGE_PMD_HUGE _AC(0x0100000000000000,UL) /* Huge page */ +#define _PAGE_PUD_HUGE _PAGE_PMD_HUGE + +/* SUN4U pte bits... */ +#define _PAGE_SZ4MB_4U _AC(0x6000000000000000,UL) /* 4MB Page */ +#define _PAGE_SZ512K_4U _AC(0x4000000000000000,UL) /* 512K Page */ +#define _PAGE_SZ64K_4U _AC(0x2000000000000000,UL) /* 64K Page */ +#define _PAGE_SZ8K_4U _AC(0x0000000000000000,UL) /* 8K Page */ +#define _PAGE_NFO_4U _AC(0x1000000000000000,UL) /* No Fault Only */ +#define _PAGE_IE_4U _AC(0x0800000000000000,UL) /* Invert Endianness */ +#define _PAGE_SOFT2_4U _AC(0x07FC000000000000,UL) /* Software bits, set 2 */ +#define _PAGE_SPECIAL_4U _AC(0x0200000000000000,UL) /* Special page */ +#define _PAGE_PMD_HUGE_4U _AC(0x0100000000000000,UL) /* Huge page */ +#define _PAGE_RES1_4U _AC(0x0002000000000000,UL) /* Reserved */ +#define _PAGE_SZ32MB_4U _AC(0x0001000000000000,UL) /* (Panther) 32MB page */ +#define _PAGE_SZ256MB_4U _AC(0x2001000000000000,UL) /* (Panther) 256MB page */ +#define _PAGE_SZALL_4U _AC(0x6001000000000000,UL) /* All pgsz bits */ +#define _PAGE_SN_4U _AC(0x0000800000000000,UL) /* (Cheetah) Snoop */ +#define _PAGE_RES2_4U _AC(0x0000780000000000,UL) /* Reserved */ +#define _PAGE_PADDR_4U _AC(0x000007FFFFFFE000,UL) /* (Cheetah) pa[42:13] */ +#define _PAGE_SOFT_4U _AC(0x0000000000001F80,UL) /* Software bits: */ +#define _PAGE_EXEC_4U _AC(0x0000000000001000,UL) /* Executable SW bit */ +#define _PAGE_MODIFIED_4U _AC(0x0000000000000800,UL) /* Modified (dirty) */ +#define _PAGE_ACCESSED_4U _AC(0x0000000000000400,UL) /* Accessed (ref'd) */ +#define _PAGE_READ_4U _AC(0x0000000000000200,UL) /* Readable SW Bit */ +#define _PAGE_WRITE_4U _AC(0x0000000000000100,UL) /* Writable SW Bit */ +#define _PAGE_PRESENT_4U _AC(0x0000000000000080,UL) /* Present */ +#define _PAGE_L_4U _AC(0x0000000000000040,UL) /* Locked TTE */ +#define _PAGE_CP_4U _AC(0x0000000000000020,UL) /* Cacheable in P-Cache */ +#define _PAGE_CV_4U _AC(0x0000000000000010,UL) /* Cacheable in V-Cache */ +#define _PAGE_E_4U _AC(0x0000000000000008,UL) /* side-Effect */ +#define _PAGE_P_4U _AC(0x0000000000000004,UL) /* Privileged Page */ +#define _PAGE_W_4U _AC(0x0000000000000002,UL) /* Writable */ + +/* SUN4V pte bits... */ +#define _PAGE_NFO_4V _AC(0x4000000000000000,UL) /* No Fault Only */ +#define _PAGE_SOFT2_4V _AC(0x3F00000000000000,UL) /* Software bits, set 2 */ +#define _PAGE_MODIFIED_4V _AC(0x2000000000000000,UL) /* Modified (dirty) */ +#define _PAGE_ACCESSED_4V _AC(0x1000000000000000,UL) /* Accessed (ref'd) */ +#define _PAGE_READ_4V _AC(0x0800000000000000,UL) /* Readable SW Bit */ +#define _PAGE_WRITE_4V _AC(0x0400000000000000,UL) /* Writable SW Bit */ +#define _PAGE_SPECIAL_4V _AC(0x0200000000000000,UL) /* Special page */ +#define _PAGE_PMD_HUGE_4V _AC(0x0100000000000000,UL) /* Huge page */ +#define _PAGE_PADDR_4V _AC(0x00FFFFFFFFFFE000,UL) /* paddr[55:13] */ +#define _PAGE_IE_4V _AC(0x0000000000001000,UL) /* Invert Endianness */ +#define _PAGE_E_4V _AC(0x0000000000000800,UL) /* side-Effect */ +#define _PAGE_CP_4V _AC(0x0000000000000400,UL) /* Cacheable in P-Cache */ +#define _PAGE_CV_4V _AC(0x0000000000000200,UL) /* Cacheable in V-Cache */ +/* Bit 9 is used to enable MCD corruption detection instead on M7 */ +#define _PAGE_MCD_4V _AC(0x0000000000000200,UL) /* Memory Corruption */ +#define _PAGE_P_4V _AC(0x0000000000000100,UL) /* Privileged Page */ +#define _PAGE_EXEC_4V _AC(0x0000000000000080,UL) /* Executable Page */ +#define _PAGE_W_4V _AC(0x0000000000000040,UL) /* Writable */ +#define _PAGE_SOFT_4V _AC(0x0000000000000030,UL) /* Software bits */ +#define _PAGE_PRESENT_4V _AC(0x0000000000000010,UL) /* Present */ +#define _PAGE_RESV_4V _AC(0x0000000000000008,UL) /* Reserved */ +#define _PAGE_SZ16GB_4V _AC(0x0000000000000007,UL) /* 16GB Page */ +#define _PAGE_SZ2GB_4V _AC(0x0000000000000006,UL) /* 2GB Page */ +#define _PAGE_SZ256MB_4V _AC(0x0000000000000005,UL) /* 256MB Page */ +#define _PAGE_SZ32MB_4V _AC(0x0000000000000004,UL) /* 32MB Page */ +#define _PAGE_SZ4MB_4V _AC(0x0000000000000003,UL) /* 4MB Page */ +#define _PAGE_SZ512K_4V _AC(0x0000000000000002,UL) /* 512K Page */ +#define _PAGE_SZ64K_4V _AC(0x0000000000000001,UL) /* 64K Page */ +#define _PAGE_SZ8K_4V _AC(0x0000000000000000,UL) /* 8K Page */ +#define _PAGE_SZALL_4V _AC(0x0000000000000007,UL) /* All pgsz bits */ + +#define _PAGE_SZBITS_4U _PAGE_SZ8K_4U +#define _PAGE_SZBITS_4V _PAGE_SZ8K_4V + +#if REAL_HPAGE_SHIFT != 22 +#error REAL_HPAGE_SHIFT and _PAGE_SZHUGE_foo must match up +#endif + +#define _PAGE_SZHUGE_4U _PAGE_SZ4MB_4U +#define _PAGE_SZHUGE_4V _PAGE_SZ4MB_4V + +/* We borrow bit 20 to store the exclusive marker in swap PTEs. */ +#define _PAGE_SWP_EXCLUSIVE _AC(0x0000000000100000, UL) + +#ifndef __ASSEMBLER__ + +pte_t mk_pte_io(unsigned long, pgprot_t, int, unsigned long); + +unsigned long pte_sz_bits(unsigned long size); + +extern pgprot_t PAGE_KERNEL; +extern pgprot_t PAGE_KERNEL_LOCKED; +extern pgprot_t PAGE_COPY; +extern pgprot_t PAGE_SHARED; + +/* XXX This ugliness is for the atyfb driver's sparc mmap() support. XXX */ +extern unsigned long _PAGE_IE; +extern unsigned long _PAGE_E; +extern unsigned long _PAGE_CACHE; + +extern unsigned long pg_iobits; +extern unsigned long _PAGE_ALL_SZ_BITS; + +/* PFNs are real physical page numbers. However, mem_map only begins to record + * per-page information starting at pfn_base. This is to handle systems where + * the first physical page in the machine is at some huge physical address, + * such as 4GB. This is common on a partitioned E10000, for example. + */ +static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) +{ + unsigned long paddr = pfn << PAGE_SHIFT; + + BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL); + return __pte(paddr | pgprot_val(prot)); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) +{ + pte_t pte = pfn_pte(page_nr, pgprot); + + return __pmd(pte_val(pte)); +} +#endif + +/* This one can be done with two shifts. */ +static inline unsigned long pte_pfn(pte_t pte) +{ + unsigned long ret; + + __asm__ __volatile__( + "\n661: sllx %1, %2, %0\n" + " srlx %0, %3, %0\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sllx %1, %4, %0\n" + " srlx %0, %5, %0\n" + " .previous\n" + : "=r" (ret) + : "r" (pte_val(pte)), + "i" (21), "i" (21 + PAGE_SHIFT), + "i" (8), "i" (8 + PAGE_SHIFT)); + + return ret; +} +#define pte_page(x) pfn_to_page(pte_pfn(x)) + +static inline pte_t pte_modify(pte_t pte, pgprot_t prot) +{ + unsigned long mask, tmp; + + /* SUN4U: 0x630107ffffffec38 (negated == 0x9cfef800000013c7) + * SUN4V: 0x33ffffffffffee07 (negated == 0xcc000000000011f8) + * + * Even if we use negation tricks the result is still a 6 + * instruction sequence, so don't try to play fancy and just + * do the most straightforward implementation. + * + * Note: We encode this into 3 sun4v 2-insn patch sequences. + */ + + BUILD_BUG_ON(_PAGE_SZBITS_4U != 0UL || _PAGE_SZBITS_4V != 0UL); + __asm__ __volatile__( + "\n661: sethi %%uhi(%2), %1\n" + " sethi %%hi(%2), %0\n" + "\n662: or %1, %%ulo(%2), %1\n" + " or %0, %%lo(%2), %0\n" + "\n663: sllx %1, 32, %1\n" + " or %0, %1, %0\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%3), %1\n" + " sethi %%hi(%3), %0\n" + " .word 662b\n" + " or %1, %%ulo(%3), %1\n" + " or %0, %%lo(%3), %0\n" + " .word 663b\n" + " sllx %1, 32, %1\n" + " or %0, %1, %0\n" + " .previous\n" + " .section .sun_m7_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%4), %1\n" + " sethi %%hi(%4), %0\n" + " .word 662b\n" + " or %1, %%ulo(%4), %1\n" + " or %0, %%lo(%4), %0\n" + " .word 663b\n" + " sllx %1, 32, %1\n" + " or %0, %1, %0\n" + " .previous\n" + : "=r" (mask), "=r" (tmp) + : "i" (_PAGE_PADDR_4U | _PAGE_MODIFIED_4U | _PAGE_ACCESSED_4U | + _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_E_4U | + _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4U), + "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | + _PAGE_CP_4V | _PAGE_CV_4V | _PAGE_E_4V | + _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V), + "i" (_PAGE_PADDR_4V | _PAGE_MODIFIED_4V | _PAGE_ACCESSED_4V | + _PAGE_CP_4V | _PAGE_E_4V | + _PAGE_SPECIAL | _PAGE_PMD_HUGE | _PAGE_SZALL_4V)); + + return __pte((pte_val(pte) & mask) | (pgprot_val(prot) & ~mask)); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_modify(pte, newprot); + + return __pmd(pte_val(pte)); +} +#endif + +static inline pgprot_t pgprot_noncached(pgprot_t prot) +{ + unsigned long val = pgprot_val(prot); + + __asm__ __volatile__( + "\n661: andn %0, %2, %0\n" + " or %0, %3, %0\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " andn %0, %4, %0\n" + " or %0, %5, %0\n" + " .previous\n" + " .section .sun_m7_2insn_patch, \"ax\"\n" + " .word 661b\n" + " andn %0, %6, %0\n" + " or %0, %5, %0\n" + " .previous\n" + : "=r" (val) + : "0" (val), "i" (_PAGE_CP_4U | _PAGE_CV_4U), "i" (_PAGE_E_4U), + "i" (_PAGE_CP_4V | _PAGE_CV_4V), "i" (_PAGE_E_4V), + "i" (_PAGE_CP_4V)); + + return __pgprot(val); +} +/* Various pieces of code check for platform support by ifdef testing + * on "pgprot_noncached". That's broken and should be fixed, but for + * now... + */ +#define pgprot_noncached pgprot_noncached + +static inline unsigned long pte_dirty(pte_t pte) +{ + unsigned long mask; + + __asm__ __volatile__( + "\n661: mov %1, %0\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V)); + + return (pte_val(pte) & mask); +} + +static inline unsigned long pte_write(pte_t pte) +{ + unsigned long mask; + + __asm__ __volatile__( + "\n661: mov %1, %0\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V)); + + return (pte_val(pte) & mask); +} + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags); +#define arch_make_huge_pte arch_make_huge_pte +static inline unsigned long __pte_default_huge_mask(void) +{ + unsigned long mask; + + __asm__ __volatile__( + "\n661: sethi %%uhi(%1), %0\n" + " sllx %0, 32, %0\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " mov %2, %0\n" + " nop\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_SZHUGE_4U), "i" (_PAGE_SZHUGE_4V)); + + return mask; +} + +static inline pte_t pte_mkhuge(pte_t pte) +{ + return __pte(pte_val(pte) | __pte_default_huge_mask()); +} + +static inline bool is_default_hugetlb_pte(pte_t pte) +{ + unsigned long mask = __pte_default_huge_mask(); + + return (pte_val(pte) & mask) == mask; +} + +static inline bool is_hugetlb_pmd(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_PMD_HUGE); +} + +static inline bool is_hugetlb_pud(pud_t pud) +{ + return !!(pud_val(pud) & _PAGE_PUD_HUGE); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline pmd_t pmd_mkhuge(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkhuge(pte); + pte_val(pte) |= _PAGE_PMD_HUGE; + + return __pmd(pte_val(pte)); +} +#endif +#else +static inline bool is_hugetlb_pte(pte_t pte) +{ + return false; +} +#endif + +static inline pte_t __pte_mkhwwrite(pte_t pte) +{ + unsigned long val = pte_val(pte); + + /* + * Note: we only want to set the HW writable bit if the SW writable bit + * and the SW dirty bit are set. + */ + __asm__ __volatile__( + "\n661: or %0, %2, %0\n" + " .section .sun4v_1insn_patch, \"ax\"\n" + " .word 661b\n" + " or %0, %3, %0\n" + " .previous\n" + : "=r" (val) + : "0" (val), "i" (_PAGE_W_4U), "i" (_PAGE_W_4V)); + + return __pte(val); +} + +static inline pte_t pte_mkdirty(pte_t pte) +{ + unsigned long val = pte_val(pte), mask; + + __asm__ __volatile__( + "\n661: mov %1, %0\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_MODIFIED_4U), "i" (_PAGE_MODIFIED_4V)); + + pte = __pte(val | mask); + return pte_write(pte) ? __pte_mkhwwrite(pte) : pte; +} + +static inline pte_t pte_mkclean(pte_t pte) +{ + unsigned long val = pte_val(pte), tmp; + + __asm__ __volatile__( + "\n661: andn %0, %3, %0\n" + " nop\n" + "\n662: nop\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%4), %1\n" + " sllx %1, 32, %1\n" + " .word 662b\n" + " or %1, %%lo(%4), %1\n" + " andn %0, %1, %0\n" + " .previous\n" + : "=r" (val), "=r" (tmp) + : "0" (val), "i" (_PAGE_MODIFIED_4U | _PAGE_W_4U), + "i" (_PAGE_MODIFIED_4V | _PAGE_W_4V)); + + return __pte(val); +} + +static inline pte_t pte_mkwrite_novma(pte_t pte) +{ + unsigned long val = pte_val(pte), mask; + + __asm__ __volatile__( + "\n661: mov %1, %0\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_WRITE_4U), "i" (_PAGE_WRITE_4V)); + + pte = __pte(val | mask); + return pte_dirty(pte) ? __pte_mkhwwrite(pte) : pte; +} + +static inline pte_t pte_wrprotect(pte_t pte) +{ + unsigned long val = pte_val(pte), tmp; + + __asm__ __volatile__( + "\n661: andn %0, %3, %0\n" + " nop\n" + "\n662: nop\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%4), %1\n" + " sllx %1, 32, %1\n" + " .word 662b\n" + " or %1, %%lo(%4), %1\n" + " andn %0, %1, %0\n" + " .previous\n" + : "=r" (val), "=r" (tmp) + : "0" (val), "i" (_PAGE_WRITE_4U | _PAGE_W_4U), + "i" (_PAGE_WRITE_4V | _PAGE_W_4V)); + + return __pte(val); +} + +static inline pte_t pte_mkold(pte_t pte) +{ + unsigned long mask; + + __asm__ __volatile__( + "\n661: mov %1, %0\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V)); + + mask |= _PAGE_R; + + return __pte(pte_val(pte) & ~mask); +} + +static inline pte_t pte_mkyoung(pte_t pte) +{ + unsigned long mask; + + __asm__ __volatile__( + "\n661: mov %1, %0\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V)); + + mask |= _PAGE_R; + + return __pte(pte_val(pte) | mask); +} + +static inline pte_t pte_mkspecial(pte_t pte) +{ + pte_val(pte) |= _PAGE_SPECIAL; + return pte; +} + +static inline pte_t pte_mkmcd(pte_t pte) +{ + pte_val(pte) |= _PAGE_MCD_4V; + return pte; +} + +static inline pte_t pte_mknotmcd(pte_t pte) +{ + pte_val(pte) &= ~_PAGE_MCD_4V; + return pte; +} + +static inline unsigned long pte_young(pte_t pte) +{ + unsigned long mask; + + __asm__ __volatile__( + "\n661: mov %1, %0\n" + " nop\n" + " .section .sun4v_2insn_patch, \"ax\"\n" + " .word 661b\n" + " sethi %%uhi(%2), %0\n" + " sllx %0, 32, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_ACCESSED_4U), "i" (_PAGE_ACCESSED_4V)); + + return (pte_val(pte) & mask); +} + +static inline unsigned long pte_exec(pte_t pte) +{ + unsigned long mask; + + __asm__ __volatile__( + "\n661: sethi %%hi(%1), %0\n" + " .section .sun4v_1insn_patch, \"ax\"\n" + " .word 661b\n" + " mov %2, %0\n" + " .previous\n" + : "=r" (mask) + : "i" (_PAGE_EXEC_4U), "i" (_PAGE_EXEC_4V)); + + return (pte_val(pte) & mask); +} + +static inline unsigned long pte_present(pte_t pte) +{ + unsigned long val = pte_val(pte); + + __asm__ __volatile__( + "\n661: and %0, %2, %0\n" + " .section .sun4v_1insn_patch, \"ax\"\n" + " .word 661b\n" + " and %0, %3, %0\n" + " .previous\n" + : "=r" (val) + : "0" (val), "i" (_PAGE_PRESENT_4U), "i" (_PAGE_PRESENT_4V)); + + return val; +} + +#define pte_accessible pte_accessible +static inline unsigned long pte_accessible(struct mm_struct *mm, pte_t a) +{ + return pte_val(a) & _PAGE_VALID; +} + +static inline unsigned long pte_special(pte_t pte) +{ + return pte_val(pte) & _PAGE_SPECIAL; +} + +#define pmd_leaf pmd_leaf +static inline bool pmd_leaf(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_val(pte) & _PAGE_PMD_HUGE; +} + +static inline unsigned long pmd_pfn(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_pfn(pte); +} + +#define pmd_write pmd_write +static inline unsigned long pmd_write(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_write(pte); +} + +#define pud_write(pud) pte_write(__pte(pud_val(pud))) + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#define pmd_dirty pmd_dirty +static inline unsigned long pmd_dirty(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_dirty(pte); +} + +#define pmd_young pmd_young +static inline unsigned long pmd_young(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_young(pte); +} + +static inline unsigned long pmd_trans_huge(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + return pte_val(pte) & _PAGE_PMD_HUGE; +} + +static inline pmd_t pmd_mkold(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkold(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_wrprotect(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_wrprotect(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_mkdirty(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkdirty(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_mkclean(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkclean(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_mkyoung(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkyoung(pte); + + return __pmd(pte_val(pte)); +} + +static inline pmd_t pmd_mkwrite_novma(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + + pte = pte_mkwrite_novma(pte); + + return __pmd(pte_val(pte)); +} + +#define pmd_pgprot pmd_pgprot +static inline pgprot_t pmd_pgprot(pmd_t entry) +{ + unsigned long val = pmd_val(entry); + + return __pgprot(val); +} +#endif + +static inline int pmd_present(pmd_t pmd) +{ + return pmd_val(pmd) != 0UL; +} + +#define pmd_none(pmd) (!pmd_val(pmd)) + +/* pmd_bad() is only called on non-trans-huge PMDs. Our encoding is + * very simple, it's just the physical address. PTE tables are of + * size PAGE_SIZE so make sure the sub-PAGE_SIZE bits are clear and + * the top bits outside of the range of any physical address size we + * support are clear as well. We also validate the physical itself. + */ +#define pmd_bad(pmd) (pmd_val(pmd) & ~PAGE_MASK) + +#define pud_none(pud) (!pud_val(pud)) + +#define pud_bad(pud) (pud_val(pud) & ~PAGE_MASK) + +#define p4d_none(p4d) (!p4d_val(p4d)) + +#define p4d_bad(p4d) (p4d_val(p4d) & ~PAGE_MASK) + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd); +#else +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + *pmdp = pmd; +} +#endif + +static inline void pmd_set(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep) +{ + unsigned long val = __pa((unsigned long) (ptep)); + + pmd_val(*pmdp) = val; +} + +#define pud_set(pudp, pmdp) \ + (pud_val(*(pudp)) = (__pa((unsigned long) (pmdp)))) +static inline unsigned long pmd_page_vaddr(pmd_t pmd) +{ + pte_t pte = __pte(pmd_val(pmd)); + unsigned long pfn; + + pfn = pte_pfn(pte); + + return ((unsigned long) __va(pfn << PAGE_SHIFT)); +} + +static inline pmd_t *pud_pgtable(pud_t pud) +{ + pte_t pte = __pte(pud_val(pud)); + unsigned long pfn; + + pfn = pte_pfn(pte); + + return ((pmd_t *) __va(pfn << PAGE_SHIFT)); +} + +#define pmd_page(pmd) virt_to_page((void *)pmd_page_vaddr(pmd)) +#define pud_page(pud) virt_to_page((void *)pud_pgtable(pud)) +#define pmd_clear(pmdp) (pmd_val(*(pmdp)) = 0UL) +#define pud_present(pud) (pud_val(pud) != 0U) +#define pud_clear(pudp) (pud_val(*(pudp)) = 0UL) +#define p4d_pgtable(p4d) \ + ((pud_t *) __va(p4d_val(p4d))) +#define p4d_present(p4d) (p4d_val(p4d) != 0U) +#define p4d_clear(p4dp) (p4d_val(*(p4dp)) = 0UL) + +/* only used by the stubbed out hugetlb gup code, should never be called */ +#define p4d_page(p4d) NULL + +#define pud_leaf pud_leaf +static inline bool pud_leaf(pud_t pud) +{ + pte_t pte = __pte(pud_val(pud)); + + return pte_val(pte) & _PAGE_PMD_HUGE; +} + +#define pud_pfn pud_pfn +static inline unsigned long pud_pfn(pud_t pud) +{ + pte_t pte = __pte(pud_val(pud)); + + return pte_pfn(pte); +} + +/* Same in both SUN4V and SUN4U. */ +#define pte_none(pte) (!pte_val(pte)) + +#define p4d_set(p4dp, pudp) \ + (p4d_val(*(p4dp)) = (__pa((unsigned long) (pudp)))) + +/* We cannot include at this point yet: */ +extern struct mm_struct init_mm; + +/* Actual page table PTE updates. */ +void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift); + +static void maybe_tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift) +{ + /* It is more efficient to let flush_tlb_kernel_range() + * handle init_mm tlb flushes. + * + * SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U + * and SUN4V pte layout, so this inline test is fine. + */ + if (likely(mm != &init_mm) && pte_accessible(mm, orig)) + tlb_batch_add(mm, vaddr, ptep, orig, fullmm, hugepage_shift); +} + +#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR +static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, + unsigned long addr, + pmd_t *pmdp) +{ + pmd_t pmd = *pmdp; + set_pmd_at(mm, addr, pmdp, __pmd(0UL)); + return pmd; +} + +static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, int fullmm) +{ + pte_t orig = *ptep; + + *ptep = pte; + maybe_tlb_batch_add(mm, addr, ptep, orig, fullmm, PAGE_SHIFT); +} + +#define PFN_PTE_SHIFT PAGE_SHIFT + +static inline void set_ptes(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte, unsigned int nr) +{ + for (;;) { + __set_pte_at(mm, addr, ptep, pte, 0); + if (--nr == 0) + break; + ptep++; + pte_val(pte) += PAGE_SIZE; + addr += PAGE_SIZE; + } +} +#define set_ptes set_ptes + +#define pte_clear(mm,addr,ptep) \ + set_pte_at((mm), (addr), (ptep), __pte(0UL)) + +#ifdef DCACHE_ALIASING_POSSIBLE +#define __HAVE_ARCH_MOVE_PTE +#define move_pte(pte, old_addr, new_addr) \ +({ \ + pte_t newpte = (pte); \ + if (tlb_type != hypervisor && pte_present(pte)) { \ + unsigned long this_pfn = pte_pfn(pte); \ + \ + if (pfn_valid(this_pfn) && \ + (((old_addr) ^ (new_addr)) & (1 << 13))) \ + flush_dcache_folio_all(current->mm, \ + page_folio(pfn_to_page(this_pfn))); \ + } \ + newpte; \ +}) +#endif + +extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; + +void paging_init(void); +unsigned long find_ecache_flush_span(unsigned long size); + +struct seq_file; +void mmu_info(struct seq_file *); + +struct vm_area_struct; +void update_mmu_cache_range(struct vm_fault *, struct vm_area_struct *, + unsigned long addr, pte_t *ptep, unsigned int nr); +#define update_mmu_cache(vma, addr, ptep) \ + update_mmu_cache_range(NULL, vma, addr, ptep, 1) +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, + pmd_t *pmd); + +#define __HAVE_ARCH_PMDP_INVALIDATE +extern pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp); + +#define __HAVE_ARCH_PGTABLE_DEPOSIT +void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable); + +#define __HAVE_ARCH_PGTABLE_WITHDRAW +pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); +#endif + +/* + * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that + * are !pte_none() && !pte_present(). + * + * Format of swap PTEs: + * + * 6 6 6 6 5 5 5 5 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3 3 + * 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 + * <--------------------------- offset --------------------------- + * + * 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * --------------------> E <-- type ---> <------- zeroes --------> + */ +#define __swp_type(entry) (((entry).val >> PAGE_SHIFT) & 0x7fUL) +#define __swp_offset(entry) ((entry).val >> (PAGE_SHIFT + 8UL)) +#define __swp_entry(type, offset) \ + ( (swp_entry_t) \ + { \ + ((((long)(type) & 0x7fUL) << PAGE_SHIFT) | \ + ((long)(offset) << (PAGE_SHIFT + 8UL))) \ + } ) +#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) +#define __swp_entry_to_pte(x) ((pte_t) { (x).val }) + +static inline bool pte_swp_exclusive(pte_t pte) +{ + return pte_val(pte) & _PAGE_SWP_EXCLUSIVE; +} + +static inline pte_t pte_swp_mkexclusive(pte_t pte) +{ + return __pte(pte_val(pte) | _PAGE_SWP_EXCLUSIVE); +} + +static inline pte_t pte_swp_clear_exclusive(pte_t pte) +{ + return __pte(pte_val(pte) & ~_PAGE_SWP_EXCLUSIVE); +} + +int page_in_phys_avail(unsigned long paddr); + +/* + * For sparc32&64, the pfn in io_remap_pfn_range() carries in + * its high 4 bits. These macros/functions put it there or get it from there. + */ +#define MK_IOSPACE_PFN(space, pfn) (pfn | (space << (BITS_PER_LONG - 4))) +#define GET_IOSPACE(pfn) (pfn >> (BITS_PER_LONG - 4)) +#define GET_PFN(pfn) (pfn & 0x0fffffffffffffffUL) + +void adi_restore_tags(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, pte_t pte); + +int adi_save_tags(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, pte_t oldpte); + +#define __HAVE_ARCH_DO_SWAP_PAGE +static inline void arch_do_swap_page(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr, + pte_t pte, pte_t oldpte) +{ + /* If this is a new page being mapped in, there can be no + * ADI tags stored away for this page. Skip looking for + * stored tags + */ + if (pte_none(oldpte)) + return; + + if (adi_state.enabled && (pte_val(pte) & _PAGE_MCD_4V)) + adi_restore_tags(mm, vma, addr, pte); +} + +#define __HAVE_ARCH_UNMAP_ONE +static inline int arch_unmap_one(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr, pte_t oldpte) +{ + if (adi_state.enabled && (pte_val(oldpte) & _PAGE_MCD_4V)) + return adi_save_tags(mm, vma, addr, oldpte); + return 0; +} + +static inline unsigned long io_remap_pfn_range_pfn(unsigned long pfn, + unsigned long size) +{ + unsigned long offset = GET_PFN(pfn) << PAGE_SHIFT; + int space = GET_IOSPACE(pfn); + unsigned long phys_base; + + phys_base = offset | (((unsigned long) space) << 32UL); + + return phys_base >> PAGE_SHIFT; +} +#define io_remap_pfn_range_pfn io_remap_pfn_range_pfn + +static inline unsigned long __untagged_addr(unsigned long start) +{ + if (adi_capable()) { + long addr = start; + + /* If userspace has passed a versioned address, kernel + * will not find it in the VMAs since it does not store + * the version tags in the list of VMAs. Storing version + * tags in list of VMAs is impractical since they can be + * changed any time from userspace without dropping into + * kernel. Any address search in VMAs will be done with + * non-versioned addresses. Ensure the ADI version bits + * are dropped here by sign extending the last bit before + * ADI bits. IOMMU does not implement version tags. + */ + return (addr << (long)adi_nbits()) >> (long)adi_nbits(); + } + + return start; +} +#define untagged_addr(addr) \ + ((__typeof__(addr))(__untagged_addr((unsigned long)(addr)))) + +static inline bool pte_access_permitted(pte_t pte, bool write) +{ + u64 prot; + + if (tlb_type == hypervisor) { + prot = _PAGE_PRESENT_4V | _PAGE_P_4V; + if (write) + prot |= _PAGE_WRITE_4V; + } else { + prot = _PAGE_PRESENT_4U | _PAGE_P_4U; + if (write) + prot |= _PAGE_WRITE_4U; + } + + return (pte_val(pte) & (prot | _PAGE_SPECIAL)) == prot; +} +#define pte_access_permitted pte_access_permitted + +/* We provide our own get_unmapped_area to cope with VA holes and + * SHM area cache aliasing for userland. + */ +#define HAVE_ARCH_UNMAPPED_AREA +#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN + +/* We provide a special get_unmapped_area for framebuffer mmaps to try and use + * the largest alignment possible such that larget PTEs can be used. + */ +unsigned long get_fb_unmapped_area(struct file *filp, unsigned long, + unsigned long, unsigned long, + unsigned long); +#define HAVE_ARCH_FB_UNMAPPED_AREA + +void sun4v_register_fault_status(void); +void sun4v_ktsb_register(void); +void __init cheetah_ecache_flush_init(void); +void sun4v_patch_tlb_handlers(void); + +extern unsigned long cmdline_memory_size; + +asmlinkage void do_sparc64_fault(struct pt_regs *regs); + +#define pmd_pgtable(PMD) ((pte_t *)pmd_page_vaddr(PMD)) + +#ifdef CONFIG_HUGETLB_PAGE + +#define pud_leaf_size pud_leaf_size +extern unsigned long pud_leaf_size(pud_t pud); + +#define pmd_leaf_size pmd_leaf_size +extern unsigned long pmd_leaf_size(pmd_t pmd); + +#define pte_leaf_size pte_leaf_size +extern unsigned long pte_leaf_size(pte_t pte); + +#endif /* CONFIG_HUGETLB_PAGE */ + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(_SPARC64_PGTABLE_H) */ diff --git a/arch/sparc/include/asm/pgtsrmmu.h b/arch/sparc/include/asm/pgtsrmmu.h new file mode 100644 index 000000000..a265822a4 --- /dev/null +++ b/arch/sparc/include/asm/pgtsrmmu.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * pgtsrmmu.h: SRMMU page table defines and code. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_PGTSRMMU_H +#define _SPARC_PGTSRMMU_H + +#include + +#ifdef __ASSEMBLER__ +#include /* TI_UWINMASK for WINDOW_FLUSH */ +#endif + +/* Number of contexts is implementation-dependent; 64k is the most we support */ +#define SRMMU_MAX_CONTEXTS 65536 + +#define SRMMU_PTE_TABLE_SIZE (PTRS_PER_PTE*4) +#define SRMMU_PMD_TABLE_SIZE (PTRS_PER_PMD*4) +#define SRMMU_PGD_TABLE_SIZE (PTRS_PER_PGD*4) + +/* Definition of the values in the ET field of PTD's and PTE's */ +#define SRMMU_ET_MASK 0x3 +#define SRMMU_ET_INVALID 0x0 +#define SRMMU_ET_PTD 0x1 +#define SRMMU_ET_PTE 0x2 +#define SRMMU_ET_REPTE 0x3 /* AIEEE, SuperSparc II reverse endian page! */ + +/* Physical page extraction from PTP's and PTE's. */ +#define SRMMU_CTX_PMASK 0xfffffff0 +#define SRMMU_PTD_PMASK 0xfffffff0 +#define SRMMU_PTE_PMASK 0xffffff00 + +/* The pte non-page bits. Some notes: + * 1) cache, dirty, valid, and ref are frobbable + * for both supervisor and user pages. + * 2) exec and write will only give the desired effect + * on user pages + * 3) use priv and priv_readonly for changing the + * characteristics of supervisor ptes + */ +#define SRMMU_CACHE 0x80 +#define SRMMU_DIRTY 0x40 +#define SRMMU_REF 0x20 +#define SRMMU_NOREAD 0x10 +#define SRMMU_EXEC 0x08 +#define SRMMU_WRITE 0x04 +#define SRMMU_VALID 0x02 /* SRMMU_ET_PTE */ +#define SRMMU_PRIV 0x1c +#define SRMMU_PRIV_RDONLY 0x18 + +#define SRMMU_CHG_MASK (0xffffff00 | SRMMU_REF | SRMMU_DIRTY) + +/* SRMMU swap entry encoding */ +#define SRMMU_SWP_TYPE_MASK 0x1f +#define SRMMU_SWP_TYPE_SHIFT 7 +#define SRMMU_SWP_OFF_MASK 0xfffff +#define SRMMU_SWP_OFF_SHIFT (SRMMU_SWP_TYPE_SHIFT + 5) +/* We borrow bit 6 to store the exclusive marker in swap PTEs. */ +#define SRMMU_SWP_EXCLUSIVE SRMMU_DIRTY + +/* Some day I will implement true fine grained access bits for + * user pages because the SRMMU gives us the capabilities to + * enforce all the protection levels that vma's can have. + * XXX But for now... + */ +#define SRMMU_PAGE_NONE __pgprot(SRMMU_CACHE | \ + SRMMU_PRIV | SRMMU_REF) +#define SRMMU_PAGE_SHARED __pgprot(SRMMU_VALID | SRMMU_CACHE | \ + SRMMU_EXEC | SRMMU_WRITE | SRMMU_REF) +#define SRMMU_PAGE_COPY __pgprot(SRMMU_VALID | SRMMU_CACHE | \ + SRMMU_EXEC | SRMMU_REF) +#define SRMMU_PAGE_RDONLY __pgprot(SRMMU_VALID | SRMMU_CACHE | \ + SRMMU_EXEC | SRMMU_REF) +#define SRMMU_PAGE_KERNEL __pgprot(SRMMU_VALID | SRMMU_CACHE | SRMMU_PRIV | \ + SRMMU_DIRTY | SRMMU_REF) + +/* SRMMU Register addresses in ASI 0x4. These are valid for all + * current SRMMU implementations that exist. + */ +#define SRMMU_CTRL_REG 0x00000000 +#define SRMMU_CTXTBL_PTR 0x00000100 +#define SRMMU_CTX_REG 0x00000200 +#define SRMMU_FAULT_STATUS 0x00000300 +#define SRMMU_FAULT_ADDR 0x00000400 + +#define WINDOW_FLUSH(tmp1, tmp2) \ + mov 0, tmp1; \ +98: ld [%g6 + TI_UWINMASK], tmp2; \ + orcc %g0, tmp2, %g0; \ + add tmp1, 1, tmp1; \ + bne 98b; \ + save %sp, -64, %sp; \ +99: subcc tmp1, 1, tmp1; \ + bne 99b; \ + restore %g0, %g0, %g0; + +#ifndef __ASSEMBLER__ +extern unsigned long last_valid_pfn; + +/* This makes sense. Honest it does - Anton */ +/* XXX Yes but it's ugly as sin. FIXME. -KMW */ +extern void *srmmu_nocache_pool; +#define __nocache_pa(VADDR) (((unsigned long)VADDR) - SRMMU_NOCACHE_VADDR + __pa((unsigned long)srmmu_nocache_pool)) +#define __nocache_va(PADDR) (__va((unsigned long)PADDR) - (unsigned long)srmmu_nocache_pool + SRMMU_NOCACHE_VADDR) +#define __nocache_fix(VADDR) ((__typeof__(VADDR))__va(__nocache_pa(VADDR))) + +/* Accessing the MMU control register. */ +unsigned int srmmu_get_mmureg(void); +void srmmu_set_mmureg(unsigned long regval); +void srmmu_set_ctable_ptr(unsigned long paddr); +void srmmu_set_context(int context); +int srmmu_get_context(void); +unsigned int srmmu_get_fstatus(void); +unsigned int srmmu_get_faddr(void); + +/* This is guaranteed on all SRMMU's. */ +static inline void srmmu_flush_whole_tlb(void) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t": : + "r" (0x400), /* Flush entire TLB!! */ + "i" (ASI_M_FLUSH_PROBE) : "memory"); + +} + +static inline int +srmmu_get_pte (unsigned long addr) +{ + register unsigned long entry; + + __asm__ __volatile__("\n\tlda [%1] %2,%0\n\t" : + "=r" (entry): + "r" ((addr & 0xfffff000) | 0x400), "i" (ASI_M_FLUSH_PROBE)); + return entry; +} + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(_SPARC_PGTSRMMU_H) */ diff --git a/arch/sparc/include/asm/pil.h b/arch/sparc/include/asm/pil.h new file mode 100644 index 000000000..4003c3530 --- /dev/null +++ b/arch/sparc/include/asm/pil.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_PIL_H +#define _SPARC64_PIL_H + +/* To avoid some locking problems, we hard allocate certain PILs + * for SMP cross call messages that must do a etrap/rtrap. + * + * A local_irq_disable() does not block the cross call delivery, so + * when SMP locking is an issue we reschedule the event into a PIL + * interrupt which is blocked by local_irq_disable(). + * + * In fact any XCALL which has to etrap/rtrap has a problem because + * it is difficult to prevent rtrap from running BH's, and that would + * need to be done if the XCALL arrived while %pil==PIL_NORMAL_MAX. + * + * Finally, in order to handle profiling events even when a + * local_irq_disable() is in progress, we only disable up to level 14 + * interrupts. Profile counter overflow interrupts arrive at level + * 15. + */ +#define PIL_SMP_CALL_FUNC 1 +#define PIL_SMP_RECEIVE_SIGNAL 2 +#define PIL_SMP_CAPTURE 3 +#define PIL_DEVICE_IRQ 5 +#define PIL_SMP_CALL_FUNC_SNGL 6 +#define PIL_DEFERRED_PCR_WORK 7 +#define PIL_KGDB_CAPTURE 8 +#define PIL_NORMAL_MAX 14 +#define PIL_NMI 15 + +#endif /* !(_SPARC64_PIL_H) */ diff --git a/arch/sparc/include/asm/processor.h b/arch/sparc/include/asm/processor.h new file mode 100644 index 000000000..e34de9565 --- /dev/null +++ b/arch/sparc/include/asm/processor.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_PROCESSOR_H +#define ___ASM_SPARC_PROCESSOR_H + +#include + +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h new file mode 100644 index 000000000..a074d313f --- /dev/null +++ b/arch/sparc/include/asm/processor_32.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* include/asm/processor.h + * + * Copyright (C) 1994 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef __ASM_SPARC_PROCESSOR_H +#define __ASM_SPARC_PROCESSOR_H + +#include +#include +#include +#include +#include + +/* Whee, this is STACK_TOP + PAGE_SIZE and the lowest kernel address too... + * That one page is used to protect kernel from intruders, so that + * we can make our access_ok test faster + */ +#define TASK_SIZE PAGE_OFFSET +#ifdef __KERNEL__ +#define STACK_TOP (PAGE_OFFSET - PAGE_SIZE) +#define STACK_TOP_MAX STACK_TOP +#endif /* __KERNEL__ */ + +struct task_struct; + +#ifdef __KERNEL__ +struct fpq { + unsigned long *insn_addr; + unsigned long insn; +}; +#endif + +/* The Sparc processor specific thread struct. */ +struct thread_struct { + struct pt_regs *kregs; + unsigned int _pad1; + + /* Special child fork kpsr/kwim values. */ + unsigned long fork_kpsr __attribute__ ((aligned (8))); + unsigned long fork_kwim; + + /* Floating point regs */ + unsigned long float_regs[32] __attribute__ ((aligned (8))); + unsigned long fsr; + unsigned long fpqdepth; + struct fpq fpqueue[16]; +}; + +#define INIT_THREAD { \ + .kregs = (struct pt_regs *)(init_stack+THREAD_SIZE)-1 \ +} + +/* Do necessary setup to start up a newly executed thread. */ +static inline void start_thread(struct pt_regs * regs, unsigned long pc, + unsigned long sp) +{ + register unsigned long zero asm("g1"); + + regs->psr = (regs->psr & (PSR_CWP)) | PSR_S; + regs->pc = ((pc & (~3)) - 4); + regs->npc = regs->pc + 4; + regs->y = 0; + zero = 0; + __asm__ __volatile__("std\t%%g0, [%0 + %3 + 0x00]\n\t" + "std\t%%g0, [%0 + %3 + 0x08]\n\t" + "std\t%%g0, [%0 + %3 + 0x10]\n\t" + "std\t%%g0, [%0 + %3 + 0x18]\n\t" + "std\t%%g0, [%0 + %3 + 0x20]\n\t" + "std\t%%g0, [%0 + %3 + 0x28]\n\t" + "std\t%%g0, [%0 + %3 + 0x30]\n\t" + "st\t%1, [%0 + %3 + 0x38]\n\t" + "st\t%%g0, [%0 + %3 + 0x3c]" + : /* no outputs */ + : "r" (regs), + "r" (sp - sizeof(struct reg_window32)), + "r" (zero), + "i" ((const unsigned long)(&((struct pt_regs *)0)->u_regs[0])) + : "memory"); +} + +unsigned long __get_wchan(struct task_struct *); + +#define task_pt_regs(tsk) ((tsk)->thread.kregs) +#define KSTK_EIP(tsk) ((tsk)->thread.kregs->pc) +#define KSTK_ESP(tsk) ((tsk)->thread.kregs->u_regs[UREG_FP]) + +#ifdef __KERNEL__ + +extern struct task_struct *last_task_used_math; +int do_mathemu(struct pt_regs *regs, struct task_struct *fpt); + +extern void (*sparc_idle)(void); + +#endif + +#endif /* __ASM_SPARC_PROCESSOR_H */ diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h new file mode 100644 index 000000000..485070495 --- /dev/null +++ b/arch/sparc/include/asm/processor_64.h @@ -0,0 +1,222 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/asm/processor.h + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef __ASM_SPARC64_PROCESSOR_H +#define __ASM_SPARC64_PROCESSOR_H + +#include +#include +#include +#include + +/* + * User lives in his very own context, and cannot reference us. Note + * that TASK_SIZE is a misnomer, it really gives maximum user virtual + * address that the kernel will allocate out. + * + * XXX No longer using virtual page tables, kill this upper limit... + */ +#define VA_BITS 44 +#ifndef __ASSEMBLER__ +#define VPTE_SIZE (1UL << (VA_BITS - PAGE_SHIFT + 3)) +#else +#define VPTE_SIZE (1 << (VA_BITS - PAGE_SHIFT + 3)) +#endif + +#define TASK_SIZE_OF(tsk) \ + (test_tsk_thread_flag(tsk,TIF_32BIT) ? \ + (1UL << 32UL) : ((unsigned long)-VPTE_SIZE)) +#define TASK_SIZE \ + (test_thread_flag(TIF_32BIT) ? \ + (1UL << 32UL) : ((unsigned long)-VPTE_SIZE)) +#ifdef __KERNEL__ + +#define STACK_TOP32 ((1UL << 32UL) - PAGE_SIZE) +#define STACK_TOP64 (0x0000080000000000UL - (1UL << 32UL)) + +#define STACK_TOP (test_thread_flag(TIF_32BIT) ? \ + STACK_TOP32 : STACK_TOP64) + +#define STACK_TOP_MAX STACK_TOP64 + +#endif + +#ifndef __ASSEMBLER__ + +/* The Sparc processor specific thread struct. */ +/* XXX This should die, everything can go into thread_info now. */ +struct thread_struct { +#ifdef CONFIG_DEBUG_SPINLOCK + /* How many spinlocks held by this thread. + * Used with spin lock debugging to catch tasks + * sleeping illegally with locks held. + */ + int smp_lock_count; + unsigned int smp_lock_pc; +#else + int dummy; /* f'in gcc bug... */ +#endif +}; + +#endif /* !(__ASSEMBLER__) */ + +#ifndef CONFIG_DEBUG_SPINLOCK +#define INIT_THREAD { \ + 0, \ +} +#else /* CONFIG_DEBUG_SPINLOCK */ +#define INIT_THREAD { \ +/* smp_lock_count, smp_lock_pc, */ \ + 0, 0, \ +} +#endif /* !(CONFIG_DEBUG_SPINLOCK) */ + +#ifndef __ASSEMBLER__ + +#include +#include + +struct task_struct; + +/* On Uniprocessor, even in RMO processes see TSO semantics */ +#ifdef CONFIG_SMP +#define TSTATE_INITIAL_MM TSTATE_TSO +#else +#define TSTATE_INITIAL_MM TSTATE_RMO +#endif + +/* Do necessary setup to start up a newly executed thread. */ +#define start_thread(regs, pc, sp) \ +do { \ + unsigned long __asi = ASI_PNF; \ + regs->tstate = (regs->tstate & (TSTATE_CWP)) | (TSTATE_INITIAL_MM|TSTATE_IE) | (__asi << 24UL); \ + regs->tpc = ((pc & (~3)) - 4); \ + regs->tnpc = regs->tpc + 4; \ + regs->y = 0; \ + set_thread_wstate(1 << 3); \ + if (current_thread_info()->utraps) { \ + if (*(current_thread_info()->utraps) < 2) \ + kfree(current_thread_info()->utraps); \ + else \ + (*(current_thread_info()->utraps))--; \ + current_thread_info()->utraps = NULL; \ + } \ + __asm__ __volatile__( \ + "stx %%g0, [%0 + %2 + 0x00]\n\t" \ + "stx %%g0, [%0 + %2 + 0x08]\n\t" \ + "stx %%g0, [%0 + %2 + 0x10]\n\t" \ + "stx %%g0, [%0 + %2 + 0x18]\n\t" \ + "stx %%g0, [%0 + %2 + 0x20]\n\t" \ + "stx %%g0, [%0 + %2 + 0x28]\n\t" \ + "stx %%g0, [%0 + %2 + 0x30]\n\t" \ + "stx %%g0, [%0 + %2 + 0x38]\n\t" \ + "stx %%g0, [%0 + %2 + 0x40]\n\t" \ + "stx %%g0, [%0 + %2 + 0x48]\n\t" \ + "stx %%g0, [%0 + %2 + 0x50]\n\t" \ + "stx %%g0, [%0 + %2 + 0x58]\n\t" \ + "stx %%g0, [%0 + %2 + 0x60]\n\t" \ + "stx %%g0, [%0 + %2 + 0x68]\n\t" \ + "stx %1, [%0 + %2 + 0x70]\n\t" \ + "stx %%g0, [%0 + %2 + 0x78]\n\t" \ + "wrpr %%g0, (1 << 3), %%wstate\n\t" \ + : \ + : "r" (regs), "r" (sp - sizeof(struct reg_window) - STACK_BIAS), \ + "i" ((const unsigned long)(&((struct pt_regs *)0)->u_regs[0]))); \ + fprs_write(0); \ + current_thread_info()->xfsr[0] = 0; \ + current_thread_info()->fpsaved[0] = 0; \ + regs->tstate &= ~TSTATE_PEF; \ +} while (0) + +#define start_thread32(regs, pc, sp) \ +do { \ + unsigned long __asi = ASI_PNF; \ + pc &= 0x00000000ffffffffUL; \ + sp &= 0x00000000ffffffffUL; \ + regs->tstate = (regs->tstate & (TSTATE_CWP))|(TSTATE_INITIAL_MM|TSTATE_IE|TSTATE_AM) | (__asi << 24UL); \ + regs->tpc = ((pc & (~3)) - 4); \ + regs->tnpc = regs->tpc + 4; \ + regs->y = 0; \ + set_thread_wstate(2 << 3); \ + if (current_thread_info()->utraps) { \ + if (*(current_thread_info()->utraps) < 2) \ + kfree(current_thread_info()->utraps); \ + else \ + (*(current_thread_info()->utraps))--; \ + current_thread_info()->utraps = NULL; \ + } \ + __asm__ __volatile__( \ + "stx %%g0, [%0 + %2 + 0x00]\n\t" \ + "stx %%g0, [%0 + %2 + 0x08]\n\t" \ + "stx %%g0, [%0 + %2 + 0x10]\n\t" \ + "stx %%g0, [%0 + %2 + 0x18]\n\t" \ + "stx %%g0, [%0 + %2 + 0x20]\n\t" \ + "stx %%g0, [%0 + %2 + 0x28]\n\t" \ + "stx %%g0, [%0 + %2 + 0x30]\n\t" \ + "stx %%g0, [%0 + %2 + 0x38]\n\t" \ + "stx %%g0, [%0 + %2 + 0x40]\n\t" \ + "stx %%g0, [%0 + %2 + 0x48]\n\t" \ + "stx %%g0, [%0 + %2 + 0x50]\n\t" \ + "stx %%g0, [%0 + %2 + 0x58]\n\t" \ + "stx %%g0, [%0 + %2 + 0x60]\n\t" \ + "stx %%g0, [%0 + %2 + 0x68]\n\t" \ + "stx %1, [%0 + %2 + 0x70]\n\t" \ + "stx %%g0, [%0 + %2 + 0x78]\n\t" \ + "wrpr %%g0, (2 << 3), %%wstate\n\t" \ + : \ + : "r" (regs), "r" (sp - sizeof(struct reg_window32)), \ + "i" ((const unsigned long)(&((struct pt_regs *)0)->u_regs[0]))); \ + fprs_write(0); \ + current_thread_info()->xfsr[0] = 0; \ + current_thread_info()->fpsaved[0] = 0; \ + regs->tstate &= ~TSTATE_PEF; \ +} while (0) + +unsigned long __get_wchan(struct task_struct *task); + +#define task_pt_regs(tsk) (task_thread_info(tsk)->kregs) +#define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc) +#define KSTK_ESP(tsk) (task_pt_regs(tsk)->u_regs[UREG_FP]) + +/* Prefetch support. This is tuned for UltraSPARC-III and later. + * UltraSPARC-I will treat these as nops, and UltraSPARC-II has + * a shallower prefetch queue than later chips. + */ +#define ARCH_HAS_PREFETCH +#define ARCH_HAS_PREFETCHW + +static inline void prefetch(const void *x) +{ + /* We do not use the read prefetch mnemonic because that + * prefetches into the prefetch-cache which only is accessible + * by floating point operations in UltraSPARC-III and later. + * By contrast, "#one_write" prefetches into the L2 cache + * in shared state. + */ + __asm__ __volatile__("prefetch [%0], #one_write" + : /* no outputs */ + : "r" (x)); +} + +static inline void prefetchw(const void *x) +{ + /* The most optimal prefetch to use for writes is + * "#n_writes". This brings the cacheline into the + * L2 cache in "owned" state. + */ + __asm__ __volatile__("prefetch [%0], #n_writes" + : /* no outputs */ + : "r" (x)); +} + +#define HAVE_ARCH_PICK_MMAP_LAYOUT + +int do_mathemu(struct pt_regs *regs, struct fpustate *f, bool illegal_insn_trap); + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(__ASM_SPARC64_PROCESSOR_H) */ diff --git a/arch/sparc/include/asm/prom.h b/arch/sparc/include/asm/prom.h new file mode 100644 index 000000000..8184575b1 --- /dev/null +++ b/arch/sparc/include/asm/prom.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#include /* linux/of.h gets to determine #include ordering */ +#ifndef _SPARC_PROM_H +#define _SPARC_PROM_H +#ifdef __KERNEL__ + +/* + * Definitions for talking to the Open Firmware PROM on + * Power Macintosh computers. + * + * Copyright (C) 1996-2005 Paul Mackerras. + * + * Updates for PPC64 by Peter Bergner & David Engebretsen, IBM Corp. + * Updates for SPARC by David S. Miller + */ +#include +#include +#include +#include +#include +#include +#include + +#define of_compat_cmp(s1, s2, l) strncmp((s1), (s2), (l)) +#define of_prop_cmp(s1, s2) strcasecmp((s1), (s2)) +#define of_node_cmp(s1, s2) strcmp((s1), (s2)) + +extern raw_spinlock_t devtree_lock; + +struct of_irq_controller { + unsigned int (*irq_build)(struct device_node *, unsigned int, void *); + void *data; +}; + +struct device_node *of_find_node_by_cpuid(int cpuid); +int of_set_property(struct device_node *node, const char *name, void *val, int len); +extern struct mutex of_set_property_mutex; +int of_getintprop_default(struct device_node *np, + const char *name, + int def); +int of_find_in_proplist(const char *list, const char *match, int len); + +void prom_build_devicetree(void); +void of_populate_present_mask(void); +void of_fill_in_cpu_data(void); + +struct resource; +void __iomem *of_ioremap(struct resource *res, unsigned long offset, unsigned long size, char *name); +void of_iounmap(struct resource *res, void __iomem *base, unsigned long size); + +extern struct device_node *of_console_device; +extern char *of_console_path; +extern char *of_console_options; + +void irq_trans_init(struct device_node *dp); +char *build_path_component(struct device_node *dp); + +#endif /* __KERNEL__ */ +#endif /* _SPARC_PROM_H */ diff --git a/arch/sparc/include/asm/psr.h b/arch/sparc/include/asm/psr.h new file mode 100644 index 000000000..5af50ccda --- /dev/null +++ b/arch/sparc/include/asm/psr.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * psr.h: This file holds the macros for masking off various parts of + * the processor status register on the Sparc. This is valid + * for Version 8. On the V9 this is renamed to the PSTATE + * register and its members are accessed as fields like + * PSTATE.PRIV for the current CPU privilege level. + * + * Copyright (C) 1994 David S. Miller (davem@caip.rutgers.edu) + */ +#ifndef __LINUX_SPARC_PSR_H +#define __LINUX_SPARC_PSR_H + +#include + + +#ifndef __ASSEMBLER__ +/* Get the %psr register. */ +static inline unsigned int get_psr(void) +{ + unsigned int psr; + __asm__ __volatile__( + "rd %%psr, %0\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + : "=r" (psr) + : /* no inputs */ + : "memory"); + + return psr; +} + +static inline void put_psr(unsigned int new_psr) +{ + __asm__ __volatile__( + "wr %0, 0x0, %%psr\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" + : /* no outputs */ + : "r" (new_psr) + : "memory", "cc"); +} + +/* Get the %fsr register. Be careful, make sure the floating point + * enable bit is set in the %psr when you execute this or you will + * incur a trap. + */ + +extern unsigned int fsr_storage; + +static inline unsigned int get_fsr(void) +{ + unsigned int fsr = 0; + + __asm__ __volatile__( + "st %%fsr, %1\n\t" + "ld %1, %0\n\t" + : "=r" (fsr) + : "m" (fsr_storage)); + + return fsr; +} + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(__LINUX_SPARC_PSR_H) */ diff --git a/arch/sparc/include/asm/ptrace.h b/arch/sparc/include/asm/ptrace.h new file mode 100644 index 000000000..8adf3fd2f --- /dev/null +++ b/arch/sparc/include/asm/ptrace.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_PTRACE_H +#define __SPARC_PTRACE_H + +#include + +#if defined(__sparc__) && defined(__arch64__) +#ifndef __ASSEMBLER__ + +#include +#include +#include + +static inline int pt_regs_trap_type(struct pt_regs *regs) +{ + return regs->magic & 0x1ff; +} + +static inline bool pt_regs_is_syscall(struct pt_regs *regs) +{ + return (regs->tstate & TSTATE_SYSCALL); +} + +static inline bool pt_regs_clear_syscall(struct pt_regs *regs) +{ + return (regs->tstate &= ~TSTATE_SYSCALL); +} + +#define arch_ptrace_stop_needed() \ +({ flush_user_windows(); \ + get_thread_wsaved() != 0; \ +}) + +#define arch_ptrace_stop() \ + synchronize_user_stack() + +#define current_pt_regs() \ + ((struct pt_regs *)((unsigned long)current_thread_info() + THREAD_SIZE) - 1) + +struct global_reg_snapshot { + unsigned long tstate; + unsigned long tpc; + unsigned long tnpc; + unsigned long o7; + unsigned long i7; + unsigned long rpc; + struct thread_info *thread; + unsigned long pad1; +}; + +struct global_pmu_snapshot { + unsigned long pcr[4]; + unsigned long pic[4]; +}; + +union global_cpu_snapshot { + struct global_reg_snapshot reg; + struct global_pmu_snapshot pmu; +}; + +extern union global_cpu_snapshot global_cpu_snapshot[NR_CPUS]; + +#define force_successful_syscall_return() set_thread_noerror(1) +#define user_mode(regs) (!((regs)->tstate & TSTATE_PRIV)) +#define instruction_pointer(regs) ((regs)->tpc) +#define instruction_pointer_set(regs, val) do { \ + (regs)->tpc = (val); \ + (regs)->tnpc = (val)+4; \ + } while (0) +#define user_stack_pointer(regs) ((regs)->u_regs[UREG_FP]) +static inline int is_syscall_success(struct pt_regs *regs) +{ + return !(regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY)); +} + +static inline long regs_return_value(struct pt_regs *regs) +{ + return regs->u_regs[UREG_I0]; +} +#ifdef CONFIG_SMP +unsigned long profile_pc(struct pt_regs *); +#else +#define profile_pc(regs) instruction_pointer(regs) +#endif + +#define MAX_REG_OFFSET (offsetof(struct pt_regs, magic)) + +int regs_query_register_offset(const char *name); +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n); + +/** + * regs_get_register() - get register value from its offset + * @regs: pt_regs from which register value is gotten + * @offset: offset number of the register. + * + * regs_get_register returns the value of a register whose + * offset from @regs. The @offset is the offset of the register + * in struct pt_regs. If @offset is bigger than MAX_REG_OFFSET, + * this returns 0. + */ +static inline unsigned long regs_get_register(struct pt_regs *regs, + unsigned long offset) +{ + if (unlikely(offset >= MAX_REG_OFFSET)) + return 0; + if (offset == PT_V9_Y) + return *(unsigned int *)((unsigned long)regs + offset); + return *(unsigned long *)((unsigned long)regs + offset); +} + +/* Valid only for Kernel mode traps. */ +static inline unsigned long kernel_stack_pointer(struct pt_regs *regs) +{ + return regs->u_regs[UREG_I6]; +} +#else /* __ASSEMBLER__ */ +#endif /* __ASSEMBLER__ */ +#else /* (defined(__sparc__) && defined(__arch64__)) */ +#ifndef __ASSEMBLER__ +#include + +static inline bool pt_regs_is_syscall(struct pt_regs *regs) +{ + return (regs->psr & PSR_SYSCALL); +} + +static inline bool pt_regs_clear_syscall(struct pt_regs *regs) +{ + return (regs->psr &= ~PSR_SYSCALL); +} + +#define arch_ptrace_stop_needed() \ +({ flush_user_windows(); \ + current_thread_info()->w_saved != 0; \ +}) + +#define arch_ptrace_stop() \ + synchronize_user_stack() + +#define current_pt_regs() \ + ((struct pt_regs *)((unsigned long)current_thread_info() + THREAD_SIZE) - 1) + +#define user_mode(regs) (!((regs)->psr & PSR_PS)) +#define instruction_pointer(regs) ((regs)->pc) +#define user_stack_pointer(regs) ((regs)->u_regs[UREG_FP]) +unsigned long profile_pc(struct pt_regs *); +#else /* (!__ASSEMBLER__) */ +#endif /* (!__ASSEMBLER__) */ +#endif /* (defined(__sparc__) && defined(__arch64__)) */ +#define STACK_BIAS 2047 + +/* global_reg_snapshot offsets */ +#define GR_SNAP_TSTATE 0x00 +#define GR_SNAP_TPC 0x08 +#define GR_SNAP_TNPC 0x10 +#define GR_SNAP_O7 0x18 +#define GR_SNAP_I7 0x20 +#define GR_SNAP_RPC 0x28 +#define GR_SNAP_THREAD 0x30 +#define GR_SNAP_PAD1 0x38 + +#endif /* !(__SPARC_PTRACE_H) */ diff --git a/arch/sparc/include/asm/qrwlock.h b/arch/sparc/include/asm/qrwlock.h new file mode 100644 index 000000000..c27772915 --- /dev/null +++ b/arch/sparc/include/asm/qrwlock.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC_QRWLOCK_H +#define _ASM_SPARC_QRWLOCK_H + +#include +#include + +#endif /* _ASM_SPARC_QRWLOCK_H */ diff --git a/arch/sparc/include/asm/qspinlock.h b/arch/sparc/include/asm/qspinlock.h new file mode 100644 index 000000000..48808f3d9 --- /dev/null +++ b/arch/sparc/include/asm/qspinlock.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC_QSPINLOCK_H +#define _ASM_SPARC_QSPINLOCK_H + +#include +#include + +#endif /* _ASM_SPARC_QSPINLOCK_H */ diff --git a/arch/sparc/include/asm/ross.h b/arch/sparc/include/asm/ross.h new file mode 100644 index 000000000..53a42b374 --- /dev/null +++ b/arch/sparc/include/asm/ross.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ross.h: Ross module specific definitions and defines. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_ROSS_H +#define _SPARC_ROSS_H + +#include +#include + +/* Ross made Hypersparcs have a %psr 'impl' field of '0001'. The 'vers' + * field has '1111'. + */ + +/* The MMU control register fields on the HyperSparc. + * + * ----------------------------------------------------------------- + * |implvers| RSV |CWR|SE|WBE| MID |BM| C|CS|MR|CM|RSV|CE|RSV|NF|ME| + * ----------------------------------------------------------------- + * 31 24 23-22 21 20 19 18-15 14 13 12 11 10 9 8 7-2 1 0 + * + * Phew, lots of fields there ;-) + * + * CWR: Cache Wrapping Enabled, if one cache wrapping is on. + * SE: Snoop Enable, turns on bus snooping for cache activity if one. + * WBE: Write Buffer Enable, one turns it on. + * MID: The ModuleID of the chip for MBus transactions. + * BM: Boot-Mode. One indicates the MMU is in boot mode. + * C: Indicates whether accesses are cachable while the MMU is + * disabled. + * CS: Cache Size -- 0 = 128k, 1 = 256k + * MR: Memory Reflection, one indicates that the memory bus connected + * to the MBus supports memory reflection. + * CM: Cache Mode -- 0 = write-through, 1 = copy-back + * CE: Cache Enable -- 0 = no caching, 1 = cache is on + * NF: No Fault -- 0 = faults trap the CPU from supervisor mode + * 1 = faults from supervisor mode do not generate traps + * ME: MMU Enable -- 0 = MMU is off, 1 = MMU is on + */ + +#define HYPERSPARC_CWENABLE 0x00200000 +#define HYPERSPARC_SBENABLE 0x00100000 +#define HYPERSPARC_WBENABLE 0x00080000 +#define HYPERSPARC_MIDMASK 0x00078000 +#define HYPERSPARC_BMODE 0x00004000 +#define HYPERSPARC_ACENABLE 0x00002000 +#define HYPERSPARC_CSIZE 0x00001000 +#define HYPERSPARC_MRFLCT 0x00000800 +#define HYPERSPARC_CMODE 0x00000400 +#define HYPERSPARC_CENABLE 0x00000100 +#define HYPERSPARC_NFAULT 0x00000002 +#define HYPERSPARC_MENABLE 0x00000001 + + +/* The ICCR instruction cache register on the HyperSparc. + * + * ----------------------------------------------- + * | | FTD | ICE | + * ----------------------------------------------- + * 31 1 0 + * + * This register is accessed using the V8 'wrasr' and 'rdasr' + * opcodes, since not all assemblers understand them and those + * that do use different semantics I will just hard code the + * instruction with a '.word' statement. + * + * FTD: If set to one flush instructions executed during an + * instruction cache hit occurs, the corresponding line + * for said cache-hit is invalidated. If FTD is zero, + * an unimplemented 'flush' trap will occur when any + * flush is executed by the processor. + * + * ICE: If set to one, the instruction cache is enabled. If + * zero, the cache will not be used for instruction fetches. + * + * All other bits are read as zeros, and writes to them have no + * effect. + * + * Wheee, not many assemblers understand the %iccr register nor + * the generic asr r/w instructions. + * + * 1000 0011 0100 0111 1100 0000 0000 0000 ! rd %iccr, %g1 + * + * 0x 8 3 4 7 c 0 0 0 ! 0x8347c000 + * + * 1011 1111 1000 0000 0110 0000 0000 0000 ! wr %g1, 0x0, %iccr + * + * 0x b f 8 0 6 0 0 0 ! 0xbf806000 + * + */ + +#define HYPERSPARC_ICCR_FTD 0x00000002 +#define HYPERSPARC_ICCR_ICE 0x00000001 + +#ifndef __ASSEMBLER__ + +static inline unsigned int get_ross_icr(void) +{ + unsigned int icreg; + + __asm__ __volatile__(".word 0x8347c000\n\t" /* rd %iccr, %g1 */ + "mov %%g1, %0\n\t" + : "=r" (icreg) + : /* no inputs */ + : "g1", "memory"); + + return icreg; +} + +static inline void put_ross_icr(unsigned int icreg) +{ + __asm__ __volatile__("or %%g0, %0, %%g1\n\t" + ".word 0xbf806000\n\t" /* wr %g1, 0x0, %iccr */ + "nop\n\t" + "nop\n\t" + "nop\n\t" + : /* no outputs */ + : "r" (icreg) + : "g1", "memory"); + + return; +} + +/* HyperSparc specific cache flushing. */ + +/* This is for the on-chip instruction cache. */ +static inline void hyper_flush_whole_icache(void) +{ + __asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" + : /* no outputs */ + : "i" (ASI_M_FLUSH_IWHOLE) + : "memory"); + return; +} + +extern int vac_cache_size; +extern int vac_line_size; + +static inline void hyper_clear_all_tags(void) +{ + unsigned long addr; + + for(addr = 0; addr < vac_cache_size; addr += vac_line_size) + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (addr), "i" (ASI_M_DATAC_TAG) + : "memory"); +} + +static inline void hyper_flush_unconditional_combined(void) +{ + unsigned long addr; + + for (addr = 0; addr < vac_cache_size; addr += vac_line_size) + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (addr), "i" (ASI_M_FLUSH_CTX) + : "memory"); +} + +static inline void hyper_flush_cache_user(void) +{ + unsigned long addr; + + for (addr = 0; addr < vac_cache_size; addr += vac_line_size) + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (addr), "i" (ASI_M_FLUSH_USER) + : "memory"); +} + +static inline void hyper_flush_cache_page(unsigned long page) +{ + unsigned long end; + + page &= PAGE_MASK; + end = page + PAGE_SIZE; + while (page < end) { + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (page), "i" (ASI_M_FLUSH_PAGE) + : "memory"); + page += vac_line_size; + } +} + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(_SPARC_ROSS_H) */ diff --git a/arch/sparc/include/asm/sbi.h b/arch/sparc/include/asm/sbi.h new file mode 100644 index 000000000..861f85b5b --- /dev/null +++ b/arch/sparc/include/asm/sbi.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sbi.h: SBI (Sbus Interface on sun4d) definitions + * + * Copyright (C) 1997 Jakub Jelinek + */ + +#ifndef _SPARC_SBI_H +#define _SPARC_SBI_H + +#include + +/* SBI */ +struct sbi_regs { +/* 0x0000 */ u32 cid; /* Component ID */ +/* 0x0004 */ u32 ctl; /* Control */ +/* 0x0008 */ u32 status; /* Status */ + u32 _unused1; + +/* 0x0010 */ u32 cfg0; /* Slot0 config reg */ +/* 0x0014 */ u32 cfg1; /* Slot1 config reg */ +/* 0x0018 */ u32 cfg2; /* Slot2 config reg */ +/* 0x001c */ u32 cfg3; /* Slot3 config reg */ + +/* 0x0020 */ u32 stb0; /* Streaming buf control for slot 0 */ +/* 0x0024 */ u32 stb1; /* Streaming buf control for slot 1 */ +/* 0x0028 */ u32 stb2; /* Streaming buf control for slot 2 */ +/* 0x002c */ u32 stb3; /* Streaming buf control for slot 3 */ + +/* 0x0030 */ u32 intr_state; /* Interrupt state */ +/* 0x0034 */ u32 intr_tid; /* Interrupt target ID */ +/* 0x0038 */ u32 intr_diag; /* Interrupt diagnostics */ +}; + +#define SBI_CID 0x02800000 +#define SBI_CTL 0x02800004 +#define SBI_STATUS 0x02800008 +#define SBI_CFG0 0x02800010 +#define SBI_CFG1 0x02800014 +#define SBI_CFG2 0x02800018 +#define SBI_CFG3 0x0280001c +#define SBI_STB0 0x02800020 +#define SBI_STB1 0x02800024 +#define SBI_STB2 0x02800028 +#define SBI_STB3 0x0280002c +#define SBI_INTR_STATE 0x02800030 +#define SBI_INTR_TID 0x02800034 +#define SBI_INTR_DIAG 0x02800038 + +/* Burst bits for 8, 16, 32, 64 are in cfgX registers at bits 2, 3, 4, 5 respectively */ +#define SBI_CFG_BURST_MASK 0x0000001e + +/* How to make devid from sbi no */ +#define SBI2DEVID(sbino) ((sbino<<4)|2) + +/* intr_state has 4 bits for slots 0 .. 3 and these bits are repeated for each sbus irq level + * + * +-------+-------+-------+-------+-------+-------+-------+-------+ + * SBUS IRQ LEVEL | 7 | 6 | 5 | 4 | 3 | 2 | 1 | | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ Reser | + * SLOT # |3|2|1|0|3|2|1|0|3|2|1|0|3|2|1|0|3|2|1|0|3|2|1|0|3|2|1|0| ved | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-------+ + * Bits 31 27 23 19 15 11 7 3 0 + */ + + +#ifndef __ASSEMBLER__ + +static inline int acquire_sbi(int devid, int mask) +{ + __asm__ __volatile__ ("swapa [%2] %3, %0" : + "=r" (mask) : + "0" (mask), + "r" (ECSR_DEV_BASE(devid) | SBI_INTR_STATE), + "i" (ASI_M_CTL)); + return mask; +} + +static inline void release_sbi(int devid, int mask) +{ + __asm__ __volatile__ ("sta %0, [%1] %2" : : + "r" (mask), + "r" (ECSR_DEV_BASE(devid) | SBI_INTR_STATE), + "i" (ASI_M_CTL)); +} + +static inline void set_sbi_tid(int devid, int targetid) +{ + __asm__ __volatile__ ("sta %0, [%1] %2" : : + "r" (targetid), + "r" (ECSR_DEV_BASE(devid) | SBI_INTR_TID), + "i" (ASI_M_CTL)); +} + +static inline int get_sbi_ctl(int devid, int cfgno) +{ + int cfg; + + __asm__ __volatile__ ("lda [%1] %2, %0" : + "=r" (cfg) : + "r" ((ECSR_DEV_BASE(devid) | SBI_CFG0) + (cfgno<<2)), + "i" (ASI_M_CTL)); + return cfg; +} + +static inline void set_sbi_ctl(int devid, int cfgno, int cfg) +{ + __asm__ __volatile__ ("sta %0, [%1] %2" : : + "r" (cfg), + "r" ((ECSR_DEV_BASE(devid) | SBI_CFG0) + (cfgno<<2)), + "i" (ASI_M_CTL)); +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* !(_SPARC_SBI_H) */ diff --git a/arch/sparc/include/asm/scratchpad.h b/arch/sparc/include/asm/scratchpad.h new file mode 100644 index 000000000..958351190 --- /dev/null +++ b/arch/sparc/include/asm/scratchpad.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_SCRATCHPAD_H +#define _SPARC64_SCRATCHPAD_H + +/* Sun4v scratchpad registers, accessed via ASI_SCRATCHPAD. */ + +#define SCRATCHPAD_MMU_MISS 0x00 /* Shared with OBP - set by OBP */ +#define SCRATCHPAD_CPUID 0x08 /* Shared with OBP - set by hypervisor */ +#define SCRATCHPAD_UTSBREG1 0x10 +#define SCRATCHPAD_UTSBREG2 0x18 + /* 0x20 and 0x28, hypervisor only... */ +#define SCRATCHPAD_UNUSED1 0x30 +#define SCRATCHPAD_UNUSED2 0x38 /* Reserved for OBP */ + +#endif /* !(_SPARC64_SCRATCHPAD_H) */ diff --git a/arch/sparc/include/asm/seccomp.h b/arch/sparc/include/asm/seccomp.h new file mode 100644 index 000000000..62d4579ef --- /dev/null +++ b/arch/sparc/include/asm/seccomp.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SECCOMP_H +#define _ASM_SECCOMP_H + +#include + +#define __NR_seccomp_sigreturn_32 __NR_sigreturn + +#include + +#endif /* _ASM_SECCOMP_H */ diff --git a/arch/sparc/include/asm/sections.h b/arch/sparc/include/asm/sections.h new file mode 100644 index 000000000..08f833453 --- /dev/null +++ b/arch/sparc/include/asm/sections.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_SECTIONS_H +#define __SPARC_SECTIONS_H + +/* nothing to see, move along */ +#include + +/* sparc entry point */ +extern char _start[]; + +extern char __leon_1insn_patch[]; +extern char __leon_1insn_patch_end[]; + +#endif diff --git a/arch/sparc/include/asm/setup.h b/arch/sparc/include/asm/setup.h new file mode 100644 index 000000000..21bed5514 --- /dev/null +++ b/arch/sparc/include/asm/setup.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Just a place holder. + */ +#ifndef _SPARC_SETUP_H +#define _SPARC_SETUP_H + +#include + +#include + +extern char reboot_command[]; + +#ifdef CONFIG_SPARC32 +/* The CPU that was used for booting + * Only sun4d + leon may have boot_cpu_id != 0 + */ +extern unsigned char boot_cpu_id; + +extern int serial_console; +static inline int con_is_present(void) +{ + return serial_console ? 0 : 1; +} + +/* from irq_32.c */ +extern volatile unsigned char *fdc_status; +extern char *pdma_vaddr; +extern unsigned long pdma_size; +extern volatile int doing_pdma; + +/* This is software state */ +extern char *pdma_base; +extern unsigned long pdma_areasize; + +int sparc_floppy_request_irq(unsigned int irq, irq_handler_t irq_handler); + +/* setup_32.c */ +extern unsigned long cmdline_memory_size; + +/* devices.c */ +void __init device_scan(void); + +/* unaligned_32.c */ +unsigned long safe_compute_effective_address(struct pt_regs *, unsigned int); + +#endif + +#ifdef CONFIG_SPARC64 +void __init start_early_boot(void); + +/* unaligned_64.c */ +int handle_ldf_stq(u32 insn, struct pt_regs *regs); +void handle_ld_nf(u32 insn, struct pt_regs *regs); + +/* init_64.c */ +extern atomic_t dcpage_flushes; +extern atomic_t dcpage_flushes_xcall; + +extern int sysctl_tsb_ratio; + +#ifdef CONFIG_SERIAL_SUNHV +void sunhv_migrate_hvcons_irq(int cpu); +#endif +#endif +void sun_do_break(void); +extern int stop_a_enabled; +extern int scons_pwroff; + +#endif /* _SPARC_SETUP_H */ diff --git a/arch/sparc/include/asm/sfafsr.h b/arch/sparc/include/asm/sfafsr.h new file mode 100644 index 000000000..9c98f4fc8 --- /dev/null +++ b/arch/sparc/include/asm/sfafsr.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_SFAFSR_H +#define _SPARC64_SFAFSR_H + +#include + +/* Spitfire Asynchronous Fault Status register, ASI=0x4C VA<63:0>=0x0 */ + +#define SFAFSR_ME (_AC(1,UL) << SFAFSR_ME_SHIFT) +#define SFAFSR_ME_SHIFT 32 +#define SFAFSR_PRIV (_AC(1,UL) << SFAFSR_PRIV_SHIFT) +#define SFAFSR_PRIV_SHIFT 31 +#define SFAFSR_ISAP (_AC(1,UL) << SFAFSR_ISAP_SHIFT) +#define SFAFSR_ISAP_SHIFT 30 +#define SFAFSR_ETP (_AC(1,UL) << SFAFSR_ETP_SHIFT) +#define SFAFSR_ETP_SHIFT 29 +#define SFAFSR_IVUE (_AC(1,UL) << SFAFSR_IVUE_SHIFT) +#define SFAFSR_IVUE_SHIFT 28 +#define SFAFSR_TO (_AC(1,UL) << SFAFSR_TO_SHIFT) +#define SFAFSR_TO_SHIFT 27 +#define SFAFSR_BERR (_AC(1,UL) << SFAFSR_BERR_SHIFT) +#define SFAFSR_BERR_SHIFT 26 +#define SFAFSR_LDP (_AC(1,UL) << SFAFSR_LDP_SHIFT) +#define SFAFSR_LDP_SHIFT 25 +#define SFAFSR_CP (_AC(1,UL) << SFAFSR_CP_SHIFT) +#define SFAFSR_CP_SHIFT 24 +#define SFAFSR_WP (_AC(1,UL) << SFAFSR_WP_SHIFT) +#define SFAFSR_WP_SHIFT 23 +#define SFAFSR_EDP (_AC(1,UL) << SFAFSR_EDP_SHIFT) +#define SFAFSR_EDP_SHIFT 22 +#define SFAFSR_UE (_AC(1,UL) << SFAFSR_UE_SHIFT) +#define SFAFSR_UE_SHIFT 21 +#define SFAFSR_CE (_AC(1,UL) << SFAFSR_CE_SHIFT) +#define SFAFSR_CE_SHIFT 20 +#define SFAFSR_ETS (_AC(0xf,UL) << SFAFSR_ETS_SHIFT) +#define SFAFSR_ETS_SHIFT 16 +#define SFAFSR_PSYND (_AC(0xffff,UL) << SFAFSR_PSYND_SHIFT) +#define SFAFSR_PSYND_SHIFT 0 + +/* UDB Error Register, ASI=0x7f VA<63:0>=0x0(High),0x18(Low) for read + * ASI=0x77 VA<63:0>=0x0(High),0x18(Low) for write + */ + +#define UDBE_UE (_AC(1,UL) << 9) +#define UDBE_CE (_AC(1,UL) << 8) +#define UDBE_E_SYNDR (_AC(0xff,UL) << 0) + +/* The trap handlers for asynchronous errors encode the AFSR and + * other pieces of information into a 64-bit argument for C code + * encoded as follows: + * + * ----------------------------------------------- + * | UDB_H | UDB_L | TL>1 | TT | AFSR | + * ----------------------------------------------- + * 63 54 53 44 42 41 33 32 0 + * + * The AFAR is passed in unchanged. + */ +#define SFSTAT_UDBH_MASK (_AC(0x3ff,UL) << SFSTAT_UDBH_SHIFT) +#define SFSTAT_UDBH_SHIFT 54 +#define SFSTAT_UDBL_MASK (_AC(0x3ff,UL) << SFSTAT_UDBH_SHIFT) +#define SFSTAT_UDBL_SHIFT 44 +#define SFSTAT_TL_GT_ONE (_AC(1,UL) << SFSTAT_TL_GT_ONE_SHIFT) +#define SFSTAT_TL_GT_ONE_SHIFT 42 +#define SFSTAT_TRAP_TYPE (_AC(0x1FF,UL) << SFSTAT_TRAP_TYPE_SHIFT) +#define SFSTAT_TRAP_TYPE_SHIFT 33 +#define SFSTAT_AFSR_MASK (_AC(0x1ffffffff,UL) << SFSTAT_AFSR_SHIFT) +#define SFSTAT_AFSR_SHIFT 0 + +/* ESTATE Error Enable Register, ASI=0x4b VA<63:0>=0x0 */ +#define ESTATE_ERR_CE 0x1 /* Correctable errors */ +#define ESTATE_ERR_NCE 0x2 /* TO, BERR, LDP, ETP, EDP, WP, UE, IVUE */ +#define ESTATE_ERR_ISAP 0x4 /* System address parity error */ +#define ESTATE_ERR_ALL (ESTATE_ERR_CE | \ + ESTATE_ERR_NCE | \ + ESTATE_ERR_ISAP) + +/* The various trap types that report using the above state. */ +#define TRAP_TYPE_IAE 0x09 /* Instruction Access Error */ +#define TRAP_TYPE_DAE 0x32 /* Data Access Error */ +#define TRAP_TYPE_CEE 0x63 /* Correctable ECC Error */ + +#endif /* _SPARC64_SFAFSR_H */ diff --git a/arch/sparc/include/asm/sfp-machine.h b/arch/sparc/include/asm/sfp-machine.h new file mode 100644 index 000000000..4a247b405 --- /dev/null +++ b/arch/sparc/include/asm/sfp-machine.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_SFP_MACHINE_H +#define ___ASM_SPARC_SFP_MACHINE_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/sfp-machine_32.h b/arch/sparc/include/asm/sfp-machine_32.h new file mode 100644 index 000000000..838c9d58f --- /dev/null +++ b/arch/sparc/include/asm/sfp-machine_32.h @@ -0,0 +1,212 @@ +/* Machine-dependent software floating-point definitions. + Sparc userland (_Q_*) version. + Copyright (C) 1997,1998,1999 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Richard Henderson (rth@cygnus.com), + Jakub Jelinek (jj@ultra.linux.cz), + David S. Miller (davem@redhat.com) and + Peter Maydell (pmaydell@chiark.greenend.org.uk). + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with the GNU C Library; see the file COPYING.LIB. If + not, write to the Free Software Foundation, Inc., + 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ + +#ifndef _SFP_MACHINE_H +#define _SFP_MACHINE_H + + +#define _FP_W_TYPE_SIZE 32 +#define _FP_W_TYPE unsigned long +#define _FP_WS_TYPE signed long +#define _FP_I_TYPE long + +#define _FP_MUL_MEAT_S(R,X,Y) \ + _FP_MUL_MEAT_1_wide(_FP_WFRACBITS_S,R,X,Y,umul_ppmm) +#define _FP_MUL_MEAT_D(R,X,Y) \ + _FP_MUL_MEAT_2_wide(_FP_WFRACBITS_D,R,X,Y,umul_ppmm) +#define _FP_MUL_MEAT_Q(R,X,Y) \ + _FP_MUL_MEAT_4_wide(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm) + +#define _FP_DIV_MEAT_S(R,X,Y) _FP_DIV_MEAT_1_udiv(S,R,X,Y) +#define _FP_DIV_MEAT_D(R,X,Y) _FP_DIV_MEAT_2_udiv(D,R,X,Y) +#define _FP_DIV_MEAT_Q(R,X,Y) _FP_DIV_MEAT_4_udiv(Q,R,X,Y) + +#define _FP_NANFRAC_S ((_FP_QNANBIT_S << 1) - 1) +#define _FP_NANFRAC_D ((_FP_QNANBIT_D << 1) - 1), -1 +#define _FP_NANFRAC_Q ((_FP_QNANBIT_Q << 1) - 1), -1, -1, -1 +#define _FP_NANSIGN_S 0 +#define _FP_NANSIGN_D 0 +#define _FP_NANSIGN_Q 0 + +#define _FP_KEEPNANFRACP 1 + +/* If one NaN is signaling and the other is not, + * we choose that one, otherwise we choose X. + */ +/* For _Qp_* and _Q_*, this should prefer X, for + * CPU instruction emulation this should prefer Y. + * (see SPAMv9 B.2.2 section). + */ +#define _FP_CHOOSENAN(fs, wc, R, X, Y, OP) \ + do { \ + if ((_FP_FRAC_HIGH_RAW_##fs(Y) & _FP_QNANBIT_##fs) \ + && !(_FP_FRAC_HIGH_RAW_##fs(X) & _FP_QNANBIT_##fs)) \ + { \ + R##_s = X##_s; \ + _FP_FRAC_COPY_##wc(R,X); \ + } \ + else \ + { \ + R##_s = Y##_s; \ + _FP_FRAC_COPY_##wc(R,Y); \ + } \ + R##_c = FP_CLS_NAN; \ + } while (0) + +/* Some assembly to speed things up. */ +#define __FP_FRAC_ADD_3(r2,r1,r0,x2,x1,x0,y2,y1,y0) \ + __asm__ ("addcc %r7,%8,%2\n\t" \ + "addxcc %r5,%6,%1\n\t" \ + "addx %r3,%4,%0\n" \ + : "=r" (r2), \ + "=&r" (r1), \ + "=&r" (r0) \ + : "%rJ" ((USItype)(x2)), \ + "rI" ((USItype)(y2)), \ + "%rJ" ((USItype)(x1)), \ + "rI" ((USItype)(y1)), \ + "%rJ" ((USItype)(x0)), \ + "rI" ((USItype)(y0)) \ + : "cc") + +#define __FP_FRAC_SUB_3(r2,r1,r0,x2,x1,x0,y2,y1,y0) \ + __asm__ ("subcc %r7,%8,%2\n\t" \ + "subxcc %r5,%6,%1\n\t" \ + "subx %r3,%4,%0\n" \ + : "=r" (r2), \ + "=&r" (r1), \ + "=&r" (r0) \ + : "%rJ" ((USItype)(x2)), \ + "rI" ((USItype)(y2)), \ + "%rJ" ((USItype)(x1)), \ + "rI" ((USItype)(y1)), \ + "%rJ" ((USItype)(x0)), \ + "rI" ((USItype)(y0)) \ + : "cc") + +#define __FP_FRAC_ADD_4(r3,r2,r1,r0,x3,x2,x1,x0,y3,y2,y1,y0) \ + do { \ + /* We need to fool gcc, as we need to pass more than 10 \ + input/outputs. */ \ + register USItype _t1 __asm__ ("g1"), _t2 __asm__ ("g2"); \ + __asm__ __volatile__ ( \ + "addcc %r8,%9,%1\n\t" \ + "addxcc %r6,%7,%0\n\t" \ + "addxcc %r4,%5,%%g2\n\t" \ + "addx %r2,%3,%%g1\n\t" \ + : "=&r" (r1), \ + "=&r" (r0) \ + : "%rJ" ((USItype)(x3)), \ + "rI" ((USItype)(y3)), \ + "%rJ" ((USItype)(x2)), \ + "rI" ((USItype)(y2)), \ + "%rJ" ((USItype)(x1)), \ + "rI" ((USItype)(y1)), \ + "%rJ" ((USItype)(x0)), \ + "rI" ((USItype)(y0)) \ + : "cc", "g1", "g2"); \ + __asm__ __volatile__ ("" : "=r" (_t1), "=r" (_t2)); \ + r3 = _t1; r2 = _t2; \ + } while (0) + +#define __FP_FRAC_SUB_4(r3,r2,r1,r0,x3,x2,x1,x0,y3,y2,y1,y0) \ + do { \ + /* We need to fool gcc, as we need to pass more than 10 \ + input/outputs. */ \ + register USItype _t1 __asm__ ("g1"), _t2 __asm__ ("g2"); \ + __asm__ __volatile__ ( \ + "subcc %r8,%9,%1\n\t" \ + "subxcc %r6,%7,%0\n\t" \ + "subxcc %r4,%5,%%g2\n\t" \ + "subx %r2,%3,%%g1\n\t" \ + : "=&r" (r1), \ + "=&r" (r0) \ + : "%rJ" ((USItype)(x3)), \ + "rI" ((USItype)(y3)), \ + "%rJ" ((USItype)(x2)), \ + "rI" ((USItype)(y2)), \ + "%rJ" ((USItype)(x1)), \ + "rI" ((USItype)(y1)), \ + "%rJ" ((USItype)(x0)), \ + "rI" ((USItype)(y0)) \ + : "cc", "g1", "g2"); \ + __asm__ __volatile__ ("" : "=r" (_t1), "=r" (_t2)); \ + r3 = _t1; r2 = _t2; \ + } while (0) + +#define __FP_FRAC_DEC_3(x2,x1,x0,y2,y1,y0) __FP_FRAC_SUB_3(x2,x1,x0,x2,x1,x0,y2,y1,y0) + +#define __FP_FRAC_DEC_4(x3,x2,x1,x0,y3,y2,y1,y0) __FP_FRAC_SUB_4(x3,x2,x1,x0,x3,x2,x1,x0,y3,y2,y1,y0) + +#define __FP_FRAC_ADDI_4(x3,x2,x1,x0,i) \ + __asm__ ("addcc %3,%4,%3\n\t" \ + "addxcc %2,%%g0,%2\n\t" \ + "addxcc %1,%%g0,%1\n\t" \ + "addx %0,%%g0,%0\n\t" \ + : "=&r" (x3), \ + "=&r" (x2), \ + "=&r" (x1), \ + "=&r" (x0) \ + : "rI" ((USItype)(i)), \ + "0" ((USItype)(x3)), \ + "1" ((USItype)(x2)), \ + "2" ((USItype)(x1)), \ + "3" ((USItype)(x0)) \ + : "cc") + +#ifndef CONFIG_SMP +extern struct task_struct *last_task_used_math; +#endif + +/* Obtain the current rounding mode. */ +#ifndef FP_ROUNDMODE +#ifdef CONFIG_SMP +#define FP_ROUNDMODE ((current->thread.fsr >> 30) & 0x3) +#else +#define FP_ROUNDMODE ((last_task_used_math->thread.fsr >> 30) & 0x3) +#endif +#endif + +/* Exception flags. */ +#define FP_EX_INVALID (1 << 4) +#define FP_EX_OVERFLOW (1 << 3) +#define FP_EX_UNDERFLOW (1 << 2) +#define FP_EX_DIVZERO (1 << 1) +#define FP_EX_INEXACT (1 << 0) + +#define FP_HANDLE_EXCEPTIONS return _fex + +#ifdef CONFIG_SMP +#define FP_INHIBIT_RESULTS ((current->thread.fsr >> 23) & _fex) +#else +#define FP_INHIBIT_RESULTS ((last_task_used_math->thread.fsr >> 23) & _fex) +#endif + +#ifdef CONFIG_SMP +#define FP_TRAPPING_EXCEPTIONS ((current->thread.fsr >> 23) & 0x1f) +#else +#define FP_TRAPPING_EXCEPTIONS ((last_task_used_math->thread.fsr >> 23) & 0x1f) +#endif + +#endif diff --git a/arch/sparc/include/asm/sfp-machine_64.h b/arch/sparc/include/asm/sfp-machine_64.h new file mode 100644 index 000000000..ca913ef40 --- /dev/null +++ b/arch/sparc/include/asm/sfp-machine_64.h @@ -0,0 +1,93 @@ +/* Machine-dependent software floating-point definitions. + Sparc64 kernel version. + Copyright (C) 1997,1998,1999 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Richard Henderson (rth@cygnus.com), + Jakub Jelinek (jj@ultra.linux.cz) and + David S. Miller (davem@redhat.com). + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with the GNU C Library; see the file COPYING.LIB. If + not, write to the Free Software Foundation, Inc., + 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ + +#ifndef _SFP_MACHINE_H +#define _SFP_MACHINE_H + +#define _FP_W_TYPE_SIZE 64 +#define _FP_W_TYPE unsigned long +#define _FP_WS_TYPE signed long +#define _FP_I_TYPE long + +#define _FP_MUL_MEAT_S(R,X,Y) \ + _FP_MUL_MEAT_1_imm(_FP_WFRACBITS_S,R,X,Y) +#define _FP_MUL_MEAT_D(R,X,Y) \ + _FP_MUL_MEAT_1_wide(_FP_WFRACBITS_D,R,X,Y,umul_ppmm) +#define _FP_MUL_MEAT_Q(R,X,Y) \ + _FP_MUL_MEAT_2_wide(_FP_WFRACBITS_Q,R,X,Y,umul_ppmm) + +#define _FP_DIV_MEAT_S(R,X,Y) _FP_DIV_MEAT_1_imm(S,R,X,Y,_FP_DIV_HELP_imm) +#define _FP_DIV_MEAT_D(R,X,Y) _FP_DIV_MEAT_1_udiv_norm(D,R,X,Y) +#define _FP_DIV_MEAT_Q(R,X,Y) _FP_DIV_MEAT_2_udiv(Q,R,X,Y) + +#define _FP_NANFRAC_S ((_FP_QNANBIT_S << 1) - 1) +#define _FP_NANFRAC_D ((_FP_QNANBIT_D << 1) - 1) +#define _FP_NANFRAC_Q ((_FP_QNANBIT_Q << 1) - 1), -1 +#define _FP_NANSIGN_S 0 +#define _FP_NANSIGN_D 0 +#define _FP_NANSIGN_Q 0 + +#define _FP_KEEPNANFRACP 1 + +/* If one NaN is signaling and the other is not, + * we choose that one, otherwise we choose X. + */ +/* For _Qp_* and _Q_*, this should prefer X, for + * CPU instruction emulation this should prefer Y. + * (see SPAMv9 B.2.2 section). + */ +#define _FP_CHOOSENAN(fs, wc, R, X, Y, OP) \ + do { \ + if ((_FP_FRAC_HIGH_RAW_##fs(Y) & _FP_QNANBIT_##fs) \ + && !(_FP_FRAC_HIGH_RAW_##fs(X) & _FP_QNANBIT_##fs)) \ + { \ + R##_s = X##_s; \ + _FP_FRAC_COPY_##wc(R,X); \ + } \ + else \ + { \ + R##_s = Y##_s; \ + _FP_FRAC_COPY_##wc(R,Y); \ + } \ + R##_c = FP_CLS_NAN; \ + } while (0) + +/* Obtain the current rounding mode. */ +#ifndef FP_ROUNDMODE +#define FP_ROUNDMODE ((current_thread_info()->xfsr[0] >> 30) & 0x3) +#endif + +/* Exception flags. */ +#define FP_EX_INVALID (1 << 4) +#define FP_EX_OVERFLOW (1 << 3) +#define FP_EX_UNDERFLOW (1 << 2) +#define FP_EX_DIVZERO (1 << 1) +#define FP_EX_INEXACT (1 << 0) + +#define FP_HANDLE_EXCEPTIONS return _fex + +#define FP_INHIBIT_RESULTS ((current_thread_info()->xfsr[0] >> 23) & _fex) + +#define FP_TRAPPING_EXCEPTIONS ((current_thread_info()->xfsr[0] >> 23) & 0x1f) + +#endif diff --git a/arch/sparc/include/asm/shmparam.h b/arch/sparc/include/asm/shmparam.h new file mode 100644 index 000000000..951a4525f --- /dev/null +++ b/arch/sparc/include/asm/shmparam.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_SHMPARAM_H +#define ___ASM_SPARC_SHMPARAM_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/shmparam_32.h b/arch/sparc/include/asm/shmparam_32.h new file mode 100644 index 000000000..9767a8b30 --- /dev/null +++ b/arch/sparc/include/asm/shmparam_32.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASMSPARC_SHMPARAM_H +#define _ASMSPARC_SHMPARAM_H + +#define __ARCH_FORCE_SHMLBA 1 + +extern int vac_cache_size; +#define SHMLBA (vac_cache_size ? vac_cache_size : PAGE_SIZE) + +#endif /* _ASMSPARC_SHMPARAM_H */ diff --git a/arch/sparc/include/asm/shmparam_64.h b/arch/sparc/include/asm/shmparam_64.h new file mode 100644 index 000000000..c0731b571 --- /dev/null +++ b/arch/sparc/include/asm/shmparam_64.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASMSPARC64_SHMPARAM_H +#define _ASMSPARC64_SHMPARAM_H + +#include + +#define __ARCH_FORCE_SHMLBA 1 +/* attach addr a multiple of this */ +#define SHMLBA ((PAGE_SIZE > L1DCACHE_SIZE) ? PAGE_SIZE : L1DCACHE_SIZE) + +#endif /* _ASMSPARC64_SHMPARAM_H */ diff --git a/arch/sparc/include/asm/sigcontext.h b/arch/sparc/include/asm/sigcontext.h new file mode 100644 index 000000000..200f95144 --- /dev/null +++ b/arch/sparc/include/asm/sigcontext.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_SIGCONTEXT_H +#define __SPARC_SIGCONTEXT_H + +#include +#include + +#ifndef __ASSEMBLER__ + +#define __SUNOS_MAXWIN 31 + +/* This is what SunOS does, so shall I unless we use new 32bit signals or rt signals. */ +struct sigcontext32 { + int sigc_onstack; /* state to restore */ + int sigc_mask; /* sigmask to restore */ + int sigc_sp; /* stack pointer */ + int sigc_pc; /* program counter */ + int sigc_npc; /* next program counter */ + int sigc_psr; /* for condition codes etc */ + int sigc_g1; /* User uses these two registers */ + int sigc_o0; /* within the trampoline code. */ + + /* Now comes information regarding the users window set + * at the time of the signal. + */ + int sigc_oswins; /* outstanding windows */ + + /* stack ptrs for each regwin buf */ + unsigned int sigc_spbuf[__SUNOS_MAXWIN]; + + /* Windows to restore after signal */ + struct reg_window32 sigc_wbuf[__SUNOS_MAXWIN]; +}; + + +/* This is what we use for 32bit new non-rt signals. */ + +typedef struct { + struct { + unsigned int psr; + unsigned int pc; + unsigned int npc; + unsigned int y; + unsigned int u_regs[16]; /* globals and ins */ + } si_regs; + int si_mask; +} __siginfo32_t; + +#define __SIGC_MAXWIN 7 + +typedef struct { + unsigned long locals[8]; + unsigned long ins[8]; +} __siginfo_reg_window; + +typedef struct { + int wsaved; + __siginfo_reg_window reg_window[__SIGC_MAXWIN]; + unsigned long rwbuf_stkptrs[__SIGC_MAXWIN]; +} __siginfo_rwin_t; + +#ifdef CONFIG_SPARC64 +typedef struct { + unsigned int si_float_regs [64]; + unsigned long si_fsr; + unsigned long si_gsr; + unsigned long si_fprs; +} __siginfo_fpu_t; + +/* This is what SunOS doesn't, so we have to write this alone + and do it properly. */ +struct sigcontext { + /* The size of this array has to match SI_MAX_SIZE from siginfo.h */ + char sigc_info[128]; + struct { + unsigned long u_regs[16]; /* globals and ins */ + unsigned long tstate; + unsigned long tpc; + unsigned long tnpc; + unsigned int y; + unsigned int fprs; + } sigc_regs; + __siginfo_fpu_t * sigc_fpu_save; + struct { + void * ss_sp; + int ss_flags; + unsigned long ss_size; + } sigc_stack; + unsigned long sigc_mask; + __siginfo_rwin_t * sigc_rwin_save; +}; + +#else + +typedef struct { + unsigned long si_float_regs [32]; + unsigned long si_fsr; + unsigned long si_fpqdepth; + struct { + unsigned long *insn_addr; + unsigned long insn; + } si_fpqueue [16]; +} __siginfo_fpu_t; +#endif /* (CONFIG_SPARC64) */ + + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(__SPARC_SIGCONTEXT_H) */ diff --git a/arch/sparc/include/asm/signal.h b/arch/sparc/include/asm/signal.h new file mode 100644 index 000000000..d93fe9354 --- /dev/null +++ b/arch/sparc/include/asm/signal.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_SIGNAL_H +#define __SPARC_SIGNAL_H + +#ifndef __ASSEMBLER__ +#include +#include +#endif +#include + +#ifndef __ASSEMBLER__ + +#define __ARCH_HAS_KA_RESTORER +#define __ARCH_HAS_SA_RESTORER + +#endif /* !(__ASSEMBLER__) */ +#endif /* !(__SPARC_SIGNAL_H) */ diff --git a/arch/sparc/include/asm/smp.h b/arch/sparc/include/asm/smp.h new file mode 100644 index 000000000..dea59f6ce --- /dev/null +++ b/arch/sparc/include/asm/smp.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_SMP_H +#define ___ASM_SPARC_SMP_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/smp_32.h b/arch/sparc/include/asm/smp_32.h new file mode 100644 index 000000000..9c6ed98fb --- /dev/null +++ b/arch/sparc/include/asm/smp_32.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* smp.h: Sparc specific SMP stuff. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_SMP_H +#define _SPARC_SMP_H + +#include +#include + +#ifndef __ASSEMBLER__ + +#include + +#endif /* __ASSEMBLER__ */ + +#ifdef CONFIG_SMP + +#ifndef __ASSEMBLER__ + +#include +#include +#include + +/* + * Private routines/data + */ + +extern unsigned char boot_cpu_id; +extern volatile unsigned long cpu_callin_map[NR_CPUS]; +extern cpumask_t smp_commenced_mask; +extern struct linux_prom_registers smp_penguin_ctable; + +void cpu_panic(void); + +/* + * General functions that each host system must provide. + */ + +void sun4m_init_smp(void); +void sun4d_init_smp(void); + +void smp_callin(void); +void smp_store_cpu_info(int); + +void smp_resched_interrupt(void); +void smp_call_function_single_interrupt(void); +void smp_call_function_interrupt(void); + +struct seq_file; +void smp_bogo(struct seq_file *); +void smp_info(struct seq_file *); + +struct sparc32_ipi_ops { + void (*cross_call)(void *func, cpumask_t mask, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4); + void (*resched)(int cpu); + void (*single)(int cpu); + void (*mask_one)(int cpu); +}; +extern const struct sparc32_ipi_ops *sparc32_ipi_ops; + +static inline void xc0(void *func) +{ + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, 0, 0, 0, 0); +} + +static inline void xc1(void *func, unsigned long arg1) +{ + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, 0, 0, 0); +} +static inline void xc2(void *func, unsigned long arg1, unsigned long arg2) +{ + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, arg1, arg2, 0, 0); +} + +static inline void xc3(void *func, unsigned long arg1, unsigned long arg2, + unsigned long arg3) +{ + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, + arg1, arg2, arg3, 0); +} + +static inline void xc4(void *func, unsigned long arg1, unsigned long arg2, + unsigned long arg3, unsigned long arg4) +{ + sparc32_ipi_ops->cross_call(func, *cpu_online_mask, + arg1, arg2, arg3, arg4); +} + +void arch_send_call_function_single_ipi(int cpu); +void arch_send_call_function_ipi_mask(const struct cpumask *mask); + +static inline int cpu_logical_map(int cpu) +{ + return cpu; +} + +int hard_smp_processor_id(void); + +#define raw_smp_processor_id() (current_thread_info()->cpu) + +void smp_setup_cpu_possible_map(void); + +#endif /* !(__ASSEMBLER__) */ + +/* Sparc specific messages. */ +#define MSG_CROSS_CALL 0x0005 /* run func on cpus */ + +/* Empirical PROM processor mailbox constants. If the per-cpu mailbox + * contains something other than one of these then the ipi is from + * Linux's active_kernel_processor. This facility exists so that + * the boot monitor can capture all the other cpus when one catches + * a watchdog reset or the user enters the monitor using L1-A keys. + */ +#define MBOX_STOPCPU 0xFB +#define MBOX_IDLECPU 0xFC +#define MBOX_IDLECPU2 0xFD +#define MBOX_STOPCPU2 0xFE + +#else /* SMP */ + +#define hard_smp_processor_id() 0 +#define smp_setup_cpu_possible_map() do { } while (0) + +#endif /* !(SMP) */ +#endif /* !(_SPARC_SMP_H) */ diff --git a/arch/sparc/include/asm/smp_64.h b/arch/sparc/include/asm/smp_64.h new file mode 100644 index 000000000..759fb4a95 --- /dev/null +++ b/arch/sparc/include/asm/smp_64.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* smp.h: Sparc64 specific SMP stuff. + * + * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net) + */ + +#ifndef _SPARC64_SMP_H +#define _SPARC64_SMP_H + +#include +#include +#include +#include + +#ifndef __ASSEMBLER__ + +#include +#include + +#endif /* !(__ASSEMBLER__) */ + +#ifdef CONFIG_SMP + +#ifndef __ASSEMBLER__ + +/* + * Private routines/data + */ + +#include +#include +#include + +DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); +extern cpumask_t cpu_core_map[NR_CPUS]; + +void smp_init_cpu_poke(void); +void scheduler_poke(void); + +void arch_send_call_function_single_ipi(int cpu); +void arch_send_call_function_ipi_mask(const struct cpumask *mask); + +/* + * General functions that each host system must provide. + */ + +int hard_smp_processor_id(void); +#define raw_smp_processor_id() (current_thread_info()->cpu) + +void smp_fill_in_sib_core_maps(void); +void __noreturn cpu_play_dead(void); + +void smp_fetch_global_regs(void); +void smp_fetch_global_pmu(void); + +struct seq_file; +void smp_bogo(struct seq_file *); +void smp_info(struct seq_file *); + +void smp_callin(void); +void cpu_panic(void); +void smp_synchronize_tick_client(void); +void smp_capture(void); +void smp_release(void); + +#ifdef CONFIG_HOTPLUG_CPU +int __cpu_disable(void); +void __cpu_die(unsigned int cpu); +#endif + +#endif /* !(__ASSEMBLER__) */ + +#else + +#define hard_smp_processor_id() 0 +#define smp_fill_in_sib_core_maps() do { } while (0) +#define smp_fetch_global_regs() do { } while (0) +#define smp_fetch_global_pmu() do { } while (0) +#define smp_init_cpu_poke() do { } while (0) +#define scheduler_poke() do { } while (0) + +#endif /* !(CONFIG_SMP) */ + +#endif /* !(_SPARC64_SMP_H) */ diff --git a/arch/sparc/include/asm/sparsemem.h b/arch/sparc/include/asm/sparsemem.h new file mode 100644 index 000000000..aa9a676bc --- /dev/null +++ b/arch/sparc/include/asm/sparsemem.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_SPARSEMEM_H +#define _SPARC64_SPARSEMEM_H + +#ifdef __KERNEL__ + +#include + +#define SECTION_SIZE_BITS 30 +#define MAX_PHYSMEM_BITS MAX_PHYS_ADDRESS_BITS + +#endif /* !(__KERNEL__) */ + +#endif /* !(_SPARC64_SPARSEMEM_H) */ diff --git a/arch/sparc/include/asm/spinlock.h b/arch/sparc/include/asm/spinlock.h new file mode 100644 index 000000000..3f4ce55bc --- /dev/null +++ b/arch/sparc/include/asm/spinlock.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_SPINLOCK_H +#define ___ASM_SPARC_SPINLOCK_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/spinlock_32.h b/arch/sparc/include/asm/spinlock_32.h new file mode 100644 index 000000000..6d6d261bf --- /dev/null +++ b/arch/sparc/include/asm/spinlock_32.h @@ -0,0 +1,188 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* spinlock.h: 32-bit Sparc spinlock support. + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef __SPARC_SPINLOCK_H +#define __SPARC_SPINLOCK_H + +#ifndef __ASSEMBLER__ + +#include +#include +#include /* for cpu_relax */ + +#define arch_spin_is_locked(lock) (*((volatile unsigned char *)(lock)) != 0) + +static inline void arch_spin_lock(arch_spinlock_t *lock) +{ + __asm__ __volatile__( + "\n1:\n\t" + "ldstub [%0], %%g2\n\t" + "orcc %%g2, 0x0, %%g0\n\t" + "bne,a 2f\n\t" + " ldub [%0], %%g2\n\t" + ".subsection 2\n" + "2:\n\t" + "orcc %%g2, 0x0, %%g0\n\t" + "bne,a 2b\n\t" + " ldub [%0], %%g2\n\t" + "b,a 1b\n\t" + ".previous\n" + : /* no outputs */ + : "r" (lock) + : "g2", "memory", "cc"); +} + +static inline int arch_spin_trylock(arch_spinlock_t *lock) +{ + unsigned int result; + __asm__ __volatile__("ldstub [%1], %0" + : "=r" (result) + : "r" (lock) + : "memory"); + return (result == 0); +} + +static inline void arch_spin_unlock(arch_spinlock_t *lock) +{ + __asm__ __volatile__("stb %%g0, [%0]" : : "r" (lock) : "memory"); +} + +/* Read-write spinlocks, allowing multiple readers + * but only one writer. + * + * NOTE! it is quite common to have readers in interrupts + * but no interrupt writers. For those circumstances we + * can "mix" irq-safe locks - any writer needs to get a + * irq-safe write-lock, but readers can get non-irqsafe + * read-locks. + * + * XXX This might create some problems with my dual spinlock + * XXX scheme, deadlocks etc. -DaveM + * + * Sort of like atomic_t's on Sparc, but even more clever. + * + * ------------------------------------ + * | 24-bit counter | wlock | arch_rwlock_t + * ------------------------------------ + * 31 8 7 0 + * + * wlock signifies the one writer is in or somebody is updating + * counter. For a writer, if he successfully acquires the wlock, + * but counter is non-zero, he has to release the lock and wait, + * till both counter and wlock are zero. + * + * Unfortunately this scheme limits us to ~16,000,000 cpus. + */ +static inline void __arch_read_lock(arch_rwlock_t *rw) +{ + register arch_rwlock_t *lp asm("g1"); + lp = rw; + __asm__ __volatile__( + "mov %%o7, %%g4\n\t" + "call ___rw_read_enter\n\t" + " ldstub [%%g1 + 3], %%g2\n" + : /* no outputs */ + : "r" (lp) + : "g2", "g4", "memory", "cc"); +} + +#define arch_read_lock(lock) \ +do { unsigned long flags; \ + local_irq_save(flags); \ + __arch_read_lock(lock); \ + local_irq_restore(flags); \ +} while(0) + +static inline void __arch_read_unlock(arch_rwlock_t *rw) +{ + register arch_rwlock_t *lp asm("g1"); + lp = rw; + __asm__ __volatile__( + "mov %%o7, %%g4\n\t" + "call ___rw_read_exit\n\t" + " ldstub [%%g1 + 3], %%g2\n" + : /* no outputs */ + : "r" (lp) + : "g2", "g4", "memory", "cc"); +} + +#define arch_read_unlock(lock) \ +do { unsigned long flags; \ + local_irq_save(flags); \ + __arch_read_unlock(lock); \ + local_irq_restore(flags); \ +} while(0) + +static inline void arch_write_lock(arch_rwlock_t *rw) +{ + register arch_rwlock_t *lp asm("g1"); + lp = rw; + __asm__ __volatile__( + "mov %%o7, %%g4\n\t" + "call ___rw_write_enter\n\t" + " ldstub [%%g1 + 3], %%g2\n" + : /* no outputs */ + : "r" (lp) + : "g2", "g4", "memory", "cc"); + *(volatile __u32 *)&lp->lock = ~0U; +} + +static inline void arch_write_unlock(arch_rwlock_t *lock) +{ + __asm__ __volatile__( +" st %%g0, [%0]" + : /* no outputs */ + : "r" (lock) + : "memory"); +} + +static inline int arch_write_trylock(arch_rwlock_t *rw) +{ + unsigned int val; + + __asm__ __volatile__("ldstub [%1 + 3], %0" + : "=r" (val) + : "r" (&rw->lock) + : "memory"); + + if (val == 0) { + val = rw->lock & ~0xff; + if (val) + ((volatile u8*)&rw->lock)[3] = 0; + else + *(volatile u32*)&rw->lock = ~0U; + } + + return (val == 0); +} + +static inline int __arch_read_trylock(arch_rwlock_t *rw) +{ + register arch_rwlock_t *lp asm("g1"); + register int res asm("o0"); + lp = rw; + __asm__ __volatile__( + "mov %%o7, %%g4\n\t" + "call ___rw_read_try\n\t" + " ldstub [%%g1 + 3], %%g2\n" + : "=r" (res) + : "r" (lp) + : "g2", "g4", "memory", "cc"); + return res; +} + +#define arch_read_trylock(lock) \ +({ unsigned long flags; \ + int res; \ + local_irq_save(flags); \ + res = __arch_read_trylock(lock); \ + local_irq_restore(flags); \ + res; \ +}) + +#endif /* !(__ASSEMBLER__) */ + +#endif /* __SPARC_SPINLOCK_H */ diff --git a/arch/sparc/include/asm/spinlock_64.h b/arch/sparc/include/asm/spinlock_64.h new file mode 100644 index 000000000..13cd15d34 --- /dev/null +++ b/arch/sparc/include/asm/spinlock_64.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* spinlock.h: 64-bit Sparc spinlock support. + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef __SPARC64_SPINLOCK_H +#define __SPARC64_SPINLOCK_H + +#ifndef __ASSEMBLER__ + +#include +#include +#include +#include + +#endif /* !(__ASSEMBLER__) */ + +#endif /* !(__SPARC64_SPINLOCK_H) */ diff --git a/arch/sparc/include/asm/spinlock_types.h b/arch/sparc/include/asm/spinlock_types.h new file mode 100644 index 000000000..ed1d569b6 --- /dev/null +++ b/arch/sparc/include/asm/spinlock_types.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_SPINLOCK_TYPES_H +#define __SPARC_SPINLOCK_TYPES_H + +#ifdef CONFIG_QUEUED_SPINLOCKS +#include +#else + +typedef struct { + volatile unsigned char lock; +} arch_spinlock_t; + +#define __ARCH_SPIN_LOCK_UNLOCKED { 0 } +#endif /* CONFIG_QUEUED_SPINLOCKS */ + +#ifdef CONFIG_QUEUED_RWLOCKS +#include +#else +typedef struct { + volatile unsigned int lock; +} arch_rwlock_t; + +#define __ARCH_RW_LOCK_UNLOCKED { 0 } +#endif /* CONFIG_QUEUED_RWLOCKS */ +#endif diff --git a/arch/sparc/include/asm/spitfire.h b/arch/sparc/include/asm/spitfire.h new file mode 100644 index 000000000..79b9dd5e9 --- /dev/null +++ b/arch/sparc/include/asm/spitfire.h @@ -0,0 +1,368 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* spitfire.h: SpitFire/BlackBird/Cheetah inline MMU operations. + * + * Copyright (C) 1996 David S. Miller (davem@davemloft.net) + */ + +#ifndef _SPARC64_SPITFIRE_H +#define _SPARC64_SPITFIRE_H + +#ifdef CONFIG_SPARC64 + +#include + +/* The following register addresses are accessible via ASI_DMMU + * and ASI_IMMU, that is there is a distinct and unique copy of + * each these registers for each TLB. + */ +#define TSB_TAG_TARGET 0x0000000000000000 /* All chips */ +#define TLB_SFSR 0x0000000000000018 /* All chips */ +#define TSB_REG 0x0000000000000028 /* All chips */ +#define TLB_TAG_ACCESS 0x0000000000000030 /* All chips */ +#define VIRT_WATCHPOINT 0x0000000000000038 /* All chips */ +#define PHYS_WATCHPOINT 0x0000000000000040 /* All chips */ +#define TSB_EXTENSION_P 0x0000000000000048 /* Ultra-III and later */ +#define TSB_EXTENSION_S 0x0000000000000050 /* Ultra-III and later, D-TLB only */ +#define TSB_EXTENSION_N 0x0000000000000058 /* Ultra-III and later */ +#define TLB_TAG_ACCESS_EXT 0x0000000000000060 /* Ultra-III+ and later */ + +/* These registers only exist as one entity, and are accessed + * via ASI_DMMU only. + */ +#define PRIMARY_CONTEXT 0x0000000000000008 +#define SECONDARY_CONTEXT 0x0000000000000010 +#define DMMU_SFAR 0x0000000000000020 +#define VIRT_WATCHPOINT 0x0000000000000038 +#define PHYS_WATCHPOINT 0x0000000000000040 + +#define SPITFIRE_HIGHEST_LOCKED_TLBENT (64 - 1) +#define CHEETAH_HIGHEST_LOCKED_TLBENT (16 - 1) + +#define L1DCACHE_SIZE 0x4000 + +#define SUN4V_CHIP_INVALID 0x00 +#define SUN4V_CHIP_NIAGARA1 0x01 +#define SUN4V_CHIP_NIAGARA2 0x02 +#define SUN4V_CHIP_NIAGARA3 0x03 +#define SUN4V_CHIP_NIAGARA4 0x04 +#define SUN4V_CHIP_NIAGARA5 0x05 +#define SUN4V_CHIP_SPARC_M6 0x06 +#define SUN4V_CHIP_SPARC_M7 0x07 +#define SUN4V_CHIP_SPARC_M8 0x08 +#define SUN4V_CHIP_SPARC64X 0x8a +#define SUN4V_CHIP_SPARC_SN 0x8b +#define SUN4V_CHIP_UNKNOWN 0xff + +/* + * The following CPU_ID_xxx constants are used + * to identify the CPU type in the setup phase + * (see head_64.S) + */ +#define CPU_ID_NIAGARA1 ('1') +#define CPU_ID_NIAGARA2 ('2') +#define CPU_ID_NIAGARA3 ('3') +#define CPU_ID_NIAGARA4 ('4') +#define CPU_ID_NIAGARA5 ('5') +#define CPU_ID_M6 ('6') +#define CPU_ID_M7 ('7') +#define CPU_ID_M8 ('8') +#define CPU_ID_SONOMA1 ('N') + +#ifndef __ASSEMBLER__ + +enum ultra_tlb_layout { + spitfire = 0, + cheetah = 1, + cheetah_plus = 2, + hypervisor = 3, +}; + +extern enum ultra_tlb_layout tlb_type; + +extern int sun4v_chip_type; + +extern int cheetah_pcache_forced_on; +void cheetah_enable_pcache(void); + +#define sparc64_highest_locked_tlbent() \ + (tlb_type == spitfire ? \ + SPITFIRE_HIGHEST_LOCKED_TLBENT : \ + CHEETAH_HIGHEST_LOCKED_TLBENT) + +extern int num_kernel_image_mappings; + +/* The data cache is write through, so this just invalidates the + * specified line. + */ +static inline void spitfire_put_dcache_tag(unsigned long addr, unsigned long tag) +{ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (tag), "r" (addr), "i" (ASI_DCACHE_TAG)); +} + +/* The instruction cache lines are flushed with this, but note that + * this does not flush the pipeline. It is possible for a line to + * get flushed but stale instructions to still be in the pipeline, + * a flush instruction (to any address) is sufficient to handle + * this issue after the line is invalidated. + */ +static inline void spitfire_put_icache_tag(unsigned long addr, unsigned long tag) +{ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (tag), "r" (addr), "i" (ASI_IC_TAG)); +} + +static inline unsigned long spitfire_get_dtlb_data(int entry) +{ + unsigned long data; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (data) + : "r" (entry << 3), "i" (ASI_DTLB_DATA_ACCESS)); + + /* Clear TTE diag bits. */ + data &= ~0x0003fe0000000000UL; + + return data; +} + +static inline unsigned long spitfire_get_dtlb_tag(int entry) +{ + unsigned long tag; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (tag) + : "r" (entry << 3), "i" (ASI_DTLB_TAG_READ)); + return tag; +} + +static inline void spitfire_put_dtlb_data(int entry, unsigned long data) +{ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (data), "r" (entry << 3), + "i" (ASI_DTLB_DATA_ACCESS)); +} + +static inline unsigned long spitfire_get_itlb_data(int entry) +{ + unsigned long data; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (data) + : "r" (entry << 3), "i" (ASI_ITLB_DATA_ACCESS)); + + /* Clear TTE diag bits. */ + data &= ~0x0003fe0000000000UL; + + return data; +} + +static inline unsigned long spitfire_get_itlb_tag(int entry) +{ + unsigned long tag; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (tag) + : "r" (entry << 3), "i" (ASI_ITLB_TAG_READ)); + return tag; +} + +static inline void spitfire_put_itlb_data(int entry, unsigned long data) +{ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (data), "r" (entry << 3), + "i" (ASI_ITLB_DATA_ACCESS)); +} + +static inline void spitfire_flush_dtlb_nucleus_page(unsigned long page) +{ + __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (page | 0x20), "i" (ASI_DMMU_DEMAP)); +} + +static inline void spitfire_flush_itlb_nucleus_page(unsigned long page) +{ + __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (page | 0x20), "i" (ASI_IMMU_DEMAP)); +} + +/* Cheetah has "all non-locked" tlb flushes. */ +static inline void cheetah_flush_dtlb_all(void) +{ + __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (0x80), "i" (ASI_DMMU_DEMAP)); +} + +static inline void cheetah_flush_itlb_all(void) +{ + __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (0x80), "i" (ASI_IMMU_DEMAP)); +} + +/* Cheetah has a 4-tlb layout so direct access is a bit different. + * The first two TLBs are fully assosciative, hold 16 entries, and are + * used only for locked and >8K sized translations. One exists for + * data accesses and one for instruction accesses. + * + * The third TLB is for data accesses to 8K non-locked translations, is + * 2 way assosciative, and holds 512 entries. The fourth TLB is for + * instruction accesses to 8K non-locked translations, is 2 way + * assosciative, and holds 128 entries. + * + * Cheetah has some bug where bogus data can be returned from + * ASI_{D,I}TLB_DATA_ACCESS loads, doing the load twice fixes + * the problem for me. -DaveM + */ +static inline unsigned long cheetah_get_ldtlb_data(int entry) +{ + unsigned long data; + + __asm__ __volatile__("ldxa [%1] %2, %%g0\n\t" + "ldxa [%1] %2, %0" + : "=r" (data) + : "r" ((0 << 16) | (entry << 3)), + "i" (ASI_DTLB_DATA_ACCESS)); + + return data; +} + +static inline unsigned long cheetah_get_litlb_data(int entry) +{ + unsigned long data; + + __asm__ __volatile__("ldxa [%1] %2, %%g0\n\t" + "ldxa [%1] %2, %0" + : "=r" (data) + : "r" ((0 << 16) | (entry << 3)), + "i" (ASI_ITLB_DATA_ACCESS)); + + return data; +} + +static inline unsigned long cheetah_get_ldtlb_tag(int entry) +{ + unsigned long tag; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (tag) + : "r" ((0 << 16) | (entry << 3)), + "i" (ASI_DTLB_TAG_READ)); + + return tag; +} + +static inline unsigned long cheetah_get_litlb_tag(int entry) +{ + unsigned long tag; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (tag) + : "r" ((0 << 16) | (entry << 3)), + "i" (ASI_ITLB_TAG_READ)); + + return tag; +} + +static inline void cheetah_put_ldtlb_data(int entry, unsigned long data) +{ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (data), + "r" ((0 << 16) | (entry << 3)), + "i" (ASI_DTLB_DATA_ACCESS)); +} + +static inline void cheetah_put_litlb_data(int entry, unsigned long data) +{ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (data), + "r" ((0 << 16) | (entry << 3)), + "i" (ASI_ITLB_DATA_ACCESS)); +} + +static inline unsigned long cheetah_get_dtlb_data(int entry, int tlb) +{ + unsigned long data; + + __asm__ __volatile__("ldxa [%1] %2, %%g0\n\t" + "ldxa [%1] %2, %0" + : "=r" (data) + : "r" ((tlb << 16) | (entry << 3)), "i" (ASI_DTLB_DATA_ACCESS)); + + return data; +} + +static inline unsigned long cheetah_get_dtlb_tag(int entry, int tlb) +{ + unsigned long tag; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (tag) + : "r" ((tlb << 16) | (entry << 3)), "i" (ASI_DTLB_TAG_READ)); + return tag; +} + +static inline void cheetah_put_dtlb_data(int entry, unsigned long data, int tlb) +{ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (data), + "r" ((tlb << 16) | (entry << 3)), + "i" (ASI_DTLB_DATA_ACCESS)); +} + +static inline unsigned long cheetah_get_itlb_data(int entry) +{ + unsigned long data; + + __asm__ __volatile__("ldxa [%1] %2, %%g0\n\t" + "ldxa [%1] %2, %0" + : "=r" (data) + : "r" ((2 << 16) | (entry << 3)), + "i" (ASI_ITLB_DATA_ACCESS)); + + return data; +} + +static inline unsigned long cheetah_get_itlb_tag(int entry) +{ + unsigned long tag; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (tag) + : "r" ((2 << 16) | (entry << 3)), "i" (ASI_ITLB_TAG_READ)); + return tag; +} + +static inline void cheetah_put_itlb_data(int entry, unsigned long data) +{ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* No outputs */ + : "r" (data), "r" ((2 << 16) | (entry << 3)), + "i" (ASI_ITLB_DATA_ACCESS)); +} + +#endif /* !(__ASSEMBLER__) */ +#endif /* CONFIG_SPARC64 */ +#endif /* !(_SPARC64_SPITFIRE_H) */ diff --git a/arch/sparc/include/asm/stacktrace.h b/arch/sparc/include/asm/stacktrace.h new file mode 100644 index 000000000..556ec5d59 --- /dev/null +++ b/arch/sparc/include/asm/stacktrace.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_STACKTRACE_H +#define _SPARC64_STACKTRACE_H + +void stack_trace_flush(void); + +#endif /* _SPARC64_STACKTRACE_H */ diff --git a/arch/sparc/include/asm/starfire.h b/arch/sparc/include/asm/starfire.h new file mode 100644 index 000000000..8e511ed78 --- /dev/null +++ b/arch/sparc/include/asm/starfire.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * starfire.h: Group all starfire specific code together. + * + * Copyright (C) 2000 Anton Blanchard (anton@samba.org) + */ + +#ifndef _SPARC64_STARFIRE_H +#define _SPARC64_STARFIRE_H + +#ifndef __ASSEMBLER__ + +extern int this_is_starfire; + +void check_if_starfire(void); +void starfire_hookup(int); +unsigned int starfire_translate(unsigned long imap, unsigned int upaid); + +#endif +#endif diff --git a/arch/sparc/include/asm/string.h b/arch/sparc/include/asm/string.h new file mode 100644 index 000000000..001a17baf --- /dev/null +++ b/arch/sparc/include/asm/string.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_STRING_H +#define ___ASM_SPARC_STRING_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif + +/* First the mem*() things. */ +#define __HAVE_ARCH_MEMMOVE +void *memmove(void *, const void *, __kernel_size_t); + +#define __HAVE_ARCH_MEMCPY +#define memcpy(t, f, n) __builtin_memcpy(t, f, n) + +#define __HAVE_ARCH_MEMSET +#define memset(s, c, count) __builtin_memset(s, c, count) + +#define __HAVE_ARCH_MEMSCAN + +#define memscan(__arg0, __char, __arg2) \ +({ \ + void *__memscan_zero(void *, size_t); \ + void *__memscan_generic(void *, int, size_t); \ + void *__retval, *__addr = (__arg0); \ + size_t __size = (__arg2); \ + \ + if(__builtin_constant_p(__char) && !(__char)) \ + __retval = __memscan_zero(__addr, __size); \ + else \ + __retval = __memscan_generic(__addr, (__char), __size); \ + \ + __retval; \ +}) + +#define __HAVE_ARCH_MEMCMP +int memcmp(const void *,const void *,__kernel_size_t); + +/* Now the str*() stuff... */ +#define __HAVE_ARCH_STRLEN +__kernel_size_t strlen(const char *); + +#define __HAVE_ARCH_STRNCMP +int strncmp(const char *, const char *, __kernel_size_t); + +#endif diff --git a/arch/sparc/include/asm/string_32.h b/arch/sparc/include/asm/string_32.h new file mode 100644 index 000000000..f488946bd --- /dev/null +++ b/arch/sparc/include/asm/string_32.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * string.h: External definitions for optimized assembly string + * routines for the Linux Kernel. + * + * Copyright (C) 1995,1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#ifndef __SPARC_STRING_H__ +#define __SPARC_STRING_H__ + +#include + +#endif /* !(__SPARC_STRING_H__) */ diff --git a/arch/sparc/include/asm/string_64.h b/arch/sparc/include/asm/string_64.h new file mode 100644 index 000000000..d5c563058 --- /dev/null +++ b/arch/sparc/include/asm/string_64.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * string.h: External definitions for optimized assembly string + * routines for the Linux Kernel. + * + * Copyright (C) 1995,1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996,1997,1999 Jakub Jelinek (jakub@redhat.com) + */ + +#ifndef __SPARC64_STRING_H__ +#define __SPARC64_STRING_H__ + +#include + +#endif /* !(__SPARC64_STRING_H__) */ diff --git a/arch/sparc/include/asm/sunbpp.h b/arch/sparc/include/asm/sunbpp.h new file mode 100644 index 000000000..55de4da70 --- /dev/null +++ b/arch/sparc/include/asm/sunbpp.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/asm/sunbpp.h + */ + +#ifndef _ASM_SPARC_SUNBPP_H +#define _ASM_SPARC_SUNBPP_H + +struct bpp_regs { + /* DMA registers */ + __volatile__ __u32 p_csr; /* DMA Control/Status Register */ + __volatile__ __u32 p_addr; /* Address Register */ + __volatile__ __u32 p_bcnt; /* Byte Count Register */ + __volatile__ __u32 p_tst_csr; /* Test Control/Status (DMA2 only) */ + /* Parallel Port registers */ + __volatile__ __u16 p_hcr; /* Hardware Configuration Register */ + __volatile__ __u16 p_ocr; /* Operation Configuration Register */ + __volatile__ __u8 p_dr; /* Parallel Data Register */ + __volatile__ __u8 p_tcr; /* Transfer Control Register */ + __volatile__ __u8 p_or; /* Output Register */ + __volatile__ __u8 p_ir; /* Input Register */ + __volatile__ __u16 p_icr; /* Interrupt Control Register */ +}; + +/* P_HCR. Time is in increments of SBus clock. */ +#define P_HCR_TEST 0x8000 /* Allows buried counters to be read */ +#define P_HCR_DSW 0x7f00 /* Data strobe width (in ticks) */ +#define P_HCR_DDS 0x007f /* Data setup before strobe (in ticks) */ + +/* P_OCR. */ +#define P_OCR_MEM_CLR 0x8000 +#define P_OCR_DATA_SRC 0x4000 /* ) */ +#define P_OCR_DS_DSEL 0x2000 /* ) Bidirectional */ +#define P_OCR_BUSY_DSEL 0x1000 /* ) selects */ +#define P_OCR_ACK_DSEL 0x0800 /* ) */ +#define P_OCR_EN_DIAG 0x0400 +#define P_OCR_BUSY_OP 0x0200 /* Busy operation */ +#define P_OCR_ACK_OP 0x0100 /* Ack operation */ +#define P_OCR_SRST 0x0080 /* Reset state machines. Not selfcleaning. */ +#define P_OCR_IDLE 0x0008 /* PP data transfer state machine is idle */ +#define P_OCR_V_ILCK 0x0002 /* Versatec faded. Zebra only. */ +#define P_OCR_EN_VER 0x0001 /* Enable Versatec (0 - enable). Zebra only. */ + +/* P_TCR */ +#define P_TCR_DIR 0x08 +#define P_TCR_BUSY 0x04 +#define P_TCR_ACK 0x02 +#define P_TCR_DS 0x01 /* Strobe */ + +/* P_OR */ +#define P_OR_V3 0x20 /* ) */ +#define P_OR_V2 0x10 /* ) on Zebra only */ +#define P_OR_V1 0x08 /* ) */ +#define P_OR_INIT 0x04 +#define P_OR_AFXN 0x02 /* Auto Feed */ +#define P_OR_SLCT_IN 0x01 + +/* P_IR */ +#define P_IR_PE 0x04 +#define P_IR_SLCT 0x02 +#define P_IR_ERR 0x01 + +/* P_ICR */ +#define P_DS_IRQ 0x8000 /* RW1 */ +#define P_ACK_IRQ 0x4000 /* RW1 */ +#define P_BUSY_IRQ 0x2000 /* RW1 */ +#define P_PE_IRQ 0x1000 /* RW1 */ +#define P_SLCT_IRQ 0x0800 /* RW1 */ +#define P_ERR_IRQ 0x0400 /* RW1 */ +#define P_DS_IRQ_EN 0x0200 /* RW Always on rising edge */ +#define P_ACK_IRQ_EN 0x0100 /* RW Always on rising edge */ +#define P_BUSY_IRP 0x0080 /* RW 1= rising edge */ +#define P_BUSY_IRQ_EN 0x0040 /* RW */ +#define P_PE_IRP 0x0020 /* RW 1= rising edge */ +#define P_PE_IRQ_EN 0x0010 /* RW */ +#define P_SLCT_IRP 0x0008 /* RW 1= rising edge */ +#define P_SLCT_IRQ_EN 0x0004 /* RW */ +#define P_ERR_IRP 0x0002 /* RW1 1= rising edge */ +#define P_ERR_IRQ_EN 0x0001 /* RW */ + +#endif /* !(_ASM_SPARC_SUNBPP_H) */ diff --git a/arch/sparc/include/asm/swift.h b/arch/sparc/include/asm/swift.h new file mode 100644 index 000000000..96f6526b9 --- /dev/null +++ b/arch/sparc/include/asm/swift.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* swift.h: Specific definitions for the _broken_ Swift SRMMU + * MMU module. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_SWIFT_H +#define _SPARC_SWIFT_H + +/* Swift is so brain damaged, here is the mmu control register. */ +#define SWIFT_ST 0x00800000 /* SW tablewalk enable */ +#define SWIFT_WP 0x00400000 /* Watchpoint enable */ + +/* Branch folding (buggy, disable on production systems!) */ +#define SWIFT_BF 0x00200000 +#define SWIFT_PMC 0x00180000 /* Page mode control */ +#define SWIFT_PE 0x00040000 /* Parity enable */ +#define SWIFT_PC 0x00020000 /* Parity control */ +#define SWIFT_AP 0x00010000 /* Graphics page mode control (TCX/SX) */ +#define SWIFT_AC 0x00008000 /* Alternate Cacheability (see viking.h) */ +#define SWIFT_BM 0x00004000 /* Boot mode */ +#define SWIFT_RC 0x00003c00 /* DRAM refresh control */ +#define SWIFT_IE 0x00000200 /* Instruction cache enable */ +#define SWIFT_DE 0x00000100 /* Data cache enable */ +#define SWIFT_SA 0x00000080 /* Store Allocate */ +#define SWIFT_NF 0x00000002 /* No fault mode */ +#define SWIFT_EN 0x00000001 /* MMU enable */ + +/* Bits [13:5] select one of 512 instruction cache tags */ +static inline void swift_inv_insn_tag(unsigned long addr) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (addr), "i" (ASI_M_TXTC_TAG) + : "memory"); +} + +/* Bits [12:4] select one of 512 data cache tags */ +static inline void swift_inv_data_tag(unsigned long addr) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (addr), "i" (ASI_M_DATAC_TAG) + : "memory"); +} + +static inline void swift_flush_dcache(void) +{ + unsigned long addr; + + for (addr = 0; addr < 0x2000; addr += 0x10) + swift_inv_data_tag(addr); +} + +static inline void swift_flush_icache(void) +{ + unsigned long addr; + + for (addr = 0; addr < 0x4000; addr += 0x20) + swift_inv_insn_tag(addr); +} + +static inline void swift_idflash_clear(void) +{ + unsigned long addr; + + for (addr = 0; addr < 0x2000; addr += 0x10) { + swift_inv_insn_tag(addr<<1); + swift_inv_data_tag(addr); + } +} + +/* Swift is so broken, it isn't even safe to use the following. */ +static inline void swift_flush_page(unsigned long page) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (page), "i" (ASI_M_FLUSH_PAGE) + : "memory"); +} + +static inline void swift_flush_segment(unsigned long addr) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (addr), "i" (ASI_M_FLUSH_SEG) + : "memory"); +} + +static inline void swift_flush_region(unsigned long addr) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (addr), "i" (ASI_M_FLUSH_REGION) + : "memory"); +} + +static inline void swift_flush_context(void) +{ + __asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" + : /* no outputs */ + : "i" (ASI_M_FLUSH_CTX) + : "memory"); +} + +#endif /* !(_SPARC_SWIFT_H) */ diff --git a/arch/sparc/include/asm/switch_to.h b/arch/sparc/include/asm/switch_to.h new file mode 100644 index 000000000..7cf1c5dc7 --- /dev/null +++ b/arch/sparc/include/asm/switch_to.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_SWITCH_TO_H +#define ___ASM_SPARC_SWITCH_TO_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/switch_to_32.h b/arch/sparc/include/asm/switch_to_32.h new file mode 100644 index 000000000..42eeafcb8 --- /dev/null +++ b/arch/sparc/include/asm/switch_to_32.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_SWITCH_TO_H +#define __SPARC_SWITCH_TO_H + +#include + +extern struct thread_info *current_set[NR_CPUS]; + +/* + * Flush windows so that the VM switch which follows + * would not pull the stack from under us. + * + * SWITCH_ENTER and SWITCH_DO_LAZY_FPU do not work yet (e.g. SMP does not work) + * XXX WTF is the above comment? Found in late teen 2.4.x. + */ +#ifdef CONFIG_SMP +#define SWITCH_ENTER(prv) \ + do { \ + if (test_tsk_thread_flag(prv, TIF_USEDFPU)) { \ + put_psr(get_psr() | PSR_EF); \ + fpsave(&(prv)->thread.float_regs[0], &(prv)->thread.fsr, \ + &(prv)->thread.fpqueue[0], &(prv)->thread.fpqdepth); \ + clear_tsk_thread_flag(prv, TIF_USEDFPU); \ + (prv)->thread.kregs->psr &= ~PSR_EF; \ + } \ + } while(0) + +#define SWITCH_DO_LAZY_FPU(next) /* */ +#else +#define SWITCH_ENTER(prv) /* */ +#define SWITCH_DO_LAZY_FPU(nxt) \ + do { \ + if (last_task_used_math != (nxt)) \ + (nxt)->thread.kregs->psr&=~PSR_EF; \ + } while(0) +#endif + +#define prepare_arch_switch(next) do { \ + __asm__ __volatile__( \ + ".globl\tflush_patch_switch\nflush_patch_switch:\n\t" \ + "save %sp, -0x40, %sp; save %sp, -0x40, %sp; save %sp, -0x40, %sp\n\t" \ + "save %sp, -0x40, %sp; save %sp, -0x40, %sp; save %sp, -0x40, %sp\n\t" \ + "save %sp, -0x40, %sp\n\t" \ + "restore; restore; restore; restore; restore; restore; restore"); \ +} while(0) + + /* Much care has gone into this code, do not touch it. + * + * We need to loadup regs l0/l1 for the newly forked child + * case because the trap return path relies on those registers + * holding certain values, gcc is told that they are clobbered. + * Gcc needs registers for 3 values in and 1 value out, so we + * clobber every non-fixed-usage register besides l2/l3/o4/o5. -DaveM + * + * Hey Dave, that do not touch sign is too much of an incentive + * - Anton & Pete + */ +#define switch_to(prev, next, last) do { \ + SWITCH_ENTER(prev); \ + SWITCH_DO_LAZY_FPU(next); \ + cpumask_set_cpu(smp_processor_id(), mm_cpumask(next->active_mm)); \ + __asm__ __volatile__( \ + "sethi %%hi(here - 0x8), %%o7\n\t" \ + "mov %%g6, %%g3\n\t" \ + "or %%o7, %%lo(here - 0x8), %%o7\n\t" \ + "rd %%psr, %%g4\n\t" \ + "std %%sp, [%%g6 + %4]\n\t" \ + "rd %%wim, %%g5\n\t" \ + "wr %%g4, 0x20, %%psr\n\t" \ + "nop\n\t" \ + "std %%g4, [%%g6 + %3]\n\t" \ + "ldd [%2 + %3], %%g4\n\t" \ + "mov %2, %%g6\n\t" \ + ".globl patchme_store_new_current\n" \ +"patchme_store_new_current:\n\t" \ + "st %2, [%1]\n\t" \ + "wr %%g4, 0x20, %%psr\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + "nop\n\t" /* LEON needs all 3 nops: load to %sp depends on CWP. */ \ + "ldd [%%g6 + %4], %%sp\n\t" \ + "wr %%g5, 0x0, %%wim\n\t" \ + "ldd [%%sp + 0x00], %%l0\n\t" \ + "ldd [%%sp + 0x38], %%i6\n\t" \ + "wr %%g4, 0x0, %%psr\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + "jmpl %%o7 + 0x8, %%g0\n\t" \ + " ld [%%g3 + %5], %0\n\t" \ + "here:\n" \ + : "=&r" (last) \ + : "r" (&(current_set[hard_smp_processor_id()])), \ + "r" (task_thread_info(next)), \ + "i" (TI_KPSR), \ + "i" (TI_KSP), \ + "i" (TI_TASK) \ + : "g1", "g2", "g3", "g4", "g5", "g7", \ + "l0", "l1", "l3", "l4", "l5", "l6", "l7", \ + "i0", "i1", "i2", "i3", "i4", "i5", \ + "o0", "o1", "o2", "o3", "o7"); \ + } while(0) + +void fpsave(unsigned long *fpregs, unsigned long *fsr, + void *fpqueue, unsigned long *fpqdepth); +void synchronize_user_stack(void); + +#endif /* __SPARC_SWITCH_TO_H */ diff --git a/arch/sparc/include/asm/switch_to_64.h b/arch/sparc/include/asm/switch_to_64.h new file mode 100644 index 000000000..d93963ff7 --- /dev/null +++ b/arch/sparc/include/asm/switch_to_64.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC64_SWITCH_TO_64_H +#define __SPARC64_SWITCH_TO_64_H + +#include + +#define prepare_arch_switch(next) \ +do { \ + flushw_all(); \ +} while (0) + + /* See what happens when you design the chip correctly? + * + * We tell gcc we clobber all non-fixed-usage registers except + * for l0/l1. It will use one for 'next' and the other to hold + * the output value of 'last'. 'next' is not referenced again + * past the invocation of switch_to in the scheduler, so we need + * not preserve its value. Hairy, but it lets us remove 2 loads + * and 2 stores in this critical code path. -DaveM + */ +#define switch_to(prev, next, last) \ +do { save_and_clear_fpu(); \ + __asm__ __volatile__("wr %%g0, %0, %%asi" \ + : : "r" (ASI_AIUS)); \ + trap_block[current_thread_info()->cpu].thread = \ + task_thread_info(next); \ + __asm__ __volatile__( \ + "mov %%g4, %%g7\n\t" \ + "stx %%i6, [%%sp + 2047 + 0x70]\n\t" \ + "stx %%i7, [%%sp + 2047 + 0x78]\n\t" \ + "rdpr %%wstate, %%o5\n\t" \ + "stx %%o6, [%%g6 + %6]\n\t" \ + "stb %%o5, [%%g6 + %5]\n\t" \ + "rdpr %%cwp, %%o5\n\t" \ + "stb %%o5, [%%g6 + %8]\n\t" \ + "wrpr %%g0, 15, %%pil\n\t" \ + "mov %4, %%g6\n\t" \ + "ldub [%4 + %8], %%g1\n\t" \ + "wrpr %%g1, %%cwp\n\t" \ + "ldx [%%g6 + %6], %%o6\n\t" \ + "ldub [%%g6 + %5], %%o5\n\t" \ + "ldub [%%g6 + %7], %%o7\n\t" \ + "wrpr %%o5, 0x0, %%wstate\n\t" \ + "ldx [%%sp + 2047 + 0x70], %%i6\n\t" \ + "ldx [%%sp + 2047 + 0x78], %%i7\n\t" \ + "ldx [%%g6 + %9], %%g4\n\t" \ + "wrpr %%g0, 14, %%pil\n\t" \ + "brz,pt %%o7, switch_to_pc\n\t" \ + " mov %%g7, %0\n\t" \ + "sethi %%hi(ret_from_fork), %%g1\n\t" \ + "jmpl %%g1 + %%lo(ret_from_fork), %%g0\n\t" \ + " nop\n\t" \ + ".globl switch_to_pc\n\t" \ + "switch_to_pc:\n\t" \ + : "=&r" (last), "=r" (current), "=r" (current_thread_info_reg), \ + "=r" (__local_per_cpu_offset) \ + : "0" (task_thread_info(next)), \ + "i" (TI_WSTATE), "i" (TI_KSP), "i" (TI_NEW_CHILD), \ + "i" (TI_CWP), "i" (TI_TASK) \ + : "cc", \ + "g1", "g2", "g3", "g7", \ + "l1", "l2", "l3", "l4", "l5", "l6", "l7", \ + "i0", "i1", "i2", "i3", "i4", "i5", \ + "o0", "o1", "o2", "o3", "o4", "o5", "o7"); \ +} while(0) + +void synchronize_user_stack(void); +struct pt_regs; +void fault_in_user_windows(struct pt_regs *); + +#endif /* __SPARC64_SWITCH_TO_64_H */ diff --git a/arch/sparc/include/asm/syscall.h b/arch/sparc/include/asm/syscall.h new file mode 100644 index 000000000..b0233924d --- /dev/null +++ b/arch/sparc/include/asm/syscall.h @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_SPARC_SYSCALL_H +#define __ASM_SPARC_SYSCALL_H + +#include +#include +#include +#include +#include +#include + +/* + * The syscall table always contains 32 bit pointers since we know that the + * address of the function to be called is (way) below 4GB. So the "int" + * type here is what we want [need] for both 32 bit and 64 bit systems. + */ +extern const unsigned int sys_call_table[]; + +/* The system call number is given by the user in %g1 */ +static inline long syscall_get_nr(struct task_struct *task, + struct pt_regs *regs) +{ + int syscall_p = pt_regs_is_syscall(regs); + + return (syscall_p ? regs->u_regs[UREG_G1] : -1L); +} + +static inline void syscall_set_nr(struct task_struct *task, + struct pt_regs *regs, + int nr) +{ + /* + * Unlike syscall_get_nr(), syscall_set_nr() can be called only when + * the target task is stopped for tracing on entering syscall, so + * there is no need to have the same check syscall_get_nr() has. + */ + regs->u_regs[UREG_G1] = nr; +} + +static inline void syscall_rollback(struct task_struct *task, + struct pt_regs *regs) +{ + /* XXX This needs some thought. On Sparc we don't + * XXX save away the original %o0 value somewhere. + * XXX Instead we hold it in register %l5 at the top + * XXX level trap frame and pass this down to the signal + * XXX dispatch code which is the only place that value + * XXX ever was needed. + */ +} + +#ifdef CONFIG_SPARC32 +static inline bool syscall_has_error(struct pt_regs *regs) +{ + return (regs->psr & PSR_C) ? true : false; +} +static inline void syscall_set_error(struct pt_regs *regs) +{ + regs->psr |= PSR_C; +} +static inline void syscall_clear_error(struct pt_regs *regs) +{ + regs->psr &= ~PSR_C; +} +#else +static inline bool syscall_has_error(struct pt_regs *regs) +{ + return (regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY)) ? true : false; +} +static inline void syscall_set_error(struct pt_regs *regs) +{ + regs->tstate |= (TSTATE_XCARRY | TSTATE_ICARRY); +} +static inline void syscall_clear_error(struct pt_regs *regs) +{ + regs->tstate &= ~(TSTATE_XCARRY | TSTATE_ICARRY); +} +#endif + +static inline long syscall_get_error(struct task_struct *task, + struct pt_regs *regs) +{ + long val = regs->u_regs[UREG_I0]; + + return (syscall_has_error(regs) ? -val : 0); +} + +static inline long syscall_get_return_value(struct task_struct *task, + struct pt_regs *regs) +{ + long val = regs->u_regs[UREG_I0]; + + return val; +} + +static inline void syscall_set_return_value(struct task_struct *task, + struct pt_regs *regs, + int error, long val) +{ + if (error) { + syscall_set_error(regs); + regs->u_regs[UREG_I0] = -error; + } else { + syscall_clear_error(regs); + regs->u_regs[UREG_I0] = val; + } +} + +static inline void syscall_get_arguments(struct task_struct *task, + struct pt_regs *regs, + unsigned long *args) +{ + int zero_extend = 0; + unsigned int j; + unsigned int n = 6; + +#ifdef CONFIG_SPARC64 + if (test_tsk_thread_flag(task, TIF_32BIT)) + zero_extend = 1; +#endif + + for (j = 0; j < n; j++) { + unsigned long val = regs->u_regs[UREG_I0 + j]; + + if (zero_extend) + args[j] = (u32) val; + else + args[j] = val; + } +} + +static inline void syscall_set_arguments(struct task_struct *task, + struct pt_regs *regs, + const unsigned long *args) +{ + unsigned int i; + + for (i = 0; i < 6; i++) + regs->u_regs[UREG_I0 + i] = args[i]; +} + +static inline int syscall_get_arch(struct task_struct *task) +{ +#if defined(CONFIG_SPARC64) && defined(CONFIG_COMPAT) + return test_tsk_thread_flag(task, TIF_32BIT) + ? AUDIT_ARCH_SPARC : AUDIT_ARCH_SPARC64; +#elif defined(CONFIG_SPARC64) + return AUDIT_ARCH_SPARC64; +#else + return AUDIT_ARCH_SPARC; +#endif +} + +#endif /* __ASM_SPARC_SYSCALL_H */ diff --git a/arch/sparc/include/asm/syscalls.h b/arch/sparc/include/asm/syscalls.h new file mode 100644 index 000000000..282e62b66 --- /dev/null +++ b/arch/sparc/include/asm/syscalls.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_SYSCALLS_H +#define _SPARC64_SYSCALLS_H + +struct pt_regs; + +asmlinkage long sparc_fork(struct pt_regs *regs); +asmlinkage long sparc_vfork(struct pt_regs *regs); +asmlinkage long sparc_clone(struct pt_regs *regs); +asmlinkage long sparc_clone3(struct pt_regs *regs); + +#endif /* _SPARC64_SYSCALLS_H */ diff --git a/arch/sparc/include/asm/termbits.h b/arch/sparc/include/asm/termbits.h new file mode 100644 index 000000000..fa9de4a46 --- /dev/null +++ b/arch/sparc/include/asm/termbits.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_TERMBITS_H +#define _SPARC_TERMBITS_H + +#include + +#define VMIN 16 +#define VTIME 17 +#endif /* !(_SPARC_TERMBITS_H) */ diff --git a/arch/sparc/include/asm/thread_info.h b/arch/sparc/include/asm/thread_info.h new file mode 100644 index 000000000..740b1743e --- /dev/null +++ b/arch/sparc/include/asm/thread_info.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_THREAD_INFO_H +#define ___ASM_SPARC_THREAD_INFO_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/thread_info_32.h b/arch/sparc/include/asm/thread_info_32.h new file mode 100644 index 000000000..fdaf7b171 --- /dev/null +++ b/arch/sparc/include/asm/thread_info_32.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * thread_info.h: sparc low-level thread information + * adapted from the ppc version by Pete Zaitcev, which was + * adapted from the i386 version by Paul Mackerras + * + * Copyright (C) 2002 David Howells (dhowells@redhat.com) + * Copyright (c) 2002 Pete Zaitcev (zaitcev@yahoo.com) + * - Incorporating suggestions made by Linus Torvalds and Dave Miller + */ + +#ifndef _ASM_THREAD_INFO_H +#define _ASM_THREAD_INFO_H + +#ifdef __KERNEL__ + +#ifndef __ASSEMBLER__ + +#include +#include + +/* + * Low level task data. + * + * If you change this, change the TI_* offsets below to match. + */ +#define NSWINS 8 +struct thread_info { + unsigned long uwinmask; + struct task_struct *task; /* main task structure */ + unsigned long flags; /* low level flags */ + int cpu; /* cpu we're on */ + int preempt_count; /* 0 => preemptable, + <0 => BUG */ + int softirq_count; + int hardirq_count; + + u32 __unused; + + /* Context switch saved kernel state. */ + unsigned long ksp; /* ... ksp __attribute__ ((aligned (8))); */ + unsigned long kpc; + unsigned long kpsr; + unsigned long kwim; + + /* A place to store user windows and stack pointers + * when the stack needs inspection. + */ + struct reg_window32 reg_window[NSWINS]; /* align for ldd! */ + unsigned long rwbuf_stkptrs[NSWINS]; + unsigned long w_saved; +}; + +/* + * macros/functions for gaining access to the thread information structure + */ +#define INIT_THREAD_INFO(tsk) \ +{ \ + .uwinmask = 0, \ + .task = &tsk, \ + .flags = 0, \ + .cpu = 0, \ + .preempt_count = INIT_PREEMPT_COUNT, \ +} + +/* how to get the thread information struct from C */ +register struct thread_info *current_thread_info_reg asm("g6"); +#define current_thread_info() (current_thread_info_reg) + +/* + * thread information allocation + */ +#define THREAD_SIZE_ORDER 1 + +#endif /* __ASSEMBLER__ */ + +/* Size of kernel stack for each process */ +#define THREAD_SIZE (2 * PAGE_SIZE) + +/* + * Offsets in thread_info structure, used in assembly code + * The "#define REGWIN_SZ 0x40" was abolished, so no multiplications. + */ +#define TI_UWINMASK 0x00 /* uwinmask */ +#define TI_TASK 0x04 +#define TI_FLAGS 0x08 +#define TI_CPU 0x0c +#define TI_PREEMPT 0x10 /* preempt_count */ +#define TI_SOFTIRQ 0x14 /* softirq_count */ +#define TI_HARDIRQ 0x18 /* hardirq_count */ +#define TI_KSP 0x20 /* ksp */ +#define TI_KPC 0x24 /* kpc (ldd'ed with kpc) */ +#define TI_KPSR 0x28 /* kpsr */ +#define TI_KWIM 0x2c /* kwim (ldd'ed with kpsr) */ +#define TI_REG_WINDOW 0x30 +#define TI_RWIN_SPTRS 0x230 +#define TI_W_SAVED 0x250 + +/* + * thread information flag bit numbers + */ +#define TIF_SYSCALL_TRACE 0 /* syscall trace active */ +#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */ +#define TIF_SIGPENDING 2 /* signal pending */ +#define TIF_NEED_RESCHED 3 /* rescheduling necessary */ +#define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */ +#define TIF_NOTIFY_SIGNAL 5 /* signal notifications exist */ +#define TIF_USEDFPU 8 /* FPU was used by this task + * this quantum (SMP) */ +#define TIF_POLLING_NRFLAG 9 /* true if poll_idle() is polling + * TIF_NEED_RESCHED */ +#define TIF_MEMDIE 10 /* is terminating due to OOM killer */ + +/* as above, but as bit values */ +#define _TIF_SYSCALL_TRACE (1< + +#ifndef __ASSEMBLER__ + +#include +#include + +struct task_struct; + +struct thread_info { + /* D$ line 1 */ + struct task_struct *task; + unsigned long flags; + __u8 fpsaved[7]; + __u8 status; + unsigned long ksp; + + /* D$ line 2 */ + unsigned long fault_address; + struct pt_regs *kregs; + int preempt_count; /* 0 => preemptable, <0 => BUG */ + __u8 new_child; + __u8 __pad; + __u16 cpu; + + unsigned long *utraps; + + struct reg_window reg_window[NSWINS]; + unsigned long rwbuf_stkptrs[NSWINS]; + + unsigned long gsr[7]; + unsigned long xfsr[7]; + + struct pt_regs *kern_una_regs; + unsigned int kern_una_insn; + + unsigned long fpregs[(7 * 256) / sizeof(unsigned long)] + __attribute__ ((aligned(64))); +}; + +#endif /* !(__ASSEMBLER__) */ + +/* offsets into the thread_info struct for assembly code access */ +#define TI_TASK 0x00000000 +#define TI_FLAGS 0x00000008 +#define TI_FAULT_CODE (TI_FLAGS + TI_FLAG_BYTE_FAULT_CODE) +#define TI_WSTATE (TI_FLAGS + TI_FLAG_BYTE_WSTATE) +#define TI_CWP (TI_FLAGS + TI_FLAG_BYTE_CWP) +#define TI_FPDEPTH (TI_FLAGS + TI_FLAG_BYTE_FPDEPTH) +#define TI_WSAVED (TI_FLAGS + TI_FLAG_BYTE_WSAVED) +#define TI_SYS_NOERROR (TI_FLAGS + TI_FLAG_BYTE_NOERROR) +#define TI_FPSAVED 0x00000010 +#define TI_KSP 0x00000018 +#define TI_FAULT_ADDR 0x00000020 +#define TI_KREGS 0x00000028 +#define TI_PRE_COUNT 0x00000030 +#define TI_NEW_CHILD 0x00000034 +#define TI_CPU 0x00000036 +#define TI_UTRAPS 0x00000038 +#define TI_REG_WINDOW 0x00000040 +#define TI_RWIN_SPTRS 0x000003c0 +#define TI_GSR 0x000003f8 +#define TI_XFSR 0x00000430 +#define TI_KUNA_REGS 0x00000468 +#define TI_KUNA_INSN 0x00000470 +#define TI_FPREGS 0x00000480 + +/* We embed this in the uppermost byte of thread_info->flags */ +#define FAULT_CODE_WRITE 0x01 /* Write access, implies D-TLB */ +#define FAULT_CODE_DTLB 0x02 /* Miss happened in D-TLB */ +#define FAULT_CODE_ITLB 0x04 /* Miss happened in I-TLB */ +#define FAULT_CODE_WINFIXUP 0x08 /* Miss happened during spill/fill */ +#define FAULT_CODE_BLKCOMMIT 0x10 /* Use blk-commit ASI in copy_page */ +#define FAULT_CODE_BAD_RA 0x20 /* Bad RA for sun4v */ + +#if PAGE_SHIFT == 13 +#define THREAD_SIZE (2*PAGE_SIZE) +#define THREAD_SHIFT (PAGE_SHIFT + 1) +#else /* PAGE_SHIFT == 13 */ +#define THREAD_SIZE PAGE_SIZE +#define THREAD_SHIFT PAGE_SHIFT +#endif /* PAGE_SHIFT == 13 */ + +/* + * macros/functions for gaining access to the thread information structure + */ +#ifndef __ASSEMBLER__ + +#define INIT_THREAD_INFO(tsk) \ +{ \ + .task = &tsk, \ + .preempt_count = INIT_PREEMPT_COUNT, \ + .kregs = (struct pt_regs *)(init_stack+THREAD_SIZE)-1 \ +} + +/* how to get the thread information struct from C */ +#ifndef BUILD_VDSO +register struct thread_info *current_thread_info_reg asm("g6"); +#define current_thread_info() (current_thread_info_reg) +#else +extern struct thread_info *current_thread_info(void); +#endif + +/* thread information allocation */ +#if PAGE_SHIFT == 13 +#define THREAD_SIZE_ORDER 1 +#else /* PAGE_SHIFT == 13 */ +#define THREAD_SIZE_ORDER 0 +#endif /* PAGE_SHIFT == 13 */ + +#define __thread_flag_byte_ptr(ti) \ + ((unsigned char *)(&((ti)->flags))) +#define __cur_thread_flag_byte_ptr __thread_flag_byte_ptr(current_thread_info()) + +#define get_thread_fault_code() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_FAULT_CODE]) +#define set_thread_fault_code(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_FAULT_CODE] = (val)) +#define get_thread_wstate() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_WSTATE]) +#define set_thread_wstate(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_WSTATE] = (val)) +#define get_thread_cwp() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_CWP]) +#define set_thread_cwp(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_CWP] = (val)) +#define get_thread_noerror() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_NOERROR]) +#define set_thread_noerror(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_NOERROR] = (val)) +#define get_thread_fpdepth() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_FPDEPTH]) +#define set_thread_fpdepth(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_FPDEPTH] = (val)) +#define get_thread_wsaved() (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_WSAVED]) +#define set_thread_wsaved(val) (__cur_thread_flag_byte_ptr[TI_FLAG_BYTE_WSAVED] = (val)) +#endif /* !(__ASSEMBLER__) */ + +/* + * Thread information flags, only 16 bits are available as we encode + * other values into the upper 6 bytes. + * + * On trap return we need to test several values: + * + * user: need_resched, notify_resume, sigpending, wsaved + * kernel: fpdepth + * + * So to check for work in the kernel case we simply load the fpdepth + * byte out of the flags and test it. For the user case we encode the + * lower 3 bytes of flags as follows: + * ---------------------------------------- + * | wsaved | flags byte 1 | flags byte 2 | + * ---------------------------------------- + * This optimizes the user test into: + * ldx [%g6 + TI_FLAGS], REG1 + * sethi %hi(_TIF_USER_WORK_MASK), REG2 + * or REG2, %lo(_TIF_USER_WORK_MASK), REG2 + * andcc REG1, REG2, %g0 + * be,pt no_work_to_do + * nop + */ +#define TIF_SYSCALL_TRACE 0 /* syscall trace active */ +#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */ +#define TIF_SIGPENDING 2 /* signal pending */ +#define TIF_NEED_RESCHED 3 /* rescheduling necessary */ +#define TIF_NOTIFY_SIGNAL 4 /* signal notifications exist */ +#define TIF_UNALIGNED 5 /* allowed to do unaligned accesses */ +#define TIF_UPROBE 6 /* breakpointed or singlestepped */ +#define TIF_32BIT 7 /* 32-bit binary */ +#define TIF_NOHZ 8 /* in adaptive nohz mode */ +#define TIF_SECCOMP 9 /* secure computing */ +#define TIF_SYSCALL_AUDIT 10 /* syscall auditing active */ +#define TIF_SYSCALL_TRACEPOINT 11 /* syscall tracepoint instrumentation */ +/* NOTE: Thread flags >= 12 should be ones we have no interest + * in using in assembly, else we can't use the mask as + * an immediate value in instructions such as andcc. + */ +#define TIF_MCDPER 12 /* Precise MCD exception */ +#define TIF_MEMDIE 13 /* is terminating due to OOM killer */ +#define TIF_POLLING_NRFLAG 14 + +#define _TIF_SYSCALL_TRACE (1< +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/timer_32.h b/arch/sparc/include/asm/timer_32.h new file mode 100644 index 000000000..eecd26969 --- /dev/null +++ b/arch/sparc/include/asm/timer_32.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * timer.h: Definitions for the timer chips on the Sparc. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + + +#ifndef _SPARC_TIMER_H +#define _SPARC_TIMER_H + +#include +#include + +#include + +#include /* For SUN4M_NCPUS */ + +#define SBUS_CLOCK_RATE 2000000 /* 2MHz */ +#define TIMER_VALUE_SHIFT 9 +#define TIMER_VALUE_MASK 0x3fffff +#define TIMER_LIMIT_BIT (1 << 31) /* Bit 31 in Counter-Timer register */ + +/* The counter timer register has the value offset by 9 bits. + * From sun4m manual: + * When a counter reaches the value in the corresponding limit register, + * the Limit bit is set and the counter is set to 500 nS (i.e. 0x00000200). + * + * To compensate for this add one to the value. + */ +static inline unsigned int timer_value(unsigned int value) +{ + return (value + 1) << TIMER_VALUE_SHIFT; +} + +extern volatile u32 __iomem *master_l10_counter; + +irqreturn_t notrace timer_interrupt(int dummy, void *dev_id); + +#ifdef CONFIG_SMP +DECLARE_PER_CPU(struct clock_event_device, sparc32_clockevent); +void register_percpu_ce(int cpu); +#endif + +#endif /* !(_SPARC_TIMER_H) */ diff --git a/arch/sparc/include/asm/timer_64.h b/arch/sparc/include/asm/timer_64.h new file mode 100644 index 000000000..ffff52c8b --- /dev/null +++ b/arch/sparc/include/asm/timer_64.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* timer.h: System timer definitions for sun5. + * + * Copyright (C) 1997, 2008 David S. Miller (davem@davemloft.net) + */ + +#ifndef _SPARC64_TIMER_H +#define _SPARC64_TIMER_H + +#include +#include +#include + +/* The most frequently accessed fields should be first, + * to fit into the same cacheline. + */ +struct sparc64_tick_ops { + unsigned long ticks_per_nsec_quotient; + unsigned long offset; + unsigned long long (*get_tick)(void); + int (*add_compare)(unsigned long); + unsigned long softint_mask; + void (*disable_irq)(void); + + void (*init_tick)(void); + unsigned long (*add_tick)(unsigned long); + unsigned long (*get_frequency)(void); + unsigned long frequency; + + char *name; +}; + +extern struct sparc64_tick_ops *tick_ops; + +unsigned long sparc64_get_clock_tick(unsigned int cpu); +void setup_sparc64_timer(void); + +#define TICK_PRIV_BIT BIT(63) +#define TICKCMP_IRQ_BIT BIT(63) + +#define HBIRD_STICKCMP_ADDR 0x1fe0000f060UL +#define HBIRD_STICK_ADDR 0x1fe0000f070UL + +#define GET_TICK_NINSTR 13 +struct get_tick_patch { + unsigned int addr; + unsigned int tick[GET_TICK_NINSTR]; + unsigned int stick[GET_TICK_NINSTR]; +}; + +extern struct get_tick_patch __get_tick_patch; +extern struct get_tick_patch __get_tick_patch_end; + +static inline unsigned long get_tick(void) +{ + unsigned long tick, tmp1, tmp2; + + __asm__ __volatile__( + /* read hbtick 13 instructions */ + "661:\n" + " mov 0x1fe, %1\n" + " sllx %1, 0x20, %1\n" + " sethi %%hi(0xf000), %2\n" + " or %2, 0x70, %2\n" + " or %1, %2, %1\n" /* %1 = HBIRD_STICK_ADDR */ + " add %1, 8, %2\n" + " ldxa [%2]%3, %0\n" + " ldxa [%1]%3, %1\n" + " ldxa [%2]%3, %2\n" + " sub %2, %0, %0\n" /* don't modify %xcc */ + " brnz,pn %0, 661b\n" /* restart to save one register */ + " sllx %2, 32, %2\n" + " or %2, %1, %0\n" + /* Common/not patched code */ + " sllx %0, 1, %0\n" + " srlx %0, 1, %0\n" /* Clear TICK_PRIV_BIT */ + /* Beginning of patch section */ + " .section .get_tick_patch, \"ax\"\n" + " .word 661b\n" + /* read tick 2 instructions and 11 skipped */ + " ba 1f\n" + " rd %%tick, %0\n" + " .skip 4 * (%4 - 2)\n" + "1:\n" + /* read stick 2 instructions and 11 skipped */ + " ba 1f\n" + " rd %%asr24, %0\n" + " .skip 4 * (%4 - 2)\n" + "1:\n" + /* End of patch section */ + " .previous\n" + : "=&r" (tick), "=&r" (tmp1), "=&r" (tmp2) + : "i" (ASI_PHYS_BYPASS_EC_E), "i" (GET_TICK_NINSTR)); + + return tick; +} + +#endif /* _SPARC64_TIMER_H */ diff --git a/arch/sparc/include/asm/timex.h b/arch/sparc/include/asm/timex.h new file mode 100644 index 000000000..f0f980581 --- /dev/null +++ b/arch/sparc/include/asm/timex.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_TIMEX_H +#define ___ASM_SPARC_TIMEX_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/timex_64.h b/arch/sparc/include/asm/timex_64.h new file mode 100644 index 000000000..fa004ef3b --- /dev/null +++ b/arch/sparc/include/asm/timex_64.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * linux/include/asm/timex.h + * + * sparc64 architecture timex specifications + */ +#ifndef _ASMsparc64_TIMEX_H +#define _ASMsparc64_TIMEX_H + +#include + +/* Getting on the cycle counter on sparc64. */ +typedef unsigned long cycles_t; +#define get_cycles() tick_ops->get_tick() + +#endif diff --git a/arch/sparc/include/asm/tlb.h b/arch/sparc/include/asm/tlb.h new file mode 100644 index 000000000..7146a5772 --- /dev/null +++ b/arch/sparc/include/asm/tlb.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_TLB_H +#define ___ASM_SPARC_TLB_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/tlb_32.h b/arch/sparc/include/asm/tlb_32.h new file mode 100644 index 000000000..5cd28a879 --- /dev/null +++ b/arch/sparc/include/asm/tlb_32.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_TLB_H +#define _SPARC_TLB_H + +#include + +#endif /* _SPARC_TLB_H */ diff --git a/arch/sparc/include/asm/tlb_64.h b/arch/sparc/include/asm/tlb_64.h new file mode 100644 index 000000000..303718748 --- /dev/null +++ b/arch/sparc/include/asm/tlb_64.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_TLB_H +#define _SPARC64_TLB_H + +#include +#include +#include +#include + +#ifdef CONFIG_SMP +void smp_flush_tlb_pending(struct mm_struct *, + unsigned long, unsigned long *); +#endif + +#ifdef CONFIG_SMP +void smp_flush_tlb_mm(struct mm_struct *mm); +#define do_flush_tlb_mm(mm) smp_flush_tlb_mm(mm) +#else +#define do_flush_tlb_mm(mm) __flush_tlb_mm(CTX_HWBITS(mm->context), SECONDARY_CONTEXT) +#endif + +void __flush_tlb_pending(unsigned long, unsigned long, unsigned long *); +void flush_tlb_pending(void); + +#define tlb_flush(tlb) flush_tlb_pending() + +/* + * SPARC64's hardware TLB fill does not use the Linux page-tables + * and therefore we don't need a TLBI when freeing page-table pages. + */ + +#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE +#define tlb_needs_table_invalidate() (false) +#endif + +#include + +#endif /* _SPARC64_TLB_H */ diff --git a/arch/sparc/include/asm/tlbflush.h b/arch/sparc/include/asm/tlbflush.h new file mode 100644 index 000000000..30ae3ea21 --- /dev/null +++ b/arch/sparc/include/asm/tlbflush.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_TLBFLUSH_H +#define ___ASM_SPARC_TLBFLUSH_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/tlbflush_32.h b/arch/sparc/include/asm/tlbflush_32.h new file mode 100644 index 000000000..470531991 --- /dev/null +++ b/arch/sparc/include/asm/tlbflush_32.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_TLBFLUSH_H +#define _SPARC_TLBFLUSH_H + +#include + +#define flush_tlb_all() \ + sparc32_cachetlb_ops->tlb_all() +#define flush_tlb_mm(mm) \ + sparc32_cachetlb_ops->tlb_mm(mm) +#define flush_tlb_range(vma, start, end) \ + sparc32_cachetlb_ops->tlb_range(vma, start, end) +#define flush_tlb_page(vma, addr) \ + sparc32_cachetlb_ops->tlb_page(vma, addr) + +/* + * This is a kludge, until I know better. --zaitcev XXX + */ +static inline void flush_tlb_kernel_range(unsigned long start, + unsigned long end) +{ + flush_tlb_all(); +} + +#endif /* _SPARC_TLBFLUSH_H */ diff --git a/arch/sparc/include/asm/tlbflush_64.h b/arch/sparc/include/asm/tlbflush_64.h new file mode 100644 index 000000000..6133306ba --- /dev/null +++ b/arch/sparc/include/asm/tlbflush_64.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_TLBFLUSH_H +#define _SPARC64_TLBFLUSH_H + +#include + +/* TSB flush operations. */ + +#define TLB_BATCH_NR 192 + +struct tlb_batch { + unsigned int hugepage_shift; + struct mm_struct *mm; + unsigned long tlb_nr; + unsigned long vaddrs[TLB_BATCH_NR]; +}; + +void flush_tsb_kernel_range(unsigned long start, unsigned long end); +void flush_tsb_user(struct tlb_batch *tb); +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, + unsigned int hugepage_shift); + +/* TLB flush operations. */ + +static inline void flush_tlb_mm(struct mm_struct *mm) +{ +} + +static inline void flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ +} + +static inline void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ +} + +void flush_tlb_kernel_range(unsigned long start, unsigned long end); + +void flush_tlb_pending(void); +void arch_enter_lazy_mmu_mode(void); +void arch_flush_lazy_mmu_mode(void); +void arch_leave_lazy_mmu_mode(void); + +/* Local cpu only. */ +void __flush_tlb_all(void); +void __flush_tlb_page(unsigned long context, unsigned long vaddr); +void __flush_tlb_kernel_range(unsigned long start, unsigned long end); + +#ifndef CONFIG_SMP + +static inline void global_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr) +{ + __flush_tlb_page(CTX_HWBITS(mm->context), vaddr); +} + +#else /* CONFIG_SMP */ + +void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end); +void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr); + +#define global_flush_tlb_page(mm, vaddr) \ + smp_flush_tlb_page(mm, vaddr) + +#endif /* ! CONFIG_SMP */ + +#endif /* _SPARC64_TLBFLUSH_H */ diff --git a/arch/sparc/include/asm/topology.h b/arch/sparc/include/asm/topology.h new file mode 100644 index 000000000..ba7b9d9d9 --- /dev/null +++ b/arch/sparc/include/asm/topology.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_TOPOLOGY_H +#define ___ASM_SPARC_TOPOLOGY_H +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif +#endif diff --git a/arch/sparc/include/asm/topology_32.h b/arch/sparc/include/asm/topology_32.h new file mode 100644 index 000000000..66c4f9ff7 --- /dev/null +++ b/arch/sparc/include/asm/topology_32.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC_TOPOLOGY_H +#define _ASM_SPARC_TOPOLOGY_H + +#include + +#endif /* _ASM_SPARC_TOPOLOGY_H */ diff --git a/arch/sparc/include/asm/topology_64.h b/arch/sparc/include/asm/topology_64.h new file mode 100644 index 000000000..34c628a22 --- /dev/null +++ b/arch/sparc/include/asm/topology_64.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_SPARC64_TOPOLOGY_H +#define _ASM_SPARC64_TOPOLOGY_H + +#ifdef CONFIG_NUMA + +#include + +static inline int cpu_to_node(int cpu) +{ + return numa_cpu_lookup_table[cpu]; +} + +#define cpumask_of_node(node) ((node) == -1 ? \ + cpu_all_mask : \ + &numa_cpumask_lookup_table[node]) + +struct pci_bus; +#ifdef CONFIG_PCI +int pcibus_to_node(struct pci_bus *pbus); +#else +static inline int pcibus_to_node(struct pci_bus *pbus) +{ + return -1; +} +#endif + +#define cpumask_of_pcibus(bus) \ + (pcibus_to_node(bus) == -1 ? \ + cpu_all_mask : \ + cpumask_of_node(pcibus_to_node(bus))) + +int __node_distance(int, int); +#define node_distance(a, b) __node_distance(a, b) + +#else /* CONFIG_NUMA */ + +#include + +#endif /* !(CONFIG_NUMA) */ + +#ifdef CONFIG_SMP + +#include + +#define topology_physical_package_id(cpu) (cpu_data(cpu).proc_id) +#define topology_core_id(cpu) (cpu_data(cpu).core_id) +#define topology_core_cpumask(cpu) (&cpu_core_sib_map[cpu]) +#define topology_core_cache_cpumask(cpu) (&cpu_core_sib_cache_map[cpu]) +#define topology_sibling_cpumask(cpu) (&per_cpu(cpu_sibling_map, cpu)) +#endif /* CONFIG_SMP */ + +extern cpumask_t cpu_core_map[NR_CPUS]; +extern cpumask_t cpu_core_sib_map[NR_CPUS]; +extern cpumask_t cpu_core_sib_cache_map[NR_CPUS]; + +/** + * Return cores that shares the last level cache. + */ +static inline const struct cpumask *cpu_coregroup_mask(int cpu) +{ + return &cpu_core_sib_cache_map[cpu]; +} + +#endif /* _ASM_SPARC64_TOPOLOGY_H */ diff --git a/arch/sparc/include/asm/trap_block.h b/arch/sparc/include/asm/trap_block.h new file mode 100644 index 000000000..6cf2a60a0 --- /dev/null +++ b/arch/sparc/include/asm/trap_block.h @@ -0,0 +1,217 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_TRAP_BLOCK_H +#define _SPARC_TRAP_BLOCK_H + +#include + +#include +#include + +#ifndef __ASSEMBLER__ + +/* Trap handling code needs to get at a few critical values upon + * trap entry and to process TSB misses. These cannot be in the + * per_cpu() area as we really need to lock them into the TLB and + * thus make them part of the main kernel image. As a result we + * try to make this as small as possible. + * + * This is padded out and aligned to 64-bytes to avoid false sharing + * on SMP. + */ + +/* If you modify the size of this structure, please update + * TRAP_BLOCK_SZ_SHIFT below. + */ +struct thread_info; +struct trap_per_cpu { +/* D-cache line 1: Basic thread information, cpu and device mondo queues */ + struct thread_info *thread; + unsigned long pgd_paddr; + unsigned long cpu_mondo_pa; + unsigned long dev_mondo_pa; + +/* D-cache line 2: Error Mondo Queue and kernel buffer pointers */ + unsigned long resum_mondo_pa; + unsigned long resum_kernel_buf_pa; + unsigned long nonresum_mondo_pa; + unsigned long nonresum_kernel_buf_pa; + +/* Dcache lines 3, 4, 5, and 6: Hypervisor Fault Status */ + struct hv_fault_status fault_info; + +/* Dcache line 7: Physical addresses of CPU send mondo block and CPU list. */ + unsigned long cpu_mondo_block_pa; + unsigned long cpu_list_pa; + unsigned long tsb_huge; + unsigned long tsb_huge_temp; + +/* Dcache line 8: IRQ work list, and keep trap_block a power-of-2 in size. */ + unsigned long irq_worklist_pa; + unsigned int cpu_mondo_qmask; + unsigned int dev_mondo_qmask; + unsigned int resum_qmask; + unsigned int nonresum_qmask; + unsigned long __per_cpu_base; +} __attribute__((aligned(64))); +extern struct trap_per_cpu trap_block[NR_CPUS]; +void init_cur_cpu_trap(struct thread_info *); +void setup_tba(void); +extern int ncpus_probed; +extern u64 cpu_mondo_counter[NR_CPUS]; + +unsigned long real_hard_smp_processor_id(void); + +struct cpuid_patch_entry { + unsigned int addr; + unsigned int cheetah_safari[4]; + unsigned int cheetah_jbus[4]; + unsigned int starfire[4]; + unsigned int sun4v[4]; +}; +extern struct cpuid_patch_entry __cpuid_patch, __cpuid_patch_end; + +struct sun4v_1insn_patch_entry { + unsigned int addr; + unsigned int insn; +}; +extern struct sun4v_1insn_patch_entry __sun4v_1insn_patch, + __sun4v_1insn_patch_end; +extern struct sun4v_1insn_patch_entry __fast_win_ctrl_1insn_patch, + __fast_win_ctrl_1insn_patch_end; +extern struct sun4v_1insn_patch_entry __sun_m7_1insn_patch, + __sun_m7_1insn_patch_end; + +struct sun4v_2insn_patch_entry { + unsigned int addr; + unsigned int insns[2]; +}; +extern struct sun4v_2insn_patch_entry __sun4v_2insn_patch, + __sun4v_2insn_patch_end; +extern struct sun4v_2insn_patch_entry __sun_m7_2insn_patch, + __sun_m7_2insn_patch_end; + + +#endif /* !(__ASSEMBLER__) */ + +#define TRAP_PER_CPU_THREAD 0x00 +#define TRAP_PER_CPU_PGD_PADDR 0x08 +#define TRAP_PER_CPU_CPU_MONDO_PA 0x10 +#define TRAP_PER_CPU_DEV_MONDO_PA 0x18 +#define TRAP_PER_CPU_RESUM_MONDO_PA 0x20 +#define TRAP_PER_CPU_RESUM_KBUF_PA 0x28 +#define TRAP_PER_CPU_NONRESUM_MONDO_PA 0x30 +#define TRAP_PER_CPU_NONRESUM_KBUF_PA 0x38 +#define TRAP_PER_CPU_FAULT_INFO 0x40 +#define TRAP_PER_CPU_CPU_MONDO_BLOCK_PA 0xc0 +#define TRAP_PER_CPU_CPU_LIST_PA 0xc8 +#define TRAP_PER_CPU_TSB_HUGE 0xd0 +#define TRAP_PER_CPU_TSB_HUGE_TEMP 0xd8 +#define TRAP_PER_CPU_IRQ_WORKLIST_PA 0xe0 +#define TRAP_PER_CPU_CPU_MONDO_QMASK 0xe8 +#define TRAP_PER_CPU_DEV_MONDO_QMASK 0xec +#define TRAP_PER_CPU_RESUM_QMASK 0xf0 +#define TRAP_PER_CPU_NONRESUM_QMASK 0xf4 +#define TRAP_PER_CPU_PER_CPU_BASE 0xf8 + +#define TRAP_BLOCK_SZ_SHIFT 8 + +#include + +#define __GET_CPUID(REG) \ + /* Spitfire implementation (default). */ \ +661: ldxa [%g0] ASI_UPA_CONFIG, REG; \ + srlx REG, 17, REG; \ + and REG, 0x1f, REG; \ + nop; \ + .section .cpuid_patch, "ax"; \ + /* Instruction location. */ \ + .word 661b; \ + /* Cheetah Safari implementation. */ \ + ldxa [%g0] ASI_SAFARI_CONFIG, REG; \ + srlx REG, 17, REG; \ + and REG, 0x3ff, REG; \ + nop; \ + /* Cheetah JBUS implementation. */ \ + ldxa [%g0] ASI_JBUS_CONFIG, REG; \ + srlx REG, 17, REG; \ + and REG, 0x1f, REG; \ + nop; \ + /* Starfire implementation. */ \ + sethi %hi(0x1fff40000d0 >> 9), REG; \ + sllx REG, 9, REG; \ + or REG, 0xd0, REG; \ + lduwa [REG] ASI_PHYS_BYPASS_EC_E, REG;\ + /* sun4v implementation. */ \ + mov SCRATCHPAD_CPUID, REG; \ + ldxa [REG] ASI_SCRATCHPAD, REG; \ + nop; \ + nop; \ + .previous; + +#ifdef CONFIG_SMP + +#define TRAP_LOAD_TRAP_BLOCK(DEST, TMP) \ + __GET_CPUID(TMP) \ + sethi %hi(trap_block), DEST; \ + sllx TMP, TRAP_BLOCK_SZ_SHIFT, TMP; \ + or DEST, %lo(trap_block), DEST; \ + add DEST, TMP, DEST; \ + +/* Clobbers TMP, current address space PGD phys address into DEST. */ +#define TRAP_LOAD_PGD_PHYS(DEST, TMP) \ + TRAP_LOAD_TRAP_BLOCK(DEST, TMP) \ + ldx [DEST + TRAP_PER_CPU_PGD_PADDR], DEST; + +/* Clobbers TMP, loads local processor's IRQ work area into DEST. */ +#define TRAP_LOAD_IRQ_WORK_PA(DEST, TMP) \ + TRAP_LOAD_TRAP_BLOCK(DEST, TMP) \ + add DEST, TRAP_PER_CPU_IRQ_WORKLIST_PA, DEST; + +/* Clobbers TMP, loads DEST with current thread info pointer. */ +#define TRAP_LOAD_THREAD_REG(DEST, TMP) \ + TRAP_LOAD_TRAP_BLOCK(DEST, TMP) \ + ldx [DEST + TRAP_PER_CPU_THREAD], DEST; + +/* Given the current thread info pointer in THR, load the per-cpu + * area base of the current processor into DEST. REG1, REG2, and REG3 are + * clobbered. + * + * You absolutely cannot use DEST as a temporary in this code. The + * reason is that traps can happen during execution, and return from + * trap will load the fully resolved DEST per-cpu base. This can corrupt + * the calculations done by the macro mid-stream. + */ +#define LOAD_PER_CPU_BASE(DEST, THR, REG1, REG2, REG3) \ + lduh [THR + TI_CPU], REG1; \ + sethi %hi(trap_block), REG2; \ + sllx REG1, TRAP_BLOCK_SZ_SHIFT, REG1; \ + or REG2, %lo(trap_block), REG2; \ + add REG2, REG1, REG2; \ + ldx [REG2 + TRAP_PER_CPU_PER_CPU_BASE], DEST; + +#else + +#define TRAP_LOAD_TRAP_BLOCK(DEST, TMP) \ + sethi %hi(trap_block), DEST; \ + or DEST, %lo(trap_block), DEST; \ + +/* Uniprocessor versions, we know the cpuid is zero. */ +#define TRAP_LOAD_PGD_PHYS(DEST, TMP) \ + TRAP_LOAD_TRAP_BLOCK(DEST, TMP) \ + ldx [DEST + TRAP_PER_CPU_PGD_PADDR], DEST; + +/* Clobbers TMP, loads local processor's IRQ work area into DEST. */ +#define TRAP_LOAD_IRQ_WORK_PA(DEST, TMP) \ + TRAP_LOAD_TRAP_BLOCK(DEST, TMP) \ + add DEST, TRAP_PER_CPU_IRQ_WORKLIST_PA, DEST; + +#define TRAP_LOAD_THREAD_REG(DEST, TMP) \ + TRAP_LOAD_TRAP_BLOCK(DEST, TMP) \ + ldx [DEST + TRAP_PER_CPU_THREAD], DEST; + +/* No per-cpu areas on uniprocessor, so no need to load DEST. */ +#define LOAD_PER_CPU_BASE(DEST, THR, REG1, REG2, REG3) + +#endif /* !(CONFIG_SMP) */ + +#endif /* _SPARC_TRAP_BLOCK_H */ diff --git a/arch/sparc/include/asm/traps.h b/arch/sparc/include/asm/traps.h new file mode 100644 index 000000000..e4e10b0e7 --- /dev/null +++ b/arch/sparc/include/asm/traps.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * traps.h: Format of entries for the Sparc trap table. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ +#ifndef _SPARC_TRAPS_H +#define _SPARC_TRAPS_H + +#include + +#ifndef __ASSEMBLER__ +/* This is for V8 compliant Sparc CPUS */ +struct tt_entry { + unsigned long inst_one; + unsigned long inst_two; + unsigned long inst_three; + unsigned long inst_four; +}; + +/* We set this to _start in system setup. */ +extern struct tt_entry *sparc_ttable; + +#endif /* !(__ASSEMBLER__) */ +#endif /* !(_SPARC_TRAPS_H) */ diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h new file mode 100644 index 000000000..239be259e --- /dev/null +++ b/arch/sparc/include/asm/tsb.h @@ -0,0 +1,380 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_TSB_H +#define _SPARC64_TSB_H + +/* The sparc64 TSB is similar to the powerpc hashtables. It's a + * power-of-2 sized table of TAG/PTE pairs. The cpu precomputes + * pointers into this table for 8K and 64K page sizes, and also a + * comparison TAG based upon the virtual address and context which + * faults. + * + * TLB miss trap handler software does the actual lookup via something + * of the form: + * + * ldxa [%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1 + * ldxa [%g0] ASI_{D,I}MMU, %g6 + * sllx %g6, 22, %g6 + * srlx %g6, 22, %g6 + * ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4 + * cmp %g4, %g6 + * bne,pn %xcc, tsb_miss_{d,i}tlb + * mov FAULT_CODE_{D,I}TLB, %g3 + * stxa %g5, [%g0] ASI_{D,I}TLB_DATA_IN + * retry + * + * + * Each 16-byte slot of the TSB is the 8-byte tag and then the 8-byte + * PTE. The TAG is of the same layout as the TLB TAG TARGET mmu + * register which is: + * + * ------------------------------------------------- + * | - | CONTEXT | - | VADDR bits 63:22 | + * ------------------------------------------------- + * 63 61 60 48 47 42 41 0 + * + * But actually, since we use per-mm TSB's, we zero out the CONTEXT + * field. + * + * Like the powerpc hashtables we need to use locking in order to + * synchronize while we update the entries. PTE updates need locking + * as well. + * + * We need to carefully choose a lock bits for the TSB entry. We + * choose to use bit 47 in the tag. Also, since we never map anything + * at page zero in context zero, we use zero as an invalid tag entry. + * When the lock bit is set, this forces a tag comparison failure. + */ + +#define TSB_TAG_LOCK_BIT 47 +#define TSB_TAG_LOCK_HIGH (1 << (TSB_TAG_LOCK_BIT - 32)) + +#define TSB_TAG_INVALID_BIT 46 +#define TSB_TAG_INVALID_HIGH (1 << (TSB_TAG_INVALID_BIT - 32)) + +/* Some cpus support physical address quad loads. We want to use + * those if possible so we don't need to hard-lock the TSB mapping + * into the TLB. We encode some instruction patching in order to + * support this. + * + * The kernel TSB is locked into the TLB by virtue of being in the + * kernel image, so we don't play these games for swapper_tsb access. + */ +#ifndef __ASSEMBLER__ +struct tsb_ldquad_phys_patch_entry { + unsigned int addr; + unsigned int sun4u_insn; + unsigned int sun4v_insn; +}; +extern struct tsb_ldquad_phys_patch_entry __tsb_ldquad_phys_patch, + __tsb_ldquad_phys_patch_end; + +struct tsb_phys_patch_entry { + unsigned int addr; + unsigned int insn; +}; +extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; +#endif +#define TSB_LOAD_QUAD(TSB, REG) \ +661: ldda [TSB] ASI_NUCLEUS_QUAD_LDD, REG; \ + .section .tsb_ldquad_phys_patch, "ax"; \ + .word 661b; \ + ldda [TSB] ASI_QUAD_LDD_PHYS, REG; \ + ldda [TSB] ASI_QUAD_LDD_PHYS_4V, REG; \ + .previous + +#define TSB_LOAD_TAG_HIGH(TSB, REG) \ +661: lduwa [TSB] ASI_N, REG; \ + .section .tsb_phys_patch, "ax"; \ + .word 661b; \ + lduwa [TSB] ASI_PHYS_USE_EC, REG; \ + .previous + +#define TSB_LOAD_TAG(TSB, REG) \ +661: ldxa [TSB] ASI_N, REG; \ + .section .tsb_phys_patch, "ax"; \ + .word 661b; \ + ldxa [TSB] ASI_PHYS_USE_EC, REG; \ + .previous + +#define TSB_CAS_TAG_HIGH(TSB, REG1, REG2) \ +661: casa [TSB] ASI_N, REG1, REG2; \ + .section .tsb_phys_patch, "ax"; \ + .word 661b; \ + casa [TSB] ASI_PHYS_USE_EC, REG1, REG2; \ + .previous + +#define TSB_CAS_TAG(TSB, REG1, REG2) \ +661: casxa [TSB] ASI_N, REG1, REG2; \ + .section .tsb_phys_patch, "ax"; \ + .word 661b; \ + casxa [TSB] ASI_PHYS_USE_EC, REG1, REG2; \ + .previous + +#define TSB_STORE(ADDR, VAL) \ +661: stxa VAL, [ADDR] ASI_N; \ + .section .tsb_phys_patch, "ax"; \ + .word 661b; \ + stxa VAL, [ADDR] ASI_PHYS_USE_EC; \ + .previous + +#define TSB_LOCK_TAG(TSB, REG1, REG2) \ +99: TSB_LOAD_TAG_HIGH(TSB, REG1); \ + sethi %hi(TSB_TAG_LOCK_HIGH), REG2;\ + andcc REG1, REG2, %g0; \ + bne,pn %icc, 99b; \ + nop; \ + TSB_CAS_TAG_HIGH(TSB, REG1, REG2); \ + cmp REG1, REG2; \ + bne,pn %icc, 99b; \ + nop; \ + +#define TSB_WRITE(TSB, TTE, TAG) \ + add TSB, 0x8, TSB; \ + TSB_STORE(TSB, TTE); \ + sub TSB, 0x8, TSB; \ + TSB_STORE(TSB, TAG); + + /* Do a kernel page table walk. Leaves valid PTE value in + * REG1. Jumps to FAIL_LABEL on early page table walk + * termination. VADDR will not be clobbered, but REG2 will. + * + * There are two masks we must apply to propagate bits from + * the virtual address into the PTE physical address field + * when dealing with huge pages. This is because the page + * table boundaries do not match the huge page size(s) the + * hardware supports. + * + * In these cases we propagate the bits that are below the + * page table level where we saw the huge page mapping, but + * are still within the relevant physical bits for the huge + * page size in question. So for PMD mappings (which fall on + * bit 23, for 8MB per PMD) we must propagate bit 22 for a + * 4MB huge page. For huge PUDs (which fall on bit 33, for + * 8GB per PUD), we have to accommodate 256MB and 2GB huge + * pages. So for those we propagate bits 32 to 28. + */ +#define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \ + sethi %hi(swapper_pg_dir), REG1; \ + or REG1, %lo(swapper_pg_dir), REG1; \ + sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldx [REG1 + REG2], REG1; \ + brz,pn REG1, FAIL_LABEL; \ + sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + brz,pn REG1, FAIL_LABEL; \ + sethi %uhi(_PAGE_PUD_HUGE), REG2; \ + brz,pn REG1, FAIL_LABEL; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + sethi %hi(0xf8000000), REG2; \ + bne,pt %xcc, 697f; \ + sllx REG2, 1, REG2; \ + sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + sethi %uhi(_PAGE_PMD_HUGE), REG2; \ + brz,pn REG1, FAIL_LABEL; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + be,pn %xcc, 698f; \ + sethi %hi(0x400000), REG2; \ +697: brgez,pn REG1, FAIL_LABEL; \ + andn REG1, REG2, REG1; \ + and VADDR, REG2, REG2; \ + ba,pt %xcc, 699f; \ + or REG1, REG2, REG1; \ +698: sllx VADDR, 64 - PMD_SHIFT, REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + brgez,pn REG1, FAIL_LABEL; \ + nop; \ +699: + + /* PUD has been loaded into REG1, interpret the value, seeing + * if it is a HUGE PUD or a normal one. If it is not valid + * then jump to FAIL_LABEL. If it is a HUGE PUD, and it + * translates to a valid PTE, branch to PTE_LABEL. + * + * We have to propagate bits [32:22] from the virtual address + * to resolve at 4M granularity. + */ +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +#define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ +700: ba 700f; \ + nop; \ + .section .pud_huge_patch, "ax"; \ + .word 700b; \ + nop; \ + .previous; \ + brz,pn REG1, FAIL_LABEL; \ + sethi %uhi(_PAGE_PUD_HUGE), REG2; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + be,pt %xcc, 700f; \ + sethi %hi(0xffe00000), REG2; \ + sllx REG2, 1, REG2; \ + brgez,pn REG1, FAIL_LABEL; \ + andn REG1, REG2, REG1; \ + and VADDR, REG2, REG2; \ + brlz,pt REG1, PTE_LABEL; \ + or REG1, REG2, REG1; \ +700: +#else +#define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ + brz,pn REG1, FAIL_LABEL; \ + nop; +#endif + + /* PMD has been loaded into REG1, interpret the value, seeing + * if it is a HUGE PMD or a normal one. If it is not valid + * then jump to FAIL_LABEL. If it is a HUGE PMD, and it + * translates to a valid PTE, branch to PTE_LABEL. + * + * We have to propagate the 4MB bit of the virtual address + * because we are fabricating 8MB pages using 4MB hw pages. + */ +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ + brz,pn REG1, FAIL_LABEL; \ + sethi %uhi(_PAGE_PMD_HUGE), REG2; \ + sllx REG2, 32, REG2; \ + andcc REG1, REG2, %g0; \ + be,pt %xcc, 700f; \ + sethi %hi(4 * 1024 * 1024), REG2; \ + brgez,pn REG1, FAIL_LABEL; \ + andn REG1, REG2, REG1; \ + and VADDR, REG2, REG2; \ + brlz,pt REG1, PTE_LABEL; \ + or REG1, REG2, REG1; \ +700: +#else +#define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ + brz,pn REG1, FAIL_LABEL; \ + nop; +#endif + + /* Do a user page table walk in MMU globals. Leaves final, + * valid, PTE value in REG1. Jumps to FAIL_LABEL on early + * page table walk termination or if the PTE is not valid. + * + * Physical base of page tables is in PHYS_PGD which will not + * be modified. + * + * VADDR will not be clobbered, but REG1 and REG2 will. + */ +#define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \ + sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \ + brz,pn REG1, FAIL_LABEL; \ + sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \ + brz,pn REG1, FAIL_LABEL; \ + sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ + USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \ + sllx VADDR, 64 - PMD_SHIFT, REG2; \ + srlx REG2, 64 - PAGE_SHIFT, REG2; \ + andn REG2, 0x7, REG2; \ + add REG1, REG2, REG1; \ + ldxa [REG1] ASI_PHYS_USE_EC, REG1; \ + brgez,pn REG1, FAIL_LABEL; \ + nop; \ +800: + +/* Lookup a OBP mapping on VADDR in the prom_trans[] table at TL>0. + * If no entry is found, FAIL_LABEL will be branched to. On success + * the resulting PTE value will be left in REG1. VADDR is preserved + * by this routine. + */ +#define OBP_TRANS_LOOKUP(VADDR, REG1, REG2, REG3, FAIL_LABEL) \ + sethi %hi(prom_trans), REG1; \ + or REG1, %lo(prom_trans), REG1; \ +97: ldx [REG1 + 0x00], REG2; \ + brz,pn REG2, FAIL_LABEL; \ + nop; \ + ldx [REG1 + 0x08], REG3; \ + add REG2, REG3, REG3; \ + cmp REG2, VADDR; \ + bgu,pt %xcc, 98f; \ + cmp VADDR, REG3; \ + bgeu,pt %xcc, 98f; \ + ldx [REG1 + 0x10], REG3; \ + sub VADDR, REG2, REG2; \ + ba,pt %xcc, 99f; \ + add REG3, REG2, REG1; \ +98: ba,pt %xcc, 97b; \ + add REG1, (3 * 8), REG1; \ +99: + + /* We use a 32K TSB for the whole kernel, this allows to + * handle about 16MB of modules and vmalloc mappings without + * incurring many hash conflicts. + */ +#define KERNEL_TSB_SIZE_BYTES (32 * 1024) +#define KERNEL_TSB_NENTRIES \ + (KERNEL_TSB_SIZE_BYTES / 16) +#define KERNEL_TSB4M_NENTRIES 4096 + + /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL + * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries + * and the found TTE will be left in REG1. REG3 and REG4 must + * be an even/odd pair of registers. + * + * VADDR and TAG will be preserved and not clobbered by this macro. + */ +#define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ +661: sethi %uhi(swapper_tsb), REG1; \ + sethi %hi(swapper_tsb), REG2; \ + or REG1, %ulo(swapper_tsb), REG1; \ + or REG2, %lo(swapper_tsb), REG2; \ + .section .swapper_tsb_phys_patch, "ax"; \ + .word 661b; \ + .previous; \ + sllx REG1, 32, REG1; \ + or REG1, REG2, REG1; \ + srlx VADDR, PAGE_SHIFT, REG2; \ + and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \ + sllx REG2, 4, REG2; \ + add REG1, REG2, REG2; \ + TSB_LOAD_QUAD(REG2, REG3); \ + cmp REG3, TAG; \ + be,a,pt %xcc, OK_LABEL; \ + mov REG4, REG1; + +#ifndef CONFIG_DEBUG_PAGEALLOC + /* This version uses a trick, the TAG is already (VADDR >> 22) so + * we can make use of that for the index computation. + */ +#define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \ +661: sethi %uhi(swapper_4m_tsb), REG1; \ + sethi %hi(swapper_4m_tsb), REG2; \ + or REG1, %ulo(swapper_4m_tsb), REG1; \ + or REG2, %lo(swapper_4m_tsb), REG2; \ + .section .swapper_4m_tsb_phys_patch, "ax"; \ + .word 661b; \ + .previous; \ + sllx REG1, 32, REG1; \ + or REG1, REG2, REG1; \ + and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \ + sllx REG2, 4, REG2; \ + add REG1, REG2, REG2; \ + TSB_LOAD_QUAD(REG2, REG3); \ + cmp REG3, TAG; \ + be,a,pt %xcc, OK_LABEL; \ + mov REG4, REG1; +#endif + +#endif /* !(_SPARC64_TSB_H) */ diff --git a/arch/sparc/include/asm/tsunami.h b/arch/sparc/include/asm/tsunami.h new file mode 100644 index 000000000..acaf014ef --- /dev/null +++ b/arch/sparc/include/asm/tsunami.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * tsunami.h: Module specific definitions for Tsunami V8 Sparcs + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_TSUNAMI_H +#define _SPARC_TSUNAMI_H + +#include + +/* The MMU control register on the Tsunami: + * + * ----------------------------------------------------------------------- + * | implvers |SW|AV|DV|MV| RSV |PC|ITD|ALC| RSV |PE| RC |IE|DE|RSV|NF|ME| + * ----------------------------------------------------------------------- + * 31 24 23 22 21 20 19-18 17 16 14 13-12 11 10-9 8 7 6-2 1 0 + * + * SW: Enable Software Table Walks 0=off 1=on + * AV: Address View bit + * DV: Data View bit + * MV: Memory View bit + * PC: Parity Control + * ITD: ITBR disable + * ALC: Alternate Cacheable + * PE: Parity Enable 0=off 1=on + * RC: Refresh Control + * IE: Instruction cache Enable 0=off 1=on + * DE: Data cache Enable 0=off 1=on + * NF: No Fault, same as all other SRMMUs + * ME: MMU Enable, same as all other SRMMUs + */ + +#define TSUNAMI_SW 0x00800000 +#define TSUNAMI_AV 0x00400000 +#define TSUNAMI_DV 0x00200000 +#define TSUNAMI_MV 0x00100000 +#define TSUNAMI_PC 0x00020000 +#define TSUNAMI_ITD 0x00010000 +#define TSUNAMI_ALC 0x00008000 +#define TSUNAMI_PE 0x00001000 +#define TSUNAMI_RCMASK 0x00000C00 +#define TSUNAMI_IENAB 0x00000200 +#define TSUNAMI_DENAB 0x00000100 +#define TSUNAMI_NF 0x00000002 +#define TSUNAMI_ME 0x00000001 + +static inline void tsunami_flush_icache(void) +{ + __asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" + : /* no outputs */ + : "i" (ASI_M_IC_FLCLEAR) + : "memory"); +} + +static inline void tsunami_flush_dcache(void) +{ + __asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" + : /* no outputs */ + : "i" (ASI_M_DC_FLCLEAR) + : "memory"); +} + +#endif /* !(_SPARC_TSUNAMI_H) */ diff --git a/arch/sparc/include/asm/ttable.h b/arch/sparc/include/asm/ttable.h new file mode 100644 index 000000000..b32d3068c --- /dev/null +++ b/arch/sparc/include/asm/ttable.h @@ -0,0 +1,694 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_TTABLE_H +#define _SPARC64_TTABLE_H + +#include +#include + +#ifdef __ASSEMBLER__ +#include +#endif + +#define BOOT_KERNEL b sparc64_boot; nop; nop; nop; nop; nop; nop; nop; + +/* We need a "cleaned" instruction... */ +#define CLEAN_WINDOW \ + rdpr %cleanwin, %l0; add %l0, 1, %l0; \ + wrpr %l0, 0x0, %cleanwin; \ + clr %o0; clr %o1; clr %o2; clr %o3; \ + clr %o4; clr %o5; clr %o6; clr %o7; \ + clr %l0; clr %l1; clr %l2; clr %l3; \ + clr %l4; clr %l5; clr %l6; clr %l7; \ + retry; \ + nop;nop;nop;nop;nop;nop;nop;nop;nop;nop;nop;nop; + +#define TRAP(routine) \ + sethi %hi(109f), %g7; \ + ba,pt %xcc, etrap; \ +109: or %g7, %lo(109b), %g7; \ + call routine; \ + add %sp, PTREGS_OFF, %o0; \ + ba,pt %xcc, rtrap; \ + nop; \ + nop; + +#define TRAP_7INSNS(routine) \ + sethi %hi(109f), %g7; \ + ba,pt %xcc, etrap; \ +109: or %g7, %lo(109b), %g7; \ + call routine; \ + add %sp, PTREGS_OFF, %o0; \ + ba,pt %xcc, rtrap; \ + nop; + +#define TRAP_SAVEFPU(routine) \ + sethi %hi(109f), %g7; \ + ba,pt %xcc, do_fptrap; \ +109: or %g7, %lo(109b), %g7; \ + call routine; \ + add %sp, PTREGS_OFF, %o0; \ + ba,pt %xcc, rtrap; \ + nop; \ + nop; + +#define TRAP_NOSAVE(routine) \ + ba,pt %xcc, routine; \ + nop; \ + nop; nop; nop; nop; nop; nop; + +#define TRAP_NOSAVE_7INSNS(routine) \ + ba,pt %xcc, routine; \ + nop; \ + nop; nop; nop; nop; nop; + +#define TRAPTL1(routine) \ + sethi %hi(109f), %g7; \ + ba,pt %xcc, etraptl1; \ +109: or %g7, %lo(109b), %g7; \ + call routine; \ + add %sp, PTREGS_OFF, %o0; \ + ba,pt %xcc, rtrap; \ + nop; \ + nop; + +#define TRAP_ARG(routine, arg) \ + sethi %hi(109f), %g7; \ + ba,pt %xcc, etrap; \ +109: or %g7, %lo(109b), %g7; \ + add %sp, PTREGS_OFF, %o0; \ + call routine; \ + mov arg, %o1; \ + ba,pt %xcc, rtrap; \ + nop; + +#define TRAPTL1_ARG(routine, arg) \ + sethi %hi(109f), %g7; \ + ba,pt %xcc, etraptl1; \ +109: or %g7, %lo(109b), %g7; \ + add %sp, PTREGS_OFF, %o0; \ + call routine; \ + mov arg, %o1; \ + ba,pt %xcc, rtrap; \ + nop; + +#define SYSCALL_TRAP(routine, systbl) \ + rdpr %pil, %g2; \ + mov TSTATE_SYSCALL, %g3; \ + sethi %hi(109f), %g7; \ + ba,pt %xcc, etrap_syscall; \ +109: or %g7, %lo(109b), %g7; \ + sethi %hi(systbl), %l7; \ + ba,pt %xcc, routine; \ + or %l7, %lo(systbl), %l7; + +#define TRAP_UTRAP(handler,lvl) \ + mov handler, %g3; \ + ba,pt %xcc, utrap_trap; \ + mov lvl, %g4; \ + nop; \ + nop; \ + nop; \ + nop; \ + nop; + +#ifdef CONFIG_COMPAT +#define LINUX_32BIT_SYSCALL_TRAP SYSCALL_TRAP(linux_sparc_syscall32, sys_call_table32) +#else +#define LINUX_32BIT_SYSCALL_TRAP BTRAP(0x110) +#endif +#define LINUX_64BIT_SYSCALL_TRAP SYSCALL_TRAP(linux_sparc_syscall, sys_call_table64) +#define GETCC_TRAP TRAP(getcc) +#define SETCC_TRAP TRAP(setcc) +#define BREAKPOINT_TRAP TRAP(breakpoint_trap) + +#ifdef CONFIG_TRACE_IRQFLAGS + +#define TRAP_IRQ(routine, level) \ + rdpr %pil, %g2; \ + wrpr %g0, PIL_NORMAL_MAX, %pil; \ + sethi %hi(1f-4), %g7; \ + ba,pt %xcc, etrap_irq; \ + or %g7, %lo(1f-4), %g7; \ + nop; \ + nop; \ + nop; \ + .subsection 2; \ +1: call trace_hardirqs_off; \ + nop; \ + mov level, %o0; \ + call routine; \ + add %sp, PTREGS_OFF, %o1; \ + ba,a,pt %xcc, rtrap_irq; \ + .previous; + +#else + +#define TRAP_IRQ(routine, level) \ + rdpr %pil, %g2; \ + wrpr %g0, PIL_NORMAL_MAX, %pil; \ + ba,pt %xcc, etrap_irq; \ + rd %pc, %g7; \ + mov level, %o0; \ + call routine; \ + add %sp, PTREGS_OFF, %o1; \ + ba,a,pt %xcc, rtrap_irq; + +#endif + +#define TRAP_NMI_IRQ(routine, level) \ + rdpr %pil, %g2; \ + wrpr %g0, PIL_NMI, %pil; \ + ba,pt %xcc, etrap_irq; \ + rd %pc, %g7; \ + mov level, %o0; \ + call routine; \ + add %sp, PTREGS_OFF, %o1; \ + ba,a,pt %xcc, rtrap_nmi; + +#define TRAP_IVEC TRAP_NOSAVE(do_ivec) + +#define BTRAP(lvl) TRAP_ARG(bad_trap, lvl) + +#define BTRAPTL1(lvl) TRAPTL1_ARG(bad_trap_tl1, lvl) + +#define FLUSH_WINDOW_TRAP \ + ba,pt %xcc, etrap; \ + rd %pc, %g7; \ + flushw; \ + ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1; \ + add %l1, 4, %l2; \ + stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC]; \ + ba,pt %xcc, rtrap; \ + stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC]; + +#ifdef CONFIG_KPROBES +#define KPROBES_TRAP(lvl) TRAP_IRQ(kprobe_trap, lvl) +#else +#define KPROBES_TRAP(lvl) TRAP_ARG(bad_trap, lvl) +#endif + +#ifdef CONFIG_UPROBES +#define UPROBES_TRAP(lvl) TRAP_ARG(uprobe_trap, lvl) +#else +#define UPROBES_TRAP(lvl) TRAP_ARG(bad_trap, lvl) +#endif + +#ifdef CONFIG_KGDB +#define KGDB_TRAP(lvl) TRAP_IRQ(kgdb_trap, lvl) +#else +#define KGDB_TRAP(lvl) TRAP_ARG(bad_trap, lvl) +#endif + +#define SUN4V_ITSB_MISS \ + ldxa [%g0] ASI_SCRATCHPAD, %g2; \ + ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4; \ + ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5; \ + srlx %g4, 22, %g6; \ + ba,pt %xcc, sun4v_itsb_miss; \ + nop; \ + nop; \ + nop; + +#define SUN4V_DTSB_MISS \ + ldxa [%g0] ASI_SCRATCHPAD, %g2; \ + ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4; \ + ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5; \ + srlx %g4, 22, %g6; \ + ba,pt %xcc, sun4v_dtsb_miss; \ + nop; \ + nop; \ + nop; + +#define SUN4V_MCD_PRECISE \ + ldxa [%g0] ASI_SCRATCHPAD, %g2; \ + ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4; \ + ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5; \ + ba,pt %xcc, etrap; \ + rd %pc, %g7; \ + ba,pt %xcc, sun4v_mcd_detect_precise; \ + nop; \ + nop; + +/* Before touching these macros, you owe it to yourself to go and + * see how arch/sparc64/kernel/winfixup.S works... -DaveM + * + * For the user cases we used to use the %asi register, but + * it turns out that the "wr xxx, %asi" costs ~5 cycles, so + * now we use immediate ASI loads and stores instead. Kudos + * to Greg Onufer for pointing out this performance anomaly. + * + * Further note that we cannot use the g2, g4, g5, and g7 alternate + * globals in the spill routines, check out the save instruction in + * arch/sparc64/kernel/etrap.S to see what I mean about g2, and + * g4/g5 are the globals which are preserved by etrap processing + * for the caller of it. The g7 register is the return pc for + * etrap. Finally, g6 is the current thread register so we cannot + * us it in the spill handlers either. Most of these rules do not + * apply to fill processing, only g6 is not usable. + */ + +/* Normal kernel spill */ +#define SPILL_0_NORMAL \ + stx %l0, [%sp + STACK_BIAS + 0x00]; \ + stx %l1, [%sp + STACK_BIAS + 0x08]; \ + stx %l2, [%sp + STACK_BIAS + 0x10]; \ + stx %l3, [%sp + STACK_BIAS + 0x18]; \ + stx %l4, [%sp + STACK_BIAS + 0x20]; \ + stx %l5, [%sp + STACK_BIAS + 0x28]; \ + stx %l6, [%sp + STACK_BIAS + 0x30]; \ + stx %l7, [%sp + STACK_BIAS + 0x38]; \ + stx %i0, [%sp + STACK_BIAS + 0x40]; \ + stx %i1, [%sp + STACK_BIAS + 0x48]; \ + stx %i2, [%sp + STACK_BIAS + 0x50]; \ + stx %i3, [%sp + STACK_BIAS + 0x58]; \ + stx %i4, [%sp + STACK_BIAS + 0x60]; \ + stx %i5, [%sp + STACK_BIAS + 0x68]; \ + stx %i6, [%sp + STACK_BIAS + 0x70]; \ + stx %i7, [%sp + STACK_BIAS + 0x78]; \ + saved; retry; nop; nop; nop; nop; nop; nop; \ + nop; nop; nop; nop; nop; nop; nop; nop; + +#define SPILL_0_NORMAL_ETRAP \ +etrap_kernel_spill: \ + stx %l0, [%sp + STACK_BIAS + 0x00]; \ + stx %l1, [%sp + STACK_BIAS + 0x08]; \ + stx %l2, [%sp + STACK_BIAS + 0x10]; \ + stx %l3, [%sp + STACK_BIAS + 0x18]; \ + stx %l4, [%sp + STACK_BIAS + 0x20]; \ + stx %l5, [%sp + STACK_BIAS + 0x28]; \ + stx %l6, [%sp + STACK_BIAS + 0x30]; \ + stx %l7, [%sp + STACK_BIAS + 0x38]; \ + stx %i0, [%sp + STACK_BIAS + 0x40]; \ + stx %i1, [%sp + STACK_BIAS + 0x48]; \ + stx %i2, [%sp + STACK_BIAS + 0x50]; \ + stx %i3, [%sp + STACK_BIAS + 0x58]; \ + stx %i4, [%sp + STACK_BIAS + 0x60]; \ + stx %i5, [%sp + STACK_BIAS + 0x68]; \ + stx %i6, [%sp + STACK_BIAS + 0x70]; \ + stx %i7, [%sp + STACK_BIAS + 0x78]; \ + saved; \ + sub %g1, 2, %g1; \ + ba,pt %xcc, etrap_save; \ + wrpr %g1, %cwp; \ + nop; nop; nop; nop; nop; nop; nop; nop; \ + nop; nop; nop; nop; + +/* Normal 64bit spill */ +#define SPILL_1_GENERIC(ASI) \ + add %sp, STACK_BIAS + 0x00, %g1; \ + stxa %l0, [%g1 + %g0] ASI; \ + mov 0x08, %g3; \ + stxa %l1, [%g1 + %g3] ASI; \ + add %g1, 0x10, %g1; \ + stxa %l2, [%g1 + %g0] ASI; \ + stxa %l3, [%g1 + %g3] ASI; \ + add %g1, 0x10, %g1; \ + stxa %l4, [%g1 + %g0] ASI; \ + stxa %l5, [%g1 + %g3] ASI; \ + add %g1, 0x10, %g1; \ + stxa %l6, [%g1 + %g0] ASI; \ + stxa %l7, [%g1 + %g3] ASI; \ + add %g1, 0x10, %g1; \ + stxa %i0, [%g1 + %g0] ASI; \ + stxa %i1, [%g1 + %g3] ASI; \ + add %g1, 0x10, %g1; \ + stxa %i2, [%g1 + %g0] ASI; \ + stxa %i3, [%g1 + %g3] ASI; \ + add %g1, 0x10, %g1; \ + stxa %i4, [%g1 + %g0] ASI; \ + stxa %i5, [%g1 + %g3] ASI; \ + add %g1, 0x10, %g1; \ + stxa %i6, [%g1 + %g0] ASI; \ + stxa %i7, [%g1 + %g3] ASI; \ + saved; \ + retry; nop; nop; \ + b,a,pt %xcc, spill_fixup_dax; \ + b,a,pt %xcc, spill_fixup_mna; \ + b,a,pt %xcc, spill_fixup; + +#define SPILL_1_GENERIC_ETRAP \ +etrap_user_spill_64bit: \ + stxa %l0, [%sp + STACK_BIAS + 0x00] %asi; \ + stxa %l1, [%sp + STACK_BIAS + 0x08] %asi; \ + stxa %l2, [%sp + STACK_BIAS + 0x10] %asi; \ + stxa %l3, [%sp + STACK_BIAS + 0x18] %asi; \ + stxa %l4, [%sp + STACK_BIAS + 0x20] %asi; \ + stxa %l5, [%sp + STACK_BIAS + 0x28] %asi; \ + stxa %l6, [%sp + STACK_BIAS + 0x30] %asi; \ + stxa %l7, [%sp + STACK_BIAS + 0x38] %asi; \ + stxa %i0, [%sp + STACK_BIAS + 0x40] %asi; \ + stxa %i1, [%sp + STACK_BIAS + 0x48] %asi; \ + stxa %i2, [%sp + STACK_BIAS + 0x50] %asi; \ + stxa %i3, [%sp + STACK_BIAS + 0x58] %asi; \ + stxa %i4, [%sp + STACK_BIAS + 0x60] %asi; \ + stxa %i5, [%sp + STACK_BIAS + 0x68] %asi; \ + stxa %i6, [%sp + STACK_BIAS + 0x70] %asi; \ + stxa %i7, [%sp + STACK_BIAS + 0x78] %asi; \ + saved; \ + sub %g1, 2, %g1; \ + ba,pt %xcc, etrap_save; \ + wrpr %g1, %cwp; \ + nop; nop; nop; nop; nop; \ + nop; nop; nop; nop; \ + ba,a,pt %xcc, etrap_spill_fixup_64bit; \ + ba,a,pt %xcc, etrap_spill_fixup_64bit; \ + ba,a,pt %xcc, etrap_spill_fixup_64bit; + +#define SPILL_1_GENERIC_ETRAP_FIXUP \ +etrap_spill_fixup_64bit: \ + ldub [%g6 + TI_WSAVED], %g1; \ + sll %g1, 3, %g3; \ + add %g6, %g3, %g3; \ + stx %sp, [%g3 + TI_RWIN_SPTRS]; \ + sll %g1, 7, %g3; \ + add %g6, %g3, %g3; \ + stx %l0, [%g3 + TI_REG_WINDOW + 0x00]; \ + stx %l1, [%g3 + TI_REG_WINDOW + 0x08]; \ + stx %l2, [%g3 + TI_REG_WINDOW + 0x10]; \ + stx %l3, [%g3 + TI_REG_WINDOW + 0x18]; \ + stx %l4, [%g3 + TI_REG_WINDOW + 0x20]; \ + stx %l5, [%g3 + TI_REG_WINDOW + 0x28]; \ + stx %l6, [%g3 + TI_REG_WINDOW + 0x30]; \ + stx %l7, [%g3 + TI_REG_WINDOW + 0x38]; \ + stx %i0, [%g3 + TI_REG_WINDOW + 0x40]; \ + stx %i1, [%g3 + TI_REG_WINDOW + 0x48]; \ + stx %i2, [%g3 + TI_REG_WINDOW + 0x50]; \ + stx %i3, [%g3 + TI_REG_WINDOW + 0x58]; \ + stx %i4, [%g3 + TI_REG_WINDOW + 0x60]; \ + stx %i5, [%g3 + TI_REG_WINDOW + 0x68]; \ + stx %i6, [%g3 + TI_REG_WINDOW + 0x70]; \ + stx %i7, [%g3 + TI_REG_WINDOW + 0x78]; \ + add %g1, 1, %g1; \ + stb %g1, [%g6 + TI_WSAVED]; \ + saved; \ + rdpr %cwp, %g1; \ + sub %g1, 2, %g1; \ + ba,pt %xcc, etrap_save; \ + wrpr %g1, %cwp; \ + nop; nop; nop + +/* Normal 32bit spill */ +#define SPILL_2_GENERIC(ASI) \ + and %sp, 1, %g3; \ + brnz,pn %g3, (. - (128 + 4)); \ + srl %sp, 0, %sp; \ + stwa %l0, [%sp + %g0] ASI; \ + mov 0x04, %g3; \ + stwa %l1, [%sp + %g3] ASI; \ + add %sp, 0x08, %g1; \ + stwa %l2, [%g1 + %g0] ASI; \ + stwa %l3, [%g1 + %g3] ASI; \ + add %g1, 0x08, %g1; \ + stwa %l4, [%g1 + %g0] ASI; \ + stwa %l5, [%g1 + %g3] ASI; \ + add %g1, 0x08, %g1; \ + stwa %l6, [%g1 + %g0] ASI; \ + stwa %l7, [%g1 + %g3] ASI; \ + add %g1, 0x08, %g1; \ + stwa %i0, [%g1 + %g0] ASI; \ + stwa %i1, [%g1 + %g3] ASI; \ + add %g1, 0x08, %g1; \ + stwa %i2, [%g1 + %g0] ASI; \ + stwa %i3, [%g1 + %g3] ASI; \ + add %g1, 0x08, %g1; \ + stwa %i4, [%g1 + %g0] ASI; \ + stwa %i5, [%g1 + %g3] ASI; \ + add %g1, 0x08, %g1; \ + stwa %i6, [%g1 + %g0] ASI; \ + stwa %i7, [%g1 + %g3] ASI; \ + saved; \ + retry; \ + b,a,pt %xcc, spill_fixup_dax; \ + b,a,pt %xcc, spill_fixup_mna; \ + b,a,pt %xcc, spill_fixup; + +#define SPILL_2_GENERIC_ETRAP \ +etrap_user_spill_32bit: \ + and %sp, 1, %g3; \ + brnz,pn %g3, etrap_user_spill_64bit; \ + srl %sp, 0, %sp; \ + stwa %l0, [%sp + 0x00] %asi; \ + stwa %l1, [%sp + 0x04] %asi; \ + stwa %l2, [%sp + 0x08] %asi; \ + stwa %l3, [%sp + 0x0c] %asi; \ + stwa %l4, [%sp + 0x10] %asi; \ + stwa %l5, [%sp + 0x14] %asi; \ + stwa %l6, [%sp + 0x18] %asi; \ + stwa %l7, [%sp + 0x1c] %asi; \ + stwa %i0, [%sp + 0x20] %asi; \ + stwa %i1, [%sp + 0x24] %asi; \ + stwa %i2, [%sp + 0x28] %asi; \ + stwa %i3, [%sp + 0x2c] %asi; \ + stwa %i4, [%sp + 0x30] %asi; \ + stwa %i5, [%sp + 0x34] %asi; \ + stwa %i6, [%sp + 0x38] %asi; \ + stwa %i7, [%sp + 0x3c] %asi; \ + saved; \ + sub %g1, 2, %g1; \ + ba,pt %xcc, etrap_save; \ + wrpr %g1, %cwp; \ + nop; nop; nop; nop; \ + nop; nop; \ + ba,a,pt %xcc, etrap_spill_fixup_32bit; \ + ba,a,pt %xcc, etrap_spill_fixup_32bit; \ + ba,a,pt %xcc, etrap_spill_fixup_32bit; + +#define SPILL_2_GENERIC_ETRAP_FIXUP \ +etrap_spill_fixup_32bit: \ + ldub [%g6 + TI_WSAVED], %g1; \ + sll %g1, 3, %g3; \ + add %g6, %g3, %g3; \ + stx %sp, [%g3 + TI_RWIN_SPTRS]; \ + sll %g1, 7, %g3; \ + add %g6, %g3, %g3; \ + stw %l0, [%g3 + TI_REG_WINDOW + 0x00]; \ + stw %l1, [%g3 + TI_REG_WINDOW + 0x04]; \ + stw %l2, [%g3 + TI_REG_WINDOW + 0x08]; \ + stw %l3, [%g3 + TI_REG_WINDOW + 0x0c]; \ + stw %l4, [%g3 + TI_REG_WINDOW + 0x10]; \ + stw %l5, [%g3 + TI_REG_WINDOW + 0x14]; \ + stw %l6, [%g3 + TI_REG_WINDOW + 0x18]; \ + stw %l7, [%g3 + TI_REG_WINDOW + 0x1c]; \ + stw %i0, [%g3 + TI_REG_WINDOW + 0x20]; \ + stw %i1, [%g3 + TI_REG_WINDOW + 0x24]; \ + stw %i2, [%g3 + TI_REG_WINDOW + 0x28]; \ + stw %i3, [%g3 + TI_REG_WINDOW + 0x2c]; \ + stw %i4, [%g3 + TI_REG_WINDOW + 0x30]; \ + stw %i5, [%g3 + TI_REG_WINDOW + 0x34]; \ + stw %i6, [%g3 + TI_REG_WINDOW + 0x38]; \ + stw %i7, [%g3 + TI_REG_WINDOW + 0x3c]; \ + add %g1, 1, %g1; \ + stb %g1, [%g6 + TI_WSAVED]; \ + saved; \ + rdpr %cwp, %g1; \ + sub %g1, 2, %g1; \ + ba,pt %xcc, etrap_save; \ + wrpr %g1, %cwp; \ + nop; nop; nop + +#define SPILL_1_NORMAL SPILL_1_GENERIC(ASI_AIUP) +#define SPILL_2_NORMAL SPILL_2_GENERIC(ASI_AIUP) +#define SPILL_3_NORMAL SPILL_0_NORMAL +#define SPILL_4_NORMAL SPILL_0_NORMAL +#define SPILL_5_NORMAL SPILL_0_NORMAL +#define SPILL_6_NORMAL SPILL_0_NORMAL +#define SPILL_7_NORMAL SPILL_0_NORMAL + +#define SPILL_0_OTHER SPILL_0_NORMAL +#define SPILL_1_OTHER SPILL_1_GENERIC(ASI_AIUS) +#define SPILL_2_OTHER SPILL_2_GENERIC(ASI_AIUS) +#define SPILL_3_OTHER SPILL_3_NORMAL +#define SPILL_4_OTHER SPILL_4_NORMAL +#define SPILL_5_OTHER SPILL_5_NORMAL +#define SPILL_6_OTHER SPILL_6_NORMAL +#define SPILL_7_OTHER SPILL_7_NORMAL + +/* Normal kernel fill */ +#define FILL_0_NORMAL \ + ldx [%sp + STACK_BIAS + 0x00], %l0; \ + ldx [%sp + STACK_BIAS + 0x08], %l1; \ + ldx [%sp + STACK_BIAS + 0x10], %l2; \ + ldx [%sp + STACK_BIAS + 0x18], %l3; \ + ldx [%sp + STACK_BIAS + 0x20], %l4; \ + ldx [%sp + STACK_BIAS + 0x28], %l5; \ + ldx [%sp + STACK_BIAS + 0x30], %l6; \ + ldx [%sp + STACK_BIAS + 0x38], %l7; \ + ldx [%sp + STACK_BIAS + 0x40], %i0; \ + ldx [%sp + STACK_BIAS + 0x48], %i1; \ + ldx [%sp + STACK_BIAS + 0x50], %i2; \ + ldx [%sp + STACK_BIAS + 0x58], %i3; \ + ldx [%sp + STACK_BIAS + 0x60], %i4; \ + ldx [%sp + STACK_BIAS + 0x68], %i5; \ + ldx [%sp + STACK_BIAS + 0x70], %i6; \ + ldx [%sp + STACK_BIAS + 0x78], %i7; \ + restored; retry; nop; nop; nop; nop; nop; nop; \ + nop; nop; nop; nop; nop; nop; nop; nop; + +#define FILL_0_NORMAL_RTRAP \ +kern_rtt_fill: \ + rdpr %cwp, %g1; \ + sub %g1, 1, %g1; \ + wrpr %g1, %cwp; \ + ldx [%sp + STACK_BIAS + 0x00], %l0; \ + ldx [%sp + STACK_BIAS + 0x08], %l1; \ + ldx [%sp + STACK_BIAS + 0x10], %l2; \ + ldx [%sp + STACK_BIAS + 0x18], %l3; \ + ldx [%sp + STACK_BIAS + 0x20], %l4; \ + ldx [%sp + STACK_BIAS + 0x28], %l5; \ + ldx [%sp + STACK_BIAS + 0x30], %l6; \ + ldx [%sp + STACK_BIAS + 0x38], %l7; \ + ldx [%sp + STACK_BIAS + 0x40], %i0; \ + ldx [%sp + STACK_BIAS + 0x48], %i1; \ + ldx [%sp + STACK_BIAS + 0x50], %i2; \ + ldx [%sp + STACK_BIAS + 0x58], %i3; \ + ldx [%sp + STACK_BIAS + 0x60], %i4; \ + ldx [%sp + STACK_BIAS + 0x68], %i5; \ + ldx [%sp + STACK_BIAS + 0x70], %i6; \ + ldx [%sp + STACK_BIAS + 0x78], %i7; \ + restored; \ + add %g1, 1, %g1; \ + ba,pt %xcc, kern_rtt_restore; \ + wrpr %g1, %cwp; \ + nop; nop; nop; nop; nop; \ + nop; nop; nop; nop; + + +/* Normal 64bit fill */ +#define FILL_1_GENERIC(ASI) \ + add %sp, STACK_BIAS + 0x00, %g1; \ + ldxa [%g1 + %g0] ASI, %l0; \ + mov 0x08, %g2; \ + mov 0x10, %g3; \ + ldxa [%g1 + %g2] ASI, %l1; \ + mov 0x18, %g5; \ + ldxa [%g1 + %g3] ASI, %l2; \ + ldxa [%g1 + %g5] ASI, %l3; \ + add %g1, 0x20, %g1; \ + ldxa [%g1 + %g0] ASI, %l4; \ + ldxa [%g1 + %g2] ASI, %l5; \ + ldxa [%g1 + %g3] ASI, %l6; \ + ldxa [%g1 + %g5] ASI, %l7; \ + add %g1, 0x20, %g1; \ + ldxa [%g1 + %g0] ASI, %i0; \ + ldxa [%g1 + %g2] ASI, %i1; \ + ldxa [%g1 + %g3] ASI, %i2; \ + ldxa [%g1 + %g5] ASI, %i3; \ + add %g1, 0x20, %g1; \ + ldxa [%g1 + %g0] ASI, %i4; \ + ldxa [%g1 + %g2] ASI, %i5; \ + ldxa [%g1 + %g3] ASI, %i6; \ + ldxa [%g1 + %g5] ASI, %i7; \ + restored; \ + retry; nop; nop; nop; nop; \ + b,a,pt %xcc, fill_fixup_dax; \ + b,a,pt %xcc, fill_fixup_mna; \ + b,a,pt %xcc, fill_fixup; + +#define FILL_1_GENERIC_RTRAP \ +user_rtt_fill_64bit: \ + ldxa [%sp + STACK_BIAS + 0x00] %asi, %l0; \ + ldxa [%sp + STACK_BIAS + 0x08] %asi, %l1; \ + ldxa [%sp + STACK_BIAS + 0x10] %asi, %l2; \ + ldxa [%sp + STACK_BIAS + 0x18] %asi, %l3; \ + ldxa [%sp + STACK_BIAS + 0x20] %asi, %l4; \ + ldxa [%sp + STACK_BIAS + 0x28] %asi, %l5; \ + ldxa [%sp + STACK_BIAS + 0x30] %asi, %l6; \ + ldxa [%sp + STACK_BIAS + 0x38] %asi, %l7; \ + ldxa [%sp + STACK_BIAS + 0x40] %asi, %i0; \ + ldxa [%sp + STACK_BIAS + 0x48] %asi, %i1; \ + ldxa [%sp + STACK_BIAS + 0x50] %asi, %i2; \ + ldxa [%sp + STACK_BIAS + 0x58] %asi, %i3; \ + ldxa [%sp + STACK_BIAS + 0x60] %asi, %i4; \ + ldxa [%sp + STACK_BIAS + 0x68] %asi, %i5; \ + ldxa [%sp + STACK_BIAS + 0x70] %asi, %i6; \ + ldxa [%sp + STACK_BIAS + 0x78] %asi, %i7; \ + ba,pt %xcc, user_rtt_pre_restore; \ + restored; \ + nop; nop; nop; nop; nop; nop; \ + nop; nop; nop; nop; nop; \ + ba,a,pt %xcc, user_rtt_fill_fixup_dax; \ + ba,a,pt %xcc, user_rtt_fill_fixup_mna; \ + ba,a,pt %xcc, user_rtt_fill_fixup; + + +/* Normal 32bit fill */ +#define FILL_2_GENERIC(ASI) \ + and %sp, 1, %g3; \ + brnz,pn %g3, (. - (128 + 4)); \ + srl %sp, 0, %sp; \ + lduwa [%sp + %g0] ASI, %l0; \ + mov 0x04, %g2; \ + mov 0x08, %g3; \ + lduwa [%sp + %g2] ASI, %l1; \ + mov 0x0c, %g5; \ + lduwa [%sp + %g3] ASI, %l2; \ + lduwa [%sp + %g5] ASI, %l3; \ + add %sp, 0x10, %g1; \ + lduwa [%g1 + %g0] ASI, %l4; \ + lduwa [%g1 + %g2] ASI, %l5; \ + lduwa [%g1 + %g3] ASI, %l6; \ + lduwa [%g1 + %g5] ASI, %l7; \ + add %g1, 0x10, %g1; \ + lduwa [%g1 + %g0] ASI, %i0; \ + lduwa [%g1 + %g2] ASI, %i1; \ + lduwa [%g1 + %g3] ASI, %i2; \ + lduwa [%g1 + %g5] ASI, %i3; \ + add %g1, 0x10, %g1; \ + lduwa [%g1 + %g0] ASI, %i4; \ + lduwa [%g1 + %g2] ASI, %i5; \ + lduwa [%g1 + %g3] ASI, %i6; \ + lduwa [%g1 + %g5] ASI, %i7; \ + restored; \ + retry; nop; nop; \ + b,a,pt %xcc, fill_fixup_dax; \ + b,a,pt %xcc, fill_fixup_mna; \ + b,a,pt %xcc, fill_fixup; + +#define FILL_2_GENERIC_RTRAP \ +user_rtt_fill_32bit: \ + and %sp, 1, %g3; \ + brnz,pn %g3, user_rtt_fill_64bit; \ + srl %sp, 0, %sp; \ + lduwa [%sp + 0x00] %asi, %l0; \ + lduwa [%sp + 0x04] %asi, %l1; \ + lduwa [%sp + 0x08] %asi, %l2; \ + lduwa [%sp + 0x0c] %asi, %l3; \ + lduwa [%sp + 0x10] %asi, %l4; \ + lduwa [%sp + 0x14] %asi, %l5; \ + lduwa [%sp + 0x18] %asi, %l6; \ + lduwa [%sp + 0x1c] %asi, %l7; \ + lduwa [%sp + 0x20] %asi, %i0; \ + lduwa [%sp + 0x24] %asi, %i1; \ + lduwa [%sp + 0x28] %asi, %i2; \ + lduwa [%sp + 0x2c] %asi, %i3; \ + lduwa [%sp + 0x30] %asi, %i4; \ + lduwa [%sp + 0x34] %asi, %i5; \ + lduwa [%sp + 0x38] %asi, %i6; \ + lduwa [%sp + 0x3c] %asi, %i7; \ + ba,pt %xcc, user_rtt_pre_restore; \ + restored; \ + nop; nop; nop; nop; nop; \ + nop; nop; nop; \ + ba,a,pt %xcc, user_rtt_fill_fixup_dax; \ + ba,a,pt %xcc, user_rtt_fill_fixup_mna; \ + ba,a,pt %xcc, user_rtt_fill_fixup; + + +#define FILL_1_NORMAL FILL_1_GENERIC(ASI_AIUP) +#define FILL_2_NORMAL FILL_2_GENERIC(ASI_AIUP) +#define FILL_3_NORMAL FILL_0_NORMAL +#define FILL_4_NORMAL FILL_0_NORMAL +#define FILL_5_NORMAL FILL_0_NORMAL +#define FILL_6_NORMAL FILL_0_NORMAL +#define FILL_7_NORMAL FILL_0_NORMAL + +#define FILL_0_OTHER FILL_0_NORMAL +#define FILL_1_OTHER FILL_1_GENERIC(ASI_AIUS) +#define FILL_2_OTHER FILL_2_GENERIC(ASI_AIUS) +#define FILL_3_OTHER FILL_3_NORMAL +#define FILL_4_OTHER FILL_4_NORMAL +#define FILL_5_OTHER FILL_5_NORMAL +#define FILL_6_OTHER FILL_6_NORMAL +#define FILL_7_OTHER FILL_7_NORMAL + +#endif /* !(_SPARC64_TTABLE_H) */ diff --git a/arch/sparc/include/asm/turbosparc.h b/arch/sparc/include/asm/turbosparc.h new file mode 100644 index 000000000..5f73263b6 --- /dev/null +++ b/arch/sparc/include/asm/turbosparc.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * turbosparc.h: Defines specific to the TurboSparc module. + * This is SRMMU stuff. + * + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ +#ifndef _SPARC_TURBOSPARC_H +#define _SPARC_TURBOSPARC_H + +#include +#include + +/* Bits in the SRMMU control register for TurboSparc modules. + * + * ------------------------------------------------------------------- + * |impl-vers| RSV| PMC |PE|PC| RSV |BM| RFR |IC|DC|PSO|RSV|ICS|NF|ME| + * ------------------------------------------------------------------- + * 31 24 23-21 20-19 18 17 16-15 14 13-10 9 8 7 6-3 2 1 0 + * + * BM: Boot Mode -- 0 = not in boot mode, 1 = in boot mode + * + * This indicates whether the TurboSparc is in boot-mode or not. + * + * IC: Instruction Cache -- 0 = off, 1 = on + * DC: Data Cache -- 0 = off, 1 = 0n + * + * These bits enable the on-cpu TurboSparc split I/D caches. + * + * ICS: ICache Snooping -- 0 = disable, 1 = enable snooping of icache + * NF: No Fault -- 0 = faults generate traps, 1 = faults don't trap + * ME: MMU enable -- 0 = mmu not translating, 1 = mmu translating + * + */ + +#define TURBOSPARC_MMUENABLE 0x00000001 +#define TURBOSPARC_NOFAULT 0x00000002 +#define TURBOSPARC_ICSNOOP 0x00000004 +#define TURBOSPARC_PSO 0x00000080 +#define TURBOSPARC_DCENABLE 0x00000100 /* Enable data cache */ +#define TURBOSPARC_ICENABLE 0x00000200 /* Enable instruction cache */ +#define TURBOSPARC_BMODE 0x00004000 +#define TURBOSPARC_PARITYODD 0x00020000 /* Parity odd, if enabled */ +#define TURBOSPARC_PCENABLE 0x00040000 /* Enable parity checking */ + +/* Bits in the CPU configuration register for TurboSparc modules. + * + * ------------------------------------------------------- + * |IOClk|SNP|AXClk| RAH | WS | RSV |SBC|WT|uS2|SE|SCC| + * ------------------------------------------------------- + * 31 30 29-28 27-26 25-23 22-8 7-6 5 4 3 2-0 + * + */ + +#define TURBOSPARC_SCENABLE 0x00000008 /* Secondary cache enable */ +#define TURBOSPARC_uS2 0x00000010 /* Swift compatibility mode */ +#define TURBOSPARC_WTENABLE 0x00000020 /* Write thru for dcache */ +#define TURBOSPARC_SNENABLE 0x40000000 /* DVMA snoop enable */ + +#ifndef __ASSEMBLER__ + +/* Bits [13:5] select one of 512 instruction cache tags */ +static inline void turbosparc_inv_insn_tag(unsigned long addr) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (addr), "i" (ASI_M_TXTC_TAG) + : "memory"); +} + +/* Bits [13:5] select one of 512 data cache tags */ +static inline void turbosparc_inv_data_tag(unsigned long addr) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (addr), "i" (ASI_M_DATAC_TAG) + : "memory"); +} + +static inline void turbosparc_flush_icache(void) +{ + unsigned long addr; + + for (addr = 0; addr < 0x4000; addr += 0x20) + turbosparc_inv_insn_tag(addr); +} + +static inline void turbosparc_flush_dcache(void) +{ + unsigned long addr; + + for (addr = 0; addr < 0x4000; addr += 0x20) + turbosparc_inv_data_tag(addr); +} + +static inline void turbosparc_idflash_clear(void) +{ + unsigned long addr; + + for (addr = 0; addr < 0x4000; addr += 0x20) { + turbosparc_inv_insn_tag(addr); + turbosparc_inv_data_tag(addr); + } +} + +static inline void turbosparc_set_ccreg(unsigned long regval) +{ + __asm__ __volatile__("sta %0, [%1] %2\n\t" + : /* no outputs */ + : "r" (regval), "r" (0x600), "i" (ASI_M_MMUREGS) + : "memory"); +} + +static inline unsigned long turbosparc_get_ccreg(void) +{ + unsigned long regval; + + __asm__ __volatile__("lda [%1] %2, %0\n\t" + : "=r" (regval) + : "r" (0x600), "i" (ASI_M_MMUREGS)); + return regval; +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* !(_SPARC_TURBOSPARC_H) */ diff --git a/arch/sparc/include/asm/uaccess.h b/arch/sparc/include/asm/uaccess.h new file mode 100644 index 000000000..ee75f69e3 --- /dev/null +++ b/arch/sparc/include/asm/uaccess.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ___ASM_SPARC_UACCESS_H +#define ___ASM_SPARC_UACCESS_H + +#include + +#if defined(__sparc__) && defined(__arch64__) +#include +#else +#include +#endif + +long strncpy_from_user(char *dest, const char __user *src, long count); + +#endif diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h new file mode 100644 index 000000000..5542d5b32 --- /dev/null +++ b/arch/sparc/include/asm/uaccess_32.h @@ -0,0 +1,221 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * uaccess.h: User space memore access functions. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ +#ifndef _ASM_UACCESS_H +#define _ASM_UACCESS_H + +#include +#include + +#include +#include + +/* Uh, these should become the main single-value transfer routines.. + * They automatically use the right size if we just have the right + * pointer type.. + * + * This gets kind of ugly. We want to return _two_ values in "get_user()" + * and yet we don't want to do any pointers, because that is too much + * of a performance impact. Thus we have a few rather ugly macros here, + * and hide all the ugliness from the user. + */ +#define put_user(x, ptr) ({ \ + void __user *__pu_addr = (ptr); \ + __chk_user_ptr(ptr); \ + __put_user_check((__typeof__(*(ptr)))(x), __pu_addr, sizeof(*(ptr))); \ +}) + +#define get_user(x, ptr) ({ \ + const void __user *__gu_addr = (ptr); \ + __chk_user_ptr(ptr); \ + __get_user_check((x), __gu_addr, sizeof(*(ptr)), __typeof__(*(ptr))); \ +}) + +/* + * The "__xxx" versions do not do address space checking, useful when + * doing multiple accesses to the same area (the user has to do the + * checks by hand with "access_ok()") + */ +#define __put_user(x, ptr) \ + __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) +#define __get_user(x, ptr) \ + __get_user_nocheck((x), (ptr), sizeof(*(ptr)), __typeof__(*(ptr))) + +struct __large_struct { unsigned long buf[100]; }; +#define __m(x) ((struct __large_struct __user *)(x)) + +#define __put_user_check(x, addr, size) ({ \ + register int __pu_ret; \ + if (__access_ok(addr, size)) { \ + switch (size) { \ + case 1: \ + __put_user_asm(x, b, addr, __pu_ret); \ + break; \ + case 2: \ + __put_user_asm(x, h, addr, __pu_ret); \ + break; \ + case 4: \ + __put_user_asm(x, , addr, __pu_ret); \ + break; \ + case 8: \ + __put_user_asm(x, d, addr, __pu_ret); \ + break; \ + default: \ + __pu_ret = __put_user_bad(); \ + break; \ + } \ + } else { \ + __pu_ret = -EFAULT; \ + } \ + __pu_ret; \ +}) + +#define __put_user_nocheck(x, addr, size) ({ \ + register int __pu_ret; \ + switch (size) { \ + case 1: __put_user_asm(x, b, addr, __pu_ret); break; \ + case 2: __put_user_asm(x, h, addr, __pu_ret); break; \ + case 4: __put_user_asm(x, , addr, __pu_ret); break; \ + case 8: __put_user_asm(x, d, addr, __pu_ret); break; \ + default: __pu_ret = __put_user_bad(); break; \ + } \ + __pu_ret; \ +}) + +#define __put_user_asm(x, size, addr, ret) \ +__asm__ __volatile__( \ + "/* Put user asm, inline. */\n" \ + "1:\t" "st"#size " %1, %2\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\t" \ + ".previous\n\n\t" \ + ".section __ex_table,#alloc\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\t" \ + ".previous\n\n\t" \ + : "=&r" (ret) : "r" (x), "m" (*__m(addr)), \ + "i" (-EFAULT)) + +int __put_user_bad(void); + +#define __get_user_check(x, addr, size, type) ({ \ + register int __gu_ret; \ + register unsigned long __gu_val; \ + if (__access_ok(addr, size)) { \ + switch (size) { \ + case 1: \ + __get_user_asm(__gu_val, ub, addr, __gu_ret); \ + break; \ + case 2: \ + __get_user_asm(__gu_val, uh, addr, __gu_ret); \ + break; \ + case 4: \ + __get_user_asm(__gu_val, , addr, __gu_ret); \ + break; \ + case 8: \ + __get_user_asm(__gu_val, d, addr, __gu_ret); \ + break; \ + default: \ + __gu_val = 0; \ + __gu_ret = __get_user_bad(); \ + break; \ + } \ + } else { \ + __gu_val = 0; \ + __gu_ret = -EFAULT; \ + } \ + x = (__force type) __gu_val; \ + __gu_ret; \ +}) + +#define __get_user_nocheck(x, addr, size, type) ({ \ + register int __gu_ret; \ + register unsigned long __gu_val; \ + switch (size) { \ + case 1: __get_user_asm(__gu_val, ub, addr, __gu_ret); break; \ + case 2: __get_user_asm(__gu_val, uh, addr, __gu_ret); break; \ + case 4: __get_user_asm(__gu_val, , addr, __gu_ret); break; \ + case 8: __get_user_asm(__gu_val, d, addr, __gu_ret); break; \ + default: \ + __gu_val = 0; \ + __gu_ret = __get_user_bad(); \ + break; \ + } \ + x = (__force type) __gu_val; \ + __gu_ret; \ +}) + +#define __get_user_asm(x, size, addr, ret) \ +__asm__ __volatile__( \ + "/* Get user asm, inline. */\n" \ + "1:\t" "ld"#size " %2, %1\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "clr %1\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,#alloc\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\n\t" \ + ".previous\n\t" \ + : "=&r" (ret), "=&r" (x) : "m" (*__m(addr)), \ + "i" (-EFAULT)) + +int __get_user_bad(void); + +unsigned long __copy_user(void __user *to, const void __user *from, unsigned long size); + +static inline unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return __copy_user(to, (__force void __user *) from, n); +} + +static inline unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return __copy_user((__force void __user *) to, from, n); +} + +#define INLINE_COPY_USER + +static inline unsigned long __clear_user(void __user *addr, unsigned long size) +{ + unsigned long ret; + + __asm__ __volatile__ ( + "mov %2, %%o1\n" + "call __bzero\n\t" + " mov %1, %%o0\n\t" + "mov %%o0, %0\n" + : "=r" (ret) : "r" (addr), "r" (size) : + "o0", "o1", "o2", "o3", "o4", "o5", "o7", + "g1", "g2", "g3", "g4", "g5", "g7", "cc"); + + return ret; +} + +static inline unsigned long clear_user(void __user *addr, unsigned long n) +{ + if (n && __access_ok(addr, n)) + return __clear_user(addr, n); + else + return n; +} + +__must_check long strnlen_user(const char __user *str, long n); + +#endif /* _ASM_UACCESS_H */ diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h new file mode 100644 index 000000000..e2989cfba --- /dev/null +++ b/arch/sparc/include/asm/uaccess_64.h @@ -0,0 +1,251 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_UACCESS_H +#define _ASM_UACCESS_H + +/* + * User space memory access functions + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +/* + * Sparc64 is segmented, though more like the M68K than the I386. + * We use the secondary ASI to address user memory, which references a + * completely different VM map, thus there is zero chance of the user + * doing something queer and tricking us into poking kernel memory. + */ + +/* + * Test whether a block of memory is a valid user space address. + * Returns 0 if the range is valid, nonzero otherwise. + */ +static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit) +{ + if (__builtin_constant_p(size)) + return addr > limit - size; + + addr += size; + if (addr < size) + return true; + + return addr > limit; +} + +#define __range_not_ok(addr, size, limit) \ +({ \ + __chk_user_ptr(addr); \ + __chk_range_not_ok((unsigned long __force)(addr), size, limit); \ +}) + +void __retl_efault(void); + +/* Uh, these should become the main single-value transfer routines.. + * They automatically use the right size if we just have the right + * pointer type.. + * + * This gets kind of ugly. We want to return _two_ values in "get_user()" + * and yet we don't want to do any pointers, because that is too much + * of a performance impact. Thus we have a few rather ugly macros here, + * and hide all the ugliness from the user. + */ +#define put_user(x, ptr) ({ \ + unsigned long __pu_addr = (unsigned long)(ptr); \ + __chk_user_ptr(ptr); \ + __put_user_nocheck((__typeof__(*(ptr)))(x), __pu_addr, sizeof(*(ptr)));\ +}) + +#define get_user(x, ptr) ({ \ + unsigned long __gu_addr = (unsigned long)(ptr); \ + __chk_user_ptr(ptr); \ + __get_user_nocheck((x), __gu_addr, sizeof(*(ptr)), __typeof__(*(ptr)));\ +}) + +#define __put_user(x, ptr) put_user(x, ptr) +#define __get_user(x, ptr) get_user(x, ptr) + +struct __large_struct { unsigned long buf[100]; }; +#define __m(x) ((struct __large_struct *)(x)) + +#define __put_kernel_nofault(dst, src, type, label) \ +do { \ + type *addr = (type __force *)(dst); \ + type data = *(type *)src; \ + register int __pu_ret; \ + switch (sizeof(type)) { \ + case 1: __put_kernel_asm(data, b, addr, __pu_ret); break; \ + case 2: __put_kernel_asm(data, h, addr, __pu_ret); break; \ + case 4: __put_kernel_asm(data, w, addr, __pu_ret); break; \ + case 8: __put_kernel_asm(data, x, addr, __pu_ret); break; \ + default: __pu_ret = __put_user_bad(); break; \ + } \ + if (__pu_ret) \ + goto label; \ +} while (0) + +#define __put_kernel_asm(x, size, addr, ret) \ +__asm__ __volatile__( \ + "/* Put kernel asm, inline. */\n" \ + "1:\t" "st"#size " %1, [%2]\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\t" \ + ".previous\n\n\t" \ + : "=r" (ret) : "r" (x), "r" (__m(addr)), \ + "i" (-EFAULT)) + +#define __put_user_nocheck(data, addr, size) ({ \ + register int __pu_ret; \ + switch (size) { \ + case 1: __put_user_asm(data, b, addr, __pu_ret); break; \ + case 2: __put_user_asm(data, h, addr, __pu_ret); break; \ + case 4: __put_user_asm(data, w, addr, __pu_ret); break; \ + case 8: __put_user_asm(data, x, addr, __pu_ret); break; \ + default: __pu_ret = __put_user_bad(); break; \ + } \ + __pu_ret; \ +}) + +#define __put_user_asm(x, size, addr, ret) \ +__asm__ __volatile__( \ + "/* Put user asm, inline. */\n" \ + "1:\t" "st"#size "a %1, [%2] %%asi\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\t" \ + ".previous\n\n\t" \ + : "=r" (ret) : "r" (x), "r" (__m(addr)), \ + "i" (-EFAULT)) + +int __put_user_bad(void); + +#define __get_kernel_nofault(dst, src, type, label) \ +do { \ + type *addr = (type __force *)(src); \ + register int __gu_ret; \ + register unsigned long __gu_val; \ + switch (sizeof(type)) { \ + case 1: __get_kernel_asm(__gu_val, ub, addr, __gu_ret); break; \ + case 2: __get_kernel_asm(__gu_val, uh, addr, __gu_ret); break; \ + case 4: __get_kernel_asm(__gu_val, uw, addr, __gu_ret); break; \ + case 8: __get_kernel_asm(__gu_val, x, addr, __gu_ret); break; \ + default: \ + __gu_val = 0; \ + __gu_ret = __get_user_bad(); \ + break; \ + } \ + if (__gu_ret) \ + goto label; \ + *(type *)dst = (__force type) __gu_val; \ +} while (0) +#define __get_kernel_asm(x, size, addr, ret) \ +__asm__ __volatile__( \ + "/* Get kernel asm, inline. */\n" \ + "1:\t" "ld"#size " [%2], %1\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "clr %1\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\n\t" \ + ".previous\n\t" \ + : "=r" (ret), "=r" (x) : "r" (__m(addr)), \ + "i" (-EFAULT)) + +#define __get_user_nocheck(data, addr, size, type) ({ \ + register int __gu_ret; \ + register unsigned long __gu_val; \ + switch (size) { \ + case 1: __get_user_asm(__gu_val, ub, addr, __gu_ret); break; \ + case 2: __get_user_asm(__gu_val, uh, addr, __gu_ret); break; \ + case 4: __get_user_asm(__gu_val, uw, addr, __gu_ret); break; \ + case 8: __get_user_asm(__gu_val, x, addr, __gu_ret); break; \ + default: \ + __gu_val = 0; \ + __gu_ret = __get_user_bad(); \ + break; \ + } \ + data = (__force type) __gu_val; \ + __gu_ret; \ +}) + +#define __get_user_asm(x, size, addr, ret) \ +__asm__ __volatile__( \ + "/* Get user asm, inline. */\n" \ + "1:\t" "ld"#size "a [%2] %%asi, %1\n\t" \ + "clr %0\n" \ + "2:\n\n\t" \ + ".section .fixup,#alloc,#execinstr\n\t" \ + ".align 4\n" \ + "3:\n\t" \ + "sethi %%hi(2b), %0\n\t" \ + "clr %1\n\t" \ + "jmpl %0 + %%lo(2b), %%g0\n\t" \ + " mov %3, %0\n\n\t" \ + ".previous\n\t" \ + ".section __ex_table,\"a\"\n\t" \ + ".align 4\n\t" \ + ".word 1b, 3b\n\n\t" \ + ".previous\n\t" \ + : "=r" (ret), "=r" (x) : "r" (__m(addr)), \ + "i" (-EFAULT)) + +int __get_user_bad(void); + +unsigned long __must_check raw_copy_from_user(void *to, + const void __user *from, + unsigned long size); + +unsigned long __must_check raw_copy_to_user(void __user *to, + const void *from, + unsigned long size); +#define INLINE_COPY_USER + +unsigned long __must_check raw_copy_in_user(void __user *to, + const void __user *from, + unsigned long size); + +unsigned long __must_check __clear_user(void __user *, unsigned long); + +#define clear_user __clear_user + +__must_check long strnlen_user(const char __user *str, long n); + +struct pt_regs; +unsigned long compute_effective_address(struct pt_regs *, + unsigned int insn, + unsigned int rd); + +#endif /* _ASM_UACCESS_H */ diff --git a/arch/sparc/include/asm/unistd.h b/arch/sparc/include/asm/unistd.h new file mode 100644 index 000000000..d6bc76706 --- /dev/null +++ b/arch/sparc/include/asm/unistd.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * System calls under the Sparc. + * + * Don't be scared by the ugly clobbers, it is the only way I can + * think of right now to force the arguments into fixed registers + * before the trap into the system call with gcc 'asm' statements. + * + * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) + * + * SunOS compatibility based upon preliminary work which is: + * + * Copyright (C) 1995 Adrian M. Rodriguez (adrian@remus.rutgers.edu) + */ +#ifndef _SPARC_UNISTD_H +#define _SPARC_UNISTD_H + +#include + +#define NR_syscalls __NR_syscalls + +#ifdef __32bit_syscall_numbers__ +#else +#define __NR_time 231 /* Linux sparc32 */ +#endif +#define __ARCH_WANT_NEW_STAT +#define __ARCH_WANT_OLD_READDIR +#define __ARCH_WANT_STAT64 +#define __ARCH_WANT_SYS_ALARM +#define __ARCH_WANT_SYS_GETHOSTNAME +#define __ARCH_WANT_SYS_PAUSE +#define __ARCH_WANT_SYS_SIGNAL +#define __ARCH_WANT_SYS_TIME32 +#define __ARCH_WANT_SYS_UTIME32 +#define __ARCH_WANT_SYS_WAITPID +#define __ARCH_WANT_SYS_SOCKETCALL +#define __ARCH_WANT_SYS_FADVISE64 +#define __ARCH_WANT_SYS_GETPGRP +#define __ARCH_WANT_SYS_NICE +#define __ARCH_WANT_SYS_OLDUMOUNT +#define __ARCH_WANT_SYS_SIGPENDING +#define __ARCH_WANT_SYS_SIGPROCMASK +#ifdef __32bit_syscall_numbers__ +#define __ARCH_WANT_SYS_IPC +#else +#define __ARCH_WANT_SYS_TIME +#define __ARCH_WANT_SYS_UTIME +#define __ARCH_WANT_COMPAT_SYS_SENDFILE +#define __ARCH_WANT_COMPAT_STAT +#endif + +#ifdef __32bit_syscall_numbers__ +/* Sparc 32-bit only has the "setresuid32", "getresuid32" variants, + * it never had the plain ones and there is no value to adding those + * old versions into the syscall table. + */ +#define __IGNORE_setresuid +#define __IGNORE_getresuid +#define __IGNORE_setresgid +#define __IGNORE_getresgid +#endif + +#endif /* _SPARC_UNISTD_H */ diff --git a/arch/sparc/include/asm/upa.h b/arch/sparc/include/asm/upa.h new file mode 100644 index 000000000..b1df3a7f4 --- /dev/null +++ b/arch/sparc/include/asm/upa.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_UPA_H +#define _SPARC64_UPA_H + +#include + +/* UPA level registers and defines. */ + +/* UPA Config Register */ +#define UPA_CONFIG_RESV 0xffffffffc0000000 /* Reserved. */ +#define UPA_CONFIG_PCON 0x000000003fc00000 /* Depth of various sys queues. */ +#define UPA_CONFIG_MID 0x00000000003e0000 /* Module ID. */ +#define UPA_CONFIG_PCAP 0x000000000001ffff /* Port Capabilities. */ + +/* UPA Port ID Register */ +#define UPA_PORTID_FNP 0xff00000000000000 /* Hardcoded to 0xfc on ultra. */ +#define UPA_PORTID_RESV 0x00fffff800000000 /* Reserved. */ +#define UPA_PORTID_ECCVALID 0x0000000400000000 /* Zero if mod can generate ECC */ +#define UPA_PORTID_ONEREAD 0x0000000200000000 /* Set if mod generates P_RASB */ +#define UPA_PORTID_PINTRDQ 0x0000000180000000 /* # outstanding P_INT_REQ's */ +#define UPA_PORTID_PREQDQ 0x000000007e000000 /* slave-wr's to mod supported */ +#define UPA_PORTID_PREQRD 0x0000000001e00000 /* # incoming P_REQ's supported */ +#define UPA_PORTID_UPACAP 0x00000000001f0000 /* UPA capabilities of mod */ +#define UPA_PORTID_ID 0x000000000000ffff /* Module Identification bits */ + +/* UPA I/O space accessors */ +#if defined(__KERNEL__) && !defined(__ASSEMBLER__) +static inline unsigned char _upa_readb(unsigned long addr) +{ + unsigned char ret; + + __asm__ __volatile__("lduba\t[%1] %2, %0\t/* upa_readb */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + + return ret; +} + +static inline unsigned short _upa_readw(unsigned long addr) +{ + unsigned short ret; + + __asm__ __volatile__("lduha\t[%1] %2, %0\t/* upa_readw */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + + return ret; +} + +static inline unsigned int _upa_readl(unsigned long addr) +{ + unsigned int ret; + + __asm__ __volatile__("lduwa\t[%1] %2, %0\t/* upa_readl */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + + return ret; +} + +static inline unsigned long _upa_readq(unsigned long addr) +{ + unsigned long ret; + + __asm__ __volatile__("ldxa\t[%1] %2, %0\t/* upa_readq */" + : "=r" (ret) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); + + return ret; +} + +static inline void _upa_writeb(unsigned char b, unsigned long addr) +{ + __asm__ __volatile__("stba\t%0, [%1] %2\t/* upa_writeb */" + : /* no outputs */ + : "r" (b), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); +} + +static inline void _upa_writew(unsigned short w, unsigned long addr) +{ + __asm__ __volatile__("stha\t%0, [%1] %2\t/* upa_writew */" + : /* no outputs */ + : "r" (w), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); +} + +static inline void _upa_writel(unsigned int l, unsigned long addr) +{ + __asm__ __volatile__("stwa\t%0, [%1] %2\t/* upa_writel */" + : /* no outputs */ + : "r" (l), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); +} + +static inline void _upa_writeq(unsigned long q, unsigned long addr) +{ + __asm__ __volatile__("stxa\t%0, [%1] %2\t/* upa_writeq */" + : /* no outputs */ + : "r" (q), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E)); +} + +#define upa_readb(__addr) (_upa_readb((unsigned long)(__addr))) +#define upa_readw(__addr) (_upa_readw((unsigned long)(__addr))) +#define upa_readl(__addr) (_upa_readl((unsigned long)(__addr))) +#define upa_readq(__addr) (_upa_readq((unsigned long)(__addr))) +#define upa_writeb(__b, __addr) (_upa_writeb((__b), (unsigned long)(__addr))) +#define upa_writew(__w, __addr) (_upa_writew((__w), (unsigned long)(__addr))) +#define upa_writel(__l, __addr) (_upa_writel((__l), (unsigned long)(__addr))) +#define upa_writeq(__q, __addr) (_upa_writeq((__q), (unsigned long)(__addr))) +#endif /* __KERNEL__ && !__ASSEMBLER__ */ + +#endif /* !(_SPARC64_UPA_H) */ diff --git a/arch/sparc/include/asm/uprobes.h b/arch/sparc/include/asm/uprobes.h new file mode 100644 index 000000000..79938fb94 --- /dev/null +++ b/arch/sparc/include/asm/uprobes.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _ASM_UPROBES_H +#define _ASM_UPROBES_H +/* + * User-space Probes (UProbes) for sparc + * + * Copyright (C) 2013 Oracle, Inc. + * + * Authors: + * Jose E. Marchesi + * Eric Saint Etienne + */ + +typedef u32 uprobe_opcode_t; + +#define MAX_UINSN_BYTES 4 +#define UPROBE_XOL_SLOT_BYTES (MAX_UINSN_BYTES * 2) + +#define UPROBE_SWBP_INSN_SIZE 4 +#define UPROBE_SWBP_INSN 0x91d02073 /* ta 0x73 */ +#define UPROBE_STP_INSN 0x91d02074 /* ta 0x74 */ + +#define ANNUL_BIT (1 << 29) + +struct arch_uprobe { + union { + u8 insn[MAX_UINSN_BYTES]; + u32 ixol; + }; +}; + +struct arch_uprobe_task { + u64 saved_tpc; + u64 saved_tnpc; +}; + +struct task_struct; +struct notifier_block; + +extern int arch_uprobe_analyze_insn(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long addr); +extern int arch_uprobe_pre_xol(struct arch_uprobe *aup, struct pt_regs *regs); +extern int arch_uprobe_post_xol(struct arch_uprobe *aup, struct pt_regs *regs); +extern bool arch_uprobe_xol_was_trapped(struct task_struct *tsk); +extern int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data); +extern void arch_uprobe_abort_xol(struct arch_uprobe *aup, struct pt_regs *regs); + +#endif /* _ASM_UPROBES_H */ diff --git a/arch/sparc/include/asm/user.h b/arch/sparc/include/asm/user.h new file mode 100644 index 000000000..3400ea87f --- /dev/null +++ b/arch/sparc/include/asm/user.h @@ -0,0 +1,6 @@ +#ifndef _SPARC_USER_H +#define _SPARC_USER_H + +/* Nothing to define. */ + +#endif /* !(_SPARC_USER_H) */ diff --git a/arch/sparc/include/asm/vaddrs.h b/arch/sparc/include/asm/vaddrs.h new file mode 100644 index 000000000..da567600c --- /dev/null +++ b/arch/sparc/include/asm/vaddrs.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_VADDRS_H +#define _SPARC_VADDRS_H + +#include + +/* + * asm/vaddrs.h: Here we define the virtual addresses at + * which important things will be mapped. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 2000 Anton Blanchard (anton@samba.org) + */ + +#define SRMMU_MAXMEM 0x0c000000 + +#define SRMMU_NOCACHE_VADDR (KERNBASE + SRMMU_MAXMEM) + /* = 0x0fc000000 */ +/* XXX Empiricals - this needs to go away - KMW */ +#define SRMMU_MIN_NOCACHE_PAGES (550) +#define SRMMU_MAX_NOCACHE_PAGES (1280) + +/* The following constant is used in mm/srmmu.c::srmmu_nocache_calcsize() + * to determine the amount of memory that will be reserved as nocache: + * + * 256 pages will be taken as nocache per each + * SRMMU_NOCACHE_ALCRATIO MB of system memory. + * + * limits enforced: nocache minimum = 256 pages + * nocache maximum = 1280 pages + */ +#define SRMMU_NOCACHE_ALCRATIO 64 /* 256 pages per 64MB of system RAM */ + +#ifndef __ASSEMBLER__ +#include + +enum fixed_addresses { + FIX_HOLE, +#ifdef CONFIG_HIGHMEM + FIX_KMAP_BEGIN, + FIX_KMAP_END = (KM_MAX_IDX * NR_CPUS), +#endif + __end_of_fixed_addresses +}; +#endif + +/* Leave one empty page between IO pages at 0xfd000000 and + * the top of the fixmap. + */ +#define FIXADDR_TOP (0xfcfff000UL) +#define FIXADDR_SIZE ((FIX_KMAP_END + 1) << PAGE_SHIFT) +#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) + +#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) + +#define SUN4M_IOBASE_VADDR 0xfd000000 /* Base for mapping pages */ +#define IOBASE_VADDR 0xfe000000 +#define IOBASE_END 0xfe600000 + +#define KADB_DEBUGGER_BEGVM 0xffc00000 /* Where kern debugger is in virt-mem */ +#define KADB_DEBUGGER_ENDVM 0xffd00000 +#define DEBUG_FIRSTVADDR KADB_DEBUGGER_BEGVM +#define DEBUG_LASTVADDR KADB_DEBUGGER_ENDVM + +#define LINUX_OPPROM_BEGVM 0xffd00000 +#define LINUX_OPPROM_ENDVM 0xfff00000 + +#define DVMA_VADDR 0xfff00000 /* Base area of the DVMA on suns */ +#define DVMA_END 0xfffc0000 + +#endif /* !(_SPARC_VADDRS_H) */ diff --git a/arch/sparc/include/asm/vdso.h b/arch/sparc/include/asm/vdso.h new file mode 100644 index 000000000..f08562d10 --- /dev/null +++ b/arch/sparc/include/asm/vdso.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#ifndef _ASM_SPARC_VDSO_H +#define _ASM_SPARC_VDSO_H + +struct vdso_image { + void *data; + unsigned long size; /* Always a multiple of PAGE_SIZE */ +}; + +#ifdef CONFIG_SPARC64 +extern const struct vdso_image vdso_image_64_builtin; +#endif +#ifdef CONFIG_COMPAT +extern const struct vdso_image vdso_image_32_builtin; +#endif + +#endif /* _ASM_SPARC_VDSO_H */ diff --git a/arch/sparc/include/asm/vdso/clocksource.h b/arch/sparc/include/asm/vdso/clocksource.h new file mode 100644 index 000000000..007aa8cea --- /dev/null +++ b/arch/sparc/include/asm/vdso/clocksource.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_CLOCKSOURCE_H +#define __ASM_VDSO_CLOCKSOURCE_H + +/* VDSO clocksources */ +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_TICK, \ + VDSO_CLOCKMODE_STICK + +#endif /* __ASM_VDSO_CLOCKSOURCE_H */ diff --git a/arch/sparc/include/asm/vdso/gettimeofday.h b/arch/sparc/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000..b0c80c8a2 --- /dev/null +++ b/arch/sparc/include/asm/vdso/gettimeofday.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2006 Andi Kleen, SUSE Labs. + */ + +#ifndef _ASM_SPARC_VDSO_GETTIMEOFDAY_H +#define _ASM_SPARC_VDSO_GETTIMEOFDAY_H + +#include +#include + +#include +#include +#include +#include + +#include + +#ifdef CONFIG_SPARC64 +static __always_inline u64 vread_tick(void) +{ + u64 ret; + + __asm__ __volatile__("rd %%tick, %0" : "=r" (ret)); + return ret; +} + +static __always_inline u64 vread_tick_stick(void) +{ + u64 ret; + + __asm__ __volatile__("rd %%asr24, %0" : "=r" (ret)); + return ret; +} +#else +static __always_inline u64 vdso_shift_ns(u64 val, u32 amt) +{ + u64 ret; + + __asm__ __volatile__("sllx %H1, 32, %%g1\n\t" + "srl %L1, 0, %L1\n\t" + "or %%g1, %L1, %%g1\n\t" + "srlx %%g1, %2, %L0\n\t" + "srlx %L0, 32, %H0" + : "=r" (ret) + : "r" (val), "r" (amt) + : "g1"); + return ret; +} +#define vdso_shift_ns vdso_shift_ns + +static __always_inline u64 vread_tick(void) +{ + register unsigned long long ret asm("o4"); + + __asm__ __volatile__("rd %%tick, %L0\n\t" + "srlx %L0, 32, %H0" + : "=r" (ret)); + return ret; +} + +static __always_inline u64 vread_tick_stick(void) +{ + register unsigned long long ret asm("o4"); + + __asm__ __volatile__("rd %%asr24, %L0\n\t" + "srlx %L0, 32, %H0" + : "=r" (ret)); + return ret; +} +#endif + +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, const struct vdso_time_data *vd) +{ + if (likely(clock_mode == VDSO_CLOCKMODE_STICK)) + return vread_tick_stick(); + else + return vread_tick(); +} + +#ifdef CONFIG_SPARC64 +#define SYSCALL_STRING \ + "ta 0x6d;" \ + "bcs,a 1f;" \ + " sub %%g0, %%o0, %%o0;" \ + "1:" +#else +#define SYSCALL_STRING \ + "ta 0x10;" \ + "bcs,a 1f;" \ + " sub %%g0, %%o0, %%o0;" \ + "1:" +#endif + +#define SYSCALL_CLOBBERS \ + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \ + "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \ + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \ + "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \ + "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \ + "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \ + "cc", "memory" + +#ifdef CONFIG_SPARC64 + +static __always_inline +long clock_gettime_fallback(clockid_t clock, struct __kernel_timespec *ts) +{ + register long num __asm__("g1") = __NR_clock_gettime; + register long o0 __asm__("o0") = clock; + register long o1 __asm__("o1") = (long) ts; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +#else /* !CONFIG_SPARC64 */ + +static __always_inline +long clock_gettime_fallback(clockid_t clock, struct __kernel_timespec *ts) +{ + register long num __asm__("g1") = __NR_clock_gettime64; + register long o0 __asm__("o0") = clock; + register long o1 __asm__("o1") = (long) ts; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +static __always_inline +long clock_gettime32_fallback(clockid_t clock, struct old_timespec32 *ts) +{ + register long num __asm__("g1") = __NR_clock_gettime; + register long o0 __asm__("o0") = clock; + register long o1 __asm__("o1") = (long) ts; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +#endif /* CONFIG_SPARC64 */ + +static __always_inline +long gettimeofday_fallback(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + register long num __asm__("g1") = __NR_gettimeofday; + register long o0 __asm__("o0") = (long) tv; + register long o1 __asm__("o1") = (long) tz; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void) +{ + unsigned long ret; + + /* + * SPARC does not support native PC-relative code relocations. + * Calculate the address manually, works for 32 and 64 bit code. + */ + __asm__ __volatile__( + "1:\n" + "call 3f\n" // Jump over the embedded data and set up %o7 + "nop\n" // Delay slot + "2:\n" + ".word vdso_u_time_data - .\n" // Embedded offset to external symbol + "3:\n" + "add %%o7, 2b - 1b, %%o7\n" // Point %o7 to the embedded offset + "ldsw [%%o7], %0\n" // Load the offset + "add %0, %%o7, %0\n" // Calculate the absolute address + : "=r" (ret) + : + : "o7"); + + return (const struct vdso_time_data *)ret; +} +#define __arch_get_vdso_u_time_data __arch_get_vdso_u_time_data + +#endif /* _ASM_SPARC_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/sparc/include/asm/vdso/processor.h b/arch/sparc/include/asm/vdso/processor.h new file mode 100644 index 000000000..f7a9adc80 --- /dev/null +++ b/arch/sparc/include/asm/vdso/processor.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SPARC_VDSO_PROCESSOR_H +#define _ASM_SPARC_VDSO_PROCESSOR_H + +#include + +#if defined(__arch64__) + +/* Please see the commentary in asm/backoff.h for a description of + * what these instructions are doing and how they have been chosen. + * To make a long story short, we are trying to yield the current cpu + * strand during busy loops. + */ +#ifdef BUILD_VDSO +#define cpu_relax() asm volatile("\n99:\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + ::: "memory") +#else /* ! BUILD_VDSO */ +#define cpu_relax() asm volatile("\n99:\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + ".section .pause_3insn_patch,\"ax\"\n\t"\ + ".word 99b\n\t" \ + "wr %%g0, 128, %%asr27\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + ".previous" \ + ::: "memory") +#endif /* BUILD_VDSO */ + +#else /* ! __arch64__ */ + +#define cpu_relax() barrier() + +#endif /* __arch64__ */ + +#endif /* _ASM_SPARC_VDSO_PROCESSOR_H */ diff --git a/arch/sparc/include/asm/vdso/vsyscall.h b/arch/sparc/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000..8bfe703fe --- /dev/null +++ b/arch/sparc/include/asm/vdso/vsyscall.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SPARC_VDSO_VSYSCALL_H +#define _ASM_SPARC_VDSO_VSYSCALL_H + +#define __VDSO_PAGES 4 + +#include + +#endif /* _ASM_SPARC_VDSO_VSYSCALL_H */ diff --git a/arch/sparc/include/asm/video.h b/arch/sparc/include/asm/video.h new file mode 100644 index 000000000..773717b6d --- /dev/null +++ b/arch/sparc/include/asm/video.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC_VIDEO_H_ +#define _SPARC_VIDEO_H_ + +#include +#include + +#include + +struct device; + +#ifdef CONFIG_SPARC32 +static inline pgprot_t pgprot_framebuffer(pgprot_t prot, + unsigned long vm_start, unsigned long vm_end, + unsigned long offset) +{ + return prot; +} +#define pgprot_framebuffer pgprot_framebuffer +#endif + +#ifdef CONFIG_VIDEO +bool video_is_primary_device(struct device *dev); +#define video_is_primary_device video_is_primary_device +#endif + +static inline void fb_memcpy_fromio(void *to, const volatile void __iomem *from, size_t n) +{ + sbus_memcpy_fromio(to, from, n); +} +#define fb_memcpy_fromio fb_memcpy_fromio + +static inline void fb_memcpy_toio(volatile void __iomem *to, const void *from, size_t n) +{ + sbus_memcpy_toio(to, from, n); +} +#define fb_memcpy_toio fb_memcpy_toio + +static inline void fb_memset_io(volatile void __iomem *addr, int c, size_t n) +{ + sbus_memset_io(addr, c, n); +} +#define fb_memset fb_memset_io + +#include + +#endif /* _SPARC_VIDEO_H_ */ diff --git a/arch/sparc/include/asm/viking.h b/arch/sparc/include/asm/viking.h new file mode 100644 index 000000000..bbb714de4 --- /dev/null +++ b/arch/sparc/include/asm/viking.h @@ -0,0 +1,255 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * viking.h: Defines specific to the GNU/Viking MBUS module. + * This is SRMMU stuff. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ +#ifndef _SPARC_VIKING_H +#define _SPARC_VIKING_H + +#include +#include +#include +#include + +/* Bits in the SRMMU control register for GNU/Viking modules. + * + * ----------------------------------------------------------- + * |impl-vers| RSV |TC|AC|SP|BM|PC|MBM|SB|IC|DC|PSO|RSV|NF|ME| + * ----------------------------------------------------------- + * 31 24 23-17 16 15 14 13 12 11 10 9 8 7 6-2 1 0 + * + * TC: Tablewalk Cacheable -- 0 = Twalks are not cacheable in E-cache + * 1 = Twalks are cacheable in E-cache + * + * GNU/Viking will only cache tablewalks in the E-cache (mxcc) if present + * and never caches them internally (or so states the docs). Therefore + * for machines lacking an E-cache (ie. in MBUS mode) this bit must + * remain cleared. + * + * AC: Alternate Cacheable -- 0 = Passthru physical accesses not cacheable + * 1 = Passthru physical accesses cacheable + * + * This indicates whether accesses are cacheable when no cachable bit + * is present in the pte when the processor is in boot-mode or the + * access does not need pte's for translation (ie. pass-thru ASI's). + * "Cachable" is only referring to E-cache (if present) and not the + * on chip split I/D caches of the GNU/Viking. + * + * SP: SnooP Enable -- 0 = bus snooping off, 1 = bus snooping on + * + * This enables snooping on the GNU/Viking bus. This must be on + * for the hardware cache consistency mechanisms of the GNU/Viking + * to work at all. On non-mxcc GNU/Viking modules the split I/D + * caches will snoop regardless of whether they are enabled, this + * takes care of the case where the I or D or both caches are turned + * off yet still contain valid data. Note also that this bit does + * not affect GNU/Viking store-buffer snoops, those happen if the + * store-buffer is enabled no matter what. + * + * BM: Boot Mode -- 0 = not in boot mode, 1 = in boot mode + * + * This indicates whether the GNU/Viking is in boot-mode or not, + * if it is then all instruction fetch physical addresses are + * computed as 0xff0000000 + low 28 bits of requested address. + * GNU/Viking boot-mode does not affect data accesses. Also, + * in boot mode instruction accesses bypass the split on chip I/D + * caches, they may be cached by the GNU/MXCC if present and enabled. + * + * MBM: MBus Mode -- 0 = not in MBus mode, 1 = in MBus mode + * + * This indicated the GNU/Viking configuration present. If in + * MBUS mode, the GNU/Viking lacks a GNU/MXCC E-cache. If it is + * not then the GNU/Viking is on a module VBUS connected directly + * to a GNU/MXCC cache controller. The GNU/MXCC can be thus connected + * to either an GNU/MBUS (sun4m) or the packet-switched GNU/XBus (sun4d). + * + * SB: StoreBuffer enable -- 0 = store buffer off, 1 = store buffer on + * + * The GNU/Viking store buffer allows the chip to continue execution + * after a store even if the data cannot be placed in one of the + * caches during that cycle. If disabled, all stores operations + * occur synchronously. + * + * IC: Instruction Cache -- 0 = off, 1 = on + * DC: Data Cache -- 0 = off, 1 = 0n + * + * These bits enable the on-cpu GNU/Viking split I/D caches. Note, + * as mentioned above, these caches will snoop the bus in GNU/MBUS + * configurations even when disabled to avoid data corruption. + * + * NF: No Fault -- 0 = faults generate traps, 1 = faults don't trap + * ME: MMU enable -- 0 = mmu not translating, 1 = mmu translating + * + */ + +#define VIKING_MMUENABLE 0x00000001 +#define VIKING_NOFAULT 0x00000002 +#define VIKING_PSO 0x00000080 +#define VIKING_DCENABLE 0x00000100 /* Enable data cache */ +#define VIKING_ICENABLE 0x00000200 /* Enable instruction cache */ +#define VIKING_SBENABLE 0x00000400 /* Enable store buffer */ +#define VIKING_MMODE 0x00000800 /* MBUS mode */ +#define VIKING_PCENABLE 0x00001000 /* Enable parity checking */ +#define VIKING_BMODE 0x00002000 +#define VIKING_SPENABLE 0x00004000 /* Enable bus cache snooping */ +#define VIKING_ACENABLE 0x00008000 /* Enable alternate caching */ +#define VIKING_TCENABLE 0x00010000 /* Enable table-walks to be cached */ +#define VIKING_DPENABLE 0x00040000 /* Enable the data prefetcher */ + +/* + * GNU/Viking Breakpoint Action Register fields. + */ +#define VIKING_ACTION_MIX 0x00001000 /* Enable multiple instructions */ + +/* + * GNU/Viking Cache Tags. + */ +#define VIKING_PTAG_VALID 0x01000000 /* Cache block is valid */ +#define VIKING_PTAG_DIRTY 0x00010000 /* Block has been modified */ +#define VIKING_PTAG_SHARED 0x00000100 /* Shared with some other cache */ + +#ifndef __ASSEMBLER__ + +static inline void viking_flush_icache(void) +{ + __asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" + : /* no outputs */ + : "i" (ASI_M_IC_FLCLEAR) + : "memory"); +} + +static inline void viking_flush_dcache(void) +{ + __asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" + : /* no outputs */ + : "i" (ASI_M_DC_FLCLEAR) + : "memory"); +} + +static inline void viking_unlock_icache(void) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (0x80000000), "i" (ASI_M_IC_FLCLEAR) + : "memory"); +} + +static inline void viking_unlock_dcache(void) +{ + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" + : /* no outputs */ + : "r" (0x80000000), "i" (ASI_M_DC_FLCLEAR) + : "memory"); +} + +static inline void viking_set_bpreg(unsigned long regval) +{ + __asm__ __volatile__("sta %0, [%%g0] %1\n\t" + : /* no outputs */ + : "r" (regval), "i" (ASI_M_ACTION) + : "memory"); +} + +static inline unsigned long viking_get_bpreg(void) +{ + unsigned long regval; + + __asm__ __volatile__("lda [%%g0] %1, %0\n\t" + : "=r" (regval) + : "i" (ASI_M_ACTION)); + return regval; +} + +static inline void viking_get_dcache_ptag(int set, int block, + unsigned long *data) +{ + unsigned long ptag = ((set & 0x7f) << 5) | ((block & 0x3) << 26) | + 0x80000000; + unsigned long info, page; + + __asm__ __volatile__ ("ldda [%2] %3, %%g2\n\t" + "or %%g0, %%g2, %0\n\t" + "or %%g0, %%g3, %1\n\t" + : "=r" (info), "=r" (page) + : "r" (ptag), "i" (ASI_M_DATAC_TAG) + : "g2", "g3"); + data[0] = info; + data[1] = page; +} + +static inline void viking_mxcc_turn_off_parity(unsigned long *mregp, + unsigned long *mxcc_cregp) +{ + unsigned long mreg = *mregp; + unsigned long mxcc_creg = *mxcc_cregp; + + mreg &= ~(VIKING_PCENABLE); + mxcc_creg &= ~(MXCC_CTL_PARE); + + __asm__ __volatile__ ("set 1f, %%g2\n\t" + "andcc %%g2, 4, %%g0\n\t" + "bne 2f\n\t" + " nop\n" + "1:\n\t" + "sta %0, [%%g0] %3\n\t" + "sta %1, [%2] %4\n\t" + "b 1f\n\t" + " nop\n\t" + "nop\n" + "2:\n\t" + "sta %0, [%%g0] %3\n\t" + "sta %1, [%2] %4\n" + "1:\n\t" + : /* no output */ + : "r" (mreg), "r" (mxcc_creg), + "r" (MXCC_CREG), "i" (ASI_M_MMUREGS), + "i" (ASI_M_MXCC) + : "g2", "memory", "cc"); + *mregp = mreg; + *mxcc_cregp = mxcc_creg; +} + +static inline unsigned long viking_hwprobe(unsigned long vaddr) +{ + unsigned long val; + + vaddr &= PAGE_MASK; + /* Probe all MMU entries. */ + __asm__ __volatile__("lda [%1] %2, %0\n\t" + : "=r" (val) + : "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE)); + if (!val) + return 0; + + /* Probe region. */ + __asm__ __volatile__("lda [%1] %2, %0\n\t" + : "=r" (val) + : "r" (vaddr | 0x200), "i" (ASI_M_FLUSH_PROBE)); + if ((val & SRMMU_ET_MASK) == SRMMU_ET_PTE) { + vaddr &= ~PGDIR_MASK; + vaddr >>= PAGE_SHIFT; + return val | (vaddr << 8); + } + + /* Probe segment. */ + __asm__ __volatile__("lda [%1] %2, %0\n\t" + : "=r" (val) + : "r" (vaddr | 0x100), "i" (ASI_M_FLUSH_PROBE)); + if ((val & SRMMU_ET_MASK) == SRMMU_ET_PTE) { + vaddr &= ~PMD_MASK; + vaddr >>= PAGE_SHIFT; + return val | (vaddr << 8); + } + + /* Probe page. */ + __asm__ __volatile__("lda [%1] %2, %0\n\t" + : "=r" (val) + : "r" (vaddr), "i" (ASI_M_FLUSH_PROBE)); + return val; +} + +#endif /* !__ASSEMBLER__ */ + +#endif /* !(_SPARC_VIKING_H) */ diff --git a/arch/sparc/include/asm/vio.h b/arch/sparc/include/asm/vio.h new file mode 100644 index 000000000..0ca8c3463 --- /dev/null +++ b/arch/sparc/include/asm/vio.h @@ -0,0 +1,508 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_VIO_H +#define _SPARC64_VIO_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct vio_msg_tag { + u8 type; +#define VIO_TYPE_CTRL 0x01 +#define VIO_TYPE_DATA 0x02 +#define VIO_TYPE_ERR 0x04 + + u8 stype; +#define VIO_SUBTYPE_INFO 0x01 +#define VIO_SUBTYPE_ACK 0x02 +#define VIO_SUBTYPE_NACK 0x04 + + u16 stype_env; +#define VIO_VER_INFO 0x0001 +#define VIO_ATTR_INFO 0x0002 +#define VIO_DRING_REG 0x0003 +#define VIO_DRING_UNREG 0x0004 +#define VIO_RDX 0x0005 +#define VIO_PKT_DATA 0x0040 +#define VIO_DESC_DATA 0x0041 +#define VIO_DRING_DATA 0x0042 +#define VNET_MCAST_INFO 0x0101 + + u32 sid; +}; + +struct vio_rdx { + struct vio_msg_tag tag; + u64 resv[6]; +}; + +struct vio_ver_info { + struct vio_msg_tag tag; + u16 major; + u16 minor; + u8 dev_class; +#define VDEV_NETWORK 0x01 +#define VDEV_NETWORK_SWITCH 0x02 +#define VDEV_DISK 0x03 +#define VDEV_DISK_SERVER 0x04 +#define VDEV_CONSOLE_CON 0x05 + + u8 resv1[3]; + u64 resv2[5]; +}; + +struct vio_dring_register { + struct vio_msg_tag tag; + u64 dring_ident; + u32 num_descr; + u32 descr_size; + u16 options; +#define VIO_TX_DRING 0x0001 +#define VIO_RX_DRING 0x0002 +#define VIO_RX_DRING_DATA 0x0004 + u16 resv; + u32 num_cookies; + struct ldc_trans_cookie cookies[]; +}; + +struct vio_dring_unregister { + struct vio_msg_tag tag; + u64 dring_ident; + u64 resv[5]; +}; + +/* Data transfer modes */ +#define VIO_PKT_MODE 0x01 /* Packet based transfer */ +#define VIO_DESC_MODE 0x02 /* In-band descriptors */ +#define VIO_DRING_MODE 0x03 /* Descriptor rings */ +/* in vers >= 1.2, VIO_DRING_MODE is 0x04 and transfer mode is a bitmask */ +#define VIO_NEW_DRING_MODE 0x04 + +struct vio_dring_data { + struct vio_msg_tag tag; + u64 seq; + u64 dring_ident; + u32 start_idx; + u32 end_idx; + u8 state; +#define VIO_DRING_ACTIVE 0x01 +#define VIO_DRING_STOPPED 0x02 + + u8 __pad1; + u16 __pad2; + u32 __pad3; + u64 __par4[2]; +}; + +struct vio_dring_hdr { + u8 state; +#define VIO_DESC_FREE 0x01 +#define VIO_DESC_READY 0x02 +#define VIO_DESC_ACCEPTED 0x03 +#define VIO_DESC_DONE 0x04 + u8 ack; +#define VIO_ACK_ENABLE 0x01 +#define VIO_ACK_DISABLE 0x00 + + u16 __pad1; + u32 __pad2; +}; + +/* VIO disk specific structures and defines */ +struct vio_disk_attr_info { + struct vio_msg_tag tag; + u8 xfer_mode; + u8 vdisk_type; +#define VD_DISK_TYPE_SLICE 0x01 /* Slice in block device */ +#define VD_DISK_TYPE_DISK 0x02 /* Entire block device */ + u8 vdisk_mtype; /* v1.1 */ +#define VD_MEDIA_TYPE_FIXED 0x01 /* Fixed device */ +#define VD_MEDIA_TYPE_CD 0x02 /* CD Device */ +#define VD_MEDIA_TYPE_DVD 0x03 /* DVD Device */ + u8 resv1; + u32 vdisk_block_size; + u64 operations; + u64 vdisk_size; /* v1.1 */ + u64 max_xfer_size; + u32 phys_block_size; /* v1.2 */ + u32 resv2; + u64 resv3[1]; +}; + +struct vio_disk_desc { + struct vio_dring_hdr hdr; + u64 req_id; + u8 operation; +#define VD_OP_BREAD 0x01 /* Block read */ +#define VD_OP_BWRITE 0x02 /* Block write */ +#define VD_OP_FLUSH 0x03 /* Flush disk contents */ +#define VD_OP_GET_WCE 0x04 /* Get write-cache status */ +#define VD_OP_SET_WCE 0x05 /* Enable/disable write-cache */ +#define VD_OP_GET_VTOC 0x06 /* Get VTOC */ +#define VD_OP_SET_VTOC 0x07 /* Set VTOC */ +#define VD_OP_GET_DISKGEOM 0x08 /* Get disk geometry */ +#define VD_OP_SET_DISKGEOM 0x09 /* Set disk geometry */ +#define VD_OP_SCSICMD 0x0a /* SCSI control command */ +#define VD_OP_GET_DEVID 0x0b /* Get device ID */ +#define VD_OP_GET_EFI 0x0c /* Get EFI */ +#define VD_OP_SET_EFI 0x0d /* Set EFI */ + u8 slice; + u16 resv1; + u32 status; + u64 offset; + u64 size; + u32 ncookies; + u32 resv2; + struct ldc_trans_cookie cookies[]; +}; + +#define VIO_DISK_VNAME_LEN 8 +#define VIO_DISK_ALABEL_LEN 128 +#define VIO_DISK_NUM_PART 8 + +struct vio_disk_vtoc { + u8 volume_name[VIO_DISK_VNAME_LEN]; + u16 sector_size; + u16 num_partitions; + u8 ascii_label[VIO_DISK_ALABEL_LEN]; + struct { + u16 id; + u16 perm_flags; + u32 resv; + u64 start_block; + u64 num_blocks; + } partitions[VIO_DISK_NUM_PART]; +}; + +struct vio_disk_geom { + u16 num_cyl; /* Num data cylinders */ + u16 alt_cyl; /* Num alternate cylinders */ + u16 beg_cyl; /* Cyl off of fixed head area */ + u16 num_hd; /* Num heads */ + u16 num_sec; /* Num sectors */ + u16 ifact; /* Interleave factor */ + u16 apc; /* Alts per cylinder (SCSI) */ + u16 rpm; /* Revolutions per minute */ + u16 phy_cyl; /* Num physical cylinders */ + u16 wr_skip; /* Num sects to skip, writes */ + u16 rd_skip; /* Num sects to skip, writes */ +}; + +struct vio_disk_devid { + u16 resv; + u16 type; + u32 len; + char id[]; +}; + +struct vio_disk_efi { + u64 lba; + u64 len; + char data[]; +}; + +/* VIO net specific structures and defines */ +struct vio_net_attr_info { + struct vio_msg_tag tag; + u8 xfer_mode; + u8 addr_type; +#define VNET_ADDR_ETHERMAC 0x01 + u16 ack_freq; + u8 plnk_updt; +#define PHYSLINK_UPDATE_NONE 0x00 +#define PHYSLINK_UPDATE_STATE 0x01 +#define PHYSLINK_UPDATE_STATE_ACK 0x02 +#define PHYSLINK_UPDATE_STATE_NACK 0x03 + u8 options; + u16 resv1; + u64 addr; + u64 mtu; + u16 cflags; +#define VNET_LSO_IPV4_CAPAB 0x0001 + u16 ipv4_lso_maxlen; + u32 resv2; + u64 resv3[2]; +}; + +#define VNET_NUM_MCAST 7 + +struct vio_net_mcast_info { + struct vio_msg_tag tag; + u8 set; + u8 count; + u8 mcast_addr[VNET_NUM_MCAST * 6]; + u32 resv; +}; + +struct vio_net_desc { + struct vio_dring_hdr hdr; + u32 size; + u32 ncookies; + struct ldc_trans_cookie cookies[]; +}; + +struct vio_net_dext { + u8 flags; +#define VNET_PKT_HASH 0x01 +#define VNET_PKT_HCK_IPV4_HDRCKSUM 0x02 +#define VNET_PKT_HCK_FULLCKSUM 0x04 +#define VNET_PKT_IPV4_LSO 0x08 +#define VNET_PKT_HCK_IPV4_HDRCKSUM_OK 0x10 +#define VNET_PKT_HCK_FULLCKSUM_OK 0x20 + + u8 vnet_hashval; + u16 ipv4_lso_mss; + u32 resv3; +}; + +static inline struct vio_net_dext *vio_net_ext(struct vio_net_desc *desc) +{ + return (struct vio_net_dext *)&desc->cookies[2]; +} + +#define VIO_MAX_RING_COOKIES 24 + +struct vio_dring_state { + u64 ident; + void *base; + u64 snd_nxt; + u64 rcv_nxt; + u32 entry_size; + u32 num_entries; + u32 prod; + u32 cons; + u32 pending; + int ncookies; + struct ldc_trans_cookie cookies[VIO_MAX_RING_COOKIES]; +}; + +#define VIO_TAG_SIZE (sizeof(struct vio_msg_tag)) +#define VIO_VCC_MTU_SIZE (LDC_PACKET_SIZE - VIO_TAG_SIZE) + +struct vio_vcc { + struct vio_msg_tag tag; + char data[VIO_VCC_MTU_SIZE]; +}; + +static inline void *vio_dring_cur(struct vio_dring_state *dr) +{ + return dr->base + (dr->entry_size * dr->prod); +} + +static inline void *vio_dring_entry(struct vio_dring_state *dr, + unsigned int index) +{ + return dr->base + (dr->entry_size * index); +} + +static inline u32 vio_dring_avail(struct vio_dring_state *dr, + unsigned int ring_size) +{ + return (dr->pending - + ((dr->prod - dr->cons) & (ring_size - 1)) - 1); +} + +static inline u32 vio_dring_next(struct vio_dring_state *dr, u32 index) +{ + if (++index == dr->num_entries) + index = 0; + return index; +} + +static inline u32 vio_dring_prev(struct vio_dring_state *dr, u32 index) +{ + if (index == 0) + return dr->num_entries - 1; + else + return index - 1; +} + +#define VIO_MAX_TYPE_LEN 32 +#define VIO_MAX_NAME_LEN 32 +#define VIO_MAX_COMPAT_LEN 64 + +struct vio_dev { + u64 mp; + struct device_node *dp; + + char node_name[VIO_MAX_NAME_LEN]; + char type[VIO_MAX_TYPE_LEN]; + char compat[VIO_MAX_COMPAT_LEN]; + int compat_len; + + u64 dev_no; + + unsigned long port_id; + unsigned long channel_id; + + unsigned int tx_irq; + unsigned int rx_irq; + u64 rx_ino; + u64 tx_ino; + + /* Handle to the root of "channel-devices" sub-tree in MDESC */ + u64 cdev_handle; + + /* MD specific data used to identify the vdev in MD */ + union md_node_info md_node_info; + + struct device dev; +}; + +struct vio_driver { + const char *name; + struct list_head node; + const struct vio_device_id *id_table; + int (*probe)(struct vio_dev *dev, const struct vio_device_id *id); + void (*remove)(struct vio_dev *dev); + void (*shutdown)(struct vio_dev *dev); + unsigned long driver_data; + struct device_driver driver; + bool no_irq; +}; + +struct vio_version { + u16 major; + u16 minor; +}; + +struct vio_driver_state; +struct vio_driver_ops { + int (*send_attr)(struct vio_driver_state *vio); + int (*handle_attr)(struct vio_driver_state *vio, void *pkt); + void (*handshake_complete)(struct vio_driver_state *vio); +}; + +struct vio_completion { + struct completion com; + int err; + int waiting_for; +}; + +struct vio_driver_state { + /* Protects VIO handshake and, optionally, driver private state. */ + spinlock_t lock; + + struct ldc_channel *lp; + + u32 _peer_sid; + u32 _local_sid; + struct vio_dring_state drings[2]; +#define VIO_DRIVER_TX_RING 0 +#define VIO_DRIVER_RX_RING 1 + + u8 hs_state; +#define VIO_HS_INVALID 0x00 +#define VIO_HS_GOTVERS 0x01 +#define VIO_HS_GOT_ATTR 0x04 +#define VIO_HS_SENT_DREG 0x08 +#define VIO_HS_SENT_RDX 0x10 +#define VIO_HS_GOT_RDX_ACK 0x20 +#define VIO_HS_GOT_RDX 0x40 +#define VIO_HS_SENT_RDX_ACK 0x80 +#define VIO_HS_COMPLETE (VIO_HS_GOT_RDX_ACK | VIO_HS_SENT_RDX_ACK) + + u8 dev_class; + + u8 dr_state; +#define VIO_DR_STATE_TXREG 0x01 +#define VIO_DR_STATE_RXREG 0x02 +#define VIO_DR_STATE_TXREQ 0x10 +#define VIO_DR_STATE_RXREQ 0x20 + + u8 debug; +#define VIO_DEBUG_HS 0x01 +#define VIO_DEBUG_DATA 0x02 + + void *desc_buf; + unsigned int desc_buf_len; + + struct vio_completion *cmp; + + struct vio_dev *vdev; + + struct timer_list timer; + + struct vio_version ver; + + struct vio_version *ver_table; + int ver_table_entries; + + char *name; + + struct vio_driver_ops *ops; +}; + +static inline bool vio_version_before(struct vio_driver_state *vio, + u16 major, u16 minor) +{ + u32 have = (u32)vio->ver.major << 16 | vio->ver.minor; + u32 want = (u32)major << 16 | minor; + + return have < want; +} + +static inline bool vio_version_after(struct vio_driver_state *vio, + u16 major, u16 minor) +{ + u32 have = (u32)vio->ver.major << 16 | vio->ver.minor; + u32 want = (u32)major << 16 | minor; + + return have > want; +} + +static inline bool vio_version_after_eq(struct vio_driver_state *vio, + u16 major, u16 minor) +{ + u32 have = (u32)vio->ver.major << 16 | vio->ver.minor; + u32 want = (u32)major << 16 | minor; + + return have >= want; +} + +#define viodbg(TYPE, f, a...) \ +do { if (vio->debug & VIO_DEBUG_##TYPE) \ + printk(KERN_INFO "vio: ID[%lu] " f, \ + vio->vdev->channel_id, ## a); \ +} while (0) + +int __vio_register_driver(struct vio_driver *drv, struct module *owner, + const char *mod_name); +/* + * vio_register_driver must be a macro so that KBUILD_MODNAME can be expanded + */ +#define vio_register_driver(driver) \ + __vio_register_driver(driver, THIS_MODULE, KBUILD_MODNAME) +void vio_unregister_driver(struct vio_driver *drv); + +#define to_vio_driver(__drv) container_of_const(__drv, struct vio_driver, driver) +#define to_vio_dev(__dev) container_of_const(__dev, struct vio_dev, dev) + +int vio_ldc_send(struct vio_driver_state *vio, void *data, int len); +void vio_link_state_change(struct vio_driver_state *vio, int event); +void vio_conn_reset(struct vio_driver_state *vio); +int vio_control_pkt_engine(struct vio_driver_state *vio, void *pkt); +int vio_validate_sid(struct vio_driver_state *vio, + struct vio_msg_tag *tp); +u32 vio_send_sid(struct vio_driver_state *vio); +int vio_ldc_alloc(struct vio_driver_state *vio, + struct ldc_channel_config *base_cfg, void *event_arg); +void vio_ldc_free(struct vio_driver_state *vio); +int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev, + u8 dev_class, struct vio_version *ver_table, + int ver_table_size, struct vio_driver_ops *ops, + char *name); + +void vio_port_up(struct vio_driver_state *vio); +int vio_set_intr(unsigned long dev_ino, int state); +u64 vio_vdev_node(struct mdesc_handle *hp, struct vio_dev *vdev); + +#endif /* _SPARC64_VIO_H */ diff --git a/arch/sparc/include/asm/visasm.h b/arch/sparc/include/asm/visasm.h new file mode 100644 index 000000000..71eb4e9af --- /dev/null +++ b/arch/sparc/include/asm/visasm.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_VISASM_H +#define _SPARC64_VISASM_H + +/* visasm.h: FPU saving macros for VIS routines + * + * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#include +#include + +/* Clobbers %o5, %g1, %g2, %g3, %g7, %icc, %xcc */ + +#define VISEntry \ + rd %fprs, %o5; \ + andcc %o5, (FPRS_FEF|FPRS_DU), %g0; \ + be,pt %icc, 297f; \ + sethi %hi(297f), %g7; \ + sethi %hi(VISenter), %g1; \ + jmpl %g1 + %lo(VISenter), %g0; \ + or %g7, %lo(297f), %g7; \ +297: wr %g0, FPRS_FEF, %fprs; \ + +#define VISExit \ + wr %g0, 0, %fprs; + +/* Clobbers %o5, %g1, %g2, %g3, %g7, %icc, %xcc. + * Must preserve %o5 between VISEntryHalf and VISExitHalf */ + +#define VISEntryHalf \ + VISEntry + +#define VISExitHalf \ + VISExit + +#define VISEntryHalfFast(fail_label) \ + rd %fprs, %o5; \ + andcc %o5, FPRS_FEF, %g0; \ + be,pt %icc, 297f; \ + nop; \ + ba,a,pt %xcc, fail_label; \ +297: wr %o5, FPRS_FEF, %fprs; + +#define VISExitHalfFast \ + wr %o5, 0, %fprs; + +#ifndef __ASSEMBLER__ +static inline void save_and_clear_fpu(void) { + __asm__ __volatile__ ( +" rd %%fprs, %%o5\n" +" andcc %%o5, %0, %%g0\n" +" be,pt %%icc, 299f\n" +" sethi %%hi(298f), %%g7\n" +" sethi %%hi(VISenter), %%g1\n" +" jmpl %%g1 + %%lo(VISenter), %%g0\n" +" or %%g7, %%lo(298f), %%g7\n" +" 298: wr %%g0, 0, %%fprs\n" +" 299:\n" +" " : : "i" (FPRS_FEF|FPRS_DU) : + "o5", "g1", "g2", "g3", "g7", "cc"); +} + +int vis_emul(struct pt_regs *, unsigned int); +#endif + +#endif /* _SPARC64_ASI_H */ diff --git a/arch/sparc/include/asm/vmalloc.h b/arch/sparc/include/asm/vmalloc.h new file mode 100644 index 000000000..04b8ab951 --- /dev/null +++ b/arch/sparc/include/asm/vmalloc.h @@ -0,0 +1,4 @@ +#ifndef _ASM_SPARC_VMALLOC_H +#define _ASM_SPARC_VMALLOC_H + +#endif /* _ASM_SPARC_VMALLOC_H */ diff --git a/arch/sparc/include/asm/winmacro.h b/arch/sparc/include/asm/winmacro.h new file mode 100644 index 000000000..b6e911f5d --- /dev/null +++ b/arch/sparc/include/asm/winmacro.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * winmacro.h: Window loading-unloading macros. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_WINMACRO_H +#define _SPARC_WINMACRO_H + +#include + +/* Store the register window onto the 8-byte aligned area starting + * at %reg. It might be %sp, it might not, we don't care. + */ +#define STORE_WINDOW(reg) \ + std %l0, [%reg + RW_L0]; \ + std %l2, [%reg + RW_L2]; \ + std %l4, [%reg + RW_L4]; \ + std %l6, [%reg + RW_L6]; \ + std %i0, [%reg + RW_I0]; \ + std %i2, [%reg + RW_I2]; \ + std %i4, [%reg + RW_I4]; \ + std %i6, [%reg + RW_I6]; + +/* Load a register window from the area beginning at %reg. */ +#define LOAD_WINDOW(reg) \ + ldd [%reg + RW_L0], %l0; \ + ldd [%reg + RW_L2], %l2; \ + ldd [%reg + RW_L4], %l4; \ + ldd [%reg + RW_L6], %l6; \ + ldd [%reg + RW_I0], %i0; \ + ldd [%reg + RW_I2], %i2; \ + ldd [%reg + RW_I4], %i4; \ + ldd [%reg + RW_I6], %i6; + +/* Loading and storing struct pt_reg trap frames. */ +#define LOAD_PT_INS(base_reg) \ + ldd [%base_reg + STACKFRAME_SZ + PT_I0], %i0; \ + ldd [%base_reg + STACKFRAME_SZ + PT_I2], %i2; \ + ldd [%base_reg + STACKFRAME_SZ + PT_I4], %i4; \ + ldd [%base_reg + STACKFRAME_SZ + PT_I6], %i6; + +#define LOAD_PT_GLOBALS(base_reg) \ + ld [%base_reg + STACKFRAME_SZ + PT_G1], %g1; \ + ldd [%base_reg + STACKFRAME_SZ + PT_G2], %g2; \ + ldd [%base_reg + STACKFRAME_SZ + PT_G4], %g4; \ + ldd [%base_reg + STACKFRAME_SZ + PT_G6], %g6; + +#define LOAD_PT_YREG(base_reg, scratch) \ + ld [%base_reg + STACKFRAME_SZ + PT_Y], %scratch; \ + wr %scratch, 0x0, %y; + +#define LOAD_PT_PRIV(base_reg, pt_psr, pt_pc, pt_npc) \ + ld [%base_reg + STACKFRAME_SZ + PT_PSR], %pt_psr; \ + ld [%base_reg + STACKFRAME_SZ + PT_PC], %pt_pc; \ + ld [%base_reg + STACKFRAME_SZ + PT_NPC], %pt_npc; + +#define LOAD_PT_ALL(base_reg, pt_psr, pt_pc, pt_npc, scratch) \ + LOAD_PT_YREG(base_reg, scratch) \ + LOAD_PT_INS(base_reg) \ + LOAD_PT_GLOBALS(base_reg) \ + LOAD_PT_PRIV(base_reg, pt_psr, pt_pc, pt_npc) + +#define STORE_PT_INS(base_reg) \ + std %i0, [%base_reg + STACKFRAME_SZ + PT_I0]; \ + std %i2, [%base_reg + STACKFRAME_SZ + PT_I2]; \ + std %i4, [%base_reg + STACKFRAME_SZ + PT_I4]; \ + std %i6, [%base_reg + STACKFRAME_SZ + PT_I6]; + +#define STORE_PT_GLOBALS(base_reg) \ + st %g1, [%base_reg + STACKFRAME_SZ + PT_G1]; \ + std %g2, [%base_reg + STACKFRAME_SZ + PT_G2]; \ + std %g4, [%base_reg + STACKFRAME_SZ + PT_G4]; \ + std %g6, [%base_reg + STACKFRAME_SZ + PT_G6]; + +#define STORE_PT_YREG(base_reg, scratch) \ + rd %y, %scratch; \ + st %scratch, [%base_reg + STACKFRAME_SZ + PT_Y]; + +#define STORE_PT_PRIV(base_reg, pt_psr, pt_pc, pt_npc) \ + st %pt_psr, [%base_reg + STACKFRAME_SZ + PT_PSR]; \ + st %pt_pc, [%base_reg + STACKFRAME_SZ + PT_PC]; \ + st %pt_npc, [%base_reg + STACKFRAME_SZ + PT_NPC]; + +#define STORE_PT_ALL(base_reg, reg_psr, reg_pc, reg_npc, g_scratch) \ + STORE_PT_PRIV(base_reg, reg_psr, reg_pc, reg_npc) \ + STORE_PT_GLOBALS(base_reg) \ + STORE_PT_YREG(base_reg, g_scratch) \ + STORE_PT_INS(base_reg) + +#define SAVE_BOLIXED_USER_STACK(cur_reg, scratch) \ + ld [%cur_reg + TI_W_SAVED], %scratch; \ + sll %scratch, 2, %scratch; \ + add %scratch, %cur_reg, %scratch; \ + st %sp, [%scratch + TI_RWIN_SPTRS]; \ + sub %scratch, %cur_reg, %scratch; \ + sll %scratch, 4, %scratch; \ + add %scratch, %cur_reg, %scratch; \ + STORE_WINDOW(scratch + TI_REG_WINDOW); \ + sub %scratch, %cur_reg, %scratch; \ + srl %scratch, 6, %scratch; \ + add %scratch, 1, %scratch; \ + st %scratch, [%cur_reg + TI_W_SAVED]; + +#ifdef CONFIG_SMP +#define LOAD_CURRENT(dest_reg, idreg) \ +661: rd %tbr, %idreg; \ + srl %idreg, 10, %idreg; \ + and %idreg, 0xc, %idreg; \ + .section .cpuid_patch, "ax"; \ + /* Instruction location. */ \ + .word 661b; \ + /* SUN4D implementation. */ \ + lda [%g0] ASI_M_VIKING_TMP1, %idreg; \ + sll %idreg, 2, %idreg; \ + nop; \ + /* LEON implementation. */ \ + rd %asr17, %idreg; \ + srl %idreg, 0x1c, %idreg; \ + sll %idreg, 0x02, %idreg; \ + .previous; \ + sethi %hi(current_set), %dest_reg; \ + or %dest_reg, %lo(current_set), %dest_reg;\ + ld [%idreg + %dest_reg], %dest_reg; +#else +#define LOAD_CURRENT(dest_reg, idreg) \ + sethi %hi(current_set), %idreg; \ + ld [%idreg + %lo(current_set)], %dest_reg; +#endif + +#endif /* !(_SPARC_WINMACRO_H) */ diff --git a/arch/sparc/include/uapi/asm/Kbuild b/arch/sparc/include/uapi/asm/Kbuild new file mode 100644 index 000000000..353b70b19 --- /dev/null +++ b/arch/sparc/include/uapi/asm/Kbuild @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 +generated-y += unistd_32.h +generated-y += unistd_64.h diff --git a/arch/sparc/include/uapi/asm/apc.h b/arch/sparc/include/uapi/asm/apc.h new file mode 100644 index 000000000..aeb369b87 --- /dev/null +++ b/arch/sparc/include/uapi/asm/apc.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* apc - Driver definitions for power management functions + * of Aurora Personality Chip (APC) on SPARCstation-4/5 and + * derivatives + * + * Copyright (c) 2001 Eric Brower (ebrower@usa.net) + * + */ + +#ifndef _SPARC_APC_H +#define _SPARC_APC_H + +#include + +#define APC_IOC 'A' + +#define APCIOCGFANCTL _IOR(APC_IOC, 0x00, int) /* Get fan speed */ +#define APCIOCSFANCTL _IOW(APC_IOC, 0x01, int) /* Set fan speed */ + +#define APCIOCGCPWR _IOR(APC_IOC, 0x02, int) /* Get CPOWER state */ +#define APCIOCSCPWR _IOW(APC_IOC, 0x03, int) /* Set CPOWER state */ + +#define APCIOCGBPORT _IOR(APC_IOC, 0x04, int) /* Get BPORT state */ +#define APCIOCSBPORT _IOW(APC_IOC, 0x05, int) /* Set BPORT state */ + +/* + * Register offsets + */ +#define APC_IDLE_REG 0x00 +#define APC_FANCTL_REG 0x20 +#define APC_CPOWER_REG 0x24 +#define APC_BPORT_REG 0x30 + +#define APC_REGMASK 0x01 +#define APC_BPMASK 0x03 + +/* + * IDLE - CPU standby values (set to initiate standby) + */ +#define APC_IDLE_ON 0x01 + +/* + * FANCTL - Fan speed control state values + */ +#define APC_FANCTL_HI 0x00 /* Fan speed high */ +#define APC_FANCTL_LO 0x01 /* Fan speed low */ + +/* + * CPWR - Convenience power outlet state values + */ +#define APC_CPOWER_ON 0x00 /* Conv power on */ +#define APC_CPOWER_OFF 0x01 /* Conv power off */ + +/* + * BPA/BPB - Read-Write "Bit Ports" state values (reset to 0 at power-on) + * + * WARNING: Internal usage of bit ports is platform dependent-- + * don't modify BPORT settings unless you know what you are doing. + * + * On SS5 BPA seems to toggle onboard ethernet loopback... -E + */ +#define APC_BPORT_A 0x01 /* Bit Port A */ +#define APC_BPORT_B 0x02 /* Bit Port B */ + +#endif /* !(_SPARC_APC_H) */ diff --git a/arch/sparc/include/uapi/asm/asi.h b/arch/sparc/include/uapi/asm/asi.h new file mode 100644 index 000000000..fbb30a5b0 --- /dev/null +++ b/arch/sparc/include/uapi/asm/asi.h @@ -0,0 +1,303 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_ASI_H +#define _SPARC_ASI_H + +/* asi.h: Address Space Identifier values for the sparc. + * + * Copyright (C) 1995,1996 David S. Miller (davem@caip.rutgers.edu) + * + * Pioneer work for sun4m: Paul Hatchman (paul@sfe.com.au) + * Joint edition for sun4c+sun4m: Pete A. Zaitcev + */ + +/* The first batch are for the sun4c. */ + +#define ASI_NULL1 0x00 +#define ASI_NULL2 0x01 + +/* sun4c and sun4 control registers and mmu/vac ops */ +#define ASI_CONTROL 0x02 +#define ASI_SEGMAP 0x03 +#define ASI_PTE 0x04 +#define ASI_HWFLUSHSEG 0x05 +#define ASI_HWFLUSHPAGE 0x06 +#define ASI_REGMAP 0x06 +#define ASI_HWFLUSHCONTEXT 0x07 + +#define ASI_USERTXT 0x08 +#define ASI_KERNELTXT 0x09 +#define ASI_USERDATA 0x0a +#define ASI_KERNELDATA 0x0b + +/* VAC Cache flushing on sun4c and sun4 */ +#define ASI_FLUSHSEG 0x0c +#define ASI_FLUSHPG 0x0d +#define ASI_FLUSHCTX 0x0e + +/* SPARCstation-5: only 6 bits are decoded. */ +/* wo = Write Only, rw = Read Write; */ +/* ss = Single Size, as = All Sizes; */ +#define ASI_M_RES00 0x00 /* Don't touch... */ +#define ASI_M_UNA01 0x01 /* Same here... */ +#define ASI_M_MXCC 0x02 /* Access to TI VIKING MXCC registers */ +#define ASI_M_FLUSH_PROBE 0x03 /* Reference MMU Flush/Probe; rw, ss */ +#define ASI_M_MMUREGS 0x04 /* MMU Registers; rw, ss */ +#define ASI_M_TLBDIAG 0x05 /* MMU TLB only Diagnostics */ +#define ASI_M_DIAGS 0x06 /* Reference MMU Diagnostics */ +#define ASI_M_IODIAG 0x07 /* MMU I/O TLB only Diagnostics */ +#define ASI_M_USERTXT 0x08 /* Same as ASI_USERTXT; rw, as */ +#define ASI_M_KERNELTXT 0x09 /* Same as ASI_KERNELTXT; rw, as */ +#define ASI_M_USERDATA 0x0A /* Same as ASI_USERDATA; rw, as */ +#define ASI_M_KERNELDATA 0x0B /* Same as ASI_KERNELDATA; rw, as */ +#define ASI_M_TXTC_TAG 0x0C /* Instruction Cache Tag; rw, ss */ +#define ASI_M_TXTC_DATA 0x0D /* Instruction Cache Data; rw, ss */ +#define ASI_M_DATAC_TAG 0x0E /* Data Cache Tag; rw, ss */ +#define ASI_M_DATAC_DATA 0x0F /* Data Cache Data; rw, ss */ + +/* The following cache flushing ASIs work only with the 'sta' + * instruction. Results are unpredictable for 'swap' and 'ldstuba', + * so don't do it. + */ + +/* These ASI flushes affect external caches too. */ +#define ASI_M_FLUSH_PAGE 0x10 /* Flush I&D Cache Line (page); wo, ss */ +#define ASI_M_FLUSH_SEG 0x11 /* Flush I&D Cache Line (seg); wo, ss */ +#define ASI_M_FLUSH_REGION 0x12 /* Flush I&D Cache Line (region); wo, ss */ +#define ASI_M_FLUSH_CTX 0x13 /* Flush I&D Cache Line (context); wo, ss */ +#define ASI_M_FLUSH_USER 0x14 /* Flush I&D Cache Line (user); wo, ss */ + +/* Block-copy operations are available only on certain V8 cpus. */ +#define ASI_M_BCOPY 0x17 /* Block copy */ + +/* These affect only the ICACHE and are Ross HyperSparc and TurboSparc specific. */ +#define ASI_M_IFLUSH_PAGE 0x18 /* Flush I Cache Line (page); wo, ss */ +#define ASI_M_IFLUSH_SEG 0x19 /* Flush I Cache Line (seg); wo, ss */ +#define ASI_M_IFLUSH_REGION 0x1A /* Flush I Cache Line (region); wo, ss */ +#define ASI_M_IFLUSH_CTX 0x1B /* Flush I Cache Line (context); wo, ss */ +#define ASI_M_IFLUSH_USER 0x1C /* Flush I Cache Line (user); wo, ss */ + +/* Block-fill operations are available on certain V8 cpus */ +#define ASI_M_BFILL 0x1F + +/* This allows direct access to main memory, actually 0x20 to 0x2f are + * the available ASI's for physical ram pass-through, but I don't have + * any idea what the other ones do.... + */ + +#define ASI_M_BYPASS 0x20 /* Reference MMU bypass; rw, as */ +#define ASI_M_FBMEM 0x29 /* Graphics card frame buffer access */ +#define ASI_M_VMEUS 0x2A /* VME user 16-bit access */ +#define ASI_M_VMEPS 0x2B /* VME priv 16-bit access */ +#define ASI_M_VMEUT 0x2C /* VME user 32-bit access */ +#define ASI_M_VMEPT 0x2D /* VME priv 32-bit access */ +#define ASI_M_SBUS 0x2E /* Direct SBus access */ +#define ASI_M_CTL 0x2F /* Control Space (ECC and MXCC are here) */ + + +/* This is ROSS HyperSparc only. */ +#define ASI_M_FLUSH_IWHOLE 0x31 /* Flush entire ICACHE; wo, ss */ + +/* Tsunami/Viking/TurboSparc i/d cache flash clear. */ +#define ASI_M_IC_FLCLEAR 0x36 +#define ASI_M_DC_FLCLEAR 0x37 + +#define ASI_M_DCDR 0x39 /* Data Cache Diagnostics Register rw, ss */ + +#define ASI_M_VIKING_TMP1 0x40 /* Emulation temporary 1 on Viking */ +/* only available on SuperSparc I */ +/* #define ASI_M_VIKING_TMP2 0x41 */ /* Emulation temporary 2 on Viking */ + +#define ASI_M_ACTION 0x4c /* Breakpoint Action Register (GNU/Viking) */ + +/* LEON ASI */ +#define ASI_LEON_NOCACHE 0x01 + +#define ASI_LEON_DCACHE_MISS 0x01 + +#define ASI_LEON_CACHEREGS 0x02 +#define ASI_LEON_IFLUSH 0x10 +#define ASI_LEON_DFLUSH 0x11 + +#define ASI_LEON_MMUFLUSH 0x18 +#define ASI_LEON_MMUREGS 0x19 +#define ASI_LEON_BYPASS 0x1c +#define ASI_LEON_FLUSH_PAGE 0x10 + +/* V9 Architecture mandary ASIs. */ +#define ASI_N 0x04 /* Nucleus */ +#define ASI_NL 0x0c /* Nucleus, little endian */ +#define ASI_AIUP 0x10 /* Primary, user */ +#define ASI_AIUS 0x11 /* Secondary, user */ +#define ASI_AIUPL 0x18 /* Primary, user, little endian */ +#define ASI_AIUSL 0x19 /* Secondary, user, little endian */ +#define ASI_P 0x80 /* Primary, implicit */ +#define ASI_S 0x81 /* Secondary, implicit */ +#define ASI_PNF 0x82 /* Primary, no fault */ +#define ASI_SNF 0x83 /* Secondary, no fault */ +#define ASI_PL 0x88 /* Primary, implicit, l-endian */ +#define ASI_SL 0x89 /* Secondary, implicit, l-endian */ +#define ASI_PNFL 0x8a /* Primary, no fault, l-endian */ +#define ASI_SNFL 0x8b /* Secondary, no fault, l-endian */ + +/* SpitFire and later extended ASIs. The "(III)" marker designates + * UltraSparc-III and later specific ASIs. The "(CMT)" marker designates + * Chip Multi Threading specific ASIs. "(NG)" designates Niagara specific + * ASIs, "(4V)" designates SUN4V specific ASIs. "(NG4)" designates SPARC-T4 + * and later ASIs. + */ +#define ASI_MCD_PRIV_PRIMARY 0x02 /* (NG7) Privileged MCD version VA */ +#define ASI_MCD_REAL 0x05 /* (NG7) Privileged MCD version PA */ +#define ASI_PHYS_USE_EC 0x14 /* PADDR, E-cachable */ +#define ASI_PHYS_BYPASS_EC_E 0x15 /* PADDR, E-bit */ +#define ASI_BLK_AIUP_4V 0x16 /* (4V) Prim, user, block ld/st */ +#define ASI_BLK_AIUS_4V 0x17 /* (4V) Sec, user, block ld/st */ +#define ASI_PHYS_USE_EC_L 0x1c /* PADDR, E-cachable, little endian*/ +#define ASI_PHYS_BYPASS_EC_E_L 0x1d /* PADDR, E-bit, little endian */ +#define ASI_BLK_AIUP_L_4V 0x1e /* (4V) Prim, user, block, l-endian*/ +#define ASI_BLK_AIUS_L_4V 0x1f /* (4V) Sec, user, block, l-endian */ +#define ASI_SCRATCHPAD 0x20 /* (4V) Scratch Pad Registers */ +#define ASI_MMU 0x21 /* (4V) MMU Context Registers */ +#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23 /* (NG) init-store, twin load, + * secondary, user + */ +#define ASI_NUCLEUS_QUAD_LDD 0x24 /* Cachable, qword load */ +#define ASI_QUEUE 0x25 /* (4V) Interrupt Queue Registers */ +#define ASI_QUAD_LDD_PHYS_4V 0x26 /* (4V) Physical, qword load */ +#define ASI_NUCLEUS_QUAD_LDD_L 0x2c /* Cachable, qword load, l-endian */ +#define ASI_QUAD_LDD_PHYS_L_4V 0x2e /* (4V) Phys, qword load, l-endian */ +#define ASI_PCACHE_DATA_STATUS 0x30 /* (III) PCache data stat RAM diag */ +#define ASI_PCACHE_DATA 0x31 /* (III) PCache data RAM diag */ +#define ASI_PCACHE_TAG 0x32 /* (III) PCache tag RAM diag */ +#define ASI_PCACHE_SNOOP_TAG 0x33 /* (III) PCache snoop tag RAM diag */ +#define ASI_QUAD_LDD_PHYS 0x34 /* (III+) PADDR, qword load */ +#define ASI_WCACHE_VALID_BITS 0x38 /* (III) WCache Valid Bits diag */ +#define ASI_WCACHE_DATA 0x39 /* (III) WCache data RAM diag */ +#define ASI_WCACHE_TAG 0x3a /* (III) WCache tag RAM diag */ +#define ASI_WCACHE_SNOOP_TAG 0x3b /* (III) WCache snoop tag RAM diag */ +#define ASI_QUAD_LDD_PHYS_L 0x3c /* (III+) PADDR, qw-load, l-endian */ +#define ASI_SRAM_FAST_INIT 0x40 /* (III+) Fast SRAM init */ +#define ASI_CORE_AVAILABLE 0x41 /* (CMT) LP Available */ +#define ASI_CORE_ENABLE_STAT 0x41 /* (CMT) LP Enable Status */ +#define ASI_CORE_ENABLE 0x41 /* (CMT) LP Enable RW */ +#define ASI_XIR_STEERING 0x41 /* (CMT) XIR Steering RW */ +#define ASI_CORE_RUNNING_RW 0x41 /* (CMT) LP Running RW */ +#define ASI_CORE_RUNNING_W1S 0x41 /* (CMT) LP Running Write-One Set */ +#define ASI_CORE_RUNNING_W1C 0x41 /* (CMT) LP Running Write-One Clr */ +#define ASI_CORE_RUNNING_STAT 0x41 /* (CMT) LP Running Status */ +#define ASI_CMT_ERROR_STEERING 0x41 /* (CMT) Error Steering RW */ +#define ASI_DCACHE_INVALIDATE 0x42 /* (III) DCache Invalidate diag */ +#define ASI_DCACHE_UTAG 0x43 /* (III) DCache uTag diag */ +#define ASI_DCACHE_SNOOP_TAG 0x44 /* (III) DCache snoop tag RAM diag */ +#define ASI_LSU_CONTROL 0x45 /* Load-store control unit */ +#define ASI_DCU_CONTROL_REG 0x45 /* (III) DCache Unit Control reg */ +#define ASI_DCACHE_DATA 0x46 /* DCache data-ram diag access */ +#define ASI_DCACHE_TAG 0x47 /* Dcache tag/valid ram diag access*/ +#define ASI_INTR_DISPATCH_STAT 0x48 /* IRQ vector dispatch status */ +#define ASI_INTR_RECEIVE 0x49 /* IRQ vector receive status */ +#define ASI_UPA_CONFIG 0x4a /* UPA config space */ +#define ASI_JBUS_CONFIG 0x4a /* (IIIi) JBUS Config Register */ +#define ASI_SAFARI_CONFIG 0x4a /* (III) Safari Config Register */ +#define ASI_SAFARI_ADDRESS 0x4a /* (III) Safari Address Register */ +#define ASI_ESTATE_ERROR_EN 0x4b /* E-cache error enable space */ +#define ASI_AFSR 0x4c /* Async fault status register */ +#define ASI_AFAR 0x4d /* Async fault address register */ +#define ASI_EC_TAG_DATA 0x4e /* E-cache tag/valid ram diag acc */ +#define ASI_IMMU 0x50 /* Insn-MMU main register space */ +#define ASI_IMMU_TSB_8KB_PTR 0x51 /* Insn-MMU 8KB TSB pointer reg */ +#define ASI_IMMU_TSB_64KB_PTR 0x52 /* Insn-MMU 64KB TSB pointer reg */ +#define ASI_ITLB_DATA_IN 0x54 /* Insn-MMU TLB data in reg */ +#define ASI_ITLB_DATA_ACCESS 0x55 /* Insn-MMU TLB data access reg */ +#define ASI_ITLB_TAG_READ 0x56 /* Insn-MMU TLB tag read reg */ +#define ASI_IMMU_DEMAP 0x57 /* Insn-MMU TLB demap */ +#define ASI_DMMU 0x58 /* Data-MMU main register space */ +#define ASI_DMMU_TSB_8KB_PTR 0x59 /* Data-MMU 8KB TSB pointer reg */ +#define ASI_DMMU_TSB_64KB_PTR 0x5a /* Data-MMU 16KB TSB pointer reg */ +#define ASI_DMMU_TSB_DIRECT_PTR 0x5b /* Data-MMU TSB direct pointer reg */ +#define ASI_DTLB_DATA_IN 0x5c /* Data-MMU TLB data in reg */ +#define ASI_DTLB_DATA_ACCESS 0x5d /* Data-MMU TLB data access reg */ +#define ASI_DTLB_TAG_READ 0x5e /* Data-MMU TLB tag read reg */ +#define ASI_DMMU_DEMAP 0x5f /* Data-MMU TLB demap */ +#define ASI_IIU_INST_TRAP 0x60 /* (III) Instruction Breakpoint */ +#define ASI_INTR_ID 0x63 /* (CMT) Interrupt ID register */ +#define ASI_CORE_ID 0x63 /* (CMT) LP ID register */ +#define ASI_CESR_ID 0x63 /* (CMT) CESR ID register */ +#define ASI_IC_INSTR 0x66 /* Insn cache instrucion ram diag */ +#define ASI_IC_TAG 0x67 /* Insn cache tag/valid ram diag */ +#define ASI_IC_STAG 0x68 /* (III) Insn cache snoop tag ram */ +#define ASI_IC_PRE_DECODE 0x6e /* Insn cache pre-decode ram diag */ +#define ASI_IC_NEXT_FIELD 0x6f /* Insn cache next-field ram diag */ +#define ASI_BRPRED_ARRAY 0x6f /* (III) Branch Prediction RAM diag*/ +#define ASI_BLK_AIUP 0x70 /* Primary, user, block load/store */ +#define ASI_BLK_AIUS 0x71 /* Secondary, user, block ld/st */ +#define ASI_MCU_CTRL_REG 0x72 /* (III) Memory controller regs */ +#define ASI_EC_DATA 0x74 /* (III) E-cache data staging reg */ +#define ASI_EC_CTRL 0x75 /* (III) E-cache control reg */ +#define ASI_EC_W 0x76 /* E-cache diag write access */ +#define ASI_UDB_ERROR_W 0x77 /* External UDB error regs W */ +#define ASI_UDB_CONTROL_W 0x77 /* External UDB control regs W */ +#define ASI_INTR_W 0x77 /* IRQ vector dispatch write */ +#define ASI_INTR_DATAN_W 0x77 /* (III) Out irq vector data reg N */ +#define ASI_INTR_DISPATCH_W 0x77 /* (III) Interrupt vector dispatch */ +#define ASI_BLK_AIUPL 0x78 /* Primary, user, little, blk ld/st*/ +#define ASI_BLK_AIUSL 0x79 /* Secondary, user, little, blk ld/st*/ +#define ASI_EC_R 0x7e /* E-cache diag read access */ +#define ASI_UDBH_ERROR_R 0x7f /* External UDB error regs rd hi */ +#define ASI_UDBL_ERROR_R 0x7f /* External UDB error regs rd low */ +#define ASI_UDBH_CONTROL_R 0x7f /* External UDB control regs rd hi */ +#define ASI_UDBL_CONTROL_R 0x7f /* External UDB control regs rd low*/ +#define ASI_INTR_R 0x7f /* IRQ vector dispatch read */ +#define ASI_INTR_DATAN_R 0x7f /* (III) In irq vector data reg N */ +#define ASI_MCD_PRIMARY 0x90 /* (NG7) MCD version load/store */ +#define ASI_MCD_ST_BLKINIT_PRIMARY \ + 0x92 /* (NG7) MCD store BLKINIT primary */ +#define ASI_PIC 0xb0 /* (NG4) PIC registers */ +#define ASI_PST8_P 0xc0 /* Primary, 8 8-bit, partial */ +#define ASI_PST8_S 0xc1 /* Secondary, 8 8-bit, partial */ +#define ASI_PST16_P 0xc2 /* Primary, 4 16-bit, partial */ +#define ASI_PST16_S 0xc3 /* Secondary, 4 16-bit, partial */ +#define ASI_PST32_P 0xc4 /* Primary, 2 32-bit, partial */ +#define ASI_PST32_S 0xc5 /* Secondary, 2 32-bit, partial */ +#define ASI_PST8_PL 0xc8 /* Primary, 8 8-bit, partial, L */ +#define ASI_PST8_SL 0xc9 /* Secondary, 8 8-bit, partial, L */ +#define ASI_PST16_PL 0xca /* Primary, 4 16-bit, partial, L */ +#define ASI_PST16_SL 0xcb /* Secondary, 4 16-bit, partial, L */ +#define ASI_PST32_PL 0xcc /* Primary, 2 32-bit, partial, L */ +#define ASI_PST32_SL 0xcd /* Secondary, 2 32-bit, partial, L */ +#define ASI_FL8_P 0xd0 /* Primary, 1 8-bit, fpu ld/st */ +#define ASI_FL8_S 0xd1 /* Secondary, 1 8-bit, fpu ld/st */ +#define ASI_FL16_P 0xd2 /* Primary, 1 16-bit, fpu ld/st */ +#define ASI_FL16_S 0xd3 /* Secondary, 1 16-bit, fpu ld/st */ +#define ASI_FL8_PL 0xd8 /* Primary, 1 8-bit, fpu ld/st, L */ +#define ASI_FL8_SL 0xd9 /* Secondary, 1 8-bit, fpu ld/st, L*/ +#define ASI_FL16_PL 0xda /* Primary, 1 16-bit, fpu ld/st, L */ +#define ASI_FL16_SL 0xdb /* Secondary, 1 16-bit, fpu ld/st,L*/ +#define ASI_BLK_COMMIT_P 0xe0 /* Primary, blk store commit */ +#define ASI_BLK_COMMIT_S 0xe1 /* Secondary, blk store commit */ +#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 /* (NG) init-store, twin load, + * primary, implicit + */ +#define ASI_BLK_INIT_QUAD_LDD_S 0xe3 /* (NG) init-store, twin load, + * secondary, implicit + */ +#define ASI_BLK_P 0xf0 /* Primary, blk ld/st */ +#define ASI_BLK_S 0xf1 /* Secondary, blk ld/st */ +#define ASI_ST_BLKINIT_MRU_P 0xf2 /* (NG4) init-store, twin load, + * Most-Recently-Used, primary, + * implicit + */ +#define ASI_ST_BLKINIT_MRU_S 0xf3 /* (NG4) init-store, twin load, + * Most-Recently-Used, secondary, + * implicit + */ +#define ASI_BLK_PL 0xf8 /* Primary, blk ld/st, little */ +#define ASI_BLK_SL 0xf9 /* Secondary, blk ld/st, little */ +#define ASI_ST_BLKINIT_MRU_PL 0xfa /* (NG4) init-store, twin load, + * Most-Recently-Used, primary, + * implicit, little-endian + */ +#define ASI_ST_BLKINIT_MRU_SL 0xfb /* (NG4) init-store, twin load, + * Most-Recently-Used, secondary, + * implicit, little-endian + */ + +#endif /* _SPARC_ASI_H */ diff --git a/arch/sparc/include/uapi/asm/auxvec.h b/arch/sparc/include/uapi/asm/auxvec.h new file mode 100644 index 000000000..ab8780fb9 --- /dev/null +++ b/arch/sparc/include/uapi/asm/auxvec.h @@ -0,0 +1,15 @@ +#ifndef __ASMSPARC_AUXVEC_H +#define __ASMSPARC_AUXVEC_H + +#define AT_SYSINFO_EHDR 33 + +/* Avoid overlap with other AT_* values since they are consolidated in + * glibc and any overlaps can cause problems + */ +#define AT_ADI_BLKSZ 48 +#define AT_ADI_NBITS 49 +#define AT_ADI_UEONADI 50 + +#define AT_VECTOR_SIZE_ARCH 4 + +#endif /* !(__ASMSPARC_AUXVEC_H) */ diff --git a/arch/sparc/include/uapi/asm/bitsperlong.h b/arch/sparc/include/uapi/asm/bitsperlong.h new file mode 100644 index 000000000..cd9a43227 --- /dev/null +++ b/arch/sparc/include/uapi/asm/bitsperlong.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __ASM_ALPHA_BITSPERLONG_H +#define __ASM_ALPHA_BITSPERLONG_H + +#if defined(__sparc__) && defined(__arch64__) +#define __BITS_PER_LONG 64 +#else +#define __BITS_PER_LONG 32 +#endif + +#include + +#endif /* __ASM_ALPHA_BITSPERLONG_H */ + diff --git a/arch/sparc/include/uapi/asm/byteorder.h b/arch/sparc/include/uapi/asm/byteorder.h new file mode 100644 index 000000000..216b8e593 --- /dev/null +++ b/arch/sparc/include/uapi/asm/byteorder.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_BYTEORDER_H +#define _SPARC_BYTEORDER_H + +#include + +#endif /* _SPARC_BYTEORDER_H */ diff --git a/arch/sparc/include/uapi/asm/display7seg.h b/arch/sparc/include/uapi/asm/display7seg.h new file mode 100644 index 000000000..7e9fef0c6 --- /dev/null +++ b/arch/sparc/include/uapi/asm/display7seg.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * display7seg - Driver interface for the 7-segment display + * present on Sun Microsystems CP1400 and CP1500 + * + * Copyright (c) 2000 Eric Brower + * + */ + +#ifndef __display7seg_h__ +#define __display7seg_h__ + +#define D7S_IOC 'p' + +#define D7SIOCRD _IOR(D7S_IOC, 0x45, int) /* Read device state */ +#define D7SIOCWR _IOW(D7S_IOC, 0x46, int) /* Write device state */ +#define D7SIOCTM _IO (D7S_IOC, 0x47) /* Translate mode (FLIP)*/ + +/* + * ioctl flag definitions + * + * POINT - Toggle decimal point (0=absent 1=present) + * ALARM - Toggle alarm LED (0=green 1=red) + * FLIP - Toggle inverted mode (0=normal 1=flipped) + * bits 0-4 - Character displayed (see definitions below) + * + * Display segments are defined as follows, + * subject to D7S_FLIP register state: + * + * a + * --- + * f| |b + * -g- + * e| |c + * --- + * d + */ + +#define D7S_POINT (1 << 7) /* Decimal point*/ +#define D7S_ALARM (1 << 6) /* Alarm LED */ +#define D7S_FLIP (1 << 5) /* Flip display */ + +#define D7S_0 0x00 /* Numerals 0-9 */ +#define D7S_1 0x01 +#define D7S_2 0x02 +#define D7S_3 0x03 +#define D7S_4 0x04 +#define D7S_5 0x05 +#define D7S_6 0x06 +#define D7S_7 0x07 +#define D7S_8 0x08 +#define D7S_9 0x09 +#define D7S_A 0x0A /* Letters A-F, H, L, P */ +#define D7S_B 0x0B +#define D7S_C 0x0C +#define D7S_D 0x0D +#define D7S_E 0x0E +#define D7S_F 0x0F +#define D7S_H 0x10 +#define D7S_E2 0x11 +#define D7S_L 0x12 +#define D7S_P 0x13 +#define D7S_SEGA 0x14 /* Individual segments */ +#define D7S_SEGB 0x15 +#define D7S_SEGC 0x16 +#define D7S_SEGD 0x17 +#define D7S_SEGE 0x18 +#define D7S_SEGF 0x19 +#define D7S_SEGG 0x1A +#define D7S_SEGABFG 0x1B /* Segment groupings */ +#define D7S_SEGCDEG 0x1C +#define D7S_SEGBCEF 0x1D +#define D7S_SEGADG 0x1E +#define D7S_BLANK 0x1F /* Clear all segments */ + +#define D7S_MIN_VAL 0x0 +#define D7S_MAX_VAL 0x1F + +#endif /* ifndef __display7seg_h__ */ diff --git a/arch/sparc/include/uapi/asm/envctrl.h b/arch/sparc/include/uapi/asm/envctrl.h new file mode 100644 index 000000000..cf8aa0a14 --- /dev/null +++ b/arch/sparc/include/uapi/asm/envctrl.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * envctrl.h: Definitions for access to the i2c environment + * monitoring on Ultrasparc systems. + * + * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 2000 Vinh Truong (vinh.truong@eng.sun.com) + * VT - Add all ioctl commands and environment status definitions + * VT - Add application note + */ +#ifndef _SPARC64_ENVCTRL_H +#define _SPARC64_ENVCTRL_H 1 + +#include + +/* Application note: + * + * The driver supports 4 operations: open(), close(), ioctl(), read() + * The device name is /dev/envctrl. + * Below is sample usage: + * + * fd = open("/dev/envtrl", O_RDONLY); + * if (ioctl(fd, ENVCTRL_READ_SHUTDOWN_TEMPERATURE, 0) < 0) + * printf("error\n"); + * ret = read(fd, buf, 10); + * close(fd); + * + * Notice in the case of cpu voltage and temperature, the default is + * cpu0. If we need to know the info of cpu1, cpu2, cpu3, we need to + * pass in cpu number in ioctl() last parameter. For example, to + * get the voltage of cpu2: + * + * ioctlbuf[0] = 2; + * if (ioctl(fd, ENVCTRL_READ_CPU_VOLTAGE, ioctlbuf) < 0) + * printf("error\n"); + * ret = read(fd, buf, 10); + * + * All the return values are in ascii. So check read return value + * and do appropriate conversions in your application. + */ + +/* IOCTL commands */ + +/* Note: these commands reflect possible monitor features. + * Some boards choose to support some of the features only. + */ +#define ENVCTRL_RD_CPU_TEMPERATURE _IOR('p', 0x40, int) +#define ENVCTRL_RD_CPU_VOLTAGE _IOR('p', 0x41, int) +#define ENVCTRL_RD_FAN_STATUS _IOR('p', 0x42, int) +#define ENVCTRL_RD_WARNING_TEMPERATURE _IOR('p', 0x43, int) +#define ENVCTRL_RD_SHUTDOWN_TEMPERATURE _IOR('p', 0x44, int) +#define ENVCTRL_RD_VOLTAGE_STATUS _IOR('p', 0x45, int) +#define ENVCTRL_RD_SCSI_TEMPERATURE _IOR('p', 0x46, int) +#define ENVCTRL_RD_ETHERNET_TEMPERATURE _IOR('p', 0x47, int) +#define ENVCTRL_RD_MTHRBD_TEMPERATURE _IOR('p', 0x48, int) + +#define ENVCTRL_RD_GLOBALADDRESS _IOR('p', 0x49, int) + +/* Read return values for a voltage status request. */ +#define ENVCTRL_VOLTAGE_POWERSUPPLY_GOOD 0x01 +#define ENVCTRL_VOLTAGE_BAD 0x02 +#define ENVCTRL_POWERSUPPLY_BAD 0x03 +#define ENVCTRL_VOLTAGE_POWERSUPPLY_BAD 0x04 + +/* Read return values for a fan status request. + * A failure match means either the fan fails or + * the fan is not connected. Some boards have optional + * connectors to connect extra fans. + * + * There are maximum 8 monitor fans. Some are cpu fans + * some are system fans. The mask below only indicates + * fan by order number. + * Below is a sample application: + * + * if (ioctl(fd, ENVCTRL_READ_FAN_STATUS, 0) < 0) { + * printf("ioctl fan failed\n"); + * } + * if (read(fd, rslt, 1) <= 0) { + * printf("error or fan not monitored\n"); + * } else { + * if (rslt[0] == ENVCTRL_ALL_FANS_GOOD) { + * printf("all fans good\n"); + * } else if (rslt[0] == ENVCTRL_ALL_FANS_BAD) { + * printf("all fans bad\n"); + * } else { + * if (rslt[0] & ENVCTRL_FAN0_FAILURE_MASK) { + * printf("fan 0 failed or not connected\n"); + * } + * ...... + */ + +#define ENVCTRL_ALL_FANS_GOOD 0x00 +#define ENVCTRL_FAN0_FAILURE_MASK 0x01 +#define ENVCTRL_FAN1_FAILURE_MASK 0x02 +#define ENVCTRL_FAN2_FAILURE_MASK 0x04 +#define ENVCTRL_FAN3_FAILURE_MASK 0x08 +#define ENVCTRL_FAN4_FAILURE_MASK 0x10 +#define ENVCTRL_FAN5_FAILURE_MASK 0x20 +#define ENVCTRL_FAN6_FAILURE_MASK 0x40 +#define ENVCTRL_FAN7_FAILURE_MASK 0x80 +#define ENVCTRL_ALL_FANS_BAD 0xFF + +#endif /* !(_SPARC64_ENVCTRL_H) */ diff --git a/arch/sparc/include/uapi/asm/errno.h b/arch/sparc/include/uapi/asm/errno.h new file mode 100644 index 000000000..71940ec91 --- /dev/null +++ b/arch/sparc/include/uapi/asm/errno.h @@ -0,0 +1,122 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_ERRNO_H +#define _SPARC_ERRNO_H + +/* These match the SunOS error numbering scheme. */ + +#include + +#define EWOULDBLOCK EAGAIN /* Operation would block */ +#define EINPROGRESS 36 /* Operation now in progress */ +#define EALREADY 37 /* Operation already in progress */ +#define ENOTSOCK 38 /* Socket operation on non-socket */ +#define EDESTADDRREQ 39 /* Destination address required */ +#define EMSGSIZE 40 /* Message too long */ +#define EPROTOTYPE 41 /* Protocol wrong type for socket */ +#define ENOPROTOOPT 42 /* Protocol not available */ +#define EPROTONOSUPPORT 43 /* Protocol not supported */ +#define ESOCKTNOSUPPORT 44 /* Socket type not supported */ +#define EOPNOTSUPP 45 /* Op not supported on transport endpoint */ +#define EPFNOSUPPORT 46 /* Protocol family not supported */ +#define EAFNOSUPPORT 47 /* Address family not supported by protocol */ +#define EADDRINUSE 48 /* Address already in use */ +#define EADDRNOTAVAIL 49 /* Cannot assign requested address */ +#define ENETDOWN 50 /* Network is down */ +#define ENETUNREACH 51 /* Network is unreachable */ +#define ENETRESET 52 /* Net dropped connection because of reset */ +#define ECONNABORTED 53 /* Software caused connection abort */ +#define ECONNRESET 54 /* Connection reset by peer */ +#define ENOBUFS 55 /* No buffer space available */ +#define EISCONN 56 /* Transport endpoint is already connected */ +#define ENOTCONN 57 /* Transport endpoint is not connected */ +#define ESHUTDOWN 58 /* No send after transport endpoint shutdown */ +#define ETOOMANYREFS 59 /* Too many references: cannot splice */ +#define ETIMEDOUT 60 /* Connection timed out */ +#define ECONNREFUSED 61 /* Connection refused */ +#define ELOOP 62 /* Too many symbolic links encountered */ +#define ENAMETOOLONG 63 /* File name too long */ +#define EHOSTDOWN 64 /* Host is down */ +#define EHOSTUNREACH 65 /* No route to host */ +#define ENOTEMPTY 66 /* Directory not empty */ +#define EPROCLIM 67 /* SUNOS: Too many processes */ +#define EUSERS 68 /* Too many users */ +#define EDQUOT 69 /* Quota exceeded */ +#define ESTALE 70 /* Stale file handle */ +#define EREMOTE 71 /* Object is remote */ +#define ENOSTR 72 /* Device not a stream */ +#define ETIME 73 /* Timer expired */ +#define ENOSR 74 /* Out of streams resources */ +#define ENOMSG 75 /* No message of desired type */ +#define EBADMSG 76 /* Not a data message */ +#define EFSBADCRC EBADMSG /* Bad CRC detected */ +#define EIDRM 77 /* Identifier removed */ +#define EDEADLK 78 /* Resource deadlock would occur */ +#define ENOLCK 79 /* No record locks available */ +#define ENONET 80 /* Machine is not on the network */ +#define ERREMOTE 81 /* SunOS: Too many lvls of remote in path */ +#define ENOLINK 82 /* Link has been severed */ +#define EADV 83 /* Advertise error */ +#define ESRMNT 84 /* Srmount error */ +#define ECOMM 85 /* Communication error on send */ +#define EPROTO 86 /* Protocol error */ +#define EMULTIHOP 87 /* Multihop attempted */ +#define EDOTDOT 88 /* RFS specific error */ +#define EREMCHG 89 /* Remote address changed */ +#define ENOSYS 90 /* Function not implemented */ + +/* The rest have no SunOS equivalent. */ +#define ESTRPIPE 91 /* Streams pipe error */ +#define EOVERFLOW 92 /* Value too large for defined data type */ +#define EBADFD 93 /* File descriptor in bad state */ +#define ECHRNG 94 /* Channel number out of range */ +#define EL2NSYNC 95 /* Level 2 not synchronized */ +#define EL3HLT 96 /* Level 3 halted */ +#define EL3RST 97 /* Level 3 reset */ +#define ELNRNG 98 /* Link number out of range */ +#define EUNATCH 99 /* Protocol driver not attached */ +#define ENOCSI 100 /* No CSI structure available */ +#define EL2HLT 101 /* Level 2 halted */ +#define EBADE 102 /* Invalid exchange */ +#define EBADR 103 /* Invalid request descriptor */ +#define EXFULL 104 /* Exchange full */ +#define ENOANO 105 /* No anode */ +#define EBADRQC 106 /* Invalid request code */ +#define EBADSLT 107 /* Invalid slot */ +#define EDEADLOCK 108 /* File locking deadlock error */ +#define EBFONT 109 /* Bad font file format */ +#define ELIBEXEC 110 /* Cannot exec a shared library directly */ +#define ENODATA 111 /* No data available */ +#define ELIBBAD 112 /* Accessing a corrupted shared library */ +#define ENOPKG 113 /* Package not installed */ +#define ELIBACC 114 /* Can not access a needed shared library */ +#define ENOTUNIQ 115 /* Name not unique on network */ +#define ERESTART 116 /* Interrupted syscall should be restarted */ +#define EUCLEAN 117 /* Structure needs cleaning */ +#define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */ +#define ENOTNAM 118 /* Not a XENIX named type file */ +#define ENAVAIL 119 /* No XENIX semaphores available */ +#define EISNAM 120 /* Is a named type file */ +#define EREMOTEIO 121 /* Remote I/O error */ +#define EILSEQ 122 /* Illegal byte sequence */ +#define ELIBMAX 123 /* Atmpt to link in too many shared libs */ +#define ELIBSCN 124 /* .lib section in a.out corrupted */ + +#define ENOMEDIUM 125 /* No medium found */ +#define EMEDIUMTYPE 126 /* Wrong medium type */ +#define ECANCELED 127 /* Operation Cancelled */ +#define ENOKEY 128 /* Required key not available */ +#define EKEYEXPIRED 129 /* Key has expired */ +#define EKEYREVOKED 130 /* Key has been revoked */ +#define EKEYREJECTED 131 /* Key was rejected by service */ + +/* for robust mutexes */ +#define EOWNERDEAD 132 /* Owner died */ +#define ENOTRECOVERABLE 133 /* State not recoverable */ + +#define ERFKILL 134 /* Operation not possible due to RF-kill */ + +#define EHWPOISON 135 /* Memory page has hardware error */ + +#define EFTYPE 136 /* Wrong file type for the intended operation */ + +#endif diff --git a/arch/sparc/include/uapi/asm/fbio.h b/arch/sparc/include/uapi/asm/fbio.h new file mode 100644 index 000000000..0dafe2c1e --- /dev/null +++ b/arch/sparc/include/uapi/asm/fbio.h @@ -0,0 +1,260 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI__LINUX_FBIO_H +#define _UAPI__LINUX_FBIO_H + +#include +#include + +/* Constants used for fbio SunOS compatibility */ +/* (C) 1996 Miguel de Icaza */ + +/* Frame buffer types */ +#define FBTYPE_NOTYPE -1 +#define FBTYPE_SUN1BW 0 /* mono */ +#define FBTYPE_SUN1COLOR 1 +#define FBTYPE_SUN2BW 2 +#define FBTYPE_SUN2COLOR 3 +#define FBTYPE_SUN2GP 4 +#define FBTYPE_SUN5COLOR 5 +#define FBTYPE_SUN3COLOR 6 +#define FBTYPE_MEMCOLOR 7 +#define FBTYPE_SUN4COLOR 8 + +#define FBTYPE_NOTSUN1 9 +#define FBTYPE_NOTSUN2 10 +#define FBTYPE_NOTSUN3 11 + +#define FBTYPE_SUNFAST_COLOR 12 /* cg6 */ +#define FBTYPE_SUNROP_COLOR 13 +#define FBTYPE_SUNFB_VIDEO 14 +#define FBTYPE_SUNGIFB 15 +#define FBTYPE_SUNGPLAS 16 +#define FBTYPE_SUNGP3 17 +#define FBTYPE_SUNGT 18 +#define FBTYPE_SUNLEO 19 /* zx Leo card */ +#define FBTYPE_MDICOLOR 20 /* cg14 */ +#define FBTYPE_TCXCOLOR 21 /* SUNW,tcx card */ + +#define FBTYPE_LASTPLUSONE 21 /* This is not last + 1 in fact... */ + +/* Does not seem to be listed in the Sun file either */ +#define FBTYPE_CREATOR 22 +#define FBTYPE_PCI_IGA1682 23 +#define FBTYPE_P9100COLOR 24 + +#define FBTYPE_PCI_GENERIC 1000 +#define FBTYPE_PCI_MACH64 1001 + +/* fbio ioctls */ +/* Returned by FBIOGTYPE */ +struct fbtype { + int fb_type; /* fb type, see above */ + int fb_height; /* pixels */ + int fb_width; /* pixels */ + int fb_depth; + int fb_cmsize; /* color map entries */ + int fb_size; /* fb size in bytes */ +}; +#define FBIOGTYPE _IOR('F', 0, struct fbtype) + +struct fbcmap { + int index; /* first element (0 origin) */ + int count; + unsigned char __user *red; + unsigned char __user *green; + unsigned char __user *blue; +}; + +#ifndef __KERNEL__ +#define FBIOPUTCMAP _IOW('F', 3, struct fbcmap) +#define FBIOGETCMAP _IOW('F', 4, struct fbcmap) +#endif + +/* # of device specific values */ +#define FB_ATTR_NDEVSPECIFIC 8 +/* # of possible emulations */ +#define FB_ATTR_NEMUTYPES 4 + +struct fbsattr { + int flags; + int emu_type; /* -1 if none */ + int dev_specific[FB_ATTR_NDEVSPECIFIC]; +}; + +struct fbgattr { + int real_type; /* real frame buffer type */ + int owner; /* unknown */ + struct fbtype fbtype; /* real frame buffer fbtype */ + struct fbsattr sattr; + int emu_types[FB_ATTR_NEMUTYPES]; /* supported emulations */ +}; +#define FBIOSATTR _IOW('F', 5, struct fbgattr) /* Unsupported: */ +#define FBIOGATTR _IOR('F', 6, struct fbgattr) /* supported */ + +#define FBIOSVIDEO _IOW('F', 7, int) +#define FBIOGVIDEO _IOR('F', 8, int) + +struct fbcursor { + short set; /* what to set, choose from the list above */ + short enable; /* cursor on/off */ + struct fbcurpos pos; /* cursor position */ + struct fbcurpos hot; /* cursor hot spot */ + struct fbcmap cmap; /* color map info */ + struct fbcurpos size; /* cursor bit map size */ + char __user *image; /* cursor image bits */ + char __user *mask; /* cursor mask bits */ +}; + +/* set/get cursor attributes/shape */ +#define FBIOSCURSOR _IOW('F', 24, struct fbcursor) +#define FBIOGCURSOR _IOWR('F', 25, struct fbcursor) + +/* set/get cursor position */ +#define FBIOSCURPOS _IOW('F', 26, struct fbcurpos) +#define FBIOGCURPOS _IOW('F', 27, struct fbcurpos) + +/* get max cursor size */ +#define FBIOGCURMAX _IOR('F', 28, struct fbcurpos) + +/* wid manipulation */ +struct fb_wid_alloc { +#define FB_WID_SHARED_8 0 +#define FB_WID_SHARED_24 1 +#define FB_WID_DBL_8 2 +#define FB_WID_DBL_24 3 + __u32 wa_type; + __s32 wa_index; /* Set on return */ + __u32 wa_count; +}; +struct fb_wid_item { + __u32 wi_type; + __s32 wi_index; + __u32 wi_attrs; + __u32 wi_values[32]; +}; +struct fb_wid_list { + __u32 wl_flags; + __u32 wl_count; + struct fb_wid_item *wl_list; +}; + +#define FBIO_WID_ALLOC _IOWR('F', 30, struct fb_wid_alloc) +#define FBIO_WID_FREE _IOW('F', 31, struct fb_wid_alloc) +#define FBIO_WID_PUT _IOW('F', 32, struct fb_wid_list) +#define FBIO_WID_GET _IOWR('F', 33, struct fb_wid_list) + +/* Creator ioctls */ +#define FFB_IOCTL ('F'<<8) +#define FFB_SYS_INFO (FFB_IOCTL|80) +#define FFB_CLUTREAD (FFB_IOCTL|81) +#define FFB_CLUTPOST (FFB_IOCTL|82) +#define FFB_SETDIAGMODE (FFB_IOCTL|83) +#define FFB_GETMONITORID (FFB_IOCTL|84) +#define FFB_GETVIDEOMODE (FFB_IOCTL|85) +#define FFB_SETVIDEOMODE (FFB_IOCTL|86) +#define FFB_SETSERVER (FFB_IOCTL|87) +#define FFB_SETOVCTL (FFB_IOCTL|88) +#define FFB_GETOVCTL (FFB_IOCTL|89) +#define FFB_GETSAXNUM (FFB_IOCTL|90) +#define FFB_FBDEBUG (FFB_IOCTL|91) + +/* Cg14 ioctls */ +#define MDI_IOCTL ('M'<<8) +#define MDI_RESET (MDI_IOCTL|1) +#define MDI_GET_CFGINFO (MDI_IOCTL|2) +#define MDI_SET_PIXELMODE (MDI_IOCTL|3) +# define MDI_32_PIX 32 +# define MDI_16_PIX 16 +# define MDI_8_PIX 8 + +struct mdi_cfginfo { + int mdi_ncluts; /* Number of implemented CLUTs in this MDI */ + int mdi_type; /* FBTYPE name */ + int mdi_height; /* height */ + int mdi_width; /* width */ + int mdi_size; /* available ram */ + int mdi_mode; /* 8bpp, 16bpp or 32bpp */ + int mdi_pixfreq; /* pixel clock (from PROM) */ +}; + +/* SparcLinux specific ioctl for the MDI, should be replaced for + * the SET_XLUT/SET_CLUTn ioctls instead + */ +#define MDI_CLEAR_XLUT (MDI_IOCTL|9) + +/* leo & ffb ioctls */ +struct fb_clut_alloc { + __u32 clutid; /* Set on return */ + __u32 flag; + __u32 index; +}; + +struct fb_clut { +#define FB_CLUT_WAIT 0x00000001 /* Not yet implemented */ + __u32 flag; + __u32 clutid; + __u32 offset; + __u32 count; + char * red; + char * green; + char * blue; +}; + +struct fb_clut32 { + __u32 flag; + __u32 clutid; + __u32 offset; + __u32 count; + __u32 red; + __u32 green; + __u32 blue; +}; + +#define LEO_CLUTALLOC _IOWR('L', 53, struct fb_clut_alloc) +#define LEO_CLUTFREE _IOW('L', 54, struct fb_clut_alloc) +#define LEO_CLUTREAD _IOW('L', 55, struct fb_clut) +#define LEO_CLUTPOST _IOW('L', 56, struct fb_clut) +#define LEO_SETGAMMA _IOW('L', 68, int) /* Not yet implemented */ +#define LEO_GETGAMMA _IOR('L', 69, int) /* Not yet implemented */ + + +/* These are exported to userland for applications to use */ +/* Mappable offsets for the cg14: control registers */ +#define MDI_DIRECT_MAP 0x10000000 +#define MDI_CTLREG_MAP 0x20000000 +#define MDI_CURSOR_MAP 0x30000000 +#define MDI_SHDW_VRT_MAP 0x40000000 + +/* Mappable offsets for the cg14: frame buffer resolutions */ +/* 32 bits */ +#define MDI_CHUNKY_XBGR_MAP 0x50000000 +#define MDI_CHUNKY_BGR_MAP 0x60000000 + +/* 16 bits */ +#define MDI_PLANAR_X16_MAP 0x70000000 +#define MDI_PLANAR_C16_MAP 0x80000000 + +/* 8 bit is done as CG3 MMAP offset */ +/* 32 bits, planar */ +#define MDI_PLANAR_X32_MAP 0x90000000 +#define MDI_PLANAR_B32_MAP 0xa0000000 +#define MDI_PLANAR_G32_MAP 0xb0000000 +#define MDI_PLANAR_R32_MAP 0xc0000000 + +/* Mappable offsets on leo */ +#define LEO_SS0_MAP 0x00000000 +#define LEO_LC_SS0_USR_MAP 0x00800000 +#define LEO_LD_SS0_MAP 0x00801000 +#define LEO_LX_CURSOR_MAP 0x00802000 +#define LEO_SS1_MAP 0x00803000 +#define LEO_LC_SS1_USR_MAP 0x01003000 +#define LEO_LD_SS1_MAP 0x01004000 +#define LEO_UNK_MAP 0x01005000 +#define LEO_LX_KRN_MAP 0x01006000 +#define LEO_LC_SS0_KRN_MAP 0x01007000 +#define LEO_LC_SS1_KRN_MAP 0x01008000 +#define LEO_LD_GBL_MAP 0x01009000 +#define LEO_UNK2_MAP 0x0100a000 + + +#endif /* _UAPI__LINUX_FBIO_H */ diff --git a/arch/sparc/include/uapi/asm/fcntl.h b/arch/sparc/include/uapi/asm/fcntl.h new file mode 100644 index 000000000..29c5639bc --- /dev/null +++ b/arch/sparc/include/uapi/asm/fcntl.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_FCNTL_H +#define _SPARC_FCNTL_H + +#define O_APPEND (1 << 3) +#define FASYNC (1 << 6) /* fcntl, for BSD compatibility */ +#define O_CREAT (1 << 9) /* not fcntl */ +#define O_TRUNC (1 << 10) /* not fcntl */ +#define O_EXCL (1 << 11) /* not fcntl */ +#define O_DSYNC (1 << 13) /* used to be O_SYNC, see below */ +#define O_NONBLOCK (1 << 14) +#if defined(__sparc__) && defined(__arch64__) +#define O_NDELAY (1 << 2) +#else +#define O_NDELAY ((1 << 2) | O_NONBLOCK) +#endif +#define O_NOCTTY (1 << 15) /* not fcntl */ +#define O_LARGEFILE (1 << 18) +#define O_DIRECT (1 << 20) /* direct disk access hint */ +#define O_NOATIME (1 << 21) +#define O_CLOEXEC (1 << 22) +/* + * Before Linux 2.6.33 only O_DSYNC semantics were implemented, but using + * the O_SYNC flag. We continue to use the existing numerical value + * for O_DSYNC semantics now, but using the correct symbolic name for it. + * This new value is used to request true Posix O_SYNC semantics. It is + * defined in this strange way to make sure applications compiled against + * new headers get at least O_DSYNC semantics on older kernels. + * + * This has the nice side-effect that we can simply test for O_DSYNC + * wherever we do not care if O_DSYNC or O_SYNC is used. + * + * Note: __O_SYNC must never be used directly. + */ +#define __O_SYNC (1 << 23) +#define O_SYNC (__O_SYNC|O_DSYNC) + +#define O_PATH (1 << 24) +#define __O_TMPFILE (1 << 25) + +#define F_GETOWN 5 /* for sockets. */ +#define F_SETOWN 6 /* for sockets. */ +#define F_GETLK 7 +#define F_SETLK 8 +#define F_SETLKW 9 + +/* for posix fcntl() and lockf() */ +#define F_RDLCK 1 +#define F_WRLCK 2 +#define F_UNLCK 3 + +#define __ARCH_FLOCK_PAD short __unused; +#define __ARCH_FLOCK64_PAD short __unused; + +#include + +#endif diff --git a/arch/sparc/include/uapi/asm/ioctl.h b/arch/sparc/include/uapi/asm/ioctl.h new file mode 100644 index 000000000..96c598fe7 --- /dev/null +++ b/arch/sparc/include/uapi/asm/ioctl.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_IOCTL_H +#define _SPARC_IOCTL_H + +/* + * Our DIR and SIZE overlap in order to simulteneously provide + * a non-zero _IOC_NONE (for binary compatibility) and + * 14 bits of size as on i386. Here's the layout: + * + * 0xE0000000 DIR + * 0x80000000 DIR = WRITE + * 0x40000000 DIR = READ + * 0x20000000 DIR = NONE + * 0x3FFF0000 SIZE (overlaps NONE bit) + * 0x0000FF00 TYPE + * 0x000000FF NR (CMD) + */ + +#define _IOC_NRBITS 8 +#define _IOC_TYPEBITS 8 +#define _IOC_SIZEBITS 13 /* Actually 14, see below. */ +#define _IOC_DIRBITS 3 + +#define _IOC_NRMASK ((1 << _IOC_NRBITS)-1) +#define _IOC_TYPEMASK ((1 << _IOC_TYPEBITS)-1) +#define _IOC_SIZEMASK ((1 << _IOC_SIZEBITS)-1) +#define _IOC_XSIZEMASK ((1 << (_IOC_SIZEBITS+1))-1) +#define _IOC_DIRMASK ((1 << _IOC_DIRBITS)-1) + +#define _IOC_NRSHIFT 0 +#define _IOC_TYPESHIFT (_IOC_NRSHIFT + _IOC_NRBITS) +#define _IOC_SIZESHIFT (_IOC_TYPESHIFT + _IOC_TYPEBITS) +#define _IOC_DIRSHIFT (_IOC_SIZESHIFT + _IOC_SIZEBITS) + +#define _IOC_NONE 1U +#define _IOC_READ 2U +#define _IOC_WRITE 4U + +#define _IOC(dir,type,nr,size) \ + (((dir) << _IOC_DIRSHIFT) | \ + ((type) << _IOC_TYPESHIFT) | \ + ((nr) << _IOC_NRSHIFT) | \ + ((size) << _IOC_SIZESHIFT)) + +#define _IO(type,nr) _IOC(_IOC_NONE,(type),(nr),0) +#define _IOR(type,nr,size) _IOC(_IOC_READ,(type),(nr),sizeof(size)) +#define _IOW(type,nr,size) _IOC(_IOC_WRITE,(type),(nr),sizeof(size)) +#define _IOWR(type,nr,size) _IOC(_IOC_READ|_IOC_WRITE,(type),(nr),sizeof(size)) + +/* Used to decode ioctl numbers in drivers despite the leading underscore... */ +#define _IOC_DIR(nr) \ + ( (((((nr) >> _IOC_DIRSHIFT) & _IOC_DIRMASK) & (_IOC_WRITE|_IOC_READ)) != 0)? \ + (((nr) >> _IOC_DIRSHIFT) & (_IOC_WRITE|_IOC_READ)): \ + (((nr) >> _IOC_DIRSHIFT) & _IOC_DIRMASK) ) +#define _IOC_TYPE(nr) (((nr) >> _IOC_TYPESHIFT) & _IOC_TYPEMASK) +#define _IOC_NR(nr) (((nr) >> _IOC_NRSHIFT) & _IOC_NRMASK) +#define _IOC_SIZE(nr) \ + ((((((nr) >> _IOC_DIRSHIFT) & _IOC_DIRMASK) & (_IOC_WRITE|_IOC_READ)) == 0)? \ + 0: (((nr) >> _IOC_SIZESHIFT) & _IOC_XSIZEMASK)) + +/* ...and for the PCMCIA and sound. */ +#define IOC_IN (_IOC_WRITE << _IOC_DIRSHIFT) +#define IOC_OUT (_IOC_READ << _IOC_DIRSHIFT) +#define IOC_INOUT ((_IOC_WRITE|_IOC_READ) << _IOC_DIRSHIFT) +#define IOCSIZE_MASK (_IOC_XSIZEMASK << _IOC_SIZESHIFT) +#define IOCSIZE_SHIFT (_IOC_SIZESHIFT) + +#endif /* !(_SPARC_IOCTL_H) */ diff --git a/arch/sparc/include/uapi/asm/ioctls.h b/arch/sparc/include/uapi/asm/ioctls.h new file mode 100644 index 000000000..a8bbdf987 --- /dev/null +++ b/arch/sparc/include/uapi/asm/ioctls.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_ASM_SPARC_IOCTLS_H +#define _UAPI_ASM_SPARC_IOCTLS_H + +#include + +/* Big T */ +#define TCGETA 0x40125401 /* _IOR('T', 1, struct termio) */ +#define TCSETA 0x80125402 /* _IOW('T', 2, struct termio) */ +#define TCSETAW 0x80125403 /* _IOW('T', 3, struct termio) */ +#define TCSETAF 0x80125404 /* _IOW('T', 4, struct termio) */ +#define TCSBRK _IO('T', 5) +#define TCXONC _IO('T', 6) +#define TCFLSH _IO('T', 7) +#define TCGETS _IOR('T', 8, struct termios) +#define TCSETS _IOW('T', 9, struct termios) +#define TCSETSW _IOW('T', 10, struct termios) +#define TCSETSF _IOW('T', 11, struct termios) +#define TCGETS2 _IOR('T', 12, struct termios2) +#define TCSETS2 _IOW('T', 13, struct termios2) +#define TCSETSW2 _IOW('T', 14, struct termios2) +#define TCSETSF2 _IOW('T', 15, struct termios2) +#define TIOCGDEV _IOR('T',0x32, unsigned int) /* Get primary device node of /dev/console */ +#define TIOCVHANGUP _IO('T', 0x37) +#define TIOCGPKT _IOR('T', 0x38, int) /* Get packet mode state */ +#define TIOCGPTLCK _IOR('T', 0x39, int) /* Get Pty lock state */ +#define TIOCGEXCL _IOR('T', 0x40, int) /* Get exclusive mode state */ +#define TIOCGRS485 _IOR('T', 0x41, struct serial_rs485) +#define TIOCSRS485 _IOWR('T', 0x42, struct serial_rs485) +#define TIOCGISO7816 _IOR('T', 0x43, struct serial_iso7816) +#define TIOCSISO7816 _IOWR('T', 0x44, struct serial_iso7816) + +/* Note that all the ioctls that are not available in Linux have a + * double underscore on the front to: a) avoid some programs to + * think we support some ioctls under Linux (autoconfiguration stuff) + */ +/* Little t */ +#define TIOCGETD _IOR('t', 0, int) +#define TIOCSETD _IOW('t', 1, int) +#define __TIOCHPCL _IO('t', 2) /* SunOS Specific */ +#define __TIOCMODG _IOR('t', 3, int) /* SunOS Specific */ +#define __TIOCMODS _IOW('t', 4, int) /* SunOS Specific */ +#define __TIOCGETP _IOR('t', 8, struct sgttyb) /* SunOS Specific */ +#define __TIOCSETP _IOW('t', 9, struct sgttyb) /* SunOS Specific */ +#define __TIOCSETN _IOW('t', 10, struct sgttyb) /* SunOS Specific */ +#define TIOCEXCL _IO('t', 13) +#define TIOCNXCL _IO('t', 14) +#define __TIOCFLUSH _IOW('t', 16, int) /* SunOS Specific */ +#define __TIOCSETC _IOW('t', 17, struct tchars) /* SunOS Specific */ +#define __TIOCGETC _IOR('t', 18, struct tchars) /* SunOS Specific */ +#define __TIOCTCNTL _IOW('t', 32, int) /* SunOS Specific */ +#define __TIOCSIGNAL _IOW('t', 33, int) /* SunOS Specific */ +#define __TIOCSETX _IOW('t', 34, int) /* SunOS Specific */ +#define __TIOCGETX _IOR('t', 35, int) /* SunOS Specific */ +#define TIOCCONS _IO('t', 36) +#define TIOCGSOFTCAR _IOR('t', 100, int) +#define TIOCSSOFTCAR _IOW('t', 101, int) +#define __TIOCUCNTL _IOW('t', 102, int) /* SunOS Specific */ +#define TIOCSWINSZ _IOW('t', 103, struct winsize) +#define TIOCGWINSZ _IOR('t', 104, struct winsize) +#define __TIOCREMOTE _IOW('t', 105, int) /* SunOS Specific */ +#define TIOCMGET _IOR('t', 106, int) +#define TIOCMBIC _IOW('t', 107, int) +#define TIOCMBIS _IOW('t', 108, int) +#define TIOCMSET _IOW('t', 109, int) +#define TIOCSTART _IO('t', 110) +#define TIOCSTOP _IO('t', 111) +#define TIOCPKT _IOW('t', 112, int) +#define TIOCNOTTY _IO('t', 113) +#define TIOCSTI _IOW('t', 114, char) +#define TIOCOUTQ _IOR('t', 115, int) +#define __TIOCGLTC _IOR('t', 116, struct ltchars) /* SunOS Specific */ +#define __TIOCSLTC _IOW('t', 117, struct ltchars) /* SunOS Specific */ +/* 118 is the non-posix setpgrp tty ioctl */ +/* 119 is the non-posix getpgrp tty ioctl */ +#define __TIOCCDTR _IO('t', 120) /* SunOS Specific */ +#define __TIOCSDTR _IO('t', 121) /* SunOS Specific */ +#define TIOCCBRK _IO('t', 122) +#define TIOCSBRK _IO('t', 123) +#define __TIOCLGET _IOW('t', 124, int) /* SunOS Specific */ +#define __TIOCLSET _IOW('t', 125, int) /* SunOS Specific */ +#define __TIOCLBIC _IOW('t', 126, int) /* SunOS Specific */ +#define __TIOCLBIS _IOW('t', 127, int) /* SunOS Specific */ +#define __TIOCISPACE _IOR('t', 128, int) /* SunOS Specific */ +#define __TIOCISIZE _IOR('t', 129, int) /* SunOS Specific */ +#define TIOCSPGRP _IOW('t', 130, int) +#define TIOCGPGRP _IOR('t', 131, int) +#define TIOCSCTTY _IO('t', 132) +#define TIOCGSID _IOR('t', 133, int) +/* Get minor device of a pty master's FD -- Solaris equiv is ISPTM */ +#define TIOCGPTN _IOR('t', 134, unsigned int) /* Get Pty Number */ +#define TIOCSPTLCK _IOW('t', 135, int) /* Lock/unlock PTY */ +#define TIOCSIG _IOW('t', 136, int) /* Generate signal on Pty slave */ +#define TIOCGPTPEER _IO('t', 137) /* Safely open the slave */ + +/* Little f */ +#define FIOCLEX _IO('f', 1) +#define FIONCLEX _IO('f', 2) +#define FIOASYNC _IOW('f', 125, int) +#define FIONBIO _IOW('f', 126, int) +#define FIONREAD _IOR('f', 127, int) +#define TIOCINQ FIONREAD +#define FIOQSIZE _IOR('f', 128, loff_t) + +/* SCARY Rutgers local SunOS kernel hackery, perhaps I will support it + * someday. This is completely bogus, I know... + */ +#define __TCGETSTAT _IO('T', 200) /* Rutgers specific */ +#define __TCSETSTAT _IO('T', 201) /* Rutgers specific */ + +/* Linux specific, no SunOS equivalent. */ +#define TIOCLINUX 0x541C +#define TIOCGSERIAL 0x541E +#define TIOCSSERIAL 0x541F +#define TCSBRKP 0x5425 +#define TIOCSERCONFIG 0x5453 +#define TIOCSERGWILD 0x5454 +#define TIOCSERSWILD 0x5455 +#define TIOCGLCKTRMIOS 0x5456 +#define TIOCSLCKTRMIOS 0x5457 +#define TIOCSERGSTRUCT 0x5458 /* For debugging only */ +#define TIOCSERGETLSR 0x5459 /* Get line status register */ +#define TIOCSERGETMULTI 0x545A /* Get multiport config */ +#define TIOCSERSETMULTI 0x545B /* Set multiport config */ +#define TIOCMIWAIT 0x545C /* Wait for change on serial input line(s) */ +#define TIOCGICOUNT 0x545D /* Read serial port inline interrupt counts */ + +/* Kernel definitions */ + +/* Used for packet mode */ +#define TIOCPKT_DATA 0 +#define TIOCPKT_FLUSHREAD 1 +#define TIOCPKT_FLUSHWRITE 2 +#define TIOCPKT_STOP 4 +#define TIOCPKT_START 8 +#define TIOCPKT_NOSTOP 16 +#define TIOCPKT_DOSTOP 32 +#define TIOCPKT_IOCTL 64 + +#endif /* _UAPI_ASM_SPARC_IOCTLS_H */ diff --git a/arch/sparc/include/uapi/asm/ipcbuf.h b/arch/sparc/include/uapi/asm/ipcbuf.h new file mode 100644 index 000000000..0ea1240d2 --- /dev/null +++ b/arch/sparc/include/uapi/asm/ipcbuf.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __SPARC_IPCBUF_H +#define __SPARC_IPCBUF_H + +#include + +/* + * The ipc64_perm structure for sparc/sparc64 architecture. + * Note extra padding because this structure is passed back and forth + * between kernel and user space. + * + * Pad space is left for: + * - 32-bit seq + * - on sparc for 32 bit mode (it is 32 bit on sparc64) + * - 2 miscellaneous 64-bit values + */ + +struct ipc64_perm +{ + __kernel_key_t key; + __kernel_uid32_t uid; + __kernel_gid32_t gid; + __kernel_uid32_t cuid; + __kernel_gid32_t cgid; +#ifndef __arch64__ + unsigned short __pad0; +#endif + __kernel_mode_t mode; + unsigned short __pad1; + unsigned short seq; + unsigned long long __unused1; + unsigned long long __unused2; +}; + +#endif /* __SPARC_IPCBUF_H */ diff --git a/arch/sparc/include/uapi/asm/mman.h b/arch/sparc/include/uapi/asm/mman.h new file mode 100644 index 000000000..cec9f4109 --- /dev/null +++ b/arch/sparc/include/uapi/asm/mman.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI__SPARC_MMAN_H__ +#define _UAPI__SPARC_MMAN_H__ + +#include + +/* SunOS'ified... */ + +#define PROT_ADI 0x10 /* ADI enabled */ + +#define MAP_RENAME MAP_ANONYMOUS /* In SunOS terminology */ +#define MAP_NORESERVE 0x40 /* don't reserve swap pages */ +#define MAP_INHERIT 0x80 /* SunOS doesn't do this, but... */ +#define MAP_LOCKED 0x100 /* lock the mapping */ +#define _MAP_NEW 0x80000000 /* Binary compatibility is fun... */ + +#define MAP_GROWSDOWN 0x0200 /* stack-like segment */ +#define MAP_DENYWRITE 0x0800 /* ETXTBSY */ +#define MAP_EXECUTABLE 0x1000 /* mark it as an executable */ + +#define MCL_CURRENT 0x2000 /* lock all currently mapped pages */ +#define MCL_FUTURE 0x4000 /* lock all additions to address space */ +#define MCL_ONFAULT 0x8000 /* lock all pages that are faulted in */ + +#endif /* _UAPI__SPARC_MMAN_H__ */ diff --git a/arch/sparc/include/uapi/asm/msgbuf.h b/arch/sparc/include/uapi/asm/msgbuf.h new file mode 100644 index 000000000..0954552da --- /dev/null +++ b/arch/sparc/include/uapi/asm/msgbuf.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_MSGBUF_H +#define _SPARC_MSGBUF_H + +#include + +/* + * The msqid64_ds structure for sparc64 architecture. + * Note extra padding because this structure is passed back and forth + * between kernel and user space. + * + * Pad space is left for: + * - 2 miscellaneous 32-bit values + */ +struct msqid64_ds { + struct ipc64_perm msg_perm; +#if defined(__sparc__) && defined(__arch64__) + long msg_stime; /* last msgsnd time */ + long msg_rtime; /* last msgrcv time */ + long msg_ctime; /* last change time */ +#else + unsigned long msg_stime_high; + unsigned long msg_stime; /* last msgsnd time */ + unsigned long msg_rtime_high; + unsigned long msg_rtime; /* last msgrcv time */ + unsigned long msg_ctime_high; + unsigned long msg_ctime; /* last change time */ +#endif + unsigned long msg_cbytes; /* current number of bytes on queue */ + unsigned long msg_qnum; /* number of messages in queue */ + unsigned long msg_qbytes; /* max number of bytes on queue */ + __kernel_pid_t msg_lspid; /* pid of last msgsnd */ + __kernel_pid_t msg_lrpid; /* last receive pid */ + unsigned long __unused1; + unsigned long __unused2; +}; +#endif /* _SPARC_MSGBUF_H */ diff --git a/arch/sparc/include/uapi/asm/openpromio.h b/arch/sparc/include/uapi/asm/openpromio.h new file mode 100644 index 000000000..2a73ec77a --- /dev/null +++ b/arch/sparc/include/uapi/asm/openpromio.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_OPENPROMIO_H +#define _SPARC_OPENPROMIO_H + +#include +#include + +/* + * SunOS and Solaris /dev/openprom definitions. The ioctl values + * were chosen to be exactly equal to the SunOS equivalents. + */ + +struct openpromio { + unsigned int oprom_size; /* Actual size of the oprom_array. */ + char oprom_array[]; /* Holds property names and values. */ +}; + +#define OPROMMAXPARAM 4096 /* Maximum size of oprom_array. */ + +#define OPROMGETOPT 0x20004F01 +#define OPROMSETOPT 0x20004F02 +#define OPROMNXTOPT 0x20004F03 +#define OPROMSETOPT2 0x20004F04 +#define OPROMNEXT 0x20004F05 +#define OPROMCHILD 0x20004F06 +#define OPROMGETPROP 0x20004F07 +#define OPROMNXTPROP 0x20004F08 +#define OPROMU2P 0x20004F09 +#define OPROMGETCONS 0x20004F0A +#define OPROMGETFBNAME 0x20004F0B +#define OPROMGETBOOTARGS 0x20004F0C +/* Linux extensions */ /* Arguments in oprom_array: */ +#define OPROMSETCUR 0x20004FF0 /* int node - Sets current node */ +#define OPROMPCI2NODE 0x20004FF1 /* int pci_bus, pci_devfn - Sets current node to PCI device's node */ +#define OPROMPATH2NODE 0x20004FF2 /* char path[] - Set current node from fully qualified PROM path */ + +/* + * Return values from OPROMGETCONS: + */ + +#define OPROMCONS_NOT_WSCONS 0 +#define OPROMCONS_STDIN_IS_KBD 0x1 /* stdin device is kbd */ +#define OPROMCONS_STDOUT_IS_FB 0x2 /* stdout is a framebuffer */ +#define OPROMCONS_OPENPROM 0x4 /* supports openboot */ + + +/* + * NetBSD/OpenBSD /dev/openprom definitions. + */ + +struct opiocdesc +{ + int op_nodeid; /* PROM Node ID (value-result) */ + int op_namelen; /* Length of op_name. */ + char __user *op_name; /* Pointer to the property name. */ + int op_buflen; /* Length of op_buf (value-result) */ + char __user *op_buf; /* Pointer to buffer. */ +}; + +#define OPIOCGET _IOWR('O', 1, struct opiocdesc) +#define OPIOCSET _IOW('O', 2, struct opiocdesc) +#define OPIOCNEXTPROP _IOWR('O', 3, struct opiocdesc) +#define OPIOCGETOPTNODE _IOR('O', 4, int) +#define OPIOCGETNEXT _IOWR('O', 5, int) +#define OPIOCGETCHILD _IOWR('O', 6, int) + +#endif /* _SPARC_OPENPROMIO_H */ + diff --git a/arch/sparc/include/uapi/asm/oradax.h b/arch/sparc/include/uapi/asm/oradax.h new file mode 100644 index 000000000..0dace6905 --- /dev/null +++ b/arch/sparc/include/uapi/asm/oradax.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later WITH Linux-syscall-note */ +/* + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + */ + +/* + * Oracle DAX driver API definitions + */ + +#ifndef _ORADAX_H +#define _ORADAX_H + +#include + +#define CCB_KILL 0 +#define CCB_INFO 1 +#define CCB_DEQUEUE 2 + +struct dax_command { + __u16 command; /* CCB_KILL/INFO/DEQUEUE */ + __u16 ca_offset; /* offset into mmapped completion area */ +}; + +struct ccb_kill_result { + __u16 action; /* action taken to kill ccb */ +}; + +struct ccb_info_result { + __u16 state; /* state of enqueued ccb */ + __u16 inst_num; /* dax instance number of enqueued ccb */ + __u16 q_num; /* queue number of enqueued ccb */ + __u16 q_pos; /* ccb position in queue */ +}; + +struct ccb_exec_result { + __u64 status_data; /* additional status data (e.g. bad VA) */ + __u32 status; /* one of DAX_SUBMIT_* */ +}; + +union ccb_result { + struct ccb_exec_result exec; + struct ccb_info_result info; + struct ccb_kill_result kill; +}; + +#define DAX_MMAP_LEN (16 * 1024) +#define DAX_MAX_CCBS 15 +#define DAX_CCB_BUF_MAXLEN (DAX_MAX_CCBS * 64) +#define DAX_NAME "oradax" + +/* CCB_EXEC status */ +#define DAX_SUBMIT_OK 0 +#define DAX_SUBMIT_ERR_RETRY 1 +#define DAX_SUBMIT_ERR_WOULDBLOCK 2 +#define DAX_SUBMIT_ERR_BUSY 3 +#define DAX_SUBMIT_ERR_THR_INIT 4 +#define DAX_SUBMIT_ERR_ARG_INVAL 5 +#define DAX_SUBMIT_ERR_CCB_INVAL 6 +#define DAX_SUBMIT_ERR_NO_CA_AVAIL 7 +#define DAX_SUBMIT_ERR_CCB_ARR_MMU_MISS 8 +#define DAX_SUBMIT_ERR_NOMAP 9 +#define DAX_SUBMIT_ERR_NOACCESS 10 +#define DAX_SUBMIT_ERR_TOOMANY 11 +#define DAX_SUBMIT_ERR_UNAVAIL 12 +#define DAX_SUBMIT_ERR_INTERNAL 13 + +/* CCB_INFO states - must match HV_CCB_STATE_* definitions */ +#define DAX_CCB_COMPLETED 0 +#define DAX_CCB_ENQUEUED 1 +#define DAX_CCB_INPROGRESS 2 +#define DAX_CCB_NOTFOUND 3 + +/* CCB_KILL actions - must match HV_CCB_KILL_* definitions */ +#define DAX_KILL_COMPLETED 0 +#define DAX_KILL_DEQUEUED 1 +#define DAX_KILL_KILLED 2 +#define DAX_KILL_NOTFOUND 3 + +#endif /* _ORADAX_H */ diff --git a/arch/sparc/include/uapi/asm/param.h b/arch/sparc/include/uapi/asm/param.h new file mode 100644 index 000000000..057d7135e --- /dev/null +++ b/arch/sparc/include/uapi/asm/param.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _ASMSPARC_PARAM_H +#define _ASMSPARC_PARAM_H + +#define EXEC_PAGESIZE 8192 /* Thanks for sun4's we carry baggage... */ +#include + +#endif /* _ASMSPARC_PARAM_H */ diff --git a/arch/sparc/include/uapi/asm/perfctr.h b/arch/sparc/include/uapi/asm/perfctr.h new file mode 100644 index 000000000..316b837bc --- /dev/null +++ b/arch/sparc/include/uapi/asm/perfctr.h @@ -0,0 +1,167 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/*---------------------------------------- + PERFORMANCE INSTRUMENTATION + Guillaume Thouvenin 08/10/98 + David S. Miller 10/06/98 + ---------------------------------------*/ +#ifndef PERF_COUNTER_API +#define PERF_COUNTER_API + +/* sys_perfctr() interface. First arg is operation code + * from enumeration below. The meaning of further arguments + * are determined by the operation code. + * + * NOTE: This system call is no longer provided, use the perf_events + * infrastructure. + * + * Pointers which are passed by the user are pointers to 64-bit + * integers. + * + * Once enabled, performance counter state is retained until the + * process either exits or performs an exec. That is, performance + * counters remain enabled for fork/clone children. + */ +enum perfctr_opcode { + /* Enable UltraSparc performance counters, ARG0 is pointer + * to 64-bit accumulator for D0 counter in PIC, ARG1 is pointer + * to 64-bit accumulator for D1 counter. ARG2 is a pointer to + * the initial PCR register value to use. + */ + PERFCTR_ON, + + /* Disable UltraSparc performance counters. The PCR is written + * with zero and the user counter accumulator pointers and + * working PCR register value are forgotten. + */ + PERFCTR_OFF, + + /* Add current D0 and D1 PIC values into user pointers given + * in PERFCTR_ON operation. The PIC is cleared before returning. + */ + PERFCTR_READ, + + /* Clear the PIC register. */ + PERFCTR_CLRPIC, + + /* Begin using a new PCR value, the pointer to which is passed + * in ARG0. The PIC is also cleared after the new PCR value is + * written. + */ + PERFCTR_SETPCR, + + /* Store in pointer given in ARG0 the current PCR register value + * being used. + */ + PERFCTR_GETPCR +}; + +#define PRIV 0x00000001 +#define SYS 0x00000002 +#define USR 0x00000004 + +/* Pic.S0 Selection Bit Field Encoding, Ultra-I/II */ +#define CYCLE_CNT 0x00000000 +#define INSTR_CNT 0x00000010 +#define DISPATCH0_IC_MISS 0x00000020 +#define DISPATCH0_STOREBUF 0x00000030 +#define IC_REF 0x00000080 +#define DC_RD 0x00000090 +#define DC_WR 0x000000A0 +#define LOAD_USE 0x000000B0 +#define EC_REF 0x000000C0 +#define EC_WRITE_HIT_RDO 0x000000D0 +#define EC_SNOOP_INV 0x000000E0 +#define EC_RD_HIT 0x000000F0 + +/* Pic.S0 Selection Bit Field Encoding, Ultra-III */ +#define US3_CYCLE_CNT 0x00000000 +#define US3_INSTR_CNT 0x00000010 +#define US3_DISPATCH0_IC_MISS 0x00000020 +#define US3_DISPATCH0_BR_TGT 0x00000030 +#define US3_DISPATCH0_2ND_BR 0x00000040 +#define US3_RSTALL_STOREQ 0x00000050 +#define US3_RSTALL_IU_USE 0x00000060 +#define US3_IC_REF 0x00000080 +#define US3_DC_RD 0x00000090 +#define US3_DC_WR 0x000000a0 +#define US3_EC_REF 0x000000c0 +#define US3_EC_WR_HIT_RTO 0x000000d0 +#define US3_EC_SNOOP_INV 0x000000e0 +#define US3_EC_RD_MISS 0x000000f0 +#define US3_PC_PORT0_RD 0x00000100 +#define US3_SI_SNOOP 0x00000110 +#define US3_SI_CIQ_FLOW 0x00000120 +#define US3_SI_OWNED 0x00000130 +#define US3_SW_COUNT_0 0x00000140 +#define US3_IU_BR_MISS_TAKEN 0x00000150 +#define US3_IU_BR_COUNT_TAKEN 0x00000160 +#define US3_DISP_RS_MISPRED 0x00000170 +#define US3_FA_PIPE_COMPL 0x00000180 +#define US3_MC_READS_0 0x00000200 +#define US3_MC_READS_1 0x00000210 +#define US3_MC_READS_2 0x00000220 +#define US3_MC_READS_3 0x00000230 +#define US3_MC_STALLS_0 0x00000240 +#define US3_MC_STALLS_2 0x00000250 + +/* Pic.S1 Selection Bit Field Encoding, Ultra-I/II */ +#define CYCLE_CNT_D1 0x00000000 +#define INSTR_CNT_D1 0x00000800 +#define DISPATCH0_IC_MISPRED 0x00001000 +#define DISPATCH0_FP_USE 0x00001800 +#define IC_HIT 0x00004000 +#define DC_RD_HIT 0x00004800 +#define DC_WR_HIT 0x00005000 +#define LOAD_USE_RAW 0x00005800 +#define EC_HIT 0x00006000 +#define EC_WB 0x00006800 +#define EC_SNOOP_CB 0x00007000 +#define EC_IT_HIT 0x00007800 + +/* Pic.S1 Selection Bit Field Encoding, Ultra-III */ +#define US3_CYCLE_CNT_D1 0x00000000 +#define US3_INSTR_CNT_D1 0x00000800 +#define US3_DISPATCH0_MISPRED 0x00001000 +#define US3_IC_MISS_CANCELLED 0x00001800 +#define US3_RE_ENDIAN_MISS 0x00002000 +#define US3_RE_FPU_BYPASS 0x00002800 +#define US3_RE_DC_MISS 0x00003000 +#define US3_RE_EC_MISS 0x00003800 +#define US3_IC_MISS 0x00004000 +#define US3_DC_RD_MISS 0x00004800 +#define US3_DC_WR_MISS 0x00005000 +#define US3_RSTALL_FP_USE 0x00005800 +#define US3_EC_MISSES 0x00006000 +#define US3_EC_WB 0x00006800 +#define US3_EC_SNOOP_CB 0x00007000 +#define US3_EC_IC_MISS 0x00007800 +#define US3_RE_PC_MISS 0x00008000 +#define US3_ITLB_MISS 0x00008800 +#define US3_DTLB_MISS 0x00009000 +#define US3_WC_MISS 0x00009800 +#define US3_WC_SNOOP_CB 0x0000a000 +#define US3_WC_SCRUBBED 0x0000a800 +#define US3_WC_WB_WO_READ 0x0000b000 +#define US3_PC_SOFT_HIT 0x0000c000 +#define US3_PC_SNOOP_INV 0x0000c800 +#define US3_PC_HARD_HIT 0x0000d000 +#define US3_PC_PORT1_RD 0x0000d800 +#define US3_SW_COUNT_1 0x0000e000 +#define US3_IU_STAT_BR_MIS_UNTAKEN 0x0000e800 +#define US3_IU_STAT_BR_COUNT_UNTAKEN 0x0000f000 +#define US3_PC_MS_MISSES 0x0000f800 +#define US3_MC_WRITES_0 0x00010800 +#define US3_MC_WRITES_1 0x00011000 +#define US3_MC_WRITES_2 0x00011800 +#define US3_MC_WRITES_3 0x00012000 +#define US3_MC_STALLS_1 0x00012800 +#define US3_MC_STALLS_3 0x00013000 +#define US3_RE_RAW_MISS 0x00013800 +#define US3_FM_PIPE_COMPLETION 0x00014000 + +struct vcounter_struct { + unsigned long long vcnt0; + unsigned long long vcnt1; +}; + +#endif /* !(PERF_COUNTER_API) */ diff --git a/arch/sparc/include/uapi/asm/poll.h b/arch/sparc/include/uapi/asm/poll.h new file mode 100644 index 000000000..72356c999 --- /dev/null +++ b/arch/sparc/include/uapi/asm/poll.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __SPARC_POLL_H +#define __SPARC_POLL_H + +#define POLLWRNORM POLLOUT +#define POLLWRBAND 256 +#define POLLMSG 512 +#define POLLREMOVE 1024 +#define POLLRDHUP 2048 + +#include + +#endif diff --git a/arch/sparc/include/uapi/asm/posix_types.h b/arch/sparc/include/uapi/asm/posix_types.h new file mode 100644 index 000000000..f139e0048 --- /dev/null +++ b/arch/sparc/include/uapi/asm/posix_types.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * This file is generally used by user-level software, so you need to + * be a little careful about namespace pollution etc. Also, we cannot + * assume GCC is being used. + */ + +#ifndef __SPARC_POSIX_TYPES_H +#define __SPARC_POSIX_TYPES_H + +#if defined(__sparc__) && defined(__arch64__) +/* sparc 64 bit */ + +typedef unsigned short __kernel_old_uid_t; +typedef unsigned short __kernel_old_gid_t; +#define __kernel_old_uid_t __kernel_old_uid_t + +/* Note this piece of asymmetry from the v9 ABI. */ +typedef int __kernel_suseconds_t; +#define __kernel_suseconds_t __kernel_suseconds_t + +typedef long __kernel_long_t; +typedef unsigned long __kernel_ulong_t; +#define __kernel_long_t __kernel_long_t + +struct __kernel_old_timeval { + __kernel_long_t tv_sec; + __kernel_suseconds_t tv_usec; +}; +#define __kernel_old_timeval __kernel_old_timeval + +#else +/* sparc 32 bit */ + +typedef unsigned int __kernel_size_t; +typedef int __kernel_ssize_t; +typedef long int __kernel_ptrdiff_t; +#define __kernel_size_t __kernel_size_t + +typedef unsigned short __kernel_ipc_pid_t; +#define __kernel_ipc_pid_t __kernel_ipc_pid_t + +typedef unsigned short __kernel_uid_t; +typedef unsigned short __kernel_gid_t; +#define __kernel_uid_t __kernel_uid_t + +typedef unsigned short __kernel_mode_t; +#define __kernel_mode_t __kernel_mode_t + +typedef long __kernel_daddr_t; +#define __kernel_daddr_t __kernel_daddr_t + +typedef unsigned short __kernel_old_dev_t; +#define __kernel_old_dev_t __kernel_old_dev_t + +#endif /* defined(__sparc__) && defined(__arch64__) */ + +#include + +#endif /* __SPARC_POSIX_TYPES_H */ diff --git a/arch/sparc/include/uapi/asm/psr.h b/arch/sparc/include/uapi/asm/psr.h new file mode 100644 index 000000000..e41f65f0a --- /dev/null +++ b/arch/sparc/include/uapi/asm/psr.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * psr.h: This file holds the macros for masking off various parts of + * the processor status register on the Sparc. This is valid + * for Version 8. On the V9 this is renamed to the PSTATE + * register and its members are accessed as fields like + * PSTATE.PRIV for the current CPU privilege level. + * + * Copyright (C) 1994 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _UAPI__LINUX_SPARC_PSR_H +#define _UAPI__LINUX_SPARC_PSR_H + +/* The Sparc PSR fields are laid out as the following: + * + * ------------------------------------------------------------------------ + * | impl | vers | icc | resv | EC | EF | PIL | S | PS | ET | CWP | + * | 31-28 | 27-24 | 23-20 | 19-14 | 13 | 12 | 11-8 | 7 | 6 | 5 | 4-0 | + * ------------------------------------------------------------------------ + */ +#define PSR_CWP 0x0000001f /* current window pointer */ +#define PSR_ET 0x00000020 /* enable traps field */ +#define PSR_PS 0x00000040 /* previous privilege level */ +#define PSR_S 0x00000080 /* current privilege level */ +#define PSR_PIL 0x00000f00 /* processor interrupt level */ +#define PSR_EF 0x00001000 /* enable floating point */ +#define PSR_EC 0x00002000 /* enable co-processor */ +#define PSR_SYSCALL 0x00004000 /* inside of a syscall */ +#define PSR_LE 0x00008000 /* SuperSparcII little-endian */ +#define PSR_ICC 0x00f00000 /* integer condition codes */ +#define PSR_C 0x00100000 /* carry bit */ +#define PSR_V 0x00200000 /* overflow bit */ +#define PSR_Z 0x00400000 /* zero bit */ +#define PSR_N 0x00800000 /* negative bit */ +#define PSR_VERS 0x0f000000 /* cpu-version field */ +#define PSR_IMPL 0xf0000000 /* cpu-implementation field */ + +#define PSR_VERS_SHIFT 24 +#define PSR_IMPL_SHIFT 28 +#define PSR_VERS_SHIFTED_MASK 0xf +#define PSR_IMPL_SHIFTED_MASK 0xf + +#define PSR_IMPL_TI 0x4 +#define PSR_IMPL_LEON 0xf + + +#endif /* _UAPI__LINUX_SPARC_PSR_H */ diff --git a/arch/sparc/include/uapi/asm/psrcompat.h b/arch/sparc/include/uapi/asm/psrcompat.h new file mode 100644 index 000000000..1eaffbe0d --- /dev/null +++ b/arch/sparc/include/uapi/asm/psrcompat.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC64_PSRCOMPAT_H +#define _SPARC64_PSRCOMPAT_H + +#include + +/* Old 32-bit PSR fields for the compatibility conversion code. */ +#define PSR_CWP 0x0000001f /* current window pointer */ +#define PSR_ET 0x00000020 /* enable traps field */ +#define PSR_PS 0x00000040 /* previous privilege level */ +#define PSR_S 0x00000080 /* current privilege level */ +#define PSR_PIL 0x00000f00 /* processor interrupt level */ +#define PSR_EF 0x00001000 /* enable floating point */ +#define PSR_EC 0x00002000 /* enable co-processor */ +#define PSR_SYSCALL 0x00004000 /* inside of a syscall */ +#define PSR_LE 0x00008000 /* SuperSparcII little-endian */ +#define PSR_ICC 0x00f00000 /* integer condition codes */ +#define PSR_C 0x00100000 /* carry bit */ +#define PSR_V 0x00200000 /* overflow bit */ +#define PSR_Z 0x00400000 /* zero bit */ +#define PSR_N 0x00800000 /* negative bit */ +#define PSR_VERS 0x0f000000 /* cpu-version field */ +#define PSR_IMPL 0xf0000000 /* cpu-implementation field */ + +#define PSR_V8PLUS 0xff000000 /* fake impl/ver, meaning a 64bit CPU is present */ +#define PSR_XCC 0x000f0000 /* if PSR_V8PLUS, this is %xcc */ + +static inline unsigned int tstate_to_psr(unsigned long tstate) +{ + return ((tstate & TSTATE_CWP) | + PSR_S | + ((tstate & TSTATE_ICC) >> 12) | + ((tstate & TSTATE_XCC) >> 20) | + ((tstate & TSTATE_SYSCALL) ? PSR_SYSCALL : 0) | + PSR_V8PLUS); +} + +static inline unsigned long psr_to_tstate_icc(unsigned int psr) +{ + unsigned long tstate = ((unsigned long)(psr & PSR_ICC)) << 12; + if ((psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS) + tstate |= ((unsigned long)(psr & PSR_XCC)) << 20; + return tstate; +} + +#endif /* !(_SPARC64_PSRCOMPAT_H) */ diff --git a/arch/sparc/include/uapi/asm/pstate.h b/arch/sparc/include/uapi/asm/pstate.h new file mode 100644 index 000000000..ceca96e68 --- /dev/null +++ b/arch/sparc/include/uapi/asm/pstate.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC64_PSTATE_H +#define _SPARC64_PSTATE_H + +#include + +/* The V9 PSTATE Register (with SpitFire extensions). + * + * ----------------------------------------------------------------------- + * | Resv | IG | MG | CLE | TLE | MM | RED | PEF | AM | PRIV | IE | AG | + * ----------------------------------------------------------------------- + * 63 12 11 10 9 8 7 6 5 4 3 2 1 0 + */ +/* IG on V9 conflicts with MCDE on M7. PSTATE_MCDE will only be used on + * processors that support ADI which do not use IG, hence there is no + * functional conflict + */ +#define PSTATE_IG _AC(0x0000000000000800,UL) /* Interrupt Globals. */ +#define PSTATE_MCDE _AC(0x0000000000000800,UL) /* MCD Enable */ +#define PSTATE_MG _AC(0x0000000000000400,UL) /* MMU Globals. */ +#define PSTATE_CLE _AC(0x0000000000000200,UL) /* Current Little Endian.*/ +#define PSTATE_TLE _AC(0x0000000000000100,UL) /* Trap Little Endian. */ +#define PSTATE_MM _AC(0x00000000000000c0,UL) /* Memory Model. */ +#define PSTATE_TSO _AC(0x0000000000000000,UL) /* MM: TotalStoreOrder */ +#define PSTATE_PSO _AC(0x0000000000000040,UL) /* MM: PartialStoreOrder */ +#define PSTATE_RMO _AC(0x0000000000000080,UL) /* MM: RelaxedMemoryOrder*/ +#define PSTATE_RED _AC(0x0000000000000020,UL) /* Reset Error Debug. */ +#define PSTATE_PEF _AC(0x0000000000000010,UL) /* Floating Point Enable.*/ +#define PSTATE_AM _AC(0x0000000000000008,UL) /* Address Mask. */ +#define PSTATE_PRIV _AC(0x0000000000000004,UL) /* Privilege. */ +#define PSTATE_IE _AC(0x0000000000000002,UL) /* Interrupt Enable. */ +#define PSTATE_AG _AC(0x0000000000000001,UL) /* Alternate Globals. */ + +/* The V9 TSTATE Register (with SpitFire and Linux extensions). + * + * --------------------------------------------------------------------- + * | Resv | GL | CCR | ASI | %pil | PSTATE | Resv | CWP | + * --------------------------------------------------------------------- + * 63 43 42 40 39 32 31 24 23 20 19 8 7 5 4 0 + */ +#define TSTATE_GL _AC(0x0000070000000000,UL) /* Global reg level */ +#define TSTATE_CCR _AC(0x000000ff00000000,UL) /* Condition Codes. */ +#define TSTATE_XCC _AC(0x000000f000000000,UL) /* Condition Codes. */ +#define TSTATE_XNEG _AC(0x0000008000000000,UL) /* %xcc Negative. */ +#define TSTATE_XZERO _AC(0x0000004000000000,UL) /* %xcc Zero. */ +#define TSTATE_XOVFL _AC(0x0000002000000000,UL) /* %xcc Overflow. */ +#define TSTATE_XCARRY _AC(0x0000001000000000,UL) /* %xcc Carry. */ +#define TSTATE_ICC _AC(0x0000000f00000000,UL) /* Condition Codes. */ +#define TSTATE_INEG _AC(0x0000000800000000,UL) /* %icc Negative. */ +#define TSTATE_IZERO _AC(0x0000000400000000,UL) /* %icc Zero. */ +#define TSTATE_IOVFL _AC(0x0000000200000000,UL) /* %icc Overflow. */ +#define TSTATE_ICARRY _AC(0x0000000100000000,UL) /* %icc Carry. */ +#define TSTATE_ASI _AC(0x00000000ff000000,UL) /* AddrSpace ID. */ +#define TSTATE_PIL _AC(0x0000000000f00000,UL) /* %pil (Linux traps)*/ +#define TSTATE_PSTATE _AC(0x00000000000fff00,UL) /* PSTATE. */ +/* IG on V9 conflicts with MCDE on M7. TSTATE_MCDE will only be used on + * processors that support ADI which do not support IG, hence there is + * no functional conflict + */ +#define TSTATE_IG _AC(0x0000000000080000,UL) /* Interrupt Globals.*/ +#define TSTATE_MCDE _AC(0x0000000000080000,UL) /* MCD enable. */ +#define TSTATE_MG _AC(0x0000000000040000,UL) /* MMU Globals. */ +#define TSTATE_CLE _AC(0x0000000000020000,UL) /* CurrLittleEndian. */ +#define TSTATE_TLE _AC(0x0000000000010000,UL) /* TrapLittleEndian. */ +#define TSTATE_MM _AC(0x000000000000c000,UL) /* Memory Model. */ +#define TSTATE_TSO _AC(0x0000000000000000,UL) /* MM: TSO */ +#define TSTATE_PSO _AC(0x0000000000004000,UL) /* MM: PSO */ +#define TSTATE_RMO _AC(0x0000000000008000,UL) /* MM: RMO */ +#define TSTATE_RED _AC(0x0000000000002000,UL) /* Reset Error Debug.*/ +#define TSTATE_PEF _AC(0x0000000000001000,UL) /* FPU Enable. */ +#define TSTATE_AM _AC(0x0000000000000800,UL) /* Address Mask. */ +#define TSTATE_PRIV _AC(0x0000000000000400,UL) /* Privilege. */ +#define TSTATE_IE _AC(0x0000000000000200,UL) /* Interrupt Enable. */ +#define TSTATE_AG _AC(0x0000000000000100,UL) /* Alternate Globals.*/ +#define TSTATE_SYSCALL _AC(0x0000000000000020,UL) /* in syscall trap */ +#define TSTATE_CWP _AC(0x000000000000001f,UL) /* Curr Win-Pointer. */ + +/* Floating-Point Registers State Register. + * + * -------------------------------- + * | Resv | FEF | DU | DL | + * -------------------------------- + * 63 3 2 1 0 + */ +#define FPRS_FEF _AC(0x0000000000000004,UL) /* FPU Enable. */ +#define FPRS_DU _AC(0x0000000000000002,UL) /* Dirty Upper. */ +#define FPRS_DL _AC(0x0000000000000001,UL) /* Dirty Lower. */ + +/* Version Register. + * + * ------------------------------------------------------ + * | MANUF | IMPL | MASK | Resv | MAXTL | Resv | MAXWIN | + * ------------------------------------------------------ + * 63 48 47 32 31 24 23 16 15 8 7 5 4 0 + */ +#define VERS_MANUF _AC(0xffff000000000000,UL) /* Manufacturer. */ +#define VERS_IMPL _AC(0x0000ffff00000000,UL) /* Implementation. */ +#define VERS_MASK _AC(0x00000000ff000000,UL) /* Mask Set Revision.*/ +#define VERS_MAXTL _AC(0x000000000000ff00,UL) /* Max Trap Level. */ +#define VERS_MAXWIN _AC(0x000000000000001f,UL) /* Max RegWindow Idx.*/ + +/* Compatibility Feature Register (%asr26), SPARC-T4 and later */ +#define CFR_AES _AC(0x0000000000000001,UL) /* Supports AES opcodes */ +#define CFR_DES _AC(0x0000000000000002,UL) /* Supports DES opcodes */ +#define CFR_KASUMI _AC(0x0000000000000004,UL) /* Supports KASUMI opcodes */ +#define CFR_CAMELLIA _AC(0x0000000000000008,UL) /* Supports CAMELLIA opcodes*/ +#define CFR_MD5 _AC(0x0000000000000010,UL) /* Supports MD5 opcodes */ +#define CFR_SHA1 _AC(0x0000000000000020,UL) /* Supports SHA1 opcodes */ +#define CFR_SHA256 _AC(0x0000000000000040,UL) /* Supports SHA256 opcodes */ +#define CFR_SHA512 _AC(0x0000000000000080,UL) /* Supports SHA512 opcodes */ +#define CFR_MPMUL _AC(0x0000000000000100,UL) /* Supports MPMUL opcodes */ +#define CFR_MONTMUL _AC(0x0000000000000200,UL) /* Supports MONTMUL opcodes */ +#define CFR_MONTSQR _AC(0x0000000000000400,UL) /* Supports MONTSQR opcodes */ +#define CFR_CRC32C _AC(0x0000000000000800,UL) /* Supports CRC32C opcodes */ + +#endif /* !(_SPARC64_PSTATE_H) */ diff --git a/arch/sparc/include/uapi/asm/ptrace.h b/arch/sparc/include/uapi/asm/ptrace.h new file mode 100644 index 000000000..2eb677f4e --- /dev/null +++ b/arch/sparc/include/uapi/asm/ptrace.h @@ -0,0 +1,353 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI__SPARC_PTRACE_H +#define _UAPI__SPARC_PTRACE_H + +#if defined(__sparc__) && defined(__arch64__) +/* 64 bit sparc */ +#include + +/* This struct defines the way the registers are stored on the + * stack during a system call and basically all traps. + */ + +/* This magic value must have the low 9 bits clear, + * as that is where we encode the %tt value, see below. + */ +#define PT_REGS_MAGIC 0x57ac6c00 + +#ifndef __ASSEMBLER__ + +#include + +struct pt_regs { + unsigned long u_regs[16]; /* globals and ins */ + unsigned long tstate; + unsigned long tpc; + unsigned long tnpc; + unsigned int y; + + /* We encode a magic number, PT_REGS_MAGIC, along + * with the %tt (trap type) register value at trap + * entry time. The magic number allows us to identify + * accurately a trap stack frame in the stack + * unwinder, and the %tt value allows us to test + * things like "in a system call" etc. for an arbitray + * process. + * + * The PT_REGS_MAGIC is chosen such that it can be + * loaded completely using just a sethi instruction. + */ + unsigned int magic; +}; + +struct pt_regs32 { + unsigned int psr; + unsigned int pc; + unsigned int npc; + unsigned int y; + unsigned int u_regs[16]; /* globals and ins */ +}; + +/* A V9 register window */ +struct reg_window { + unsigned long locals[8]; + unsigned long ins[8]; +}; + +/* A 32-bit register window. */ +struct reg_window32 { + unsigned int locals[8]; + unsigned int ins[8]; +}; + +/* A V9 Sparc stack frame */ +struct sparc_stackf { + unsigned long locals[8]; + unsigned long ins[6]; + struct sparc_stackf *fp; + unsigned long callers_pc; + char *structptr; + unsigned long xargs[6]; + unsigned long xxargs[1]; +}; + +/* A 32-bit Sparc stack frame */ +struct sparc_stackf32 { + unsigned int locals[8]; + unsigned int ins[6]; + unsigned int fp; + unsigned int callers_pc; + unsigned int structptr; + unsigned int xargs[6]; + unsigned int xxargs[1]; +}; + +struct sparc_trapf { + unsigned long locals[8]; + unsigned long ins[8]; + unsigned long _unused; + struct pt_regs *regs; +}; +#endif /* (!__ASSEMBLER__) */ +#else +/* 32 bit sparc */ + +#include + +/* This struct defines the way the registers are stored on the + * stack during a system call and basically all traps. + */ +#ifndef __ASSEMBLER__ + +#include + +struct pt_regs { + unsigned long psr; + unsigned long pc; + unsigned long npc; + unsigned long y; + unsigned long u_regs[16]; /* globals and ins */ +}; + +/* A 32-bit register window. */ +struct reg_window32 { + unsigned long locals[8]; + unsigned long ins[8]; +}; + +/* A Sparc stack frame */ +struct sparc_stackf { + unsigned long locals[8]; + unsigned long ins[6]; + struct sparc_stackf *fp; + unsigned long callers_pc; + char *structptr; + unsigned long xargs[6]; + unsigned long xxargs[1]; +}; +#endif /* (!__ASSEMBLER__) */ + +#endif /* (defined(__sparc__) && defined(__arch64__))*/ + +#ifndef __ASSEMBLER__ + +#define TRACEREG_SZ sizeof(struct pt_regs) +#define STACKFRAME_SZ sizeof(struct sparc_stackf) + +#define TRACEREG32_SZ sizeof(struct pt_regs32) +#define STACKFRAME32_SZ sizeof(struct sparc_stackf32) + +#endif /* (!__ASSEMBLER__) */ + +#define UREG_G0 0 +#define UREG_G1 1 +#define UREG_G2 2 +#define UREG_G3 3 +#define UREG_G4 4 +#define UREG_G5 5 +#define UREG_G6 6 +#define UREG_G7 7 +#define UREG_I0 8 +#define UREG_I1 9 +#define UREG_I2 10 +#define UREG_I3 11 +#define UREG_I4 12 +#define UREG_I5 13 +#define UREG_I6 14 +#define UREG_I7 15 +#define UREG_FP UREG_I6 +#define UREG_RETPC UREG_I7 + +#if defined(__sparc__) && defined(__arch64__) +/* 64 bit sparc */ + +#ifndef __ASSEMBLER__ + + +#else /* __ASSEMBLER__ */ +/* For assembly code. */ +#define TRACEREG_SZ 0xa0 +#define STACKFRAME_SZ 0xc0 + +#define TRACEREG32_SZ 0x50 +#define STACKFRAME32_SZ 0x60 +#endif /* __ASSEMBLER__ */ + +#else /* (defined(__sparc__) && defined(__arch64__)) */ + +/* 32 bit sparc */ + +#ifndef __ASSEMBLER__ + + +#else /* (!__ASSEMBLER__) */ +/* For assembly code. */ +#define TRACEREG_SZ 0x50 +#define STACKFRAME_SZ 0x60 +#endif /* (!__ASSEMBLER__) */ + +#endif /* (defined(__sparc__) && defined(__arch64__)) */ + + +/* These are for pt_regs. */ +#define PT_V9_G0 0x00 +#define PT_V9_G1 0x08 +#define PT_V9_G2 0x10 +#define PT_V9_G3 0x18 +#define PT_V9_G4 0x20 +#define PT_V9_G5 0x28 +#define PT_V9_G6 0x30 +#define PT_V9_G7 0x38 +#define PT_V9_I0 0x40 +#define PT_V9_I1 0x48 +#define PT_V9_I2 0x50 +#define PT_V9_I3 0x58 +#define PT_V9_I4 0x60 +#define PT_V9_I5 0x68 +#define PT_V9_I6 0x70 +#define PT_V9_FP PT_V9_I6 +#define PT_V9_I7 0x78 +#define PT_V9_TSTATE 0x80 +#define PT_V9_TPC 0x88 +#define PT_V9_TNPC 0x90 +#define PT_V9_Y 0x98 +#define PT_V9_MAGIC 0x9c +#define PT_TSTATE PT_V9_TSTATE +#define PT_TPC PT_V9_TPC +#define PT_TNPC PT_V9_TNPC + +/* These for pt_regs32. */ +#define PT_PSR 0x0 +#define PT_PC 0x4 +#define PT_NPC 0x8 +#define PT_Y 0xc +#define PT_G0 0x10 +#define PT_WIM PT_G0 +#define PT_G1 0x14 +#define PT_G2 0x18 +#define PT_G3 0x1c +#define PT_G4 0x20 +#define PT_G5 0x24 +#define PT_G6 0x28 +#define PT_G7 0x2c +#define PT_I0 0x30 +#define PT_I1 0x34 +#define PT_I2 0x38 +#define PT_I3 0x3c +#define PT_I4 0x40 +#define PT_I5 0x44 +#define PT_I6 0x48 +#define PT_FP PT_I6 +#define PT_I7 0x4c + +/* Reg_window offsets */ +#define RW_V9_L0 0x00 +#define RW_V9_L1 0x08 +#define RW_V9_L2 0x10 +#define RW_V9_L3 0x18 +#define RW_V9_L4 0x20 +#define RW_V9_L5 0x28 +#define RW_V9_L6 0x30 +#define RW_V9_L7 0x38 +#define RW_V9_I0 0x40 +#define RW_V9_I1 0x48 +#define RW_V9_I2 0x50 +#define RW_V9_I3 0x58 +#define RW_V9_I4 0x60 +#define RW_V9_I5 0x68 +#define RW_V9_I6 0x70 +#define RW_V9_I7 0x78 + +#define RW_L0 0x00 +#define RW_L1 0x04 +#define RW_L2 0x08 +#define RW_L3 0x0c +#define RW_L4 0x10 +#define RW_L5 0x14 +#define RW_L6 0x18 +#define RW_L7 0x1c +#define RW_I0 0x20 +#define RW_I1 0x24 +#define RW_I2 0x28 +#define RW_I3 0x2c +#define RW_I4 0x30 +#define RW_I5 0x34 +#define RW_I6 0x38 +#define RW_I7 0x3c + +/* Stack_frame offsets */ +#define SF_V9_L0 0x00 +#define SF_V9_L1 0x08 +#define SF_V9_L2 0x10 +#define SF_V9_L3 0x18 +#define SF_V9_L4 0x20 +#define SF_V9_L5 0x28 +#define SF_V9_L6 0x30 +#define SF_V9_L7 0x38 +#define SF_V9_I0 0x40 +#define SF_V9_I1 0x48 +#define SF_V9_I2 0x50 +#define SF_V9_I3 0x58 +#define SF_V9_I4 0x60 +#define SF_V9_I5 0x68 +#define SF_V9_FP 0x70 +#define SF_V9_PC 0x78 +#define SF_V9_RETP 0x80 +#define SF_V9_XARG0 0x88 +#define SF_V9_XARG1 0x90 +#define SF_V9_XARG2 0x98 +#define SF_V9_XARG3 0xa0 +#define SF_V9_XARG4 0xa8 +#define SF_V9_XARG5 0xb0 +#define SF_V9_XXARG 0xb8 + +#define SF_L0 0x00 +#define SF_L1 0x04 +#define SF_L2 0x08 +#define SF_L3 0x0c +#define SF_L4 0x10 +#define SF_L5 0x14 +#define SF_L6 0x18 +#define SF_L7 0x1c +#define SF_I0 0x20 +#define SF_I1 0x24 +#define SF_I2 0x28 +#define SF_I3 0x2c +#define SF_I4 0x30 +#define SF_I5 0x34 +#define SF_FP 0x38 +#define SF_PC 0x3c +#define SF_RETP 0x40 +#define SF_XARG0 0x44 +#define SF_XARG1 0x48 +#define SF_XARG2 0x4c +#define SF_XARG3 0x50 +#define SF_XARG4 0x54 +#define SF_XARG5 0x58 +#define SF_XXARG 0x5c + + +/* Stuff for the ptrace system call */ +#define PTRACE_SPARC_DETACH 11 +#define PTRACE_GETREGS 12 +#define PTRACE_SETREGS 13 +#define PTRACE_GETFPREGS 14 +#define PTRACE_SETFPREGS 15 +#define PTRACE_READDATA 16 +#define PTRACE_WRITEDATA 17 +#define PTRACE_READTEXT 18 +#define PTRACE_WRITETEXT 19 +#define PTRACE_GETFPAREGS 20 +#define PTRACE_SETFPAREGS 21 + +/* There are for debugging 64-bit processes, either from a 32 or 64 bit + * parent. Thus their complements are for debugging 32-bit processes only. + */ + +#define PTRACE_GETREGS64 22 +#define PTRACE_SETREGS64 23 +/* PTRACE_SYSCALL is 24 */ +#define PTRACE_GETFPREGS64 25 +#define PTRACE_SETFPREGS64 26 + +#endif /* _UAPI__SPARC_PTRACE_H */ diff --git a/arch/sparc/include/uapi/asm/resource.h b/arch/sparc/include/uapi/asm/resource.h new file mode 100644 index 000000000..cbe2de778 --- /dev/null +++ b/arch/sparc/include/uapi/asm/resource.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * resource.h: Resource definitions. + * + * Copyright (C) 1995,1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _SPARC_RESOURCE_H +#define _SPARC_RESOURCE_H + +/* + * These two resource limit IDs have a Sparc/Linux-specific ordering, + * the rest comes from the generic header: + */ +#define RLIMIT_NOFILE 6 /* max number of open files */ +#define RLIMIT_NPROC 7 /* max number of processes */ + +#if defined(__sparc__) && defined(__arch64__) +/* Use generic version */ +#else +/* + * SuS says limits have to be unsigned. + * We make this unsigned, but keep the + * old value for compatibility: + */ +#define RLIM_INFINITY 0x7fffffff +#endif + +#include + +#endif /* !(_SPARC_RESOURCE_H) */ diff --git a/arch/sparc/include/uapi/asm/sembuf.h b/arch/sparc/include/uapi/asm/sembuf.h new file mode 100644 index 000000000..10621d066 --- /dev/null +++ b/arch/sparc/include/uapi/asm/sembuf.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_SEMBUF_H +#define _SPARC_SEMBUF_H + +#include + +/* + * The semid64_ds structure for sparc architecture. + * Note extra padding because this structure is passed back and forth + * between kernel and user space. + * + * Pad space is left for: + * - 2 miscellaneous 32-bit values + */ + +struct semid64_ds { + struct ipc64_perm sem_perm; /* permissions .. see ipc.h */ +#if defined(__sparc__) && defined(__arch64__) + long sem_otime; /* last semop time */ + long sem_ctime; /* last change time */ +#else + unsigned long sem_otime_high; + unsigned long sem_otime; /* last semop time */ + unsigned long sem_ctime_high; + unsigned long sem_ctime; /* last change time */ +#endif + unsigned long sem_nsems; /* no. of semaphores in array */ + unsigned long __unused1; + unsigned long __unused2; +}; + +#endif /* _SPARC64_SEMBUF_H */ diff --git a/arch/sparc/include/uapi/asm/setup.h b/arch/sparc/include/uapi/asm/setup.h new file mode 100644 index 000000000..3c208a4dd --- /dev/null +++ b/arch/sparc/include/uapi/asm/setup.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * Just a place holder. + */ + +#ifndef _UAPI_SPARC_SETUP_H +#define _UAPI_SPARC_SETUP_H + +#if defined(__sparc__) && defined(__arch64__) +# define COMMAND_LINE_SIZE 2048 +#else +# define COMMAND_LINE_SIZE 256 +#endif + + +#endif /* _UAPI_SPARC_SETUP_H */ diff --git a/arch/sparc/include/uapi/asm/shmbuf.h b/arch/sparc/include/uapi/asm/shmbuf.h new file mode 100644 index 000000000..ed4f061c7 --- /dev/null +++ b/arch/sparc/include/uapi/asm/shmbuf.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_SHMBUF_H +#define _SPARC_SHMBUF_H + +#include +#include + +/* + * The shmid64_ds structure for sparc architecture. + * Note extra padding because this structure is passed back and forth + * between kernel and user space. + * + * Pad space is left for: + * - 2 miscellaneous 32-bit values + */ + +struct shmid64_ds { + struct ipc64_perm shm_perm; /* operation perms */ +#if defined(__sparc__) && defined(__arch64__) + long shm_atime; /* last attach time */ + long shm_dtime; /* last detach time */ + long shm_ctime; /* last change time */ +#else + unsigned long shm_atime_high; + unsigned long shm_atime; /* last attach time */ + unsigned long shm_dtime_high; + unsigned long shm_dtime; /* last detach time */ + unsigned long shm_ctime_high; + unsigned long shm_ctime; /* last change time */ +#endif + __kernel_size_t shm_segsz; /* size of segment (bytes) */ + __kernel_pid_t shm_cpid; /* pid of creator */ + __kernel_pid_t shm_lpid; /* pid of last operator */ + unsigned long shm_nattch; /* no. of current attaches */ + unsigned long __unused1; + unsigned long __unused2; +}; + +struct shminfo64 { + unsigned long shmmax; + unsigned long shmmin; + unsigned long shmmni; + unsigned long shmseg; + unsigned long shmall; + unsigned long __unused1; + unsigned long __unused2; + unsigned long __unused3; + unsigned long __unused4; +}; + +#endif /* _SPARC_SHMBUF_H */ diff --git a/arch/sparc/include/uapi/asm/sigcontext.h b/arch/sparc/include/uapi/asm/sigcontext.h new file mode 100644 index 000000000..043dd4b92 --- /dev/null +++ b/arch/sparc/include/uapi/asm/sigcontext.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * There isn't anything here anymore, but the file must not be empty or patch + * will delete it. + */ diff --git a/arch/sparc/include/uapi/asm/siginfo.h b/arch/sparc/include/uapi/asm/siginfo.h new file mode 100644 index 000000000..0e7c27522 --- /dev/null +++ b/arch/sparc/include/uapi/asm/siginfo.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI__SPARC_SIGINFO_H +#define _UAPI__SPARC_SIGINFO_H + +#if defined(__sparc__) && defined(__arch64__) + +#define __ARCH_SI_BAND_T int + +#endif /* defined(__sparc__) && defined(__arch64__) */ + +#include + + +#define SI_NOINFO 32767 /* no information in siginfo_t */ + +#endif /* _UAPI__SPARC_SIGINFO_H */ diff --git a/arch/sparc/include/uapi/asm/signal.h b/arch/sparc/include/uapi/asm/signal.h new file mode 100644 index 000000000..9c64d7cb8 --- /dev/null +++ b/arch/sparc/include/uapi/asm/signal.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI__SPARC_SIGNAL_H +#define _UAPI__SPARC_SIGNAL_H + +#include +#include + + +/* On the Sparc the signal handlers get passed a 'sub-signal' code + * for certain signal types, which we document here. + */ +#define SIGHUP 1 +#define SIGINT 2 +#define SIGQUIT 3 +#define SIGILL 4 +#define SUBSIG_STACK 0 +#define SUBSIG_ILLINST 2 +#define SUBSIG_PRIVINST 3 +#define SUBSIG_BADTRAP(t) (0x80 + (t)) + +#define SIGTRAP 5 +#define SIGABRT 6 +#define SIGIOT 6 + +#define SIGEMT 7 +#define SUBSIG_TAG 10 + +#define SIGFPE 8 +#define SUBSIG_FPDISABLED 0x400 +#define SUBSIG_FPERROR 0x404 +#define SUBSIG_FPINTOVFL 0x001 +#define SUBSIG_FPSTSIG 0x002 +#define SUBSIG_IDIVZERO 0x014 +#define SUBSIG_FPINEXACT 0x0c4 +#define SUBSIG_FPDIVZERO 0x0c8 +#define SUBSIG_FPUNFLOW 0x0cc +#define SUBSIG_FPOPERROR 0x0d0 +#define SUBSIG_FPOVFLOW 0x0d4 + +#define SIGKILL 9 +#define SIGBUS 10 +#define SUBSIG_BUSTIMEOUT 1 +#define SUBSIG_ALIGNMENT 2 +#define SUBSIG_MISCERROR 5 + +#define SIGSEGV 11 +#define SUBSIG_NOMAPPING 3 +#define SUBSIG_PROTECTION 4 +#define SUBSIG_SEGERROR 5 + +#define SIGSYS 12 + +#define SIGPIPE 13 +#define SIGALRM 14 +#define SIGTERM 15 +#define SIGURG 16 + +/* SunOS values which deviate from the Linux/i386 ones */ +#define SIGSTOP 17 +#define SIGTSTP 18 +#define SIGCONT 19 +#define SIGCHLD 20 +#define SIGTTIN 21 +#define SIGTTOU 22 +#define SIGIO 23 +#define SIGPOLL SIGIO /* SysV name for SIGIO */ +#define SIGXCPU 24 +#define SIGXFSZ 25 +#define SIGVTALRM 26 +#define SIGPROF 27 +#define SIGWINCH 28 +#define SIGLOST 29 +#define SIGPWR SIGLOST +#define SIGUSR1 30 +#define SIGUSR2 31 + +/* Most things should be clean enough to redefine this at will, if care + is taken to make libc match. */ + +#define __OLD_NSIG 32 +#define __NEW_NSIG 64 +#ifdef __arch64__ +#define _NSIG_BPW 64 +#else +#define _NSIG_BPW 32 +#endif +#define _NSIG_WORDS (__NEW_NSIG / _NSIG_BPW) + +#define SIGRTMIN 32 +#define SIGRTMAX __NEW_NSIG + +#if defined(__KERNEL__) || defined(__WANT_POSIX1B_SIGNALS__) +#define _NSIG __NEW_NSIG +#define __new_sigset_t sigset_t +#define __new_sigaction sigaction +#define __new_sigaction32 sigaction32 +#define __old_sigset_t old_sigset_t +#define __old_sigaction old_sigaction +#define __old_sigaction32 old_sigaction32 +#else +#define _NSIG __OLD_NSIG +#define NSIG _NSIG +#define __old_sigset_t sigset_t +#define __old_sigaction sigaction +#define __old_sigaction32 sigaction32 +#endif + +#ifndef __ASSEMBLER__ + +typedef unsigned long __old_sigset_t; /* at least 32 bits */ + +typedef struct { + unsigned long sig[_NSIG_WORDS]; +} __new_sigset_t; + +/* A SunOS sigstack */ +struct sigstack { + /* XXX 32-bit pointers pinhead XXX */ + char *the_stack; + int cur_status; +}; + +/* Sigvec flags */ +#define _SV_SSTACK 1u /* This signal handler should use sig-stack */ +#define _SV_INTR 2u /* Sig return should not restart system call */ +#define _SV_RESET 4u /* Set handler to SIG_DFL upon taken signal */ +#define _SV_IGNCHILD 8u /* Do not send SIGCHLD */ + +/* + * sa_flags values: SA_STACK is not currently supported, but will allow the + * usage of signal stacks by using the (now obsolete) sa_restorer field in + * the sigaction structure as a stack pointer. This is now possible due to + * the changes in signal handling. LBT 010493. + * SA_RESTART flag to get restarting signals (which were the default long ago) + */ +#define SA_NOCLDSTOP _SV_IGNCHILD +#define SA_STACK _SV_SSTACK +#define SA_ONSTACK _SV_SSTACK +#define SA_RESTART _SV_INTR +#define SA_RESETHAND _SV_RESET +#define SA_NODEFER 0x20u +#define SA_NOCLDWAIT 0x100u +#define SA_SIGINFO 0x200u + +#define SIG_BLOCK 0x01 /* for blocking signals */ +#define SIG_UNBLOCK 0x02 /* for unblocking signals */ +#define SIG_SETMASK 0x04 /* for setting the signal mask */ + +#define MINSIGSTKSZ 4096 +#define SIGSTKSZ 16384 + + +#include +#include + +#ifndef __KERNEL__ +struct __new_sigaction { + __sighandler_t sa_handler; + unsigned long sa_flags; + __sigrestore_t sa_restorer; /* not used by Linux/SPARC yet */ + __new_sigset_t sa_mask; +}; + +struct __old_sigaction { + __sighandler_t sa_handler; + __old_sigset_t sa_mask; + unsigned long sa_flags; + void (*sa_restorer)(void); /* not used by Linux/SPARC yet */ +}; +#endif + +typedef struct sigaltstack { + void __user *ss_sp; + int ss_flags; + __kernel_size_t ss_size; +} stack_t; + + +#endif /* !(__ASSEMBLER__) */ + +#endif /* _UAPI__SPARC_SIGNAL_H */ diff --git a/arch/sparc/include/uapi/asm/socket.h b/arch/sparc/include/uapi/asm/socket.h new file mode 100644 index 000000000..7907f3b1f --- /dev/null +++ b/arch/sparc/include/uapi/asm/socket.h @@ -0,0 +1,178 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _ASM_SOCKET_H +#define _ASM_SOCKET_H + +#include +#include + +/* For setsockopt(2) */ +#define SOL_SOCKET 0xffff + +#define SO_DEBUG 0x0001 +#define SO_PASSCRED 0x0002 +#define SO_REUSEADDR 0x0004 +#define SO_KEEPALIVE 0x0008 +#define SO_DONTROUTE 0x0010 +#define SO_BROADCAST 0x0020 +#define SO_PEERCRED 0x0040 +#define SO_LINGER 0x0080 +#define SO_OOBINLINE 0x0100 +#define SO_REUSEPORT 0x0200 +#define SO_BSDCOMPAT 0x0400 +#define SO_RCVLOWAT 0x0800 +#define SO_SNDLOWAT 0x1000 +#define SO_RCVTIMEO_OLD 0x2000 +#define SO_SNDTIMEO_OLD 0x4000 +#define SO_ACCEPTCONN 0x8000 + +#define SO_SNDBUF 0x1001 +#define SO_RCVBUF 0x1002 +#define SO_SNDBUFFORCE 0x100a +#define SO_RCVBUFFORCE 0x100b +#define SO_ERROR 0x1007 +#define SO_TYPE 0x1008 +#define SO_PROTOCOL 0x1028 +#define SO_DOMAIN 0x1029 + +/* Linux specific, keep the same. */ +#define SO_NO_CHECK 0x000b +#define SO_PRIORITY 0x000c + +#define SO_BINDTODEVICE 0x000d + +#define SO_ATTACH_FILTER 0x001a +#define SO_DETACH_FILTER 0x001b +#define SO_GET_FILTER SO_ATTACH_FILTER + +#define SO_PEERNAME 0x001c + +#define SO_PEERSEC 0x001e +#define SO_PASSSEC 0x001f + +#define SO_MARK 0x0022 + +#define SO_RXQ_OVFL 0x0024 + +#define SO_WIFI_STATUS 0x0025 +#define SCM_WIFI_STATUS SO_WIFI_STATUS +#define SO_PEEK_OFF 0x0026 + +/* Instruct lower device to use last 4-bytes of skb data as FCS */ +#define SO_NOFCS 0x0027 + +#define SO_LOCK_FILTER 0x0028 + +#define SO_SELECT_ERR_QUEUE 0x0029 + +#define SO_BUSY_POLL 0x0030 + +#define SO_MAX_PACING_RATE 0x0031 + +#define SO_BPF_EXTENSIONS 0x0032 + +#define SO_INCOMING_CPU 0x0033 + +#define SO_ATTACH_BPF 0x0034 +#define SO_DETACH_BPF SO_DETACH_FILTER + +#define SO_ATTACH_REUSEPORT_CBPF 0x0035 +#define SO_ATTACH_REUSEPORT_EBPF 0x0036 + +#define SO_CNX_ADVICE 0x0037 + +#define SCM_TIMESTAMPING_OPT_STATS 0x0038 + +#define SO_MEMINFO 0x0039 + +#define SO_INCOMING_NAPI_ID 0x003a + +#define SO_COOKIE 0x003b + +#define SCM_TIMESTAMPING_PKTINFO 0x003c + +#define SO_PEERGROUPS 0x003d + +#define SO_ZEROCOPY 0x003e + +#define SO_TXTIME 0x003f +#define SCM_TXTIME SO_TXTIME + +#define SO_BINDTOIFINDEX 0x0041 + +/* Security levels - as per NRL IPv6 - don't actually do anything */ +#define SO_SECURITY_AUTHENTICATION 0x5001 +#define SO_SECURITY_ENCRYPTION_TRANSPORT 0x5002 +#define SO_SECURITY_ENCRYPTION_NETWORK 0x5004 + +#define SO_TIMESTAMP_OLD 0x001d +#define SO_TIMESTAMPNS_OLD 0x0021 +#define SO_TIMESTAMPING_OLD 0x0023 + +#define SO_TIMESTAMP_NEW 0x0046 +#define SO_TIMESTAMPNS_NEW 0x0042 +#define SO_TIMESTAMPING_NEW 0x0043 + +#define SO_RCVTIMEO_NEW 0x0044 +#define SO_SNDTIMEO_NEW 0x0045 + +#define SO_DETACH_REUSEPORT_BPF 0x0047 + +#define SO_PREFER_BUSY_POLL 0x0048 +#define SO_BUSY_POLL_BUDGET 0x0049 + +#define SO_NETNS_COOKIE 0x0050 + +#define SO_BUF_LOCK 0x0051 + +#define SO_RESERVE_MEM 0x0052 + +#define SO_TXREHASH 0x0053 + +#define SO_RCVMARK 0x0054 + +#define SO_PASSPIDFD 0x0055 +#define SO_PEERPIDFD 0x0056 + +#define SO_DEVMEM_LINEAR 0x0057 +#define SCM_DEVMEM_LINEAR SO_DEVMEM_LINEAR +#define SO_DEVMEM_DMABUF 0x0058 +#define SCM_DEVMEM_DMABUF SO_DEVMEM_DMABUF +#define SO_DEVMEM_DONTNEED 0x0059 + +#define SCM_TS_OPT_ID 0x005a + +#define SO_RCVPRIORITY 0x005b + +#define SO_PASSRIGHTS 0x005c + +#define SO_INQ 0x005d +#define SCM_INQ SO_INQ + +#define SO_RIGHTS_NOTRUNC 0x005e + +#if !defined(__KERNEL__) + + +#if __BITS_PER_LONG == 64 +#define SO_TIMESTAMP SO_TIMESTAMP_OLD +#define SO_TIMESTAMPNS SO_TIMESTAMPNS_OLD +#define SO_TIMESTAMPING SO_TIMESTAMPING_OLD + +#define SO_RCVTIMEO SO_RCVTIMEO_OLD +#define SO_SNDTIMEO SO_SNDTIMEO_OLD +#else +#define SO_TIMESTAMP (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMP_OLD : SO_TIMESTAMP_NEW) +#define SO_TIMESTAMPNS (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMPNS_OLD : SO_TIMESTAMPNS_NEW) +#define SO_TIMESTAMPING (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_TIMESTAMPING_OLD : SO_TIMESTAMPING_NEW) + +#define SO_RCVTIMEO (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_RCVTIMEO_OLD : SO_RCVTIMEO_NEW) +#define SO_SNDTIMEO (sizeof(time_t) == sizeof(__kernel_long_t) ? SO_SNDTIMEO_OLD : SO_SNDTIMEO_NEW) +#endif + +#define SCM_TIMESTAMP SO_TIMESTAMP +#define SCM_TIMESTAMPNS SO_TIMESTAMPNS +#define SCM_TIMESTAMPING SO_TIMESTAMPING + +#endif + +#endif /* _ASM_SOCKET_H */ diff --git a/arch/sparc/include/uapi/asm/stat.h b/arch/sparc/include/uapi/asm/stat.h new file mode 100644 index 000000000..47f54133a --- /dev/null +++ b/arch/sparc/include/uapi/asm/stat.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef __SPARC_STAT_H +#define __SPARC_STAT_H + +#include + +#if defined(__sparc__) && defined(__arch64__) +/* 64 bit sparc */ +struct stat { + unsigned int st_dev; + __kernel_ino_t st_ino; + __kernel_mode_t st_mode; + short st_nlink; + __kernel_uid32_t st_uid; + __kernel_gid32_t st_gid; + unsigned int st_rdev; + long st_size; + long st_atime; + long st_mtime; + long st_ctime; + long st_blksize; + long st_blocks; + unsigned long __unused4[2]; +}; + +struct stat64 { + unsigned long st_dev; + unsigned long st_ino; + unsigned long st_nlink; + + unsigned int st_mode; + unsigned int st_uid; + unsigned int st_gid; + unsigned int __pad0; + + unsigned long st_rdev; + long st_size; + long st_blksize; + long st_blocks; + + unsigned long st_atime; + unsigned long st_atime_nsec; + unsigned long st_mtime; + unsigned long st_mtime_nsec; + unsigned long st_ctime; + unsigned long st_ctime_nsec; + long __unused[3]; +}; + +#else +/* 32 bit sparc */ +struct stat { + unsigned short st_dev; + __kernel_ino_t st_ino; + __kernel_mode_t st_mode; + short st_nlink; + unsigned short st_uid; + unsigned short st_gid; + unsigned short st_rdev; + long st_size; + long st_atime; + unsigned long st_atime_nsec; + long st_mtime; + unsigned long st_mtime_nsec; + long st_ctime; + unsigned long st_ctime_nsec; + long st_blksize; + long st_blocks; + unsigned long __unused4[2]; +}; + +#define STAT_HAVE_NSEC 1 + +struct stat64 { + unsigned long long st_dev; + + unsigned long long st_ino; + + unsigned int st_mode; + unsigned int st_nlink; + + unsigned int st_uid; + unsigned int st_gid; + + unsigned long long st_rdev; + + unsigned char __pad3[8]; + + long long st_size; + unsigned int st_blksize; + + unsigned char __pad4[8]; + unsigned int st_blocks; + + unsigned int st_atime; + unsigned int st_atime_nsec; + + unsigned int st_mtime; + unsigned int st_mtime_nsec; + + unsigned int st_ctime; + unsigned int st_ctime_nsec; + + unsigned int __unused4; + unsigned int __unused5; +}; +#endif /* defined(__sparc__) && defined(__arch64__) */ +#endif /* __SPARC_STAT_H */ diff --git a/arch/sparc/include/uapi/asm/swab.h b/arch/sparc/include/uapi/asm/swab.h new file mode 100644 index 000000000..6b1b3f1ad --- /dev/null +++ b/arch/sparc/include/uapi/asm/swab.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _SPARC_SWAB_H +#define _SPARC_SWAB_H + +#include +#include + +#if defined(__sparc__) && defined(__arch64__) +static inline __u16 __arch_swab16p(const __u16 *addr) +{ + __u16 ret; + + __asm__ __volatile__ ("lduha [%2] %3, %0" + : "=r" (ret) + : "m" (*addr), "r" (addr), "i" (ASI_PL)); + return ret; +} +#define __arch_swab16p __arch_swab16p + +static inline __u32 __arch_swab32p(const __u32 *addr) +{ + __u32 ret; + + __asm__ __volatile__ ("lduwa [%2] %3, %0" + : "=r" (ret) + : "m" (*addr), "r" (addr), "i" (ASI_PL)); + return ret; +} +#define __arch_swab32p __arch_swab32p + +static inline __u64 __arch_swab64p(const __u64 *addr) +{ + __u64 ret; + + __asm__ __volatile__ ("ldxa [%2] %3, %0" + : "=r" (ret) + : "m" (*addr), "r" (addr), "i" (ASI_PL)); + return ret; +} +#define __arch_swab64p __arch_swab64p + +#else +#define __SWAB_64_THRU_32__ +#endif /* defined(__sparc__) && defined(__arch64__) */ + +#endif /* _SPARC_SWAB_H */ diff --git a/arch/sparc/include/uapi/asm/termbits.h b/arch/sparc/include/uapi/asm/termbits.h new file mode 100644 index 000000000..0da2b1adc --- /dev/null +++ b/arch/sparc/include/uapi/asm/termbits.h @@ -0,0 +1,199 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_SPARC_TERMBITS_H +#define _UAPI_SPARC_TERMBITS_H + +#include + +#if defined(__sparc__) && defined(__arch64__) +typedef unsigned int tcflag_t; +#else +typedef unsigned long tcflag_t; +#endif + +#define NCCS 17 +struct termios { + tcflag_t c_iflag; /* input mode flags */ + tcflag_t c_oflag; /* output mode flags */ + tcflag_t c_cflag; /* control mode flags */ + tcflag_t c_lflag; /* local mode flags */ + cc_t c_line; /* line discipline */ +#ifndef __KERNEL__ + cc_t c_cc[NCCS]; /* control characters */ +#else + cc_t c_cc[NCCS+2]; /* kernel needs 2 more to hold vmin/vtime */ +#define SIZEOF_USER_TERMIOS sizeof (struct termios) - (2*sizeof (cc_t)) +#endif +}; + +struct termios2 { + tcflag_t c_iflag; /* input mode flags */ + tcflag_t c_oflag; /* output mode flags */ + tcflag_t c_cflag; /* control mode flags */ + tcflag_t c_lflag; /* local mode flags */ + cc_t c_line; /* line discipline */ + cc_t c_cc[NCCS+2]; /* control characters */ + speed_t c_ispeed; /* input speed */ + speed_t c_ospeed; /* output speed */ +}; + +struct ktermios { + tcflag_t c_iflag; /* input mode flags */ + tcflag_t c_oflag; /* output mode flags */ + tcflag_t c_cflag; /* control mode flags */ + tcflag_t c_lflag; /* local mode flags */ + cc_t c_line; /* line discipline */ + cc_t c_cc[NCCS+2]; /* control characters */ + speed_t c_ispeed; /* input speed */ + speed_t c_ospeed; /* output speed */ +}; + +/* c_cc characters */ +#define VINTR 0 +#define VQUIT 1 +#define VERASE 2 +#define VKILL 3 +#define VEOF 4 +#define VEOL 5 +#define VEOL2 6 +#define VSWTC 7 +#define VSTART 8 +#define VSTOP 9 + +#define VSUSP 10 +#define VDSUSP 11 /* SunOS POSIX nicety I do believe... */ +#define VREPRINT 12 +#define VDISCARD 13 +#define VWERASE 14 +#define VLNEXT 15 + +/* Kernel keeps vmin/vtime separated, user apps assume vmin/vtime is + * shared with eof/eol + */ +#ifndef __KERNEL__ +#define VMIN VEOF +#define VTIME VEOL +#endif + +/* c_iflag bits */ +#define IUCLC 0x0200 +#define IXON 0x0400 +#define IXOFF 0x1000 +#define IMAXBEL 0x2000 +#define IUTF8 0x4000 + +/* c_oflag bits */ +#define OLCUC 0x00002 +#define ONLCR 0x00004 +#define NLDLY 0x00100 +#define NL0 0x00000 +#define NL1 0x00100 +#define CRDLY 0x00600 +#define CR0 0x00000 +#define CR1 0x00200 +#define CR2 0x00400 +#define CR3 0x00600 +#define TABDLY 0x01800 +#define TAB0 0x00000 +#define TAB1 0x00800 +#define TAB2 0x01000 +#define TAB3 0x01800 +#define XTABS 0x01800 +#define BSDLY 0x02000 +#define BS0 0x00000 +#define BS1 0x02000 +#define VTDLY 0x04000 +#define VT0 0x00000 +#define VT1 0x04000 +#define FFDLY 0x08000 +#define FF0 0x00000 +#define FF1 0x08000 +#define PAGEOUT 0x10000 /* SUNOS specific */ +#define WRAP 0x20000 /* SUNOS specific */ + +/* c_cflag bit meaning */ +#define CBAUD 0x0000100f +#define CSIZE 0x00000030 +#define CS5 0x00000000 +#define CS6 0x00000010 +#define CS7 0x00000020 +#define CS8 0x00000030 +#define CSTOPB 0x00000040 +#define CREAD 0x00000080 +#define PARENB 0x00000100 +#define PARODD 0x00000200 +#define HUPCL 0x00000400 +#define CLOCAL 0x00000800 +#define CBAUDEX 0x00001000 +/* We'll never see these speeds with the Zilogs, but for completeness... */ +#define BOTHER 0x00001000 +#define B57600 0x00001001 +#define B115200 0x00001002 +#define B230400 0x00001003 +#define B460800 0x00001004 +/* This is what we can do with the Zilogs. */ +#define B76800 0x00001005 +/* This is what we can do with the SAB82532. */ +#define B153600 0x00001006 +#define B307200 0x00001007 +#define B614400 0x00001008 +#define B921600 0x00001009 +/* And these are the rest... */ +#define B500000 0x0000100a +#define B576000 0x0000100b +#define B1000000 0x0000100c +#define B1152000 0x0000100d +#define B1500000 0x0000100e +#define B2000000 0x0000100f +/* These have totally bogus values and nobody uses them + so far. Later on we'd have to use say 0x10000x and + adjust CBAUD constant and drivers accordingly. +#define B2500000 0x00001010 +#define B3000000 0x00001011 +#define B3500000 0x00001012 +#define B4000000 0x00001013 */ +#define CIBAUD 0x100f0000 /* input baud rate (not used) */ + +/* c_lflag bits */ +#define ISIG 0x00000001 +#define ICANON 0x00000002 +#define XCASE 0x00000004 +#define ECHO 0x00000008 +#define ECHOE 0x00000010 +#define ECHOK 0x00000020 +#define ECHONL 0x00000040 +#define NOFLSH 0x00000080 +#define TOSTOP 0x00000100 +#define ECHOCTL 0x00000200 +#define ECHOPRT 0x00000400 +#define ECHOKE 0x00000800 +#define DEFECHO 0x00001000 /* SUNOS thing, what is it? */ +#define FLUSHO 0x00002000 +#define PENDIN 0x00004000 +#define IEXTEN 0x00008000 +#define EXTPROC 0x00010000 + +/* modem lines */ +#define TIOCM_LE 0x001 +#define TIOCM_DTR 0x002 +#define TIOCM_RTS 0x004 +#define TIOCM_ST 0x008 +#define TIOCM_SR 0x010 +#define TIOCM_CTS 0x020 +#define TIOCM_CAR 0x040 +#define TIOCM_RNG 0x080 +#define TIOCM_DSR 0x100 +#define TIOCM_CD TIOCM_CAR +#define TIOCM_RI TIOCM_RNG +#define TIOCM_OUT1 0x2000 +#define TIOCM_OUT2 0x4000 +#define TIOCM_LOOP 0x8000 + +/* ioctl (fd, TIOCSERGETLSR, &result) where result may be as below */ +#define TIOCSER_TEMT 0x01 /* Transmitter physically empty */ + +/* tcsetattr uses these */ +#define TCSANOW 0 +#define TCSADRAIN 1 +#define TCSAFLUSH 2 + +#endif /* _UAPI_SPARC_TERMBITS_H */ diff --git a/arch/sparc/include/uapi/asm/termios.h b/arch/sparc/include/uapi/asm/termios.h new file mode 100644 index 000000000..cceb32260 --- /dev/null +++ b/arch/sparc/include/uapi/asm/termios.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_SPARC_TERMIOS_H +#define _UAPI_SPARC_TERMIOS_H + +#include +#include + +#if defined(__KERNEL__) || defined(__DEFINE_BSD_TERMIOS) +struct sgttyb { + char sg_ispeed; + char sg_ospeed; + char sg_erase; + char sg_kill; + short sg_flags; +}; + +struct tchars { + char t_intrc; + char t_quitc; + char t_startc; + char t_stopc; + char t_eofc; + char t_brkc; +}; + +struct ltchars { + char t_suspc; + char t_dsuspc; + char t_rprntc; + char t_flushc; + char t_werasc; + char t_lnextc; +}; +#endif /* __KERNEL__ */ + +struct winsize { + unsigned short ws_row; + unsigned short ws_col; + unsigned short ws_xpixel; + unsigned short ws_ypixel; +}; + +#define NCC 8 +struct termio { + unsigned short c_iflag; /* input mode flags */ + unsigned short c_oflag; /* output mode flags */ + unsigned short c_cflag; /* control mode flags */ + unsigned short c_lflag; /* local mode flags */ + unsigned char c_line; /* line discipline */ + unsigned char c_cc[NCC]; /* control characters */ +}; + +#endif /* _UAPI_SPARC_TERMIOS_H */ diff --git a/arch/sparc/include/uapi/asm/traps.h b/arch/sparc/include/uapi/asm/traps.h new file mode 100644 index 000000000..43fe5b8fe --- /dev/null +++ b/arch/sparc/include/uapi/asm/traps.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * traps.h: Format of entries for the Sparc trap table. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef _UAPI_SPARC_TRAPS_H +#define _UAPI_SPARC_TRAPS_H + +#define NUM_SPARC_TRAPS 255 + +#ifndef __ASSEMBLER__ +#endif /* !(__ASSEMBLER__) */ + +/* For patching the trap table at boot time, we need to know how to + * form various common Sparc instructions. Thus these macros... + */ + +#define SPARC_MOV_CONST_L3(const) (0xa6102000 | (const&0xfff)) + +/* The following assumes that the branch lies before the place we + * are branching to. This is the case for a trap vector... + * You have been warned. + */ +#define SPARC_BRANCH(dest_addr, inst_addr) \ + (0x10800000 | (((dest_addr-inst_addr)>>2)&0x3fffff)) + +#define SPARC_RD_PSR_L0 (0xa1480000) +#define SPARC_RD_WIM_L3 (0xa7500000) +#define SPARC_NOP (0x01000000) + +/* Various interesting trap levels. */ +/* First, hardware traps. */ +#define SP_TRAP_TFLT 0x1 /* Text fault */ +#define SP_TRAP_II 0x2 /* Illegal Instruction */ +#define SP_TRAP_PI 0x3 /* Privileged Instruction */ +#define SP_TRAP_FPD 0x4 /* Floating Point Disabled */ +#define SP_TRAP_WOVF 0x5 /* Window Overflow */ +#define SP_TRAP_WUNF 0x6 /* Window Underflow */ +#define SP_TRAP_MNA 0x7 /* Memory Address Unaligned */ +#define SP_TRAP_FPE 0x8 /* Floating Point Exception */ +#define SP_TRAP_DFLT 0x9 /* Data Fault */ +#define SP_TRAP_TOF 0xa /* Tag Overflow */ +#define SP_TRAP_WDOG 0xb /* Watchpoint Detected */ +#define SP_TRAP_IRQ1 0x11 /* IRQ level 1 */ +#define SP_TRAP_IRQ2 0x12 /* IRQ level 2 */ +#define SP_TRAP_IRQ3 0x13 /* IRQ level 3 */ +#define SP_TRAP_IRQ4 0x14 /* IRQ level 4 */ +#define SP_TRAP_IRQ5 0x15 /* IRQ level 5 */ +#define SP_TRAP_IRQ6 0x16 /* IRQ level 6 */ +#define SP_TRAP_IRQ7 0x17 /* IRQ level 7 */ +#define SP_TRAP_IRQ8 0x18 /* IRQ level 8 */ +#define SP_TRAP_IRQ9 0x19 /* IRQ level 9 */ +#define SP_TRAP_IRQ10 0x1a /* IRQ level 10 */ +#define SP_TRAP_IRQ11 0x1b /* IRQ level 11 */ +#define SP_TRAP_IRQ12 0x1c /* IRQ level 12 */ +#define SP_TRAP_IRQ13 0x1d /* IRQ level 13 */ +#define SP_TRAP_IRQ14 0x1e /* IRQ level 14 */ +#define SP_TRAP_IRQ15 0x1f /* IRQ level 15 Non-maskable */ +#define SP_TRAP_RACC 0x20 /* Register Access Error ??? */ +#define SP_TRAP_IACC 0x21 /* Instruction Access Error */ +#define SP_TRAP_CPDIS 0x24 /* Co-Processor Disabled */ +#define SP_TRAP_BADFL 0x25 /* Unimplemented Flush Instruction */ +#define SP_TRAP_CPEXP 0x28 /* Co-Processor Exception */ +#define SP_TRAP_DACC 0x29 /* Data Access Error */ +#define SP_TRAP_DIVZ 0x2a /* Divide By Zero */ +#define SP_TRAP_DSTORE 0x2b /* Data Store Error ??? */ +#define SP_TRAP_DMM 0x2c /* Data Access MMU Miss ??? */ +#define SP_TRAP_IMM 0x3c /* Instruction Access MMU Miss ??? */ + +/* Now the Software Traps... */ +#define SP_TRAP_SUNOS 0x80 /* SunOS System Call */ +#define SP_TRAP_SBPT 0x81 /* Software Breakpoint */ +#define SP_TRAP_SDIVZ 0x82 /* Software Divide-by-Zero trap */ +#define SP_TRAP_FWIN 0x83 /* Flush Windows */ +#define SP_TRAP_CWIN 0x84 /* Clean Windows */ +#define SP_TRAP_RCHK 0x85 /* Range Check */ +#define SP_TRAP_FUNA 0x86 /* Fix Unaligned Access */ +#define SP_TRAP_IOWFL 0x87 /* Integer Overflow */ +#define SP_TRAP_SOLARIS 0x88 /* Solaris System Call */ +#define SP_TRAP_NETBSD 0x89 /* NetBSD System Call */ +#define SP_TRAP_LINUX 0x90 /* Linux System Call */ + +/* Names used for compatibility with SunOS */ +#define ST_SYSCALL 0x00 +#define ST_BREAKPOINT 0x01 +#define ST_DIV0 0x02 +#define ST_FLUSH_WINDOWS 0x03 +#define ST_CLEAN_WINDOWS 0x04 +#define ST_RANGE_CHECK 0x05 +#define ST_FIX_ALIGN 0x06 +#define ST_INT_OVERFLOW 0x07 + +/* Special traps... */ +#define SP_TRAP_KBPT1 0xfe /* KADB/PROM Breakpoint one */ +#define SP_TRAP_KBPT2 0xff /* KADB/PROM Breakpoint two */ + +/* Handy Macros */ +/* Is this a trap we never expect to get? */ +#define BAD_TRAP_P(level) \ + ((level > SP_TRAP_WDOG && level < SP_TRAP_IRQ1) || \ + (level > SP_TRAP_IACC && level < SP_TRAP_CPDIS) || \ + (level > SP_TRAP_BADFL && level < SP_TRAP_CPEXP) || \ + (level > SP_TRAP_DMM && level < SP_TRAP_IMM) || \ + (level > SP_TRAP_IMM && level < SP_TRAP_SUNOS) || \ + (level > SP_TRAP_LINUX && level < SP_TRAP_KBPT1)) + +/* Is this a Hardware trap? */ +#define HW_TRAP_P(level) ((level > 0) && (level < SP_TRAP_SUNOS)) + +/* Is this a Software trap? */ +#define SW_TRAP_P(level) ((level >= SP_TRAP_SUNOS) && (level <= SP_TRAP_KBPT2)) + +/* Is this a system call for some OS we know about? */ +#define SCALL_TRAP_P(level) ((level == SP_TRAP_SUNOS) || \ + (level == SP_TRAP_SOLARIS) || \ + (level == SP_TRAP_NETBSD) || \ + (level == SP_TRAP_LINUX)) + +#endif /* _UAPI_SPARC_TRAPS_H */ diff --git a/arch/sparc/include/uapi/asm/ucontext.h b/arch/sparc/include/uapi/asm/ucontext.h new file mode 100644 index 000000000..e9b5dbbed --- /dev/null +++ b/arch/sparc/include/uapi/asm/ucontext.h @@ -0,0 +1,3 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ + +#include diff --git a/arch/sparc/include/uapi/asm/uctx.h b/arch/sparc/include/uapi/asm/uctx.h new file mode 100644 index 000000000..13a13198e --- /dev/null +++ b/arch/sparc/include/uapi/asm/uctx.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * uctx.h: Sparc64 {set,get}context() register state layouts. + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#ifndef __SPARC64_UCTX_H +#define __SPARC64_UCTX_H + +#define MC_TSTATE 0 +#define MC_PC 1 +#define MC_NPC 2 +#define MC_Y 3 +#define MC_G1 4 +#define MC_G2 5 +#define MC_G3 6 +#define MC_G4 7 +#define MC_G5 8 +#define MC_G6 9 +#define MC_G7 10 +#define MC_O0 11 +#define MC_O1 12 +#define MC_O2 13 +#define MC_O3 14 +#define MC_O4 15 +#define MC_O5 16 +#define MC_O6 17 +#define MC_O7 18 +#define MC_NGREG 19 + +typedef unsigned long mc_greg_t; +typedef mc_greg_t mc_gregset_t[MC_NGREG]; + +#define MC_MAXFPQ 16 +struct mc_fq { + unsigned long *mcfq_addr; + unsigned int mcfq_insn; +}; + +struct mc_fpu { + union { + unsigned int sregs[32]; + unsigned long dregs[32]; + long double qregs[16]; + } mcfpu_fregs; + unsigned long mcfpu_fsr; + unsigned long mcfpu_fprs; + unsigned long mcfpu_gsr; + struct mc_fq *mcfpu_fq; + unsigned char mcfpu_qcnt; + unsigned char mcfpu_qentsz; + unsigned char mcfpu_enab; +}; +typedef struct mc_fpu mc_fpu_t; + +typedef struct { + mc_gregset_t mc_gregs; + mc_greg_t mc_fp; + mc_greg_t mc_i7; + mc_fpu_t mc_fpregs; +} mcontext_t; + +struct ucontext { + struct ucontext *uc_link; + unsigned long uc_flags; + sigset_t uc_sigmask; + mcontext_t uc_mcontext; +}; +typedef struct ucontext ucontext_t; + +#endif /* __SPARC64_UCTX_H */ diff --git a/arch/sparc/include/uapi/asm/unistd.h b/arch/sparc/include/uapi/asm/unistd.h new file mode 100644 index 000000000..7f5d773b8 --- /dev/null +++ b/arch/sparc/include/uapi/asm/unistd.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * System calls under the Sparc. + * + * Don't be scared by the ugly clobbers, it is the only way I can + * think of right now to force the arguments into fixed registers + * before the trap into the system call with gcc 'asm' statements. + * + * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) + * + * SunOS compatibility based upon preliminary work which is: + * + * Copyright (C) 1995 Adrian M. Rodriguez (adrian@remus.rutgers.edu) + */ +#ifndef _UAPI_SPARC_UNISTD_H +#define _UAPI_SPARC_UNISTD_H + +#ifndef __32bit_syscall_numbers__ +#ifndef __arch64__ +#define __32bit_syscall_numbers__ +#endif +#endif + +#ifdef __arch64__ +#include +#else +#include +#endif + +/* Bitmask values returned from kern_features system call. */ +#define KERN_FEATURE_MIXED_MODE_STACK 0x00000001 + +#endif /* _UAPI_SPARC_UNISTD_H */ diff --git a/arch/sparc/include/uapi/asm/utrap.h b/arch/sparc/include/uapi/asm/utrap.h new file mode 100644 index 000000000..a489b08b6 --- /dev/null +++ b/arch/sparc/include/uapi/asm/utrap.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * include/asm/utrap.h + * + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#ifndef __ASM_SPARC64_UTRAP_H +#define __ASM_SPARC64_UTRAP_H + +#define UT_INSTRUCTION_EXCEPTION 1 +#define UT_INSTRUCTION_ERROR 2 +#define UT_INSTRUCTION_PROTECTION 3 +#define UT_ILLTRAP_INSTRUCTION 4 +#define UT_ILLEGAL_INSTRUCTION 5 +#define UT_PRIVILEGED_OPCODE 6 +#define UT_FP_DISABLED 7 +#define UT_FP_EXCEPTION_IEEE_754 8 +#define UT_FP_EXCEPTION_OTHER 9 +#define UT_TAG_OVERVIEW 10 +#define UT_DIVISION_BY_ZERO 11 +#define UT_DATA_EXCEPTION 12 +#define UT_DATA_ERROR 13 +#define UT_DATA_PROTECTION 14 +#define UT_MEM_ADDRESS_NOT_ALIGNED 15 +#define UT_PRIVILEGED_ACTION 16 +#define UT_ASYNC_DATA_ERROR 17 +#define UT_TRAP_INSTRUCTION_16 18 +#define UT_TRAP_INSTRUCTION_17 19 +#define UT_TRAP_INSTRUCTION_18 20 +#define UT_TRAP_INSTRUCTION_19 21 +#define UT_TRAP_INSTRUCTION_20 22 +#define UT_TRAP_INSTRUCTION_21 23 +#define UT_TRAP_INSTRUCTION_22 24 +#define UT_TRAP_INSTRUCTION_23 25 +#define UT_TRAP_INSTRUCTION_24 26 +#define UT_TRAP_INSTRUCTION_25 27 +#define UT_TRAP_INSTRUCTION_26 28 +#define UT_TRAP_INSTRUCTION_27 29 +#define UT_TRAP_INSTRUCTION_28 30 +#define UT_TRAP_INSTRUCTION_29 31 +#define UT_TRAP_INSTRUCTION_30 32 +#define UT_TRAP_INSTRUCTION_31 33 + +#define UTH_NOCHANGE (-1) + +#ifndef __ASSEMBLER__ +typedef int utrap_entry_t; +typedef void *utrap_handler_t; +#endif /* __ASSEMBLER__ */ + +#endif /* !(__ASM_SPARC64_PROCESSOR_H) */ diff --git a/arch/sparc/include/uapi/asm/watchdog.h b/arch/sparc/include/uapi/asm/watchdog.h new file mode 100644 index 000000000..497ac19a9 --- /dev/null +++ b/arch/sparc/include/uapi/asm/watchdog.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * + * watchdog - Driver interface for the hardware watchdog timers + * present on Sun Microsystems boardsets + * + * Copyright (c) 2000 Eric Brower + * + */ + +#ifndef _SPARC64_WATCHDOG_H +#define _SPARC64_WATCHDOG_H + +#include + +/* Solaris compatibility ioctls-- + * Ref. for standard linux watchdog ioctls + */ +#define WIOCSTART _IO (WATCHDOG_IOCTL_BASE, 10) /* Start Timer */ +#define WIOCSTOP _IO (WATCHDOG_IOCTL_BASE, 11) /* Stop Timer */ +#define WIOCGSTAT _IOR(WATCHDOG_IOCTL_BASE, 12, int)/* Get Timer Status */ + +/* Status flags from WIOCGSTAT ioctl + */ +#define WD_FREERUN 0x01 /* timer is running, interrupts disabled */ +#define WD_EXPIRED 0x02 /* timer has expired */ +#define WD_RUNNING 0x04 /* timer is running, interrupts enabled */ +#define WD_STOPPED 0x08 /* timer has not been started */ +#define WD_SERVICED 0x10 /* timer interrupt was serviced */ + +#endif /* ifndef _SPARC64_WATCHDOG_H */ + diff --git a/arch/sparc/kernel/.gitignore b/arch/sparc/kernel/.gitignore new file mode 100644 index 000000000..bbb90f92d --- /dev/null +++ b/arch/sparc/kernel/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +vmlinux.lds diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile new file mode 100644 index 000000000..497b5714f --- /dev/null +++ b/arch/sparc/kernel/Makefile @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: GPL-2.0 + +# +# Makefile for the linux kernel. +# + +# Undefine sparc when processing vmlinux.lds - it is used +# And teach CPP we are doing $(BITS) builds (for this case) +CPPFLAGS_vmlinux.lds := -Usparc -m$(BITS) +always-$(KBUILD_BUILTIN) += vmlinux.lds + +ifdef CONFIG_FUNCTION_TRACER +# Do not profile debug and lowlevel utilities +CFLAGS_REMOVE_ftrace.o := -pg +CFLAGS_REMOVE_time_$(BITS).o := -pg +CFLAGS_REMOVE_perf_event.o := -pg +CFLAGS_REMOVE_pcr.o := -pg +endif + +obj-y := head_$(BITS).o +obj-$(CONFIG_SPARC64) += urtt_fill.o +obj-$(CONFIG_SPARC32) += entry.o wof.o wuf.o +obj-$(CONFIG_SPARC32) += etrap_32.o +obj-$(CONFIG_SPARC32) += rtrap_32.o +obj-y += traps_$(BITS).o + +# IRQ +obj-y += irq_$(BITS).o +obj-$(CONFIG_SPARC32) += sun4m_irq.o sun4d_irq.o + +obj-y += process_$(BITS).o +obj-y += process.o +obj-y += signal_$(BITS).o +obj-y += sigutil_$(BITS).o +obj-$(CONFIG_SPARC32) += ioport.o +obj-y += setup.o +obj-y += setup_$(BITS).o +obj-y += idprom.o +obj-y += sys_sparc_$(BITS).o +obj-$(CONFIG_SPARC32) += systbls_32.o +obj-y += time_$(BITS).o +obj-$(CONFIG_SPARC32) += windows.o +obj-y += cpu.o +obj-$(CONFIG_SPARC32) += devices.o +obj-y += ptrace_$(BITS).o +obj-y += unaligned_$(BITS).o +obj-y += una_asm_$(BITS).o +obj-y += prom_common.o +obj-y += prom_$(BITS).o +obj-y += of_device_common.o +obj-y += of_device_$(BITS).o +obj-$(CONFIG_SPARC64) += prom_irqtrans.o + +obj-$(CONFIG_SPARC32) += leon_kernel.o +obj-$(CONFIG_SPARC32) += leon_pmc.o + +obj-$(CONFIG_SPARC64) += reboot.o +obj-$(CONFIG_SPARC64) += sysfs.o +obj-$(CONFIG_SPARC64) += iommu.o iommu-common.o +obj-$(CONFIG_SPARC64) += central.o +obj-$(CONFIG_SPARC64) += starfire.o +obj-$(CONFIG_SPARC64) += power.o +obj-$(CONFIG_SPARC64) += sbus.o +obj-$(CONFIG_SPARC64) += ebus.o +obj-$(CONFIG_SPARC64) += visemul.o +obj-$(CONFIG_SPARC64) += hvapi.o +obj-$(CONFIG_SPARC64) += sstate.o +obj-$(CONFIG_SPARC64) += mdesc.o +obj-$(CONFIG_SPARC64) += adi_64.o +obj-$(CONFIG_SPARC64) += pcr.o +obj-$(CONFIG_SPARC64) += nmi.o +obj-$(CONFIG_SPARC64_SMP) += cpumap.o + +obj-$(CONFIG_PCIC_PCI) += pcic.o +obj-$(CONFIG_LEON_PCI) += leon_pci.o +obj-$(CONFIG_SPARC_GRPCI2)+= leon_pci_grpci2.o +obj-$(CONFIG_SPARC_GRPCI1)+= leon_pci_grpci1.o + +obj-$(CONFIG_SMP) += trampoline_$(BITS).o smp_$(BITS).o +obj-$(CONFIG_SPARC32_SMP) += sun4m_smp.o sun4d_smp.o leon_smp.o +obj-$(CONFIG_SPARC64_SMP) += hvtramp.o + +obj-y += auxio_$(BITS).o +obj-$(CONFIG_SUN_PM) += apc.o pmc.o + +obj-y += termios.o + +obj-$(CONFIG_MODULES) += module.o +obj-$(CONFIG_MODULES) += sparc_ksyms.o +obj-$(CONFIG_SPARC_LED) += led.o +obj-$(CONFIG_KGDB) += kgdb_$(BITS).o + +obj-$(CONFIG_DYNAMIC_FTRACE) += ftrace.o +obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o + +obj-$(CONFIG_EARLYFB) += btext.o +obj-$(CONFIG_STACKTRACE) += stacktrace.o +# sparc64 PCI +obj-$(CONFIG_SPARC64_PCI) += pci.o pci_common.o psycho_common.o +obj-$(CONFIG_SPARC64_PCI) += pci_psycho.o pci_sabre.o pci_schizo.o +obj-$(CONFIG_SPARC64_PCI) += pci_sun4v.o pci_sun4v_asm.o pci_fire.o +obj-$(CONFIG_SPARC64_PCI_MSI) += pci_msi.o + + +obj-$(CONFIG_COMPAT) += sys32.o sys_sparc32.o signal32.o + +obj-$(CONFIG_US3_MC) += chmc.o + +obj-$(CONFIG_KPROBES) += kprobes.o +obj-$(CONFIG_SUN_LDOMS) += ldc.o vio.o viohs.o ds.o + +obj-$(CONFIG_AUDIT) += audit.o +audit--$(CONFIG_AUDIT) := compat_audit.o +obj-$(CONFIG_COMPAT) += $(audit--y) + +pc--$(CONFIG_PERF_EVENTS) := perf_event.o +obj-$(CONFIG_SPARC64) += $(pc--y) + +obj-$(CONFIG_UPROBES) += uprobes.o +obj-$(CONFIG_JUMP_LABEL) += jump_label.o diff --git a/arch/sparc/kernel/adi_64.c b/arch/sparc/kernel/adi_64.c new file mode 100644 index 000000000..18036a43c --- /dev/null +++ b/arch/sparc/kernel/adi_64.c @@ -0,0 +1,396 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* adi_64.c: support for ADI (Application Data Integrity) feature on + * sparc m7 and newer processors. This feature is also known as + * SSM (Silicon Secured Memory). + * + * Copyright (C) 2016 Oracle and/or its affiliates. All rights reserved. + * Author: Khalid Aziz (khalid.aziz@oracle.com) + */ +#include +#include +#include +#include +#include +#include +#include + +/* Each page of storage for ADI tags can accommodate tags for 128 + * pages. When ADI enabled pages are being swapped out, it would be + * prudent to allocate at least enough tag storage space to accommodate + * SWAPFILE_CLUSTER number of pages. Allocate enough tag storage to + * store tags for four SWAPFILE_CLUSTER pages to reduce need for + * further allocations for same vma. + */ +#define TAG_STORAGE_PAGES 8 + +struct adi_config adi_state; +EXPORT_SYMBOL(adi_state); + +/* mdesc_adi_init() : Parse machine description provided by the + * hypervisor to detect ADI capabilities + * + * Hypervisor reports ADI capabilities of platform in "hwcap-list" property + * for "cpu" node. If the platform supports ADI, "hwcap-list" property + * contains the keyword "adp". If the platform supports ADI, "platform" + * node will contain "adp-blksz", "adp-nbits" and "ue-on-adp" properties + * to describe the ADI capabilities. + */ +void __init mdesc_adi_init(void) +{ + struct mdesc_handle *hp = mdesc_grab(); + const char *prop; + u64 pn, *val; + int len; + + if (!hp) + goto adi_not_found; + + pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "cpu"); + if (pn == MDESC_NODE_NULL) + goto adi_not_found; + + prop = mdesc_get_property(hp, pn, "hwcap-list", &len); + if (!prop) + goto adi_not_found; + + /* + * Look for "adp" keyword in hwcap-list which would indicate + * ADI support + */ + adi_state.enabled = false; + while (len) { + int plen; + + if (!strcmp(prop, "adp")) { + adi_state.enabled = true; + break; + } + + plen = strlen(prop) + 1; + prop += plen; + len -= plen; + } + + if (!adi_state.enabled) + goto adi_not_found; + + /* Find the ADI properties in "platform" node. If all ADI + * properties are not found, ADI support is incomplete and + * do not enable ADI in the kernel. + */ + pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); + if (pn == MDESC_NODE_NULL) + goto adi_not_found; + + val = (u64 *) mdesc_get_property(hp, pn, "adp-blksz", &len); + if (!val) + goto adi_not_found; + adi_state.caps.blksz = *val; + + val = (u64 *) mdesc_get_property(hp, pn, "adp-nbits", &len); + if (!val) + goto adi_not_found; + adi_state.caps.nbits = *val; + + val = (u64 *) mdesc_get_property(hp, pn, "ue-on-adp", &len); + if (!val) + goto adi_not_found; + adi_state.caps.ue_on_adi = *val; + + /* Some of the code to support swapping ADI tags is written + * assumption that two ADI tags can fit inside one byte. If + * this assumption is broken by a future architecture change, + * that code will have to be revisited. If that were to happen, + * disable ADI support so we do not get unpredictable results + * with programs trying to use ADI and their pages getting + * swapped out + */ + if (adi_state.caps.nbits > 4) { + pr_warn("WARNING: ADI tag size >4 on this platform. Disabling AADI support\n"); + adi_state.enabled = false; + } + + mdesc_release(hp); + return; + +adi_not_found: + adi_state.enabled = false; + adi_state.caps.blksz = 0; + adi_state.caps.nbits = 0; + if (hp) + mdesc_release(hp); +} + +static tag_storage_desc_t *find_tag_store(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr) +{ + tag_storage_desc_t *tag_desc = NULL; + unsigned long i, max_desc, flags; + + /* Check if this vma already has tag storage descriptor + * allocated for it. + */ + max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t); + if (mm->context.tag_store) { + tag_desc = mm->context.tag_store; + spin_lock_irqsave(&mm->context.tag_lock, flags); + for (i = 0; i < max_desc; i++) { + if ((addr >= tag_desc->start) && + ((addr + PAGE_SIZE - 1) <= tag_desc->end)) + break; + tag_desc++; + } + spin_unlock_irqrestore(&mm->context.tag_lock, flags); + + /* If no matching entries were found, this must be a + * freshly allocated page + */ + if (i >= max_desc) + tag_desc = NULL; + } + + return tag_desc; +} + +static tag_storage_desc_t *alloc_tag_store(struct mm_struct *mm, + struct vm_area_struct *vma, + unsigned long addr) +{ + unsigned char *tags; + unsigned long i, size, max_desc, flags; + tag_storage_desc_t *tag_desc, *open_desc; + unsigned long end_addr, hole_start, hole_end; + + max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t); + open_desc = NULL; + hole_start = 0; + hole_end = ULONG_MAX; + end_addr = addr + PAGE_SIZE - 1; + + /* Check if this vma already has tag storage descriptor + * allocated for it. + */ + spin_lock_irqsave(&mm->context.tag_lock, flags); + if (mm->context.tag_store) { + tag_desc = mm->context.tag_store; + + /* Look for a matching entry for this address. While doing + * that, look for the first open slot as well and find + * the hole in already allocated range where this request + * will fit in. + */ + for (i = 0; i < max_desc; i++) { + if (tag_desc->tag_users == 0) { + if (open_desc == NULL) + open_desc = tag_desc; + } else { + if ((addr >= tag_desc->start) && + (tag_desc->end >= (addr + PAGE_SIZE - 1))) { + tag_desc->tag_users++; + goto out; + } + } + if ((tag_desc->start > end_addr) && + (tag_desc->start < hole_end)) + hole_end = tag_desc->start; + if ((tag_desc->end < addr) && + (tag_desc->end > hole_start)) + hole_start = tag_desc->end; + tag_desc++; + } + + } else { + size = sizeof(tag_storage_desc_t)*max_desc; + mm->context.tag_store = kzalloc(size, GFP_NOWAIT); + if (mm->context.tag_store == NULL) { + tag_desc = NULL; + goto out; + } + tag_desc = mm->context.tag_store; + for (i = 0; i < max_desc; i++, tag_desc++) + tag_desc->tag_users = 0; + open_desc = mm->context.tag_store; + i = 0; + } + + /* Check if we ran out of tag storage descriptors */ + if (open_desc == NULL) { + tag_desc = NULL; + goto out; + } + + /* Mark this tag descriptor slot in use and then initialize it */ + tag_desc = open_desc; + tag_desc->tag_users = 1; + + /* Tag storage has not been allocated for this vma and space + * is available in tag storage descriptor. Since this page is + * being swapped out, there is high probability subsequent pages + * in the VMA will be swapped out as well. Allocate pages to + * store tags for as many pages in this vma as possible but not + * more than TAG_STORAGE_PAGES. Each byte in tag space holds + * two ADI tags since each ADI tag is 4 bits. Each ADI tag + * covers adi_blksize() worth of addresses. Check if the hole is + * big enough to accommodate full address range for using + * TAG_STORAGE_PAGES number of tag pages. + */ + size = TAG_STORAGE_PAGES * PAGE_SIZE; + end_addr = addr + (size*2*adi_blksize()) - 1; + /* Check for overflow. If overflow occurs, allocate only one page */ + if (end_addr < addr) { + size = PAGE_SIZE; + end_addr = addr + (size*2*adi_blksize()) - 1; + /* If overflow happens with the minimum tag storage + * allocation as well, adjust ending address for this + * tag storage. + */ + if (end_addr < addr) + end_addr = ULONG_MAX; + } + if (hole_end < end_addr) { + /* Available hole is too small on the upper end of + * address. Can we expand the range towards the lower + * address and maximize use of this slot? + */ + unsigned long tmp_addr; + + end_addr = hole_end - 1; + tmp_addr = end_addr - (size*2*adi_blksize()) + 1; + /* Check for underflow. If underflow occurs, allocate + * only one page for storing ADI tags + */ + if (tmp_addr > addr) { + size = PAGE_SIZE; + tmp_addr = end_addr - (size*2*adi_blksize()) - 1; + /* If underflow happens with the minimum tag storage + * allocation as well, adjust starting address for + * this tag storage. + */ + if (tmp_addr > addr) + tmp_addr = 0; + } + if (tmp_addr < hole_start) { + /* Available hole is restricted on lower address + * end as well + */ + tmp_addr = hole_start + 1; + } + addr = tmp_addr; + size = (end_addr + 1 - addr)/(2*adi_blksize()); + size = (size + (PAGE_SIZE-adi_blksize()))/PAGE_SIZE; + size = size * PAGE_SIZE; + } + tags = kzalloc(size, GFP_NOWAIT); + if (tags == NULL) { + tag_desc->tag_users = 0; + tag_desc = NULL; + goto out; + } + tag_desc->start = addr; + tag_desc->tags = tags; + tag_desc->end = end_addr; + +out: + spin_unlock_irqrestore(&mm->context.tag_lock, flags); + return tag_desc; +} + +static void del_tag_store(tag_storage_desc_t *tag_desc, struct mm_struct *mm) +{ + unsigned long flags; + unsigned char *tags = NULL; + + spin_lock_irqsave(&mm->context.tag_lock, flags); + tag_desc->tag_users--; + if (tag_desc->tag_users == 0) { + tag_desc->start = tag_desc->end = 0; + /* Do not free up the tag storage space allocated + * by the first descriptor. This is persistent + * emergency tag storage space for the task. + */ + if (tag_desc != mm->context.tag_store) { + tags = tag_desc->tags; + tag_desc->tags = NULL; + } + } + spin_unlock_irqrestore(&mm->context.tag_lock, flags); + kfree(tags); +} + +#define tag_start(addr, tag_desc) \ + ((tag_desc)->tags + ((addr - (tag_desc)->start)/(2*adi_blksize()))) + +/* Retrieve any saved ADI tags for the page being swapped back in and + * restore these tags to the newly allocated physical page. + */ +void adi_restore_tags(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, pte_t pte) +{ + unsigned char *tag; + tag_storage_desc_t *tag_desc; + unsigned long paddr, tmp, version1, version2; + + /* Check if the swapped out page has an ADI version + * saved. If yes, restore version tag to the newly + * allocated page. + */ + tag_desc = find_tag_store(mm, vma, addr); + if (tag_desc == NULL) + return; + + tag = tag_start(addr, tag_desc); + paddr = pte_val(pte) & _PAGE_PADDR_4V; + for (tmp = paddr; tmp < (paddr+PAGE_SIZE); tmp += adi_blksize()) { + version1 = (*tag) >> 4; + version2 = (*tag) & 0x0f; + *tag++ = 0; + asm volatile("stxa %0, [%1] %2\n\t" + : + : "r" (version1), "r" (tmp), + "i" (ASI_MCD_REAL)); + tmp += adi_blksize(); + asm volatile("stxa %0, [%1] %2\n\t" + : + : "r" (version2), "r" (tmp), + "i" (ASI_MCD_REAL)); + } + asm volatile("membar #Sync\n\t"); + + /* Check and mark this tag space for release later if + * the swapped in page was the last user of tag space + */ + del_tag_store(tag_desc, mm); +} + +/* A page is about to be swapped out. Save any ADI tags associated with + * this physical page so they can be restored later when the page is swapped + * back in. + */ +int adi_save_tags(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, pte_t oldpte) +{ + unsigned char *tag; + tag_storage_desc_t *tag_desc; + unsigned long version1, version2, paddr, tmp; + + tag_desc = alloc_tag_store(mm, vma, addr); + if (tag_desc == NULL) + return -1; + + tag = tag_start(addr, tag_desc); + paddr = pte_val(oldpte) & _PAGE_PADDR_4V; + for (tmp = paddr; tmp < (paddr+PAGE_SIZE); tmp += adi_blksize()) { + asm volatile("ldxa [%1] %2, %0\n\t" + : "=r" (version1) + : "r" (tmp), "i" (ASI_MCD_REAL)); + tmp += adi_blksize(); + asm volatile("ldxa [%1] %2, %0\n\t" + : "=r" (version2) + : "r" (tmp), "i" (ASI_MCD_REAL)); + *tag = (version1 << 4) | version2; + tag++; + } + + return 0; +} diff --git a/arch/sparc/kernel/apc.c b/arch/sparc/kernel/apc.c new file mode 100644 index 000000000..849db20e7 --- /dev/null +++ b/arch/sparc/kernel/apc.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* apc - Driver implementation for power management functions + * of Aurora Personality Chip (APC) on SPARCstation-4/5 and + * derivatives. + * + * Copyright (c) 2002 Eric Brower (ebrower@usa.net) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +/* Debugging + * + * #define APC_DEBUG_LED + */ + +#define APC_OBPNAME "power-management" +#define APC_DEVNAME "apc" + +static u8 __iomem *regs; +static int apc_no_idle = 0; + +#define apc_readb(offs) (sbus_readb(regs+offs)) +#define apc_writeb(val, offs) (sbus_writeb(val, regs+offs)) + +/* Specify "apc=noidle" on the kernel command line to + * disable APC CPU standby support. Certain prototype + * systems (SPARCstation-Fox) do not play well with APC + * CPU idle, so disable this if your system has APC and + * crashes randomly. + */ +static int __init apc_setup(char *str) +{ + if(!strncmp(str, "noidle", strlen("noidle"))) { + apc_no_idle = 1; + return 1; + } + return 0; +} +__setup("apc=", apc_setup); + +/* + * CPU idle callback function + * See .../arch/sparc/kernel/process.c + */ +static void apc_swift_idle(void) +{ +#ifdef APC_DEBUG_LED + set_auxio(0x00, AUXIO_LED); +#endif + + apc_writeb(apc_readb(APC_IDLE_REG) | APC_IDLE_ON, APC_IDLE_REG); + +#ifdef APC_DEBUG_LED + set_auxio(AUXIO_LED, 0x00); +#endif +} + +static inline void apc_free(struct platform_device *op) +{ + of_iounmap(&op->resource[0], regs, resource_size(&op->resource[0])); +} + +static int apc_open(struct inode *inode, struct file *f) +{ + return 0; +} + +static int apc_release(struct inode *inode, struct file *f) +{ + return 0; +} + +static long apc_ioctl(struct file *f, unsigned int cmd, unsigned long __arg) +{ + __u8 inarg, __user *arg = (__u8 __user *) __arg; + + switch (cmd) { + case APCIOCGFANCTL: + if (put_user(apc_readb(APC_FANCTL_REG) & APC_REGMASK, arg)) + return -EFAULT; + break; + + case APCIOCGCPWR: + if (put_user(apc_readb(APC_CPOWER_REG) & APC_REGMASK, arg)) + return -EFAULT; + break; + + case APCIOCGBPORT: + if (put_user(apc_readb(APC_BPORT_REG) & APC_BPMASK, arg)) + return -EFAULT; + break; + + case APCIOCSFANCTL: + if (get_user(inarg, arg)) + return -EFAULT; + apc_writeb(inarg & APC_REGMASK, APC_FANCTL_REG); + break; + + case APCIOCSCPWR: + if (get_user(inarg, arg)) + return -EFAULT; + apc_writeb(inarg & APC_REGMASK, APC_CPOWER_REG); + break; + + case APCIOCSBPORT: + if (get_user(inarg, arg)) + return -EFAULT; + apc_writeb(inarg & APC_BPMASK, APC_BPORT_REG); + break; + + default: + return -EINVAL; + } + + return 0; +} + +static const struct file_operations apc_fops = { + .unlocked_ioctl = apc_ioctl, + .open = apc_open, + .release = apc_release, + .llseek = noop_llseek, +}; + +static struct miscdevice apc_miscdev = { MISC_DYNAMIC_MINOR, APC_DEVNAME, &apc_fops }; + +static int apc_probe(struct platform_device *op) +{ + int err; + + regs = of_ioremap(&op->resource[0], 0, + resource_size(&op->resource[0]), APC_OBPNAME); + if (!regs) { + printk(KERN_ERR "%s: unable to map registers\n", APC_DEVNAME); + return -ENODEV; + } + + err = misc_register(&apc_miscdev); + if (err) { + printk(KERN_ERR "%s: unable to register device\n", APC_DEVNAME); + apc_free(op); + return -ENODEV; + } + + /* Assign power management IDLE handler */ + if (!apc_no_idle) + sparc_idle = apc_swift_idle; + + printk(KERN_INFO "%s: power management initialized%s\n", + APC_DEVNAME, apc_no_idle ? " (CPU idle disabled)" : ""); + + return 0; +} + +static const struct of_device_id apc_match[] = { + { + .name = APC_OBPNAME, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, apc_match); + +static struct platform_driver apc_driver = { + .driver = { + .name = "apc", + .of_match_table = apc_match, + }, + .probe = apc_probe, +}; + +static int __init apc_init(void) +{ + return platform_driver_register(&apc_driver); +} + +/* This driver is not critical to the boot process + * and is easiest to ioremap when SBus is already + * initialized, so we install ourselves thusly: + */ +__initcall(apc_init); diff --git a/arch/sparc/kernel/asm-offsets.c b/arch/sparc/kernel/asm-offsets.c new file mode 100644 index 000000000..6e660bde4 --- /dev/null +++ b/arch/sparc/kernel/asm-offsets.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * This program is used to generate definitions needed by + * assembly language modules. + * + * We use the technique used in the OSF Mach kernel code: + * generate asm statements containing #defines, + * compile this file to assembler, and then extract the + * #defines from the assembly-language output. + * + * On sparc, thread_info data is static and TI_XXX offsets are computed by hand. + */ +#define COMPILE_OFFSETS + +#include +#include +// #include +#include + +#include + +#ifdef CONFIG_SPARC32 +static int __used sparc32_foo(void) +{ + DEFINE(AOFF_thread_fork_kpsr, + offsetof(struct thread_struct, fork_kpsr)); + return 0; +} +#else +static int __used sparc64_foo(void) +{ +#ifdef CONFIG_HIBERNATION + BLANK(); + OFFSET(SC_REG_FP, saved_context, fp); + OFFSET(SC_REG_CWP, saved_context, cwp); + OFFSET(SC_REG_WSTATE, saved_context, wstate); + + OFFSET(SC_REG_TICK, saved_context, tick); + OFFSET(SC_REG_PSTATE, saved_context, pstate); + + OFFSET(SC_REG_G4, saved_context, g4); + OFFSET(SC_REG_G5, saved_context, g5); + OFFSET(SC_REG_G6, saved_context, g6); +#endif + return 0; +} +#endif + +static int __used foo(void) +{ + BLANK(); + DEFINE(AOFF_task_thread, offsetof(struct task_struct, thread)); + BLANK(); + DEFINE(AOFF_mm_context, offsetof(struct mm_struct, context)); + BLANK(); + DEFINE(VMA_VM_MM, offsetof(struct vm_area_struct, vm_mm)); + + /* DEFINE(NUM_USER_SEGMENTS, TASK_SIZE>>28); */ + return 0; +} + diff --git a/arch/sparc/kernel/audit.c b/arch/sparc/kernel/audit.c new file mode 100644 index 000000000..b092274ec --- /dev/null +++ b/arch/sparc/kernel/audit.c @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +#include "kernel.h" + +static unsigned int dir_class[] = { +#include +~0U +}; + +static unsigned int read_class[] = { +#include +~0U +}; + +static unsigned int write_class[] = { +#include +~0U +}; + +static unsigned int chattr_class[] = { +#include +~0U +}; + +static unsigned int signal_class[] = { +#include +~0U +}; + +int audit_classify_arch(int arch) +{ +#ifdef CONFIG_COMPAT + if (arch == AUDIT_ARCH_SPARC) + return 1; +#endif + return 0; +} + +int audit_classify_syscall(int abi, unsigned int syscall) +{ +#ifdef CONFIG_COMPAT + if (abi == AUDIT_ARCH_SPARC) + return sparc32_classify_syscall(syscall); +#endif + switch(syscall) { + case __NR_open: + return AUDITSC_OPEN; + case __NR_openat: + return AUDITSC_OPENAT; + case __NR_socketcall: + return AUDITSC_SOCKETCALL; + case __NR_execve: + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; + default: + return AUDITSC_NATIVE; + } +} + +static int __init audit_classes_init(void) +{ +#ifdef CONFIG_COMPAT + audit_register_class(AUDIT_CLASS_WRITE_32, sparc32_write_class); + audit_register_class(AUDIT_CLASS_READ_32, sparc32_read_class); + audit_register_class(AUDIT_CLASS_DIR_WRITE_32, sparc32_dir_class); + audit_register_class(AUDIT_CLASS_CHATTR_32, sparc32_chattr_class); + audit_register_class(AUDIT_CLASS_SIGNAL_32, sparc32_signal_class); +#endif + audit_register_class(AUDIT_CLASS_WRITE, write_class); + audit_register_class(AUDIT_CLASS_READ, read_class); + audit_register_class(AUDIT_CLASS_DIR_WRITE, dir_class); + audit_register_class(AUDIT_CLASS_CHATTR, chattr_class); + audit_register_class(AUDIT_CLASS_SIGNAL, signal_class); + return 0; +} + +__initcall(audit_classes_init); diff --git a/arch/sparc/kernel/auxio_32.c b/arch/sparc/kernel/auxio_32.c new file mode 100644 index 000000000..989860e89 --- /dev/null +++ b/arch/sparc/kernel/auxio_32.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0 +/* auxio.c: Probing for the Sparc AUXIO register at boot time. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include /* memset(), Linux has no bzero() */ +#include + +#include "kernel.h" + +/* Probe and map in the Auxiliary I/O register */ + +/* auxio_register is not static because it is referenced + * in entry.S::floppy_tdone + */ +void __iomem *auxio_register = NULL; +static DEFINE_SPINLOCK(auxio_lock); + +void __init auxio_probe(void) +{ + phandle node, auxio_nd; + struct linux_prom_registers auxregs[1]; + struct resource r; + + switch (sparc_cpu_model) { + case sparc_leon: + case sun4d: + return; + default: + break; + } + node = prom_getchild(prom_root_node); + auxio_nd = prom_searchsiblings(node, "auxiliary-io"); + if(!auxio_nd) { + node = prom_searchsiblings(node, "obio"); + node = prom_getchild(node); + auxio_nd = prom_searchsiblings(node, "auxio"); + if(!auxio_nd) { +#ifdef CONFIG_PCI + /* There may be auxio on Ebus */ + return; +#else + if(prom_searchsiblings(node, "leds")) { + /* VME chassis sun4m machine, no auxio exists. */ + return; + } + prom_printf("Cannot find auxio node, cannot continue...\n"); + prom_halt(); +#endif + } + } + if(prom_getproperty(auxio_nd, "reg", (char *) auxregs, sizeof(auxregs)) <= 0) + return; + prom_apply_obio_ranges(auxregs, 0x1); + /* Map the register both read and write */ + r.flags = auxregs[0].which_io & 0xF; + r.start = auxregs[0].phys_addr; + r.end = auxregs[0].phys_addr + auxregs[0].reg_size - 1; + auxio_register = of_ioremap(&r, 0, auxregs[0].reg_size, "auxio"); + /* Fix the address on sun4m. */ + if ((((unsigned long) auxregs[0].phys_addr) & 3) == 3) + auxio_register += (3 - ((unsigned long)auxio_register & 3)); + + set_auxio(AUXIO_LED, 0); +} + +unsigned char get_auxio(void) +{ + if(auxio_register) + return sbus_readb(auxio_register); + return 0; +} +EXPORT_SYMBOL(get_auxio); + +void set_auxio(unsigned char bits_on, unsigned char bits_off) +{ + unsigned char regval; + unsigned long flags; + spin_lock_irqsave(&auxio_lock, flags); + switch (sparc_cpu_model) { + case sun4m: + if(!auxio_register) + break; /* VME chassis sun4m, no auxio. */ + regval = sbus_readb(auxio_register); + sbus_writeb(((regval | bits_on) & ~bits_off) | AUXIO_ORMEIN4M, + auxio_register); + break; + case sun4d: + break; + default: + panic("Can't set AUXIO register on this machine."); + } + spin_unlock_irqrestore(&auxio_lock, flags); +} +EXPORT_SYMBOL(set_auxio); + +/* sun4m power control register (AUXIO2) */ + +volatile u8 __iomem *auxio_power_register = NULL; + +void __init auxio_power_probe(void) +{ + struct linux_prom_registers regs; + phandle node; + struct resource r; + + /* Attempt to find the sun4m power control node. */ + node = prom_getchild(prom_root_node); + node = prom_searchsiblings(node, "obio"); + node = prom_getchild(node); + node = prom_searchsiblings(node, "power"); + if (node == 0 || (s32)node == -1) + return; + + /* Map the power control register. */ + if (prom_getproperty(node, "reg", (char *)®s, sizeof(regs)) <= 0) + return; + prom_apply_obio_ranges(®s, 1); + memset(&r, 0, sizeof(r)); + r.flags = regs.which_io & 0xF; + r.start = regs.phys_addr; + r.end = regs.phys_addr + regs.reg_size - 1; + auxio_power_register = + (u8 __iomem *)of_ioremap(&r, 0, regs.reg_size, "auxpower"); + + /* Display a quick message on the console. */ + if (auxio_power_register) + printk(KERN_INFO "Power off control detected.\n"); +} diff --git a/arch/sparc/kernel/auxio_64.c b/arch/sparc/kernel/auxio_64.c new file mode 100644 index 000000000..2a2800d21 --- /dev/null +++ b/arch/sparc/kernel/auxio_64.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 +/* auxio.c: Probing for the Sparc AUXIO register at boot time. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * + * Refactoring for unified NCR/PCIO support 2002 Eric Brower (ebrower@usa.net) + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +void __iomem *auxio_register = NULL; +EXPORT_SYMBOL(auxio_register); + +enum auxio_type { + AUXIO_TYPE_NODEV, + AUXIO_TYPE_SBUS, + AUXIO_TYPE_EBUS +}; + +static enum auxio_type auxio_devtype = AUXIO_TYPE_NODEV; +static DEFINE_SPINLOCK(auxio_lock); + +static void __auxio_rmw(u8 bits_on, u8 bits_off, int ebus) +{ + if (auxio_register) { + unsigned long flags; + u8 regval, newval; + + spin_lock_irqsave(&auxio_lock, flags); + + regval = (ebus ? + (u8) readl(auxio_register) : + sbus_readb(auxio_register)); + newval = regval | bits_on; + newval &= ~bits_off; + if (!ebus) + newval &= ~AUXIO_AUX1_MASK; + if (ebus) + writel((u32) newval, auxio_register); + else + sbus_writeb(newval, auxio_register); + + spin_unlock_irqrestore(&auxio_lock, flags); + } +} + +static void __auxio_set_bit(u8 bit, int on, int ebus) +{ + u8 bits_on = (ebus ? AUXIO_PCIO_LED : AUXIO_AUX1_LED); + u8 bits_off = 0; + + if (!on) { + u8 tmp = bits_off; + bits_off = bits_on; + bits_on = tmp; + } + __auxio_rmw(bits_on, bits_off, ebus); +} + +void auxio_set_led(int on) +{ + int ebus = auxio_devtype == AUXIO_TYPE_EBUS; + u8 bit; + + bit = (ebus ? AUXIO_PCIO_LED : AUXIO_AUX1_LED); + __auxio_set_bit(bit, on, ebus); +} +EXPORT_SYMBOL(auxio_set_led); + +static void __auxio_sbus_set_lte(int on) +{ + __auxio_set_bit(AUXIO_AUX1_LTE, on, 0); +} + +void auxio_set_lte(int on) +{ + switch(auxio_devtype) { + case AUXIO_TYPE_SBUS: + __auxio_sbus_set_lte(on); + break; + case AUXIO_TYPE_EBUS: + default: + break; + } +} +EXPORT_SYMBOL(auxio_set_lte); + +static const struct of_device_id auxio_match[] = { + { + .name = "auxio", + }, + {}, +}; + +MODULE_DEVICE_TABLE(of, auxio_match); + +static int auxio_probe(struct platform_device *dev) +{ + struct device_node *dp = dev->dev.of_node; + unsigned long size; + + if (of_node_name_eq(dp->parent, "ebus")) { + auxio_devtype = AUXIO_TYPE_EBUS; + size = sizeof(u32); + } else if (of_node_name_eq(dp->parent, "sbus")) { + auxio_devtype = AUXIO_TYPE_SBUS; + size = 1; + } else { + printk("auxio: Unknown parent bus type [%pOFn]\n", + dp->parent); + return -ENODEV; + } + auxio_register = of_ioremap(&dev->resource[0], 0, size, "auxio"); + if (!auxio_register) + return -ENODEV; + + printk(KERN_INFO "AUXIO: Found device at %pOF\n", dp); + + if (auxio_devtype == AUXIO_TYPE_EBUS) + auxio_set_led(AUXIO_LED_ON); + + return 0; +} + +static struct platform_driver auxio_driver = { + .probe = auxio_probe, + .driver = { + .name = "auxio", + .of_match_table = auxio_match, + }, +}; + +static int __init auxio_init(void) +{ + return platform_driver_register(&auxio_driver); +} + +/* Must be after subsys_initcall() so that busses are probed. Must + * be before device_initcall() because things like the floppy driver + * need to use the AUXIO register. + */ +fs_initcall(auxio_init); diff --git a/arch/sparc/kernel/btext.c b/arch/sparc/kernel/btext.c new file mode 100644 index 000000000..2bf558a0c --- /dev/null +++ b/arch/sparc/kernel/btext.c @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Procedures for drawing on the screen early on in the boot process. + * + * Benjamin Herrenschmidt + */ +#include +#include +#include +#include +#include + +#include +#include +#include + +#define NO_SCROLL + +#ifndef NO_SCROLL +static void scrollscreen(void); +#endif + +static void draw_byte(unsigned char c, long locX, long locY); +static void draw_byte_32(const unsigned char *bits, unsigned int *base, int rb); +static void draw_byte_16(const unsigned char *bits, unsigned int *base, int rb); +static void draw_byte_8(const unsigned char *bits, unsigned int *base, int rb); + +#define __force_data __section(".data") + +static int g_loc_X __force_data; +static int g_loc_Y __force_data; +static int g_max_loc_X __force_data; +static int g_max_loc_Y __force_data; + +static int dispDeviceRowBytes __force_data; +static int dispDeviceDepth __force_data; +static int dispDeviceRect[4] __force_data; +static unsigned char *dispDeviceBase __force_data; + +static int __init btext_initialize(phandle node) +{ + unsigned int width, height, depth, pitch; + unsigned long address = 0; + u32 prop; + + if (prom_getproperty(node, "width", (char *)&width, 4) < 0) + return -EINVAL; + if (prom_getproperty(node, "height", (char *)&height, 4) < 0) + return -EINVAL; + if (prom_getproperty(node, "depth", (char *)&depth, 4) < 0) + return -EINVAL; + pitch = width * ((depth + 7) / 8); + + if (prom_getproperty(node, "linebytes", (char *)&prop, 4) >= 0 && + prop != 0xffffffffu) + pitch = prop; + + if (pitch == 1) + pitch = 0x1000; + + if (prom_getproperty(node, "address", (char *)&prop, 4) >= 0) + address = prop; + + /* FIXME: Add support for PCI reg properties. Right now, only + * reliable on macs + */ + if (address == 0) + return -EINVAL; + + g_loc_X = 0; + g_loc_Y = 0; + g_max_loc_X = width / 8; + g_max_loc_Y = height / 16; + dispDeviceBase = (unsigned char *)address; + dispDeviceRowBytes = pitch; + dispDeviceDepth = depth == 15 ? 16 : depth; + dispDeviceRect[0] = dispDeviceRect[1] = 0; + dispDeviceRect[2] = width; + dispDeviceRect[3] = height; + + return 0; +} + +/* Calc the base address of a given point (x,y) */ +static unsigned char * calc_base(int x, int y) +{ + unsigned char *base = dispDeviceBase; + + base += (x + dispDeviceRect[0]) * (dispDeviceDepth >> 3); + base += (y + dispDeviceRect[1]) * dispDeviceRowBytes; + return base; +} + +static void btext_clearscreen(void) +{ + unsigned int *base = (unsigned int *)calc_base(0, 0); + unsigned long width = ((dispDeviceRect[2] - dispDeviceRect[0]) * + (dispDeviceDepth >> 3)) >> 2; + int i,j; + + for (i=0; i<(dispDeviceRect[3] - dispDeviceRect[1]); i++) + { + unsigned int *ptr = base; + for(j=width; j; --j) + *(ptr++) = 0; + base += (dispDeviceRowBytes >> 2); + } +} + +#ifndef NO_SCROLL +static void scrollscreen(void) +{ + unsigned int *src = (unsigned int *)calc_base(0,16); + unsigned int *dst = (unsigned int *)calc_base(0,0); + unsigned long width = ((dispDeviceRect[2] - dispDeviceRect[0]) * + (dispDeviceDepth >> 3)) >> 2; + int i,j; + + for (i=0; i<(dispDeviceRect[3] - dispDeviceRect[1] - 16); i++) + { + unsigned int *src_ptr = src; + unsigned int *dst_ptr = dst; + for(j=width; j; --j) + *(dst_ptr++) = *(src_ptr++); + src += (dispDeviceRowBytes >> 2); + dst += (dispDeviceRowBytes >> 2); + } + for (i=0; i<16; i++) + { + unsigned int *dst_ptr = dst; + for(j=width; j; --j) + *(dst_ptr++) = 0; + dst += (dispDeviceRowBytes >> 2); + } +} +#endif /* ndef NO_SCROLL */ + +static void btext_drawchar(char c) +{ + int cline = 0; +#ifdef NO_SCROLL + int x; +#endif + switch (c) { + case '\b': + if (g_loc_X > 0) + --g_loc_X; + break; + case '\t': + g_loc_X = (g_loc_X & -8) + 8; + break; + case '\r': + g_loc_X = 0; + break; + case '\n': + g_loc_X = 0; + g_loc_Y++; + cline = 1; + break; + default: + draw_byte(c, g_loc_X++, g_loc_Y); + } + if (g_loc_X >= g_max_loc_X) { + g_loc_X = 0; + g_loc_Y++; + cline = 1; + } +#ifndef NO_SCROLL + while (g_loc_Y >= g_max_loc_Y) { + scrollscreen(); + g_loc_Y--; + } +#else + /* wrap around from bottom to top of screen so we don't + waste time scrolling each line. -- paulus. */ + if (g_loc_Y >= g_max_loc_Y) + g_loc_Y = 0; + if (cline) { + for (x = 0; x < g_max_loc_X; ++x) + draw_byte(' ', x, g_loc_Y); + } +#endif +} + +static void btext_drawtext(const char *c, unsigned int len) +{ + while (len--) + btext_drawchar(*c++); +} + +static void draw_byte(unsigned char c, long locX, long locY) +{ + unsigned char *base = calc_base(locX << 3, locY << 4); + unsigned int font_index = c * 16; + const unsigned char *font = font_sun_8x16.data + font_index; + int rb = dispDeviceRowBytes; + + switch(dispDeviceDepth) { + case 24: + case 32: + draw_byte_32(font, (unsigned int *)base, rb); + break; + case 15: + case 16: + draw_byte_16(font, (unsigned int *)base, rb); + break; + case 8: + draw_byte_8(font, (unsigned int *)base, rb); + break; + } +} + +static unsigned int expand_bits_8[16] = { + 0x00000000, + 0x000000ff, + 0x0000ff00, + 0x0000ffff, + 0x00ff0000, + 0x00ff00ff, + 0x00ffff00, + 0x00ffffff, + 0xff000000, + 0xff0000ff, + 0xff00ff00, + 0xff00ffff, + 0xffff0000, + 0xffff00ff, + 0xffffff00, + 0xffffffff +}; + +static unsigned int expand_bits_16[4] = { + 0x00000000, + 0x0000ffff, + 0xffff0000, + 0xffffffff +}; + + +static void draw_byte_32(const unsigned char *font, unsigned int *base, int rb) +{ + int l, bits; + int fg = 0xFFFFFFFFUL; + int bg = 0x00000000UL; + + for (l = 0; l < 16; ++l) + { + bits = *font++; + base[0] = (-(bits >> 7) & fg) ^ bg; + base[1] = (-((bits >> 6) & 1) & fg) ^ bg; + base[2] = (-((bits >> 5) & 1) & fg) ^ bg; + base[3] = (-((bits >> 4) & 1) & fg) ^ bg; + base[4] = (-((bits >> 3) & 1) & fg) ^ bg; + base[5] = (-((bits >> 2) & 1) & fg) ^ bg; + base[6] = (-((bits >> 1) & 1) & fg) ^ bg; + base[7] = (-(bits & 1) & fg) ^ bg; + base = (unsigned int *) ((char *)base + rb); + } +} + +static void draw_byte_16(const unsigned char *font, unsigned int *base, int rb) +{ + int l, bits; + int fg = 0xFFFFFFFFUL; + int bg = 0x00000000UL; + unsigned int *eb = (int *)expand_bits_16; + + for (l = 0; l < 16; ++l) + { + bits = *font++; + base[0] = (eb[bits >> 6] & fg) ^ bg; + base[1] = (eb[(bits >> 4) & 3] & fg) ^ bg; + base[2] = (eb[(bits >> 2) & 3] & fg) ^ bg; + base[3] = (eb[bits & 3] & fg) ^ bg; + base = (unsigned int *) ((char *)base + rb); + } +} + +static void draw_byte_8(const unsigned char *font, unsigned int *base, int rb) +{ + int l, bits; + int fg = 0x0F0F0F0FUL; + int bg = 0x00000000UL; + unsigned int *eb = (int *)expand_bits_8; + + for (l = 0; l < 16; ++l) + { + bits = *font++; + base[0] = (eb[bits >> 4] & fg) ^ bg; + base[1] = (eb[bits & 0xf] & fg) ^ bg; + base = (unsigned int *) ((char *)base + rb); + } +} + +static void btext_console_write(struct console *con, const char *s, + unsigned int n) +{ + btext_drawtext(s, n); +} + +static struct console btext_console = { + .name = "btext", + .write = btext_console_write, + .flags = CON_PRINTBUFFER | CON_ENABLED | CON_BOOT | CON_ANYTIME, + .index = 0, +}; + +int __init btext_find_display(void) +{ + phandle node; + char type[32]; + int ret; + + node = prom_inst2pkg(prom_stdout); + if (prom_getproperty(node, "device_type", type, 32) < 0) + return -ENODEV; + if (strcmp(type, "display")) + return -ENODEV; + + ret = btext_initialize(node); + if (!ret) { + btext_clearscreen(); + register_console(&btext_console); + } + return ret; +} diff --git a/arch/sparc/kernel/central.c b/arch/sparc/kernel/central.c new file mode 100644 index 000000000..16b45d172 --- /dev/null +++ b/arch/sparc/kernel/central.c @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: GPL-2.0 +/* central.c: Central FHC driver for Sunfire/Starfire/Wildfire. + * + * Copyright (C) 1997, 1999, 2008 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct clock_board { + void __iomem *clock_freq_regs; + void __iomem *clock_regs; + void __iomem *clock_ver_reg; + int num_slots; + struct resource leds_resource; + struct platform_device leds_pdev; +}; + +struct fhc { + void __iomem *pregs; + bool central; + bool jtag_master; + int board_num; + struct resource leds_resource; + struct platform_device leds_pdev; +}; + +static int clock_board_calc_nslots(struct clock_board *p) +{ + u8 reg = upa_readb(p->clock_regs + CLOCK_STAT1) & 0xc0; + + switch (reg) { + case 0x40: + return 16; + + case 0xc0: + return 8; + + case 0x80: + reg = 0; + if (p->clock_ver_reg) + reg = upa_readb(p->clock_ver_reg); + if (reg) { + if (reg & 0x80) + return 4; + else + return 5; + } + fallthrough; + default: + return 4; + } +} + +static int clock_board_probe(struct platform_device *op) +{ + struct clock_board *p = kzalloc_obj(*p); + int err = -ENOMEM; + + if (!p) { + printk(KERN_ERR "clock_board: Cannot allocate struct clock_board\n"); + goto out; + } + + p->clock_freq_regs = of_ioremap(&op->resource[0], 0, + resource_size(&op->resource[0]), + "clock_board_freq"); + if (!p->clock_freq_regs) { + printk(KERN_ERR "clock_board: Cannot map clock_freq_regs\n"); + goto out_free; + } + + p->clock_regs = of_ioremap(&op->resource[1], 0, + resource_size(&op->resource[1]), + "clock_board_regs"); + if (!p->clock_regs) { + printk(KERN_ERR "clock_board: Cannot map clock_regs\n"); + goto out_unmap_clock_freq_regs; + } + + if (op->resource[2].flags) { + p->clock_ver_reg = of_ioremap(&op->resource[2], 0, + resource_size(&op->resource[2]), + "clock_ver_reg"); + if (!p->clock_ver_reg) { + printk(KERN_ERR "clock_board: Cannot map clock_ver_reg\n"); + goto out_unmap_clock_regs; + } + } + + p->num_slots = clock_board_calc_nslots(p); + + p->leds_resource.start = (unsigned long) + (p->clock_regs + CLOCK_CTRL); + p->leds_resource.end = p->leds_resource.start; + p->leds_resource.name = "leds"; + + p->leds_pdev.name = "sunfire-clockboard-leds"; + p->leds_pdev.id = -1; + p->leds_pdev.resource = &p->leds_resource; + p->leds_pdev.num_resources = 1; + p->leds_pdev.dev.parent = &op->dev; + + err = platform_device_register(&p->leds_pdev); + if (err) { + printk(KERN_ERR "clock_board: Could not register LEDS " + "platform device\n"); + goto out_unmap_clock_ver_reg; + } + + printk(KERN_INFO "clock_board: Detected %d slot Enterprise system.\n", + p->num_slots); + + err = 0; +out: + return err; + +out_unmap_clock_ver_reg: + if (p->clock_ver_reg) + of_iounmap(&op->resource[2], p->clock_ver_reg, + resource_size(&op->resource[2])); + +out_unmap_clock_regs: + of_iounmap(&op->resource[1], p->clock_regs, + resource_size(&op->resource[1])); + +out_unmap_clock_freq_regs: + of_iounmap(&op->resource[0], p->clock_freq_regs, + resource_size(&op->resource[0])); + +out_free: + kfree(p); + goto out; +} + +static const struct of_device_id clock_board_match[] = { + { + .name = "clock-board", + }, + {}, +}; + +static struct platform_driver clock_board_driver = { + .probe = clock_board_probe, + .driver = { + .name = "clock_board", + .of_match_table = clock_board_match, + }, +}; + +static int fhc_probe(struct platform_device *op) +{ + struct fhc *p = kzalloc_obj(*p); + int err = -ENOMEM; + u32 reg; + + if (!p) { + printk(KERN_ERR "fhc: Cannot allocate struct fhc\n"); + goto out; + } + + if (of_node_name_eq(op->dev.of_node->parent, "central")) + p->central = true; + + p->pregs = of_ioremap(&op->resource[0], 0, + resource_size(&op->resource[0]), + "fhc_pregs"); + if (!p->pregs) { + printk(KERN_ERR "fhc: Cannot map pregs\n"); + goto out_free; + } + + if (p->central) { + reg = upa_readl(p->pregs + FHC_PREGS_BSR); + p->board_num = ((reg >> 16) & 1) | ((reg >> 12) & 0x0e); + } else { + p->board_num = of_getintprop_default(op->dev.of_node, "board#", -1); + if (p->board_num == -1) { + printk(KERN_ERR "fhc: No board# property\n"); + goto out_unmap_pregs; + } + if (upa_readl(p->pregs + FHC_PREGS_JCTRL) & FHC_JTAG_CTRL_MENAB) + p->jtag_master = true; + } + + if (!p->central) { + p->leds_resource.start = (unsigned long) + (p->pregs + FHC_PREGS_CTRL); + p->leds_resource.end = p->leds_resource.start; + p->leds_resource.name = "leds"; + + p->leds_pdev.name = "sunfire-fhc-leds"; + p->leds_pdev.id = p->board_num; + p->leds_pdev.resource = &p->leds_resource; + p->leds_pdev.num_resources = 1; + p->leds_pdev.dev.parent = &op->dev; + + err = platform_device_register(&p->leds_pdev); + if (err) { + printk(KERN_ERR "fhc: Could not register LEDS " + "platform device\n"); + goto out_unmap_pregs; + } + } + reg = upa_readl(p->pregs + FHC_PREGS_CTRL); + + if (!p->central) + reg |= FHC_CONTROL_IXIST; + + reg &= ~(FHC_CONTROL_AOFF | + FHC_CONTROL_BOFF | + FHC_CONTROL_SLINE); + + upa_writel(reg, p->pregs + FHC_PREGS_CTRL); + upa_readl(p->pregs + FHC_PREGS_CTRL); + + reg = upa_readl(p->pregs + FHC_PREGS_ID); + printk(KERN_INFO "fhc: Board #%d, Version[%x] PartID[%x] Manuf[%x] %s\n", + p->board_num, + (reg & FHC_ID_VERS) >> 28, + (reg & FHC_ID_PARTID) >> 12, + (reg & FHC_ID_MANUF) >> 1, + (p->jtag_master ? + "(JTAG Master)" : + (p->central ? "(Central)" : ""))); + + err = 0; + +out: + return err; + +out_unmap_pregs: + of_iounmap(&op->resource[0], p->pregs, resource_size(&op->resource[0])); + +out_free: + kfree(p); + goto out; +} + +static const struct of_device_id fhc_match[] = { + { + .name = "fhc", + }, + {}, +}; + +static struct platform_driver fhc_driver = { + .probe = fhc_probe, + .driver = { + .name = "fhc", + .of_match_table = fhc_match, + }, +}; + +static int __init sunfire_init(void) +{ + (void) platform_driver_register(&fhc_driver); + (void) platform_driver_register(&clock_board_driver); + return 0; +} + +fs_initcall(sunfire_init); diff --git a/arch/sparc/kernel/cherrs.S b/arch/sparc/kernel/cherrs.S new file mode 100644 index 000000000..7f3d3d264 --- /dev/null +++ b/arch/sparc/kernel/cherrs.S @@ -0,0 +1,576 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + /* These get patched into the trap table at boot time + * once we know we have a cheetah processor. + */ + .globl cheetah_fecc_trap_vector + .type cheetah_fecc_trap_vector,#function +cheetah_fecc_trap_vector: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + andn %g1, DCU_DC | DCU_IC, %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + sethi %hi(cheetah_fast_ecc), %g2 + jmpl %g2 + %lo(cheetah_fast_ecc), %g0 + mov 0, %g1 + .size cheetah_fecc_trap_vector,.-cheetah_fecc_trap_vector + + .globl cheetah_fecc_trap_vector_tl1 + .type cheetah_fecc_trap_vector_tl1,#function +cheetah_fecc_trap_vector_tl1: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + andn %g1, DCU_DC | DCU_IC, %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + sethi %hi(cheetah_fast_ecc), %g2 + jmpl %g2 + %lo(cheetah_fast_ecc), %g0 + mov 1, %g1 + .size cheetah_fecc_trap_vector_tl1,.-cheetah_fecc_trap_vector_tl1 + + .globl cheetah_cee_trap_vector + .type cheetah_cee_trap_vector,#function +cheetah_cee_trap_vector: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + andn %g1, DCU_IC, %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + sethi %hi(cheetah_cee), %g2 + jmpl %g2 + %lo(cheetah_cee), %g0 + mov 0, %g1 + .size cheetah_cee_trap_vector,.-cheetah_cee_trap_vector + + .globl cheetah_cee_trap_vector_tl1 + .type cheetah_cee_trap_vector_tl1,#function +cheetah_cee_trap_vector_tl1: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + andn %g1, DCU_IC, %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + sethi %hi(cheetah_cee), %g2 + jmpl %g2 + %lo(cheetah_cee), %g0 + mov 1, %g1 + .size cheetah_cee_trap_vector_tl1,.-cheetah_cee_trap_vector_tl1 + + .globl cheetah_deferred_trap_vector + .type cheetah_deferred_trap_vector,#function +cheetah_deferred_trap_vector: + membar #Sync + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1; + andn %g1, DCU_DC | DCU_IC, %g1; + stxa %g1, [%g0] ASI_DCU_CONTROL_REG; + membar #Sync; + sethi %hi(cheetah_deferred_trap), %g2 + jmpl %g2 + %lo(cheetah_deferred_trap), %g0 + mov 0, %g1 + .size cheetah_deferred_trap_vector,.-cheetah_deferred_trap_vector + + .globl cheetah_deferred_trap_vector_tl1 + .type cheetah_deferred_trap_vector_tl1,#function +cheetah_deferred_trap_vector_tl1: + membar #Sync; + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1; + andn %g1, DCU_DC | DCU_IC, %g1; + stxa %g1, [%g0] ASI_DCU_CONTROL_REG; + membar #Sync; + sethi %hi(cheetah_deferred_trap), %g2 + jmpl %g2 + %lo(cheetah_deferred_trap), %g0 + mov 1, %g1 + .size cheetah_deferred_trap_vector_tl1,.-cheetah_deferred_trap_vector_tl1 + + /* Cheetah+ specific traps. These are for the new I/D cache parity + * error traps. The first argument to cheetah_plus_parity_handler + * is encoded as follows: + * + * Bit0: 0=dcache,1=icache + * Bit1: 0=recoverable,1=unrecoverable + */ + .globl cheetah_plus_dcpe_trap_vector + .type cheetah_plus_dcpe_trap_vector,#function +cheetah_plus_dcpe_trap_vector: + membar #Sync + sethi %hi(do_cheetah_plus_data_parity), %g7 + jmpl %g7 + %lo(do_cheetah_plus_data_parity), %g0 + nop + nop + nop + nop + nop + .size cheetah_plus_dcpe_trap_vector,.-cheetah_plus_dcpe_trap_vector + + .type do_cheetah_plus_data_parity,#function +do_cheetah_plus_data_parity: + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov 0x0, %o0 + call cheetah_plus_parity_error + add %sp, PTREGS_OFF, %o1 + ba,a,pt %xcc, rtrap_irq + .size do_cheetah_plus_data_parity,.-do_cheetah_plus_data_parity + + .globl cheetah_plus_dcpe_trap_vector_tl1 + .type cheetah_plus_dcpe_trap_vector_tl1,#function +cheetah_plus_dcpe_trap_vector_tl1: + membar #Sync + wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate + sethi %hi(do_dcpe_tl1), %g3 + jmpl %g3 + %lo(do_dcpe_tl1), %g0 + nop + nop + nop + nop + .size cheetah_plus_dcpe_trap_vector_tl1,.-cheetah_plus_dcpe_trap_vector_tl1 + + .globl cheetah_plus_icpe_trap_vector + .type cheetah_plus_icpe_trap_vector,#function +cheetah_plus_icpe_trap_vector: + membar #Sync + sethi %hi(do_cheetah_plus_insn_parity), %g7 + jmpl %g7 + %lo(do_cheetah_plus_insn_parity), %g0 + nop + nop + nop + nop + nop + .size cheetah_plus_icpe_trap_vector,.-cheetah_plus_icpe_trap_vector + + .type do_cheetah_plus_insn_parity,#function +do_cheetah_plus_insn_parity: + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov 0x1, %o0 + call cheetah_plus_parity_error + add %sp, PTREGS_OFF, %o1 + ba,a,pt %xcc, rtrap_irq + .size do_cheetah_plus_insn_parity,.-do_cheetah_plus_insn_parity + + .globl cheetah_plus_icpe_trap_vector_tl1 + .type cheetah_plus_icpe_trap_vector_tl1,#function +cheetah_plus_icpe_trap_vector_tl1: + membar #Sync + wrpr PSTATE_IG | PSTATE_PEF | PSTATE_PRIV, %pstate + sethi %hi(do_icpe_tl1), %g3 + jmpl %g3 + %lo(do_icpe_tl1), %g0 + nop + nop + nop + nop + .size cheetah_plus_icpe_trap_vector_tl1,.-cheetah_plus_icpe_trap_vector_tl1 + + /* If we take one of these traps when tl >= 1, then we + * jump to interrupt globals. If some trap level above us + * was also using interrupt globals, we cannot recover. + * We may use all interrupt global registers except %g6. + */ + .globl do_dcpe_tl1 + .type do_dcpe_tl1,#function +do_dcpe_tl1: + rdpr %tl, %g1 ! Save original trap level + mov 1, %g2 ! Setup TSTATE checking loop + sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit +1: wrpr %g2, %tl ! Set trap level to check + rdpr %tstate, %g4 ! Read TSTATE for this level + andcc %g4, %g3, %g0 ! Interrupt globals in use? + bne,a,pn %xcc, do_dcpe_tl1_fatal ! Yep, irrecoverable + wrpr %g1, %tl ! Restore original trap level + add %g2, 1, %g2 ! Next trap level + cmp %g2, %g1 ! Hit them all yet? + ble,pt %icc, 1b ! Not yet + nop + wrpr %g1, %tl ! Restore original trap level +do_dcpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ + sethi %hi(dcache_parity_tl1_occurred), %g2 + lduw [%g2 + %lo(dcache_parity_tl1_occurred)], %g1 + add %g1, 1, %g1 + stw %g1, [%g2 + %lo(dcache_parity_tl1_occurred)] + /* Reset D-cache parity */ + sethi %hi(1 << 16), %g1 ! D-cache size + mov (1 << 5), %g2 ! D-cache line size + sub %g1, %g2, %g1 ! Move down 1 cacheline +1: srl %g1, 14, %g3 ! Compute UTAG + membar #Sync + stxa %g3, [%g1] ASI_DCACHE_UTAG + membar #Sync + sub %g2, 8, %g3 ! 64-bit data word within line +2: membar #Sync + stxa %g0, [%g1 + %g3] ASI_DCACHE_DATA + membar #Sync + subcc %g3, 8, %g3 ! Next 64-bit data word + bge,pt %icc, 2b + nop + subcc %g1, %g2, %g1 ! Next cacheline + bge,pt %icc, 1b + nop + ba,a,pt %xcc, dcpe_icpe_tl1_common + +do_dcpe_tl1_fatal: + sethi %hi(1f), %g7 + ba,pt %xcc, etraptl1 +1: or %g7, %lo(1b), %g7 + mov 0x2, %o0 + call cheetah_plus_parity_error + add %sp, PTREGS_OFF, %o1 + ba,a,pt %xcc, rtrap + .size do_dcpe_tl1,.-do_dcpe_tl1 + + .globl do_icpe_tl1 + .type do_icpe_tl1,#function +do_icpe_tl1: + rdpr %tl, %g1 ! Save original trap level + mov 1, %g2 ! Setup TSTATE checking loop + sethi %hi(TSTATE_IG), %g3 ! TSTATE mask bit +1: wrpr %g2, %tl ! Set trap level to check + rdpr %tstate, %g4 ! Read TSTATE for this level + andcc %g4, %g3, %g0 ! Interrupt globals in use? + bne,a,pn %xcc, do_icpe_tl1_fatal ! Yep, irrecoverable + wrpr %g1, %tl ! Restore original trap level + add %g2, 1, %g2 ! Next trap level + cmp %g2, %g1 ! Hit them all yet? + ble,pt %icc, 1b ! Not yet + nop + wrpr %g1, %tl ! Restore original trap level +do_icpe_tl1_nonfatal: /* Ok we may use interrupt globals safely. */ + sethi %hi(icache_parity_tl1_occurred), %g2 + lduw [%g2 + %lo(icache_parity_tl1_occurred)], %g1 + add %g1, 1, %g1 + stw %g1, [%g2 + %lo(icache_parity_tl1_occurred)] + /* Flush I-cache */ + sethi %hi(1 << 15), %g1 ! I-cache size + mov (1 << 5), %g2 ! I-cache line size + sub %g1, %g2, %g1 +1: or %g1, (2 << 3), %g3 + stxa %g0, [%g3] ASI_IC_TAG + membar #Sync + subcc %g1, %g2, %g1 + bge,pt %icc, 1b + nop + ba,a,pt %xcc, dcpe_icpe_tl1_common + +do_icpe_tl1_fatal: + sethi %hi(1f), %g7 + ba,pt %xcc, etraptl1 +1: or %g7, %lo(1b), %g7 + mov 0x3, %o0 + call cheetah_plus_parity_error + add %sp, PTREGS_OFF, %o1 + ba,a,pt %xcc, rtrap + .size do_icpe_tl1,.-do_icpe_tl1 + + .type dcpe_icpe_tl1_common,#function +dcpe_icpe_tl1_common: + /* Flush D-cache, re-enable D/I caches in DCU and finally + * retry the trapping instruction. + */ + sethi %hi(1 << 16), %g1 ! D-cache size + mov (1 << 5), %g2 ! D-cache line size + sub %g1, %g2, %g1 +1: stxa %g0, [%g1] ASI_DCACHE_TAG + membar #Sync + subcc %g1, %g2, %g1 + bge,pt %icc, 1b + nop + ldxa [%g0] ASI_DCU_CONTROL_REG, %g1 + or %g1, (DCU_DC | DCU_IC), %g1 + stxa %g1, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + retry + .size dcpe_icpe_tl1_common,.-dcpe_icpe_tl1_common + + /* Capture I/D/E-cache state into per-cpu error scoreboard. + * + * %g1: (TL>=0) ? 1 : 0 + * %g2: scratch + * %g3: scratch + * %g4: AFSR + * %g5: AFAR + * %g6: unused, will have current thread ptr after etrap + * %g7: scratch + */ + .type __cheetah_log_error,#function +__cheetah_log_error: + /* Put "TL1" software bit into AFSR. */ + and %g1, 0x1, %g1 + sllx %g1, 63, %g2 + or %g4, %g2, %g4 + + /* Get log entry pointer for this cpu at this trap level. */ + BRANCH_IF_JALAPENO(g2,g3,50f) + ldxa [%g0] ASI_SAFARI_CONFIG, %g2 + srlx %g2, 17, %g2 + ba,pt %xcc, 60f + and %g2, 0x3ff, %g2 + +50: ldxa [%g0] ASI_JBUS_CONFIG, %g2 + srlx %g2, 17, %g2 + and %g2, 0x1f, %g2 + +60: sllx %g2, 9, %g2 + sethi %hi(cheetah_error_log), %g3 + ldx [%g3 + %lo(cheetah_error_log)], %g3 + brz,pn %g3, 80f + nop + + add %g3, %g2, %g3 + sllx %g1, 8, %g1 + add %g3, %g1, %g1 + + /* %g1 holds pointer to the top of the logging scoreboard */ + ldx [%g1 + 0x0], %g7 + cmp %g7, -1 + bne,pn %xcc, 80f + nop + + stx %g4, [%g1 + 0x0] + stx %g5, [%g1 + 0x8] + add %g1, 0x10, %g1 + + /* %g1 now points to D-cache logging area */ + set 0x3ff8, %g2 /* DC_addr mask */ + and %g5, %g2, %g2 /* DC_addr bits of AFAR */ + srlx %g5, 12, %g3 + or %g3, 1, %g3 /* PHYS tag + valid */ + +10: ldxa [%g2] ASI_DCACHE_TAG, %g7 + cmp %g3, %g7 /* TAG match? */ + bne,pt %xcc, 13f + nop + + /* Yep, what we want, capture state. */ + stx %g2, [%g1 + 0x20] + stx %g7, [%g1 + 0x28] + + /* A membar Sync is required before and after utag access. */ + membar #Sync + ldxa [%g2] ASI_DCACHE_UTAG, %g7 + membar #Sync + stx %g7, [%g1 + 0x30] + ldxa [%g2] ASI_DCACHE_SNOOP_TAG, %g7 + stx %g7, [%g1 + 0x38] + clr %g3 + +12: ldxa [%g2 + %g3] ASI_DCACHE_DATA, %g7 + stx %g7, [%g1] + add %g3, (1 << 5), %g3 + cmp %g3, (4 << 5) + bl,pt %xcc, 12b + add %g1, 0x8, %g1 + + ba,pt %xcc, 20f + add %g1, 0x20, %g1 + +13: sethi %hi(1 << 14), %g7 + add %g2, %g7, %g2 + srlx %g2, 14, %g7 + cmp %g7, 4 + bl,pt %xcc, 10b + nop + + add %g1, 0x40, %g1 + + /* %g1 now points to I-cache logging area */ +20: set 0x1fe0, %g2 /* IC_addr mask */ + and %g5, %g2, %g2 /* IC_addr bits of AFAR */ + sllx %g2, 1, %g2 /* IC_addr[13:6]==VA[12:5] */ + srlx %g5, (13 - 8), %g3 /* Make PTAG */ + andn %g3, 0xff, %g3 /* Mask off undefined bits */ + +21: ldxa [%g2] ASI_IC_TAG, %g7 + andn %g7, 0xff, %g7 + cmp %g3, %g7 + bne,pt %xcc, 23f + nop + + /* Yep, what we want, capture state. */ + stx %g2, [%g1 + 0x40] + stx %g7, [%g1 + 0x48] + add %g2, (1 << 3), %g2 + ldxa [%g2] ASI_IC_TAG, %g7 + add %g2, (1 << 3), %g2 + stx %g7, [%g1 + 0x50] + ldxa [%g2] ASI_IC_TAG, %g7 + add %g2, (1 << 3), %g2 + stx %g7, [%g1 + 0x60] + ldxa [%g2] ASI_IC_TAG, %g7 + stx %g7, [%g1 + 0x68] + sub %g2, (3 << 3), %g2 + ldxa [%g2] ASI_IC_STAG, %g7 + stx %g7, [%g1 + 0x58] + clr %g3 + srlx %g2, 2, %g2 + +22: ldxa [%g2 + %g3] ASI_IC_INSTR, %g7 + stx %g7, [%g1] + add %g3, (1 << 3), %g3 + cmp %g3, (8 << 3) + bl,pt %xcc, 22b + add %g1, 0x8, %g1 + + ba,pt %xcc, 30f + add %g1, 0x30, %g1 + +23: sethi %hi(1 << 14), %g7 + add %g2, %g7, %g2 + srlx %g2, 14, %g7 + cmp %g7, 4 + bl,pt %xcc, 21b + nop + + add %g1, 0x70, %g1 + + /* %g1 now points to E-cache logging area */ +30: andn %g5, (32 - 1), %g2 + stx %g2, [%g1 + 0x20] + ldxa [%g2] ASI_EC_TAG_DATA, %g7 + stx %g7, [%g1 + 0x28] + ldxa [%g2] ASI_EC_R, %g0 + clr %g3 + +31: ldxa [%g3] ASI_EC_DATA, %g7 + stx %g7, [%g1 + %g3] + add %g3, 0x8, %g3 + cmp %g3, 0x20 + + bl,pt %xcc, 31b + nop +80: + rdpr %tt, %g2 + cmp %g2, 0x70 + be c_fast_ecc + cmp %g2, 0x63 + be c_cee + nop + ba,a,pt %xcc, c_deferred + .size __cheetah_log_error,.-__cheetah_log_error + + /* Cheetah FECC trap handling, we get here from tl{0,1}_fecc + * in the trap table. That code has done a memory barrier + * and has disabled both the I-cache and D-cache in the DCU + * control register. The I-cache is disabled so that we may + * capture the corrupted cache line, and the D-cache is disabled + * because corrupt data may have been placed there and we don't + * want to reference it. + * + * %g1 is one if this trap occurred at %tl >= 1. + * + * Next, we turn off error reporting so that we don't recurse. + */ + .globl cheetah_fast_ecc + .type cheetah_fast_ecc,#function +cheetah_fast_ecc: + ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2 + andn %g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2 + stxa %g2, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + /* Fetch and clear AFSR/AFAR */ + ldxa [%g0] ASI_AFSR, %g4 + ldxa [%g0] ASI_AFAR, %g5 + stxa %g4, [%g0] ASI_AFSR + membar #Sync + + ba,pt %xcc, __cheetah_log_error + nop + .size cheetah_fast_ecc,.-cheetah_fast_ecc + + .type c_fast_ecc,#function +c_fast_ecc: + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov %l4, %o1 + mov %l5, %o2 + call cheetah_fecc_handler + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap_irq + .size c_fast_ecc,.-c_fast_ecc + + /* Our caller has disabled I-cache and performed membar Sync. */ + .globl cheetah_cee + .type cheetah_cee,#function +cheetah_cee: + ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2 + andn %g2, ESTATE_ERROR_CEEN, %g2 + stxa %g2, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + /* Fetch and clear AFSR/AFAR */ + ldxa [%g0] ASI_AFSR, %g4 + ldxa [%g0] ASI_AFAR, %g5 + stxa %g4, [%g0] ASI_AFSR + membar #Sync + + ba,pt %xcc, __cheetah_log_error + nop + .size cheetah_cee,.-cheetah_cee + + .type c_cee,#function +c_cee: + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov %l4, %o1 + mov %l5, %o2 + call cheetah_cee_handler + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap_irq + .size c_cee,.-c_cee + + /* Our caller has disabled I-cache+D-cache and performed membar Sync. */ + .globl cheetah_deferred_trap + .type cheetah_deferred_trap,#function +cheetah_deferred_trap: + ldxa [%g0] ASI_ESTATE_ERROR_EN, %g2 + andn %g2, ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN, %g2 + stxa %g2, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + /* Fetch and clear AFSR/AFAR */ + ldxa [%g0] ASI_AFSR, %g4 + ldxa [%g0] ASI_AFAR, %g5 + stxa %g4, [%g0] ASI_AFSR + membar #Sync + + ba,pt %xcc, __cheetah_log_error + nop + .size cheetah_deferred_trap,.-cheetah_deferred_trap + + .type c_deferred,#function +c_deferred: + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov %l4, %o1 + mov %l5, %o2 + call cheetah_deferred_handler + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap_irq + .size c_deferred,.-c_deferred diff --git a/arch/sparc/kernel/chmc.c b/arch/sparc/kernel/chmc.c new file mode 100644 index 000000000..d935177b6 --- /dev/null +++ b/arch/sparc/kernel/chmc.c @@ -0,0 +1,864 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* chmc.c: Driver for UltraSPARC-III memory controller. + * + * Copyright (C) 2001, 2007, 2008 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DRV_MODULE_NAME "chmc" +#define PFX DRV_MODULE_NAME ": " +#define DRV_MODULE_VERSION "0.2" + +MODULE_AUTHOR("David S. Miller "); +MODULE_DESCRIPTION("UltraSPARC-III memory controller driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_MODULE_VERSION); + +static int mc_type; +#define MC_TYPE_SAFARI 1 +#define MC_TYPE_JBUS 2 + +static dimm_printer_t us3mc_dimm_printer; + +#define CHMCTRL_NDGRPS 2 +#define CHMCTRL_NDIMMS 4 + +#define CHMC_DIMMS_PER_MC (CHMCTRL_NDGRPS * CHMCTRL_NDIMMS) + +/* OBP memory-layout property format. */ +struct chmc_obp_map { + unsigned char dimm_map[144]; + unsigned char pin_map[576]; +}; + +#define DIMM_LABEL_SZ 8 + +struct chmc_obp_mem_layout { + /* One max 8-byte string label per DIMM. Usually + * this matches the label on the motherboard where + * that DIMM resides. + */ + char dimm_labels[CHMC_DIMMS_PER_MC][DIMM_LABEL_SZ]; + + /* If symmetric use map[0], else it is + * asymmetric and map[1] should be used. + */ + char symmetric; + + struct chmc_obp_map map[2]; +}; + +#define CHMCTRL_NBANKS 4 + +struct chmc_bank_info { + struct chmc *p; + int bank_id; + + u64 raw_reg; + int valid; + int uk; + int um; + int lk; + int lm; + int interleave; + unsigned long base; + unsigned long size; +}; + +struct chmc { + struct list_head list; + int portid; + + struct chmc_obp_mem_layout layout_prop; + int layout_size; + + void __iomem *regs; + + u64 timing_control1; + u64 timing_control2; + u64 timing_control3; + u64 timing_control4; + u64 memaddr_control; + + struct chmc_bank_info logical_banks[CHMCTRL_NBANKS]; +}; + +#define JBUSMC_REGS_SIZE 8 + +#define JB_MC_REG1_DIMM2_BANK3 0x8000000000000000UL +#define JB_MC_REG1_DIMM1_BANK1 0x4000000000000000UL +#define JB_MC_REG1_DIMM2_BANK2 0x2000000000000000UL +#define JB_MC_REG1_DIMM1_BANK0 0x1000000000000000UL +#define JB_MC_REG1_XOR 0x0000010000000000UL +#define JB_MC_REG1_ADDR_GEN_2 0x000000e000000000UL +#define JB_MC_REG1_ADDR_GEN_2_SHIFT 37 +#define JB_MC_REG1_ADDR_GEN_1 0x0000001c00000000UL +#define JB_MC_REG1_ADDR_GEN_1_SHIFT 34 +#define JB_MC_REG1_INTERLEAVE 0x0000000001800000UL +#define JB_MC_REG1_INTERLEAVE_SHIFT 23 +#define JB_MC_REG1_DIMM2_PTYPE 0x0000000000200000UL +#define JB_MC_REG1_DIMM2_PTYPE_SHIFT 21 +#define JB_MC_REG1_DIMM1_PTYPE 0x0000000000100000UL +#define JB_MC_REG1_DIMM1_PTYPE_SHIFT 20 + +#define PART_TYPE_X8 0 +#define PART_TYPE_X4 1 + +#define INTERLEAVE_NONE 0 +#define INTERLEAVE_SAME 1 +#define INTERLEAVE_INTERNAL 2 +#define INTERLEAVE_BOTH 3 + +#define ADDR_GEN_128MB 0 +#define ADDR_GEN_256MB 1 +#define ADDR_GEN_512MB 2 +#define ADDR_GEN_1GB 3 + +#define JB_NUM_DIMM_GROUPS 2 +#define JB_NUM_DIMMS_PER_GROUP 2 +#define JB_NUM_DIMMS (JB_NUM_DIMM_GROUPS * JB_NUM_DIMMS_PER_GROUP) + +struct jbusmc_obp_map { + unsigned char dimm_map[18]; + unsigned char pin_map[144]; +}; + +struct jbusmc_obp_mem_layout { + /* One max 8-byte string label per DIMM. Usually + * this matches the label on the motherboard where + * that DIMM resides. + */ + char dimm_labels[JB_NUM_DIMMS][DIMM_LABEL_SZ]; + + /* If symmetric use map[0], else it is + * asymmetric and map[1] should be used. + */ + char symmetric; + + struct jbusmc_obp_map map; + + char _pad; +}; + +struct jbusmc_dimm_group { + struct jbusmc *controller; + int index; + u64 base_addr; + u64 size; +}; + +struct jbusmc { + void __iomem *regs; + u64 mc_reg_1; + u32 portid; + struct jbusmc_obp_mem_layout layout; + int layout_len; + int num_dimm_groups; + struct jbusmc_dimm_group dimm_groups[JB_NUM_DIMM_GROUPS]; + struct list_head list; +}; + +static DEFINE_SPINLOCK(mctrl_list_lock); +static LIST_HEAD(mctrl_list); + +static void mc_list_add(struct list_head *list) +{ + spin_lock(&mctrl_list_lock); + list_add(list, &mctrl_list); + spin_unlock(&mctrl_list_lock); +} + +static void mc_list_del(struct list_head *list) +{ + spin_lock(&mctrl_list_lock); + list_del_init(list); + spin_unlock(&mctrl_list_lock); +} + +#define SYNDROME_MIN -1 +#define SYNDROME_MAX 144 + +/* Covert syndrome code into the way the bits are positioned + * on the bus. + */ +static int syndrome_to_qword_code(int syndrome_code) +{ + if (syndrome_code < 128) + syndrome_code += 16; + else if (syndrome_code < 128 + 9) + syndrome_code -= (128 - 7); + else if (syndrome_code < (128 + 9 + 3)) + syndrome_code -= (128 + 9 - 4); + else + syndrome_code -= (128 + 9 + 3); + return syndrome_code; +} + +/* All this magic has to do with how a cache line comes over the wire + * on Safari and JBUS. A 64-bit line comes over in 1 or more quadword + * cycles, each of which transmit ECC/MTAG info as well as the actual + * data. + */ +#define L2_LINE_SIZE 64 +#define L2_LINE_ADDR_MSK (L2_LINE_SIZE - 1) +#define QW_PER_LINE 4 +#define QW_BYTES (L2_LINE_SIZE / QW_PER_LINE) +#define QW_BITS 144 +#define SAFARI_LAST_BIT (576 - 1) +#define JBUS_LAST_BIT (144 - 1) + +static void get_pin_and_dimm_str(int syndrome_code, unsigned long paddr, + int *pin_p, char **dimm_str_p, void *_prop, + int base_dimm_offset) +{ + int qword_code = syndrome_to_qword_code(syndrome_code); + int cache_line_offset; + int offset_inverse; + int dimm_map_index; + int map_val; + + if (mc_type == MC_TYPE_JBUS) { + struct jbusmc_obp_mem_layout *p = _prop; + + /* JBUS */ + cache_line_offset = qword_code; + offset_inverse = (JBUS_LAST_BIT - cache_line_offset); + dimm_map_index = offset_inverse / 8; + map_val = p->map.dimm_map[dimm_map_index]; + map_val = ((map_val >> ((7 - (offset_inverse & 7)))) & 1); + *dimm_str_p = p->dimm_labels[base_dimm_offset + map_val]; + *pin_p = p->map.pin_map[cache_line_offset]; + } else { + struct chmc_obp_mem_layout *p = _prop; + struct chmc_obp_map *mp; + int qword; + + /* Safari */ + if (p->symmetric) + mp = &p->map[0]; + else + mp = &p->map[1]; + + qword = (paddr & L2_LINE_ADDR_MSK) / QW_BYTES; + cache_line_offset = ((3 - qword) * QW_BITS) + qword_code; + offset_inverse = (SAFARI_LAST_BIT - cache_line_offset); + dimm_map_index = offset_inverse >> 2; + map_val = mp->dimm_map[dimm_map_index]; + map_val = ((map_val >> ((3 - (offset_inverse & 3)) << 1)) & 0x3); + *dimm_str_p = p->dimm_labels[base_dimm_offset + map_val]; + *pin_p = mp->pin_map[cache_line_offset]; + } +} + +static struct jbusmc_dimm_group *jbusmc_find_dimm_group(unsigned long phys_addr) +{ + struct jbusmc *p; + + list_for_each_entry(p, &mctrl_list, list) { + int i; + + for (i = 0; i < p->num_dimm_groups; i++) { + struct jbusmc_dimm_group *dp = &p->dimm_groups[i]; + + if (phys_addr < dp->base_addr || + (dp->base_addr + dp->size) <= phys_addr) + continue; + + return dp; + } + } + return NULL; +} + +static int jbusmc_print_dimm(int syndrome_code, + unsigned long phys_addr, + char *buf, int buflen) +{ + struct jbusmc_obp_mem_layout *prop; + struct jbusmc_dimm_group *dp; + struct jbusmc *p; + int first_dimm; + + dp = jbusmc_find_dimm_group(phys_addr); + if (dp == NULL || + syndrome_code < SYNDROME_MIN || + syndrome_code > SYNDROME_MAX) { + buf[0] = '?'; + buf[1] = '?'; + buf[2] = '?'; + buf[3] = '\0'; + return 0; + } + p = dp->controller; + prop = &p->layout; + + first_dimm = dp->index * JB_NUM_DIMMS_PER_GROUP; + + if (syndrome_code != SYNDROME_MIN) { + char *dimm_str; + int pin; + + get_pin_and_dimm_str(syndrome_code, phys_addr, &pin, + &dimm_str, prop, first_dimm); + sprintf(buf, "%s, pin %3d", dimm_str, pin); + } else { + int dimm; + + /* Multi-bit error, we just dump out all the + * dimm labels associated with this dimm group. + */ + for (dimm = 0; dimm < JB_NUM_DIMMS_PER_GROUP; dimm++) { + sprintf(buf, "%s ", + prop->dimm_labels[first_dimm + dimm]); + buf += strlen(buf); + } + } + + return 0; +} + +static u64 jbusmc_dimm_group_size(u64 base, + const struct linux_prom64_registers *mem_regs, + int num_mem_regs) +{ + u64 max = base + (8UL * 1024 * 1024 * 1024); + u64 max_seen = base; + int i; + + for (i = 0; i < num_mem_regs; i++) { + const struct linux_prom64_registers *ent; + u64 this_base; + u64 this_end; + + ent = &mem_regs[i]; + this_base = ent->phys_addr; + this_end = this_base + ent->reg_size; + if (base < this_base || base >= this_end) + continue; + if (this_end > max) + this_end = max; + if (this_end > max_seen) + max_seen = this_end; + } + + return max_seen - base; +} + +static void jbusmc_construct_one_dimm_group(struct jbusmc *p, + unsigned long index, + const struct linux_prom64_registers *mem_regs, + int num_mem_regs) +{ + struct jbusmc_dimm_group *dp = &p->dimm_groups[index]; + + dp->controller = p; + dp->index = index; + + dp->base_addr = (p->portid * (64UL * 1024 * 1024 * 1024)); + dp->base_addr += (index * (8UL * 1024 * 1024 * 1024)); + dp->size = jbusmc_dimm_group_size(dp->base_addr, mem_regs, num_mem_regs); +} + +static void jbusmc_construct_dimm_groups(struct jbusmc *p, + const struct linux_prom64_registers *mem_regs, + int num_mem_regs) +{ + if (p->mc_reg_1 & JB_MC_REG1_DIMM1_BANK0) { + jbusmc_construct_one_dimm_group(p, 0, mem_regs, num_mem_regs); + p->num_dimm_groups++; + } + if (p->mc_reg_1 & JB_MC_REG1_DIMM2_BANK2) { + jbusmc_construct_one_dimm_group(p, 1, mem_regs, num_mem_regs); + p->num_dimm_groups++; + } +} + +static int jbusmc_probe(struct platform_device *op) +{ + const struct linux_prom64_registers *mem_regs; + struct device_node *mem_node; + int err, len, num_mem_regs; + struct jbusmc *p; + const u32 *prop; + const void *ml; + + err = -ENODEV; + mem_node = of_find_node_by_path("/memory"); + if (!mem_node) { + printk(KERN_ERR PFX "Cannot find /memory node.\n"); + goto out; + } + mem_regs = of_get_property(mem_node, "reg", &len); + if (!mem_regs) { + printk(KERN_ERR PFX "Cannot get reg property of /memory node.\n"); + goto out; + } + num_mem_regs = len / sizeof(*mem_regs); + + err = -ENOMEM; + p = kzalloc_obj(*p); + if (!p) { + printk(KERN_ERR PFX "Cannot allocate struct jbusmc.\n"); + goto out; + } + + INIT_LIST_HEAD(&p->list); + + err = -ENODEV; + prop = of_get_property(op->dev.of_node, "portid", &len); + if (!prop || len != 4) { + printk(KERN_ERR PFX "Cannot find portid.\n"); + goto out_free; + } + + p->portid = *prop; + + prop = of_get_property(op->dev.of_node, "memory-control-register-1", &len); + if (!prop || len != 8) { + printk(KERN_ERR PFX "Cannot get memory control register 1.\n"); + goto out_free; + } + + p->mc_reg_1 = ((u64)prop[0] << 32) | (u64) prop[1]; + + err = -ENOMEM; + p->regs = of_ioremap(&op->resource[0], 0, JBUSMC_REGS_SIZE, "jbusmc"); + if (!p->regs) { + printk(KERN_ERR PFX "Cannot map jbusmc regs.\n"); + goto out_free; + } + + err = -ENODEV; + ml = of_get_property(op->dev.of_node, "memory-layout", &p->layout_len); + if (!ml) { + printk(KERN_ERR PFX "Cannot get memory layout property.\n"); + goto out_iounmap; + } + if (p->layout_len > sizeof(p->layout)) { + printk(KERN_ERR PFX "Unexpected memory-layout size %d\n", + p->layout_len); + goto out_iounmap; + } + memcpy(&p->layout, ml, p->layout_len); + + jbusmc_construct_dimm_groups(p, mem_regs, num_mem_regs); + + mc_list_add(&p->list); + + printk(KERN_INFO PFX "UltraSPARC-IIIi memory controller at %pOF\n", + op->dev.of_node); + + dev_set_drvdata(&op->dev, p); + + err = 0; + +out: + return err; + +out_iounmap: + of_iounmap(&op->resource[0], p->regs, JBUSMC_REGS_SIZE); + +out_free: + kfree(p); + goto out; +} + +/* Does BANK decode PHYS_ADDR? */ +static int chmc_bank_match(struct chmc_bank_info *bp, unsigned long phys_addr) +{ + unsigned long upper_bits = (phys_addr & PA_UPPER_BITS) >> PA_UPPER_BITS_SHIFT; + unsigned long lower_bits = (phys_addr & PA_LOWER_BITS) >> PA_LOWER_BITS_SHIFT; + + /* Bank must be enabled to match. */ + if (bp->valid == 0) + return 0; + + /* Would BANK match upper bits? */ + upper_bits ^= bp->um; /* What bits are different? */ + upper_bits = ~upper_bits; /* Invert. */ + upper_bits |= bp->uk; /* What bits don't matter for matching? */ + upper_bits = ~upper_bits; /* Invert. */ + + if (upper_bits) + return 0; + + /* Would BANK match lower bits? */ + lower_bits ^= bp->lm; /* What bits are different? */ + lower_bits = ~lower_bits; /* Invert. */ + lower_bits |= bp->lk; /* What bits don't matter for matching? */ + lower_bits = ~lower_bits; /* Invert. */ + + if (lower_bits) + return 0; + + /* I always knew you'd be the one. */ + return 1; +} + +/* Given PHYS_ADDR, search memory controller banks for a match. */ +static struct chmc_bank_info *chmc_find_bank(unsigned long phys_addr) +{ + struct chmc *p; + + list_for_each_entry(p, &mctrl_list, list) { + int bank_no; + + for (bank_no = 0; bank_no < CHMCTRL_NBANKS; bank_no++) { + struct chmc_bank_info *bp; + + bp = &p->logical_banks[bank_no]; + if (chmc_bank_match(bp, phys_addr)) + return bp; + } + } + + return NULL; +} + +/* This is the main purpose of this driver. */ +static int chmc_print_dimm(int syndrome_code, + unsigned long phys_addr, + char *buf, int buflen) +{ + struct chmc_bank_info *bp; + struct chmc_obp_mem_layout *prop; + int bank_in_controller, first_dimm; + + bp = chmc_find_bank(phys_addr); + if (bp == NULL || + syndrome_code < SYNDROME_MIN || + syndrome_code > SYNDROME_MAX) { + buf[0] = '?'; + buf[1] = '?'; + buf[2] = '?'; + buf[3] = '\0'; + return 0; + } + + prop = &bp->p->layout_prop; + bank_in_controller = bp->bank_id & (CHMCTRL_NBANKS - 1); + first_dimm = (bank_in_controller & (CHMCTRL_NDGRPS - 1)); + first_dimm *= CHMCTRL_NDIMMS; + + if (syndrome_code != SYNDROME_MIN) { + char *dimm_str; + int pin; + + get_pin_and_dimm_str(syndrome_code, phys_addr, &pin, + &dimm_str, prop, first_dimm); + sprintf(buf, "%s, pin %3d", dimm_str, pin); + } else { + int dimm; + + /* Multi-bit error, we just dump out all the + * dimm labels associated with this bank. + */ + for (dimm = 0; dimm < CHMCTRL_NDIMMS; dimm++) { + sprintf(buf, "%s ", + prop->dimm_labels[first_dimm + dimm]); + buf += strlen(buf); + } + } + return 0; +} + +/* Accessing the registers is slightly complicated. If you want + * to get at the memory controller which is on the same processor + * the code is executing, you must use special ASI load/store else + * you go through the global mapping. + */ +static u64 chmc_read_mcreg(struct chmc *p, unsigned long offset) +{ + unsigned long ret, this_cpu; + + preempt_disable(); + + this_cpu = real_hard_smp_processor_id(); + + if (p->portid == this_cpu) { + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (ret) + : "r" (offset), "i" (ASI_MCU_CTRL_REG)); + } else { + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (ret) + : "r" (p->regs + offset), + "i" (ASI_PHYS_BYPASS_EC_E)); + } + + preempt_enable(); + + return ret; +} + +#if 0 /* currently unused */ +static void chmc_write_mcreg(struct chmc *p, unsigned long offset, u64 val) +{ + if (p->portid == smp_processor_id()) { + __asm__ __volatile__("stxa %0, [%1] %2" + : : "r" (val), + "r" (offset), "i" (ASI_MCU_CTRL_REG)); + } else { + __asm__ __volatile__("ldxa %0, [%1] %2" + : : "r" (val), + "r" (p->regs + offset), + "i" (ASI_PHYS_BYPASS_EC_E)); + } +} +#endif + +static void chmc_interpret_one_decode_reg(struct chmc *p, int which_bank, u64 val) +{ + struct chmc_bank_info *bp = &p->logical_banks[which_bank]; + + bp->p = p; + bp->bank_id = (CHMCTRL_NBANKS * p->portid) + which_bank; + bp->raw_reg = val; + bp->valid = (val & MEM_DECODE_VALID) >> MEM_DECODE_VALID_SHIFT; + bp->uk = (val & MEM_DECODE_UK) >> MEM_DECODE_UK_SHIFT; + bp->um = (val & MEM_DECODE_UM) >> MEM_DECODE_UM_SHIFT; + bp->lk = (val & MEM_DECODE_LK) >> MEM_DECODE_LK_SHIFT; + bp->lm = (val & MEM_DECODE_LM) >> MEM_DECODE_LM_SHIFT; + + bp->base = (bp->um); + bp->base &= ~(bp->uk); + bp->base <<= PA_UPPER_BITS_SHIFT; + + switch(bp->lk) { + case 0xf: + default: + bp->interleave = 1; + break; + + case 0xe: + bp->interleave = 2; + break; + + case 0xc: + bp->interleave = 4; + break; + + case 0x8: + bp->interleave = 8; + break; + + case 0x0: + bp->interleave = 16; + break; + } + + /* UK[10] is reserved, and UK[11] is not set for the SDRAM + * bank size definition. + */ + bp->size = (((unsigned long)bp->uk & + ((1UL << 10UL) - 1UL)) + 1UL) << PA_UPPER_BITS_SHIFT; + bp->size /= bp->interleave; +} + +static void chmc_fetch_decode_regs(struct chmc *p) +{ + if (p->layout_size == 0) + return; + + chmc_interpret_one_decode_reg(p, 0, + chmc_read_mcreg(p, CHMCTRL_DECODE1)); + chmc_interpret_one_decode_reg(p, 1, + chmc_read_mcreg(p, CHMCTRL_DECODE2)); + chmc_interpret_one_decode_reg(p, 2, + chmc_read_mcreg(p, CHMCTRL_DECODE3)); + chmc_interpret_one_decode_reg(p, 3, + chmc_read_mcreg(p, CHMCTRL_DECODE4)); +} + +static int chmc_probe(struct platform_device *op) +{ + struct device_node *dp = op->dev.of_node; + unsigned long ver; + const void *pval; + int len, portid; + struct chmc *p; + int err; + + err = -ENODEV; + __asm__ ("rdpr %%ver, %0" : "=r" (ver)); + if ((ver >> 32UL) == __JALAPENO_ID || + (ver >> 32UL) == __SERRANO_ID) + goto out; + + portid = of_getintprop_default(dp, "portid", -1); + if (portid == -1) + goto out; + + pval = of_get_property(dp, "memory-layout", &len); + if (pval && len > sizeof(p->layout_prop)) { + printk(KERN_ERR PFX "Unexpected memory-layout property " + "size %d.\n", len); + goto out; + } + + err = -ENOMEM; + p = kzalloc_obj(*p); + if (!p) { + printk(KERN_ERR PFX "Could not allocate struct chmc.\n"); + goto out; + } + + p->portid = portid; + p->layout_size = len; + if (!pval) + p->layout_size = 0; + else + memcpy(&p->layout_prop, pval, len); + + p->regs = of_ioremap(&op->resource[0], 0, 0x48, "chmc"); + if (!p->regs) { + printk(KERN_ERR PFX "Could not map registers.\n"); + goto out_free; + } + + if (p->layout_size != 0UL) { + p->timing_control1 = chmc_read_mcreg(p, CHMCTRL_TCTRL1); + p->timing_control2 = chmc_read_mcreg(p, CHMCTRL_TCTRL2); + p->timing_control3 = chmc_read_mcreg(p, CHMCTRL_TCTRL3); + p->timing_control4 = chmc_read_mcreg(p, CHMCTRL_TCTRL4); + p->memaddr_control = chmc_read_mcreg(p, CHMCTRL_MACTRL); + } + + chmc_fetch_decode_regs(p); + + mc_list_add(&p->list); + + printk(KERN_INFO PFX "UltraSPARC-III memory controller at %pOF [%s]\n", + dp, + (p->layout_size ? "ACTIVE" : "INACTIVE")); + + dev_set_drvdata(&op->dev, p); + + err = 0; + +out: + return err; + +out_free: + kfree(p); + goto out; +} + +static int us3mc_probe(struct platform_device *op) +{ + if (mc_type == MC_TYPE_SAFARI) + return chmc_probe(op); + else if (mc_type == MC_TYPE_JBUS) + return jbusmc_probe(op); + return -ENODEV; +} + +static void chmc_destroy(struct platform_device *op, struct chmc *p) +{ + list_del(&p->list); + of_iounmap(&op->resource[0], p->regs, 0x48); + kfree(p); +} + +static void jbusmc_destroy(struct platform_device *op, struct jbusmc *p) +{ + mc_list_del(&p->list); + of_iounmap(&op->resource[0], p->regs, JBUSMC_REGS_SIZE); + kfree(p); +} + +static void us3mc_remove(struct platform_device *op) +{ + void *p = dev_get_drvdata(&op->dev); + + if (p) { + if (mc_type == MC_TYPE_SAFARI) + chmc_destroy(op, p); + else if (mc_type == MC_TYPE_JBUS) + jbusmc_destroy(op, p); + } +} + +static const struct of_device_id us3mc_match[] = { + { + .name = "memory-controller", + }, + {}, +}; +MODULE_DEVICE_TABLE(of, us3mc_match); + +static struct platform_driver us3mc_driver = { + .driver = { + .name = "us3mc", + .of_match_table = us3mc_match, + }, + .probe = us3mc_probe, + .remove = us3mc_remove, +}; + +static inline bool us3mc_platform(void) +{ + if (tlb_type == cheetah || tlb_type == cheetah_plus) + return true; + return false; +} + +static int __init us3mc_init(void) +{ + unsigned long ver; + int ret; + + if (!us3mc_platform()) + return -ENODEV; + + __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver)); + if ((ver >> 32UL) == __JALAPENO_ID || + (ver >> 32UL) == __SERRANO_ID) { + mc_type = MC_TYPE_JBUS; + us3mc_dimm_printer = jbusmc_print_dimm; + } else { + mc_type = MC_TYPE_SAFARI; + us3mc_dimm_printer = chmc_print_dimm; + } + + ret = register_dimm_printer(us3mc_dimm_printer); + + if (!ret) { + ret = platform_driver_register(&us3mc_driver); + if (ret) + unregister_dimm_printer(us3mc_dimm_printer); + } + return ret; +} + +static void __exit us3mc_cleanup(void) +{ + if (us3mc_platform()) { + unregister_dimm_printer(us3mc_dimm_printer); + platform_driver_unregister(&us3mc_driver); + } +} + +module_init(us3mc_init); +module_exit(us3mc_cleanup); diff --git a/arch/sparc/kernel/compat_audit.c b/arch/sparc/kernel/compat_audit.c new file mode 100644 index 000000000..f1ea0005a --- /dev/null +++ b/arch/sparc/kernel/compat_audit.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +#define __32bit_syscall_numbers__ +#include +#include +#include "kernel.h" + +unsigned int sparc32_dir_class[] = { +#include +~0U +}; + +unsigned int sparc32_chattr_class[] = { +#include +~0U +}; + +unsigned int sparc32_write_class[] = { +#include +~0U +}; + +unsigned int sparc32_read_class[] = { +#include +~0U +}; + +unsigned int sparc32_signal_class[] = { +#include +~0U +}; + +int sparc32_classify_syscall(unsigned int syscall) +{ + switch(syscall) { + case __NR_open: + return AUDITSC_OPEN; + case __NR_openat: + return AUDITSC_OPENAT; + case __NR_socketcall: + return AUDITSC_SOCKETCALL; + case __NR_execve: + return AUDITSC_EXECVE; + case __NR_openat2: + return AUDITSC_OPENAT2; + default: + return AUDITSC_COMPAT; + } +} diff --git a/arch/sparc/kernel/cpu.c b/arch/sparc/kernel/cpu.c new file mode 100644 index 000000000..79cd6ccfe --- /dev/null +++ b/arch/sparc/kernel/cpu.c @@ -0,0 +1,556 @@ +// SPDX-License-Identifier: GPL-2.0 +/* cpu.c: Dinky routines to look for the kind of Sparc cpu + * we are on. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" +#include "entry.h" + +DEFINE_PER_CPU(cpuinfo_sparc, __cpu_data) = { 0 }; +EXPORT_PER_CPU_SYMBOL(__cpu_data); + +int ncpus_probed; +unsigned int fsr_storage; + +struct cpu_info { + int psr_vers; + const char *name; + const char *pmu_name; +}; + +struct fpu_info { + int fp_vers; + const char *name; +}; + +#define NOCPU 8 +#define NOFPU 8 + +struct manufacturer_info { + int psr_impl; + struct cpu_info cpu_info[NOCPU]; + struct fpu_info fpu_info[NOFPU]; +}; + +#define CPU(ver, _name) \ +{ .psr_vers = ver, .name = _name } + +#define CPU_PMU(ver, _name, _pmu_name) \ +{ .psr_vers = ver, .name = _name, .pmu_name = _pmu_name } + +#define FPU(ver, _name) \ +{ .fp_vers = ver, .name = _name } + +static const struct manufacturer_info __initconst manufacturer_info[] = { +{ + 0, + /* Sun4/100, 4/200, SLC */ + .cpu_info = { + CPU(0, "Fujitsu MB86900/1A or LSI L64831 SparcKIT-40"), + /* borned STP1012PGA */ + CPU(4, "Fujitsu MB86904"), + CPU(5, "Fujitsu TurboSparc MB86907"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(0, "Fujitsu MB86910 or Weitek WTL1164/5"), + FPU(1, "Fujitsu MB86911 or Weitek WTL1164/5 or LSI L64831"), + FPU(2, "LSI Logic L64802 or Texas Instruments ACT8847"), + /* SparcStation SLC, SparcStation1 */ + FPU(3, "Weitek WTL3170/2"), + /* SPARCstation-5 */ + FPU(4, "Lsi Logic/Meiko L64804 or compatible"), + FPU(-1, NULL) + } +},{ + 1, + .cpu_info = { + /* SparcStation2, SparcServer 490 & 690 */ + CPU(0, "LSI Logic Corporation - L64811"), + /* SparcStation2 */ + CPU(1, "Cypress/ROSS CY7C601"), + /* Embedded controller */ + CPU(3, "Cypress/ROSS CY7C611"), + /* Ross Technologies HyperSparc */ + CPU(0xf, "ROSS HyperSparc RT620"), + CPU(0xe, "ROSS HyperSparc RT625 or RT626"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(0, "ROSS HyperSparc combined IU/FPU"), + FPU(1, "Lsi Logic L64814"), + FPU(2, "Texas Instruments TMS390-C602A"), + FPU(3, "Cypress CY7C602 FPU"), + FPU(-1, NULL) + } +},{ + 2, + .cpu_info = { + /* ECL Implementation, CRAY S-MP Supercomputer... AIEEE! */ + /* Someone please write the code to support this beast! ;) */ + CPU(0, "Bipolar Integrated Technology - B5010"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(-1, NULL) + } +},{ + 3, + .cpu_info = { + CPU(0, "LSI Logic Corporation - unknown-type"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(-1, NULL) + } +},{ + PSR_IMPL_TI, + .cpu_info = { + CPU(0, "Texas Instruments, Inc. - SuperSparc-(II)"), + /* SparcClassic -- borned STP1010TAB-50*/ + CPU(1, "Texas Instruments, Inc. - MicroSparc"), + CPU(2, "Texas Instruments, Inc. - MicroSparc II"), + CPU(3, "Texas Instruments, Inc. - SuperSparc 51"), + CPU(4, "Texas Instruments, Inc. - SuperSparc 61"), + CPU(5, "Texas Instruments, Inc. - unknown"), + CPU(-1, NULL) + }, + .fpu_info = { + /* SuperSparc 50 module */ + FPU(0, "SuperSparc on-chip FPU"), + /* SparcClassic */ + FPU(4, "TI MicroSparc on chip FPU"), + FPU(-1, NULL) + } +},{ + 5, + .cpu_info = { + CPU(0, "Matsushita - MN10501"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(0, "Matsushita MN10501"), + FPU(-1, NULL) + } +},{ + 6, + .cpu_info = { + CPU(0, "Philips Corporation - unknown"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(-1, NULL) + } +},{ + 7, + .cpu_info = { + CPU(0, "Harvest VLSI Design Center, Inc. - unknown"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(-1, NULL) + } +},{ + 8, + .cpu_info = { + CPU(0, "Systems and Processes Engineering Corporation (SPEC)"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(-1, NULL) + } +},{ + 9, + .cpu_info = { + /* Gallium arsenide 200MHz, BOOOOGOOOOMIPS!!! */ + CPU(0, "Fujitsu or Weitek Power-UP"), + CPU(1, "Fujitsu or Weitek Power-UP"), + CPU(2, "Fujitsu or Weitek Power-UP"), + CPU(3, "Fujitsu or Weitek Power-UP"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(3, "Fujitsu or Weitek on-chip FPU"), + FPU(-1, NULL) + } +},{ + PSR_IMPL_LEON, /* Aeroflex Gaisler */ + .cpu_info = { + CPU(3, "LEON"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(2, "GRFPU"), + FPU(3, "GRFPU-Lite"), + FPU(-1, NULL) + } +},{ + 0x17, + .cpu_info = { + CPU_PMU(0x10, "TI UltraSparc I (SpitFire)", "ultra12"), + CPU_PMU(0x11, "TI UltraSparc II (BlackBird)", "ultra12"), + CPU_PMU(0x12, "TI UltraSparc IIi (Sabre)", "ultra12"), + CPU_PMU(0x13, "TI UltraSparc IIe (Hummingbird)", "ultra12"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(0x10, "UltraSparc I integrated FPU"), + FPU(0x11, "UltraSparc II integrated FPU"), + FPU(0x12, "UltraSparc IIi integrated FPU"), + FPU(0x13, "UltraSparc IIe integrated FPU"), + FPU(-1, NULL) + } +},{ + 0x22, + .cpu_info = { + CPU_PMU(0x10, "TI UltraSparc I (SpitFire)", "ultra12"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(0x10, "UltraSparc I integrated FPU"), + FPU(-1, NULL) + } +},{ + 0x3e, + .cpu_info = { + CPU_PMU(0x14, "TI UltraSparc III (Cheetah)", "ultra3"), + CPU_PMU(0x15, "TI UltraSparc III+ (Cheetah+)", "ultra3+"), + CPU_PMU(0x16, "TI UltraSparc IIIi (Jalapeno)", "ultra3i"), + CPU_PMU(0x18, "TI UltraSparc IV (Jaguar)", "ultra3+"), + CPU_PMU(0x19, "TI UltraSparc IV+ (Panther)", "ultra4+"), + CPU_PMU(0x22, "TI UltraSparc IIIi+ (Serrano)", "ultra3i"), + CPU(-1, NULL) + }, + .fpu_info = { + FPU(0x14, "UltraSparc III integrated FPU"), + FPU(0x15, "UltraSparc III+ integrated FPU"), + FPU(0x16, "UltraSparc IIIi integrated FPU"), + FPU(0x18, "UltraSparc IV integrated FPU"), + FPU(0x19, "UltraSparc IV+ integrated FPU"), + FPU(0x22, "UltraSparc IIIi+ integrated FPU"), + FPU(-1, NULL) + } +}}; + +/* In order to get the fpu type correct, you need to take the IDPROM's + * machine type value into consideration too. I will fix this. + */ + +static const char *sparc_cpu_type; +static const char *sparc_fpu_type; +const char *sparc_pmu_type; + + +static void __init set_cpu_and_fpu(int psr_impl, int psr_vers, int fpu_vers) +{ + const struct manufacturer_info *manuf; + int i; + + sparc_cpu_type = NULL; + sparc_fpu_type = NULL; + sparc_pmu_type = NULL; + manuf = NULL; + + for (i = 0; i < ARRAY_SIZE(manufacturer_info); i++) + { + if (psr_impl == manufacturer_info[i].psr_impl) { + manuf = &manufacturer_info[i]; + break; + } + } + if (manuf != NULL) + { + const struct cpu_info *cpu; + const struct fpu_info *fpu; + + cpu = &manuf->cpu_info[0]; + while (cpu->psr_vers != -1) + { + if (cpu->psr_vers == psr_vers) { + sparc_cpu_type = cpu->name; + sparc_pmu_type = cpu->pmu_name; + sparc_fpu_type = "No FPU"; + break; + } + cpu++; + } + fpu = &manuf->fpu_info[0]; + while (fpu->fp_vers != -1) + { + if (fpu->fp_vers == fpu_vers) { + sparc_fpu_type = fpu->name; + break; + } + fpu++; + } + } + if (sparc_cpu_type == NULL) + { + printk(KERN_ERR "CPU: Unknown chip, impl[0x%x] vers[0x%x]\n", + psr_impl, psr_vers); + sparc_cpu_type = "Unknown CPU"; + } + if (sparc_fpu_type == NULL) + { + printk(KERN_ERR "FPU: Unknown chip, impl[0x%x] vers[0x%x]\n", + psr_impl, fpu_vers); + sparc_fpu_type = "Unknown FPU"; + } + if (sparc_pmu_type == NULL) + sparc_pmu_type = "Unknown PMU"; +} + +#ifdef CONFIG_SPARC32 +static int show_cpuinfo(struct seq_file *m, void *__unused) +{ + seq_printf(m, + "cpu\t\t: %s\n" + "fpu\t\t: %s\n" + "promlib\t\t: Version %d Revision %d\n" + "prom\t\t: %d.%d\n" + "type\t\t: %s\n" + "ncpus probed\t: %d\n" + "ncpus active\t: %d\n" +#ifndef CONFIG_SMP + "CPU0Bogo\t: %lu.%02lu\n" + "CPU0ClkTck\t: %ld\n" +#endif + , + sparc_cpu_type, + sparc_fpu_type , + romvec->pv_romvers, + prom_rev, + romvec->pv_printrev >> 16, + romvec->pv_printrev & 0xffff, + &cputypval[0], + ncpus_probed, + num_online_cpus() +#ifndef CONFIG_SMP + , cpu_data(0).udelay_val/(500000/HZ), + (cpu_data(0).udelay_val/(5000/HZ)) % 100, + cpu_data(0).clock_tick +#endif + ); + +#ifdef CONFIG_SMP + smp_bogo(m); +#endif + mmu_info(m); +#ifdef CONFIG_SMP + smp_info(m); +#endif + return 0; +} +#endif /* CONFIG_SPARC32 */ + +#ifdef CONFIG_SPARC64 +unsigned int dcache_parity_tl1_occurred; +unsigned int icache_parity_tl1_occurred; + + +static int show_cpuinfo(struct seq_file *m, void *__unused) +{ + seq_printf(m, + "cpu\t\t: %s\n" + "fpu\t\t: %s\n" + "pmu\t\t: %s\n" + "prom\t\t: %s\n" + "type\t\t: %s\n" + "ncpus probed\t: %d\n" + "ncpus active\t: %d\n" + "D$ parity tl1\t: %u\n" + "I$ parity tl1\t: %u\n" +#ifndef CONFIG_SMP + "Cpu0ClkTck\t: %016lx\n" +#endif + , + sparc_cpu_type, + sparc_fpu_type, + sparc_pmu_type, + prom_version, + ((tlb_type == hypervisor) ? + "sun4v" : + "sun4u"), + ncpus_probed, + num_online_cpus(), + dcache_parity_tl1_occurred, + icache_parity_tl1_occurred +#ifndef CONFIG_SMP + , cpu_data(0).clock_tick +#endif + ); + cpucap_info(m); +#ifdef CONFIG_SMP + smp_bogo(m); +#endif + mmu_info(m); +#ifdef CONFIG_SMP + smp_info(m); +#endif + return 0; +} +#endif /* CONFIG_SPARC64 */ + +static void *c_start(struct seq_file *m, loff_t *pos) +{ + /* The pointer we are returning is arbitrary, + * it just has to be non-NULL and not IS_ERR + * in the success case. + */ + return *pos == 0 ? &c_start : NULL; +} + +static void *c_next(struct seq_file *m, void *v, loff_t *pos) +{ + ++*pos; + return c_start(m, pos); +} + +static void c_stop(struct seq_file *m, void *v) +{ +} + +const struct seq_operations cpuinfo_op = { + .start =c_start, + .next = c_next, + .stop = c_stop, + .show = show_cpuinfo, +}; + +#ifdef CONFIG_SPARC32 +static int __init cpu_type_probe(void) +{ + int psr_impl, psr_vers, fpu_vers; + int psr; + + psr_impl = ((get_psr() >> PSR_IMPL_SHIFT) & PSR_IMPL_SHIFTED_MASK); + psr_vers = ((get_psr() >> PSR_VERS_SHIFT) & PSR_VERS_SHIFTED_MASK); + + psr = get_psr(); + put_psr(psr | PSR_EF); + + if (psr_impl == PSR_IMPL_LEON) + fpu_vers = get_psr() & PSR_EF ? ((get_fsr() >> 17) & 0x7) : 7; + else + fpu_vers = ((get_fsr() >> 17) & 0x7); + + put_psr(psr); + + set_cpu_and_fpu(psr_impl, psr_vers, fpu_vers); + + return 0; +} +#endif /* CONFIG_SPARC32 */ + +#ifdef CONFIG_SPARC64 +static void __init sun4v_cpu_probe(void) +{ + switch (sun4v_chip_type) { + case SUN4V_CHIP_NIAGARA1: + sparc_cpu_type = "UltraSparc T1 (Niagara)"; + sparc_fpu_type = "UltraSparc T1 integrated FPU"; + sparc_pmu_type = "niagara"; + break; + + case SUN4V_CHIP_NIAGARA2: + sparc_cpu_type = "UltraSparc T2 (Niagara2)"; + sparc_fpu_type = "UltraSparc T2 integrated FPU"; + sparc_pmu_type = "niagara2"; + break; + + case SUN4V_CHIP_NIAGARA3: + sparc_cpu_type = "UltraSparc T3 (Niagara3)"; + sparc_fpu_type = "UltraSparc T3 integrated FPU"; + sparc_pmu_type = "niagara3"; + break; + + case SUN4V_CHIP_NIAGARA4: + sparc_cpu_type = "UltraSparc T4 (Niagara4)"; + sparc_fpu_type = "UltraSparc T4 integrated FPU"; + sparc_pmu_type = "niagara4"; + break; + + case SUN4V_CHIP_NIAGARA5: + sparc_cpu_type = "UltraSparc T5 (Niagara5)"; + sparc_fpu_type = "UltraSparc T5 integrated FPU"; + sparc_pmu_type = "niagara5"; + break; + + case SUN4V_CHIP_SPARC_M6: + sparc_cpu_type = "SPARC-M6"; + sparc_fpu_type = "SPARC-M6 integrated FPU"; + sparc_pmu_type = "sparc-m6"; + break; + + case SUN4V_CHIP_SPARC_M7: + sparc_cpu_type = "SPARC-M7"; + sparc_fpu_type = "SPARC-M7 integrated FPU"; + sparc_pmu_type = "sparc-m7"; + break; + + case SUN4V_CHIP_SPARC_M8: + sparc_cpu_type = "SPARC-M8"; + sparc_fpu_type = "SPARC-M8 integrated FPU"; + sparc_pmu_type = "sparc-m8"; + break; + + case SUN4V_CHIP_SPARC_SN: + sparc_cpu_type = "SPARC-SN"; + sparc_fpu_type = "SPARC-SN integrated FPU"; + sparc_pmu_type = "sparc-sn"; + break; + + case SUN4V_CHIP_SPARC64X: + sparc_cpu_type = "SPARC64-X"; + sparc_fpu_type = "SPARC64-X integrated FPU"; + sparc_pmu_type = "sparc64-x"; + break; + + default: + printk(KERN_WARNING "CPU: Unknown sun4v cpu type [%s]\n", + prom_cpu_compatible); + sparc_cpu_type = "Unknown SUN4V CPU"; + sparc_fpu_type = "Unknown SUN4V FPU"; + sparc_pmu_type = "Unknown SUN4V PMU"; + break; + } +} + +static int __init cpu_type_probe(void) +{ + if (tlb_type == hypervisor) { + sun4v_cpu_probe(); + } else { + unsigned long ver; + int manuf, impl; + + __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver)); + + manuf = ((ver >> 48) & 0xffff); + impl = ((ver >> 32) & 0xffff); + set_cpu_and_fpu(manuf, impl, impl); + } + return 0; +} +#endif /* CONFIG_SPARC64 */ + +early_initcall(cpu_type_probe); diff --git a/arch/sparc/kernel/cpumap.c b/arch/sparc/kernel/cpumap.c new file mode 100644 index 000000000..d51ff0101 --- /dev/null +++ b/arch/sparc/kernel/cpumap.c @@ -0,0 +1,439 @@ +// SPDX-License-Identifier: GPL-2.0 +/* cpumap.c: used for optimizing CPU assignment + * + * Copyright (C) 2009 Hong H. Pham + */ + +#include +#include +#include +#include +#include +#include +#include "cpumap.h" + + +enum { + CPUINFO_LVL_ROOT = 0, + CPUINFO_LVL_NODE, + CPUINFO_LVL_CORE, + CPUINFO_LVL_PROC, + CPUINFO_LVL_MAX, +}; + +enum { + ROVER_NO_OP = 0, + /* Increment rover every time level is visited */ + ROVER_INC_ON_VISIT = 1 << 0, + /* Increment parent's rover every time rover wraps around */ + ROVER_INC_PARENT_ON_LOOP = 1 << 1, +}; + +struct cpuinfo_node { + int id; + int level; + int num_cpus; /* Number of CPUs in this hierarchy */ + int parent_index; + int child_start; /* Array index of the first child node */ + int child_end; /* Array index of the last child node */ + int rover; /* Child node iterator */ +}; + +struct cpuinfo_level { + int start_index; /* Index of first node of a level in a cpuinfo tree */ + int end_index; /* Index of last node of a level in a cpuinfo tree */ + int num_nodes; /* Number of nodes in a level in a cpuinfo tree */ +}; + +struct cpuinfo_tree { + int total_nodes; + + /* Offsets into nodes[] for each level of the tree */ + struct cpuinfo_level level[CPUINFO_LVL_MAX]; + struct cpuinfo_node nodes[] __counted_by(total_nodes); +}; + + +static struct cpuinfo_tree *cpuinfo_tree; + +static u16 cpu_distribution_map[NR_CPUS]; +static DEFINE_SPINLOCK(cpu_map_lock); + + +/* Niagara optimized cpuinfo tree traversal. */ +static const int niagara_iterate_method[] = { + [CPUINFO_LVL_ROOT] = ROVER_NO_OP, + + /* Strands (or virtual CPUs) within a core may not run concurrently + * on the Niagara, as instruction pipeline(s) are shared. Distribute + * work to strands in different cores first for better concurrency. + * Go to next NUMA node when all cores are used. + */ + [CPUINFO_LVL_NODE] = ROVER_INC_ON_VISIT|ROVER_INC_PARENT_ON_LOOP, + + /* Strands are grouped together by proc_id in cpuinfo_sparc, i.e. + * a proc_id represents an instruction pipeline. Distribute work to + * strands in different proc_id groups if the core has multiple + * instruction pipelines (e.g. the Niagara 2/2+ has two). + */ + [CPUINFO_LVL_CORE] = ROVER_INC_ON_VISIT, + + /* Pick the next strand in the proc_id group. */ + [CPUINFO_LVL_PROC] = ROVER_INC_ON_VISIT, +}; + +/* Generic cpuinfo tree traversal. Distribute work round robin across NUMA + * nodes. + */ +static const int generic_iterate_method[] = { + [CPUINFO_LVL_ROOT] = ROVER_INC_ON_VISIT, + [CPUINFO_LVL_NODE] = ROVER_NO_OP, + [CPUINFO_LVL_CORE] = ROVER_INC_PARENT_ON_LOOP, + [CPUINFO_LVL_PROC] = ROVER_INC_ON_VISIT|ROVER_INC_PARENT_ON_LOOP, +}; + + +static int cpuinfo_id(int cpu, int level) +{ + int id; + + switch (level) { + case CPUINFO_LVL_ROOT: + id = 0; + break; + case CPUINFO_LVL_NODE: + id = cpu_to_node(cpu); + break; + case CPUINFO_LVL_CORE: + id = cpu_data(cpu).core_id; + break; + case CPUINFO_LVL_PROC: + id = cpu_data(cpu).proc_id; + break; + default: + id = -EINVAL; + } + return id; +} + +/* + * Enumerate the CPU information in __cpu_data to determine the start index, + * end index, and number of nodes for each level in the cpuinfo tree. The + * total number of cpuinfo nodes required to build the tree is returned. + */ +static int enumerate_cpuinfo_nodes(struct cpuinfo_level *tree_level) +{ + int prev_id[CPUINFO_LVL_MAX]; + int i, n, num_nodes; + + for (i = CPUINFO_LVL_ROOT; i < CPUINFO_LVL_MAX; i++) { + struct cpuinfo_level *lv = &tree_level[i]; + + prev_id[i] = -1; + lv->start_index = lv->end_index = lv->num_nodes = 0; + } + + num_nodes = 1; /* Include the root node */ + + for (i = 0; i < num_possible_cpus(); i++) { + if (!cpu_online(i)) + continue; + + n = cpuinfo_id(i, CPUINFO_LVL_NODE); + if (n > prev_id[CPUINFO_LVL_NODE]) { + tree_level[CPUINFO_LVL_NODE].num_nodes++; + prev_id[CPUINFO_LVL_NODE] = n; + num_nodes++; + } + n = cpuinfo_id(i, CPUINFO_LVL_CORE); + if (n > prev_id[CPUINFO_LVL_CORE]) { + tree_level[CPUINFO_LVL_CORE].num_nodes++; + prev_id[CPUINFO_LVL_CORE] = n; + num_nodes++; + } + n = cpuinfo_id(i, CPUINFO_LVL_PROC); + if (n > prev_id[CPUINFO_LVL_PROC]) { + tree_level[CPUINFO_LVL_PROC].num_nodes++; + prev_id[CPUINFO_LVL_PROC] = n; + num_nodes++; + } + } + + tree_level[CPUINFO_LVL_ROOT].num_nodes = 1; + + n = tree_level[CPUINFO_LVL_NODE].num_nodes; + tree_level[CPUINFO_LVL_NODE].start_index = 1; + tree_level[CPUINFO_LVL_NODE].end_index = n; + + n++; + tree_level[CPUINFO_LVL_CORE].start_index = n; + n += tree_level[CPUINFO_LVL_CORE].num_nodes; + tree_level[CPUINFO_LVL_CORE].end_index = n - 1; + + tree_level[CPUINFO_LVL_PROC].start_index = n; + n += tree_level[CPUINFO_LVL_PROC].num_nodes; + tree_level[CPUINFO_LVL_PROC].end_index = n - 1; + + return num_nodes; +} + +/* Build a tree representation of the CPU hierarchy using the per CPU + * information in __cpu_data. Entries in __cpu_data[0..NR_CPUS] are + * assumed to be sorted in ascending order based on node, core_id, and + * proc_id (in order of significance). + */ +static struct cpuinfo_tree *build_cpuinfo_tree(void) +{ + struct cpuinfo_tree *new_tree; + struct cpuinfo_node *node; + struct cpuinfo_level tmp_level[CPUINFO_LVL_MAX]; + int num_cpus[CPUINFO_LVL_MAX]; + int level_rover[CPUINFO_LVL_MAX]; + int prev_id[CPUINFO_LVL_MAX]; + int n, id, cpu, prev_cpu, last_cpu, level; + + n = enumerate_cpuinfo_nodes(tmp_level); + + new_tree = kzalloc_flex(*new_tree, nodes, n, GFP_ATOMIC); + if (!new_tree) + return NULL; + + new_tree->total_nodes = n; + memcpy(&new_tree->level, tmp_level, sizeof(tmp_level)); + + prev_cpu = cpu = cpumask_first(cpu_online_mask); + + /* Initialize all levels in the tree with the first CPU */ + for (level = CPUINFO_LVL_PROC; level >= CPUINFO_LVL_ROOT; level--) { + n = new_tree->level[level].start_index; + + level_rover[level] = n; + node = &new_tree->nodes[n]; + + id = cpuinfo_id(cpu, level); + if (unlikely(id < 0)) { + kfree(new_tree); + return NULL; + } + node->id = id; + node->level = level; + node->num_cpus = 1; + + node->parent_index = (level > CPUINFO_LVL_ROOT) + ? new_tree->level[level - 1].start_index : -1; + + node->child_start = node->child_end = node->rover = + (level == CPUINFO_LVL_PROC) + ? cpu : new_tree->level[level + 1].start_index; + + prev_id[level] = node->id; + num_cpus[level] = 1; + } + + for (last_cpu = (num_possible_cpus() - 1); last_cpu >= 0; last_cpu--) { + if (cpu_online(last_cpu)) + break; + } + + while (++cpu <= last_cpu) { + if (!cpu_online(cpu)) + continue; + + for (level = CPUINFO_LVL_PROC; level >= CPUINFO_LVL_ROOT; + level--) { + id = cpuinfo_id(cpu, level); + if (unlikely(id < 0)) { + kfree(new_tree); + return NULL; + } + + if ((id != prev_id[level]) || (cpu == last_cpu)) { + prev_id[level] = id; + node = &new_tree->nodes[level_rover[level]]; + node->num_cpus = num_cpus[level]; + num_cpus[level] = 1; + + if (cpu == last_cpu) + node->num_cpus++; + + /* Connect tree node to parent */ + if (level == CPUINFO_LVL_ROOT) + node->parent_index = -1; + else + node->parent_index = + level_rover[level - 1]; + + if (level == CPUINFO_LVL_PROC) { + node->child_end = + (cpu == last_cpu) ? cpu : prev_cpu; + } else { + node->child_end = + level_rover[level + 1] - 1; + } + + /* Initialize the next node in the same level */ + n = ++level_rover[level]; + if (n <= new_tree->level[level].end_index) { + node = &new_tree->nodes[n]; + node->id = id; + node->level = level; + + /* Connect node to child */ + node->child_start = node->child_end = + node->rover = + (level == CPUINFO_LVL_PROC) + ? cpu : level_rover[level + 1]; + } + } else + num_cpus[level]++; + } + prev_cpu = cpu; + } + + return new_tree; +} + +static void increment_rover(struct cpuinfo_tree *t, int node_index, + int root_index, const int *rover_inc_table) +{ + struct cpuinfo_node *node = &t->nodes[node_index]; + int top_level, level; + + top_level = t->nodes[root_index].level; + for (level = node->level; level >= top_level; level--) { + node->rover++; + if (node->rover <= node->child_end) + return; + + node->rover = node->child_start; + /* If parent's rover does not need to be adjusted, stop here. */ + if ((level == top_level) || + !(rover_inc_table[level] & ROVER_INC_PARENT_ON_LOOP)) + return; + + node = &t->nodes[node->parent_index]; + } +} + +static int iterate_cpu(struct cpuinfo_tree *t, unsigned int root_index) +{ + const int *rover_inc_table; + int level, new_index, index = root_index; + + switch (sun4v_chip_type) { + case SUN4V_CHIP_NIAGARA1: + case SUN4V_CHIP_NIAGARA2: + case SUN4V_CHIP_NIAGARA3: + case SUN4V_CHIP_NIAGARA4: + case SUN4V_CHIP_NIAGARA5: + case SUN4V_CHIP_SPARC_M6: + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_M8: + case SUN4V_CHIP_SPARC_SN: + case SUN4V_CHIP_SPARC64X: + rover_inc_table = niagara_iterate_method; + break; + default: + rover_inc_table = generic_iterate_method; + } + + for (level = t->nodes[root_index].level; level < CPUINFO_LVL_MAX; + level++) { + new_index = t->nodes[index].rover; + if (rover_inc_table[level] & ROVER_INC_ON_VISIT) + increment_rover(t, index, root_index, rover_inc_table); + + index = new_index; + } + return index; +} + +static void _cpu_map_rebuild(void) +{ + int i; + + if (cpuinfo_tree) { + kfree(cpuinfo_tree); + cpuinfo_tree = NULL; + } + + cpuinfo_tree = build_cpuinfo_tree(); + if (!cpuinfo_tree) + return; + + /* Build CPU distribution map that spans all online CPUs. No need + * to check if the CPU is online, as that is done when the cpuinfo + * tree is being built. + */ + for (i = 0; i < cpuinfo_tree->nodes[0].num_cpus; i++) + cpu_distribution_map[i] = iterate_cpu(cpuinfo_tree, 0); +} + +/* Fallback if the cpuinfo tree could not be built. CPU mapping is linear + * round robin. + */ +static int simple_map_to_cpu(unsigned int index) +{ + int i, end, cpu_rover; + + cpu_rover = 0; + end = index % num_online_cpus(); + for (i = 0; i < num_possible_cpus(); i++) { + if (cpu_online(cpu_rover)) { + if (cpu_rover >= end) + return cpu_rover; + + cpu_rover++; + } + } + + /* Impossible, since num_online_cpus() <= num_possible_cpus() */ + return cpumask_first(cpu_online_mask); +} + +static int _map_to_cpu(unsigned int index) +{ + struct cpuinfo_node *root_node; + + if (unlikely(!cpuinfo_tree)) { + _cpu_map_rebuild(); + if (!cpuinfo_tree) + return simple_map_to_cpu(index); + } + + root_node = &cpuinfo_tree->nodes[0]; +#ifdef CONFIG_HOTPLUG_CPU + if (unlikely(root_node->num_cpus != num_online_cpus())) { + _cpu_map_rebuild(); + if (!cpuinfo_tree) + return simple_map_to_cpu(index); + } +#endif + return cpu_distribution_map[index % root_node->num_cpus]; +} + +int map_to_cpu(unsigned int index) +{ + int mapped_cpu; + unsigned long flag; + + spin_lock_irqsave(&cpu_map_lock, flag); + mapped_cpu = _map_to_cpu(index); + +#ifdef CONFIG_HOTPLUG_CPU + while (unlikely(!cpu_online(mapped_cpu))) + mapped_cpu = _map_to_cpu(index); +#endif + spin_unlock_irqrestore(&cpu_map_lock, flag); + return mapped_cpu; +} +EXPORT_SYMBOL(map_to_cpu); + +void cpu_map_rebuild(void) +{ + unsigned long flag; + + spin_lock_irqsave(&cpu_map_lock, flag); + _cpu_map_rebuild(); + spin_unlock_irqrestore(&cpu_map_lock, flag); +} diff --git a/arch/sparc/kernel/cpumap.h b/arch/sparc/kernel/cpumap.h new file mode 100644 index 000000000..7d5b77486 --- /dev/null +++ b/arch/sparc/kernel/cpumap.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _CPUMAP_H +#define _CPUMAP_H + +#ifdef CONFIG_SMP +void cpu_map_rebuild(void); +int map_to_cpu(unsigned int index); +#define cpu_map_init() cpu_map_rebuild() +#else +#define cpu_map_init() do {} while (0) +static inline int map_to_cpu(unsigned int index) +{ + return raw_smp_processor_id(); +} +#endif + +#endif diff --git a/arch/sparc/kernel/devices.c b/arch/sparc/kernel/devices.c new file mode 100644 index 000000000..23b6e50d4 --- /dev/null +++ b/arch/sparc/kernel/devices.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* devices.c: Initial scan of the prom device tree for important + * Sparc device nodes which we need to find. + * + * This is based on the sparc64 version, but sun4m doesn't always use + * the hardware MIDs, so be careful. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" + +static char *cpu_mid_prop(void) +{ + if (sparc_cpu_model == sun4d) + return "cpu-id"; + return "mid"; +} + +static int check_cpu_node(phandle nd, int *cur_inst, + int (*compare)(phandle, int, void *), void *compare_arg, + phandle *prom_node, int *mid) +{ + if (!compare(nd, *cur_inst, compare_arg)) { + if (prom_node) + *prom_node = nd; + if (mid) { + *mid = prom_getintdefault(nd, cpu_mid_prop(), 0); + if (sparc_cpu_model == sun4m) + *mid &= 3; + } + return 0; + } + + (*cur_inst)++; + + return -ENODEV; +} + +static int __cpu_find_by(int (*compare)(phandle, int, void *), + void *compare_arg, phandle *prom_node, int *mid) +{ + struct device_node *dp; + int cur_inst; + + cur_inst = 0; + for_each_node_by_type(dp, "cpu") { + int err = check_cpu_node(dp->phandle, &cur_inst, + compare, compare_arg, + prom_node, mid); + if (!err) { + of_node_put(dp); + return 0; + } + } + + return -ENODEV; +} + +static int cpu_instance_compare(phandle nd, int instance, void *_arg) +{ + int desired_instance = (int) _arg; + + if (instance == desired_instance) + return 0; + return -ENODEV; +} + +int cpu_find_by_instance(int instance, phandle *prom_node, int *mid) +{ + return __cpu_find_by(cpu_instance_compare, (void *)instance, + prom_node, mid); +} + +static int cpu_mid_compare(phandle nd, int instance, void *_arg) +{ + int desired_mid = (int) _arg; + int this_mid; + + this_mid = prom_getintdefault(nd, cpu_mid_prop(), 0); + if (this_mid == desired_mid + || (sparc_cpu_model == sun4m && (this_mid & 3) == desired_mid)) + return 0; + return -ENODEV; +} + +int cpu_find_by_mid(int mid, phandle *prom_node) +{ + return __cpu_find_by(cpu_mid_compare, (void *)mid, + prom_node, NULL); +} + +/* sun4m uses truncated mids since we base the cpuid on the ttable/irqset + * address (0-3). This gives us the true hardware mid, which might have + * some other bits set. On 4d hardware and software mids are the same. + */ +int cpu_get_hwmid(phandle prom_node) +{ + return prom_getintdefault(prom_node, cpu_mid_prop(), -ENODEV); +} + +void __init device_scan(void) +{ + printk(KERN_NOTICE "Booting Linux...\n"); + +#ifndef CONFIG_SMP + { + phandle cpu_node; + int err; + err = cpu_find_by_instance(0, &cpu_node, NULL); + if (err) { + /* Probably a sun4e, Sun is trying to trick us ;-) */ + prom_printf("No cpu nodes, cannot continue\n"); + prom_halt(); + } + cpu_data(0).clock_tick = prom_getintdefault(cpu_node, + "clock-frequency", + 0); + } +#endif /* !CONFIG_SMP */ + + auxio_probe(); + auxio_power_probe(); +} diff --git a/arch/sparc/kernel/ds.c b/arch/sparc/kernel/ds.c new file mode 100644 index 000000000..b34db8125 --- /dev/null +++ b/arch/sparc/kernel/ds.c @@ -0,0 +1,1272 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* ds.c: Domain Services driver for Logical Domains + * + * Copyright (C) 2007, 2008 David S. Miller + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "kernel.h" + +#define DRV_MODULE_NAME "ds" +#define PFX DRV_MODULE_NAME ": " +#define DRV_MODULE_VERSION "1.0" +#define DRV_MODULE_RELDATE "Jul 11, 2007" + +static char version[] = + DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; +MODULE_AUTHOR("David S. Miller "); +MODULE_DESCRIPTION("Sun LDOM domain services driver"); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRV_MODULE_VERSION); + +struct ds_msg_tag { + __u32 type; +#define DS_INIT_REQ 0x00 +#define DS_INIT_ACK 0x01 +#define DS_INIT_NACK 0x02 +#define DS_REG_REQ 0x03 +#define DS_REG_ACK 0x04 +#define DS_REG_NACK 0x05 +#define DS_UNREG_REQ 0x06 +#define DS_UNREG_ACK 0x07 +#define DS_UNREG_NACK 0x08 +#define DS_DATA 0x09 +#define DS_NACK 0x0a + + __u32 len; +}; + +/* Result codes */ +#define DS_OK 0x00 +#define DS_REG_VER_NACK 0x01 +#define DS_REG_DUP 0x02 +#define DS_INV_HDL 0x03 +#define DS_TYPE_UNKNOWN 0x04 + +struct ds_version { + __u16 major; + __u16 minor; +}; + +struct ds_ver_req { + struct ds_msg_tag tag; + struct ds_version ver; +}; + +struct ds_ver_ack { + struct ds_msg_tag tag; + __u16 minor; +}; + +struct ds_ver_nack { + struct ds_msg_tag tag; + __u16 major; +}; + +struct ds_reg_req { + struct ds_msg_tag tag; + __u64 handle; + __u16 major; + __u16 minor; + char svc_id[]; +}; + +struct ds_reg_ack { + struct ds_msg_tag tag; + __u64 handle; + __u16 minor; +}; + +struct ds_reg_nack { + struct ds_msg_tag tag; + __u64 handle; + __u16 major; +}; + +struct ds_unreg_req { + struct ds_msg_tag tag; + __u64 handle; +}; + +struct ds_unreg_ack { + struct ds_msg_tag tag; + __u64 handle; +}; + +struct ds_unreg_nack { + struct ds_msg_tag tag; + __u64 handle; +}; + +struct ds_data { + struct ds_msg_tag tag; + __u64 handle; +}; + +struct ds_data_nack { + struct ds_msg_tag tag; + __u64 handle; + __u64 result; +}; + +struct ds_info; +struct ds_cap_state { + __u64 handle; + + void (*data)(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len); + + const char *service_id; + + u8 state; +#define CAP_STATE_UNKNOWN 0x00 +#define CAP_STATE_REG_SENT 0x01 +#define CAP_STATE_REGISTERED 0x02 +}; + +static void md_update_data(struct ds_info *dp, struct ds_cap_state *cp, + void *buf, int len); +static void domain_shutdown_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len); +static void domain_panic_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len); +#ifdef CONFIG_HOTPLUG_CPU +static void dr_cpu_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len); +#endif +static void ds_pri_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len); +static void ds_var_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len); + +static struct ds_cap_state ds_states_template[] = { + { + .service_id = "md-update", + .data = md_update_data, + }, + { + .service_id = "domain-shutdown", + .data = domain_shutdown_data, + }, + { + .service_id = "domain-panic", + .data = domain_panic_data, + }, +#ifdef CONFIG_HOTPLUG_CPU + { + .service_id = "dr-cpu", + .data = dr_cpu_data, + }, +#endif + { + .service_id = "pri", + .data = ds_pri_data, + }, + { + .service_id = "var-config", + .data = ds_var_data, + }, + { + .service_id = "var-config-backup", + .data = ds_var_data, + }, +}; + +static DEFINE_SPINLOCK(ds_lock); + +struct ds_info { + struct ldc_channel *lp; + u8 hs_state; +#define DS_HS_START 0x01 +#define DS_HS_DONE 0x02 + + u64 id; + + void *rcv_buf; + int rcv_buf_len; + + struct ds_cap_state *ds_states; + int num_ds_states; + + struct ds_info *next; +}; + +static struct ds_info *ds_info_list; + +static struct ds_cap_state *find_cap(struct ds_info *dp, u64 handle) +{ + unsigned int index = handle >> 32; + + if (index >= dp->num_ds_states) + return NULL; + return &dp->ds_states[index]; +} + +static struct ds_cap_state *find_cap_by_string(struct ds_info *dp, + const char *name) +{ + int i; + + for (i = 0; i < dp->num_ds_states; i++) { + if (strcmp(dp->ds_states[i].service_id, name)) + continue; + + return &dp->ds_states[i]; + } + return NULL; +} + +static int __ds_send(struct ldc_channel *lp, void *data, int len) +{ + int err, limit = 1000; + + err = -EINVAL; + while (limit-- > 0) { + err = ldc_write(lp, data, len); + if (!err || (err != -EAGAIN)) + break; + udelay(1); + } + + return err; +} + +static int ds_send(struct ldc_channel *lp, void *data, int len) +{ + unsigned long flags; + int err; + + spin_lock_irqsave(&ds_lock, flags); + err = __ds_send(lp, data, len); + spin_unlock_irqrestore(&ds_lock, flags); + + return err; +} + +struct ds_md_update_req { + __u64 req_num; +}; + +struct ds_md_update_res { + __u64 req_num; + __u32 result; +}; + +static void md_update_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len) +{ + struct ldc_channel *lp = dp->lp; + struct ds_data *dpkt = buf; + struct ds_md_update_req *rp; + struct { + struct ds_data data; + struct ds_md_update_res res; + } pkt; + + rp = (struct ds_md_update_req *) (dpkt + 1); + + printk(KERN_INFO "ds-%llu: Machine description update.\n", dp->id); + + mdesc_update(); + + memset(&pkt, 0, sizeof(pkt)); + pkt.data.tag.type = DS_DATA; + pkt.data.tag.len = sizeof(pkt) - sizeof(struct ds_msg_tag); + pkt.data.handle = cp->handle; + pkt.res.req_num = rp->req_num; + pkt.res.result = DS_OK; + + ds_send(lp, &pkt, sizeof(pkt)); +} + +struct ds_shutdown_req { + __u64 req_num; + __u32 ms_delay; +}; + +struct ds_shutdown_res { + __u64 req_num; + __u32 result; + char reason[1]; +}; + +static void domain_shutdown_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len) +{ + struct ldc_channel *lp = dp->lp; + struct ds_data *dpkt = buf; + struct ds_shutdown_req *rp; + struct { + struct ds_data data; + struct ds_shutdown_res res; + } pkt; + + rp = (struct ds_shutdown_req *) (dpkt + 1); + + printk(KERN_ALERT "ds-%llu: Shutdown request from " + "LDOM manager received.\n", dp->id); + + memset(&pkt, 0, sizeof(pkt)); + pkt.data.tag.type = DS_DATA; + pkt.data.tag.len = sizeof(pkt) - sizeof(struct ds_msg_tag); + pkt.data.handle = cp->handle; + pkt.res.req_num = rp->req_num; + pkt.res.result = DS_OK; + pkt.res.reason[0] = 0; + + ds_send(lp, &pkt, sizeof(pkt)); + + orderly_poweroff(true); +} + +struct ds_panic_req { + __u64 req_num; +}; + +struct ds_panic_res { + __u64 req_num; + __u32 result; + char reason[1]; +}; + +static void domain_panic_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len) +{ + struct ldc_channel *lp = dp->lp; + struct ds_data *dpkt = buf; + struct ds_panic_req *rp; + struct { + struct ds_data data; + struct ds_panic_res res; + } pkt; + + rp = (struct ds_panic_req *) (dpkt + 1); + + printk(KERN_ALERT "ds-%llu: Panic request from " + "LDOM manager received.\n", dp->id); + + memset(&pkt, 0, sizeof(pkt)); + pkt.data.tag.type = DS_DATA; + pkt.data.tag.len = sizeof(pkt) - sizeof(struct ds_msg_tag); + pkt.data.handle = cp->handle; + pkt.res.req_num = rp->req_num; + pkt.res.result = DS_OK; + pkt.res.reason[0] = 0; + + ds_send(lp, &pkt, sizeof(pkt)); + + panic("PANIC requested by LDOM manager."); +} + +#ifdef CONFIG_HOTPLUG_CPU +struct dr_cpu_tag { + __u64 req_num; + __u32 type; +#define DR_CPU_CONFIGURE 0x43 +#define DR_CPU_UNCONFIGURE 0x55 +#define DR_CPU_FORCE_UNCONFIGURE 0x46 +#define DR_CPU_STATUS 0x53 + +/* Responses */ +#define DR_CPU_OK 0x6f +#define DR_CPU_ERROR 0x65 + + __u32 num_records; +}; + +struct dr_cpu_resp_entry { + __u32 cpu; + __u32 result; +#define DR_CPU_RES_OK 0x00 +#define DR_CPU_RES_FAILURE 0x01 +#define DR_CPU_RES_BLOCKED 0x02 +#define DR_CPU_RES_CPU_NOT_RESPONDING 0x03 +#define DR_CPU_RES_NOT_IN_MD 0x04 + + __u32 stat; +#define DR_CPU_STAT_NOT_PRESENT 0x00 +#define DR_CPU_STAT_UNCONFIGURED 0x01 +#define DR_CPU_STAT_CONFIGURED 0x02 + + __u32 str_off; +}; + +static void __dr_cpu_send_error(struct ds_info *dp, + struct ds_cap_state *cp, + struct ds_data *data) +{ + struct dr_cpu_tag *tag = (struct dr_cpu_tag *) (data + 1); + struct { + struct ds_data data; + struct dr_cpu_tag tag; + } pkt; + int msg_len; + + memset(&pkt, 0, sizeof(pkt)); + pkt.data.tag.type = DS_DATA; + pkt.data.handle = cp->handle; + pkt.tag.req_num = tag->req_num; + pkt.tag.type = DR_CPU_ERROR; + pkt.tag.num_records = 0; + + msg_len = (sizeof(struct ds_data) + + sizeof(struct dr_cpu_tag)); + + pkt.data.tag.len = msg_len - sizeof(struct ds_msg_tag); + + __ds_send(dp->lp, &pkt, msg_len); +} + +static void dr_cpu_send_error(struct ds_info *dp, + struct ds_cap_state *cp, + struct ds_data *data) +{ + unsigned long flags; + + spin_lock_irqsave(&ds_lock, flags); + __dr_cpu_send_error(dp, cp, data); + spin_unlock_irqrestore(&ds_lock, flags); +} + +#define CPU_SENTINEL 0xffffffff + +static void purge_dups(u32 *list, u32 num_ents) +{ + unsigned int i; + + for (i = 0; i < num_ents; i++) { + u32 cpu = list[i]; + unsigned int j; + + if (cpu == CPU_SENTINEL) + continue; + + for (j = i + 1; j < num_ents; j++) { + if (list[j] == cpu) + list[j] = CPU_SENTINEL; + } + } +} + +static int dr_cpu_size_response(int ncpus) +{ + return (sizeof(struct ds_data) + + sizeof(struct dr_cpu_tag) + + (sizeof(struct dr_cpu_resp_entry) * ncpus)); +} + +static void dr_cpu_init_response(struct ds_data *resp, u64 req_num, + u64 handle, int resp_len, int ncpus, + cpumask_t *mask, u32 default_stat) +{ + struct dr_cpu_resp_entry *ent; + struct dr_cpu_tag *tag; + int i, cpu; + + tag = (struct dr_cpu_tag *) (resp + 1); + ent = (struct dr_cpu_resp_entry *) (tag + 1); + + resp->tag.type = DS_DATA; + resp->tag.len = resp_len - sizeof(struct ds_msg_tag); + resp->handle = handle; + tag->req_num = req_num; + tag->type = DR_CPU_OK; + tag->num_records = ncpus; + + i = 0; + for_each_cpu(cpu, mask) { + ent[i].cpu = cpu; + ent[i].result = DR_CPU_RES_OK; + ent[i].stat = default_stat; + i++; + } + BUG_ON(i != ncpus); +} + +static void dr_cpu_mark(struct ds_data *resp, int cpu, int ncpus, + u32 res, u32 stat) +{ + struct dr_cpu_resp_entry *ent; + struct dr_cpu_tag *tag; + int i; + + tag = (struct dr_cpu_tag *) (resp + 1); + ent = (struct dr_cpu_resp_entry *) (tag + 1); + + for (i = 0; i < ncpus; i++) { + if (ent[i].cpu != cpu) + continue; + ent[i].result = res; + ent[i].stat = stat; + break; + } +} + +static int dr_cpu_configure(struct ds_info *dp, struct ds_cap_state *cp, + u64 req_num, cpumask_t *mask) +{ + struct ds_data *resp; + int resp_len, ncpus, cpu; + unsigned long flags; + + ncpus = cpumask_weight(mask); + resp_len = dr_cpu_size_response(ncpus); + resp = kzalloc(resp_len, GFP_KERNEL); + if (!resp) + return -ENOMEM; + + dr_cpu_init_response(resp, req_num, cp->handle, + resp_len, ncpus, mask, + DR_CPU_STAT_CONFIGURED); + + mdesc_populate_present_mask(mask); + mdesc_fill_in_cpu_data(mask); + + for_each_cpu(cpu, mask) { + int err; + + printk(KERN_INFO "ds-%llu: Starting cpu %d...\n", + dp->id, cpu); + err = add_cpu(cpu); + if (err) { + __u32 res = DR_CPU_RES_FAILURE; + __u32 stat = DR_CPU_STAT_UNCONFIGURED; + + if (!cpu_present(cpu)) { + /* CPU not present in MD */ + res = DR_CPU_RES_NOT_IN_MD; + stat = DR_CPU_STAT_NOT_PRESENT; + } else if (err == -ENODEV) { + /* CPU did not call in successfully */ + res = DR_CPU_RES_CPU_NOT_RESPONDING; + } + + printk(KERN_INFO "ds-%llu: CPU startup failed err=%d\n", + dp->id, err); + dr_cpu_mark(resp, cpu, ncpus, res, stat); + } + } + + spin_lock_irqsave(&ds_lock, flags); + __ds_send(dp->lp, resp, resp_len); + spin_unlock_irqrestore(&ds_lock, flags); + + kfree(resp); + + /* Redistribute IRQs, taking into account the new cpus. */ + fixup_irqs(); + + return 0; +} + +static int dr_cpu_unconfigure(struct ds_info *dp, + struct ds_cap_state *cp, + u64 req_num, + cpumask_t *mask) +{ + struct ds_data *resp; + int resp_len, ncpus, cpu; + unsigned long flags; + + ncpus = cpumask_weight(mask); + resp_len = dr_cpu_size_response(ncpus); + resp = kzalloc(resp_len, GFP_KERNEL); + if (!resp) + return -ENOMEM; + + dr_cpu_init_response(resp, req_num, cp->handle, + resp_len, ncpus, mask, + DR_CPU_STAT_UNCONFIGURED); + + for_each_cpu(cpu, mask) { + int err; + + printk(KERN_INFO "ds-%llu: Shutting down cpu %d...\n", + dp->id, cpu); + err = remove_cpu(cpu); + if (err) + dr_cpu_mark(resp, cpu, ncpus, + DR_CPU_RES_FAILURE, + DR_CPU_STAT_CONFIGURED); + } + + spin_lock_irqsave(&ds_lock, flags); + __ds_send(dp->lp, resp, resp_len); + spin_unlock_irqrestore(&ds_lock, flags); + + kfree(resp); + + return 0; +} + +static void dr_cpu_data(struct ds_info *dp, struct ds_cap_state *cp, void *buf, + int len) +{ + struct ds_data *data = buf; + struct dr_cpu_tag *tag = (struct dr_cpu_tag *) (data + 1); + u32 *cpu_list = (u32 *) (tag + 1); + u64 req_num = tag->req_num; + cpumask_t mask; + unsigned int i; + int err; + + switch (tag->type) { + case DR_CPU_CONFIGURE: + case DR_CPU_UNCONFIGURE: + case DR_CPU_FORCE_UNCONFIGURE: + break; + + default: + dr_cpu_send_error(dp, cp, data); + return; + } + + purge_dups(cpu_list, tag->num_records); + + cpumask_clear(&mask); + for (i = 0; i < tag->num_records; i++) { + if (cpu_list[i] == CPU_SENTINEL) + continue; + + if (cpu_list[i] < nr_cpu_ids) + cpumask_set_cpu(cpu_list[i], &mask); + } + + if (tag->type == DR_CPU_CONFIGURE) + err = dr_cpu_configure(dp, cp, req_num, &mask); + else + err = dr_cpu_unconfigure(dp, cp, req_num, &mask); + + if (err) + dr_cpu_send_error(dp, cp, data); +} +#endif /* CONFIG_HOTPLUG_CPU */ + +struct ds_pri_msg { + __u64 req_num; + __u64 type; +#define DS_PRI_REQUEST 0x00 +#define DS_PRI_DATA 0x01 +#define DS_PRI_UPDATE 0x02 +}; + +static void ds_pri_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len) +{ + struct ds_data *dpkt = buf; + struct ds_pri_msg *rp; + + rp = (struct ds_pri_msg *) (dpkt + 1); + + printk(KERN_INFO "ds-%llu: PRI REQ [%llx:%llx], len=%d\n", + dp->id, rp->req_num, rp->type, len); +} + +struct ds_var_hdr { + __u32 type; +#define DS_VAR_SET_REQ 0x00 +#define DS_VAR_DELETE_REQ 0x01 +#define DS_VAR_SET_RESP 0x02 +#define DS_VAR_DELETE_RESP 0x03 +}; + +struct ds_var_set_msg { + struct ds_var_hdr hdr; + char name_and_value[]; +}; + +struct ds_var_delete_msg { + struct ds_var_hdr hdr; + char name[]; +}; + +struct ds_var_resp { + struct ds_var_hdr hdr; + __u32 result; +#define DS_VAR_SUCCESS 0x00 +#define DS_VAR_NO_SPACE 0x01 +#define DS_VAR_INVALID_VAR 0x02 +#define DS_VAR_INVALID_VAL 0x03 +#define DS_VAR_NOT_PRESENT 0x04 +}; + +static DEFINE_MUTEX(ds_var_mutex); +static int ds_var_doorbell; +static int ds_var_response; + +static void ds_var_data(struct ds_info *dp, + struct ds_cap_state *cp, + void *buf, int len) +{ + struct ds_data *dpkt = buf; + struct ds_var_resp *rp; + + rp = (struct ds_var_resp *) (dpkt + 1); + + if (rp->hdr.type != DS_VAR_SET_RESP && + rp->hdr.type != DS_VAR_DELETE_RESP) + return; + + ds_var_response = rp->result; + wmb(); + ds_var_doorbell = 1; +} + +void ldom_set_var(const char *var, const char *value) +{ + struct ds_cap_state *cp; + struct ds_info *dp; + unsigned long flags; + + spin_lock_irqsave(&ds_lock, flags); + cp = NULL; + for (dp = ds_info_list; dp; dp = dp->next) { + struct ds_cap_state *tmp; + + tmp = find_cap_by_string(dp, "var-config"); + if (tmp && tmp->state == CAP_STATE_REGISTERED) { + cp = tmp; + break; + } + } + if (!cp) { + for (dp = ds_info_list; dp; dp = dp->next) { + struct ds_cap_state *tmp; + + tmp = find_cap_by_string(dp, "var-config-backup"); + if (tmp && tmp->state == CAP_STATE_REGISTERED) { + cp = tmp; + break; + } + } + } + spin_unlock_irqrestore(&ds_lock, flags); + + if (cp) { + union { + struct { + struct ds_data data; + struct ds_var_set_msg msg; + } header; + char all[512]; + } pkt; + char *base, *p; + int msg_len, loops; + size_t var_len, value_len; + + var_len = strlen(var) + 1; + value_len = strlen(value) + 1; + + if (var_len + value_len > sizeof(pkt) - sizeof(pkt.header)) { + pr_err(PFX + "contents length: %zu, which more than max: %lu," + "so could not set (%s) variable to (%s).\n", + var_len + value_len, + sizeof(pkt) - sizeof(pkt.header), var, value); + return; + } + + memset(&pkt, 0, sizeof(pkt)); + pkt.header.data.tag.type = DS_DATA; + pkt.header.data.handle = cp->handle; + pkt.header.msg.hdr.type = DS_VAR_SET_REQ; + base = p = &pkt.header.msg.name_and_value[0]; + strscpy(p, var, var_len); + p += var_len; + strscpy(p, value, value_len); + p += value_len; + + msg_len = (sizeof(struct ds_data) + + sizeof(struct ds_var_set_msg) + + (p - base)); + msg_len = (msg_len + 3) & ~3; + pkt.header.data.tag.len = msg_len - sizeof(struct ds_msg_tag); + + mutex_lock(&ds_var_mutex); + + spin_lock_irqsave(&ds_lock, flags); + ds_var_doorbell = 0; + ds_var_response = -1; + + __ds_send(dp->lp, &pkt, msg_len); + spin_unlock_irqrestore(&ds_lock, flags); + + loops = 1000; + while (ds_var_doorbell == 0) { + if (loops-- < 0) + break; + barrier(); + udelay(100); + } + + mutex_unlock(&ds_var_mutex); + + if (ds_var_doorbell == 0 || + ds_var_response != DS_VAR_SUCCESS) + printk(KERN_ERR "ds-%llu: var-config [%s:%s] " + "failed, response(%d).\n", + dp->id, var, value, + ds_var_response); + } else { + printk(KERN_ERR PFX "var-config not registered so " + "could not set (%s) variable to (%s).\n", + var, value); + } +} + +static char full_boot_str[256] __attribute__((aligned(32))); +static int reboot_data_supported; + +void ldom_reboot(const char *boot_command) +{ + /* Don't bother with any of this if the boot_command + * is empty. + */ + if (boot_command && strlen(boot_command)) { + unsigned long len; + + snprintf(full_boot_str, sizeof(full_boot_str), "boot %s", + boot_command); + len = strlen(full_boot_str); + + if (reboot_data_supported) { + unsigned long ra = kimage_addr_to_ra(full_boot_str); + unsigned long hv_ret; + + hv_ret = sun4v_reboot_data_set(ra, len); + if (hv_ret != HV_EOK) + pr_err("SUN4V: Unable to set reboot data " + "hv_ret=%lu\n", hv_ret); + } else { + ldom_set_var("reboot-command", full_boot_str); + } + } + sun4v_mach_sir(); +} + +void ldom_power_off(void) +{ + sun4v_mach_exit(0); +} + +static void ds_conn_reset(struct ds_info *dp) +{ + printk(KERN_ERR "ds-%llu: ds_conn_reset() from %ps\n", + dp->id, __builtin_return_address(0)); +} + +static int register_services(struct ds_info *dp) +{ + struct ldc_channel *lp = dp->lp; + int i; + + for (i = 0; i < dp->num_ds_states; i++) { + struct { + struct ds_reg_req req; + u8 id_buf[256]; + } pbuf; + struct ds_cap_state *cp = &dp->ds_states[i]; + int err, msg_len; + u64 new_count; + + if (cp->state == CAP_STATE_REGISTERED) + continue; + + new_count = sched_clock() & 0xffffffff; + cp->handle = ((u64) i << 32) | new_count; + + msg_len = (sizeof(struct ds_reg_req) + + strlen(cp->service_id)); + + memset(&pbuf, 0, sizeof(pbuf)); + pbuf.req.tag.type = DS_REG_REQ; + pbuf.req.tag.len = (msg_len - sizeof(struct ds_msg_tag)); + pbuf.req.handle = cp->handle; + pbuf.req.major = 1; + pbuf.req.minor = 0; + strscpy(pbuf.id_buf, cp->service_id); + + err = __ds_send(lp, &pbuf, msg_len); + if (err > 0) + cp->state = CAP_STATE_REG_SENT; + } + return 0; +} + +static int ds_handshake(struct ds_info *dp, struct ds_msg_tag *pkt) +{ + + if (dp->hs_state == DS_HS_START) { + if (pkt->type != DS_INIT_ACK) + goto conn_reset; + + dp->hs_state = DS_HS_DONE; + + return register_services(dp); + } + + if (dp->hs_state != DS_HS_DONE) + goto conn_reset; + + if (pkt->type == DS_REG_ACK) { + struct ds_reg_ack *ap = (struct ds_reg_ack *) pkt; + struct ds_cap_state *cp = find_cap(dp, ap->handle); + + if (!cp) { + printk(KERN_ERR "ds-%llu: REG ACK for unknown " + "handle %llx\n", dp->id, ap->handle); + return 0; + } + printk(KERN_INFO "ds-%llu: Registered %s service.\n", + dp->id, cp->service_id); + cp->state = CAP_STATE_REGISTERED; + } else if (pkt->type == DS_REG_NACK) { + struct ds_reg_nack *np = (struct ds_reg_nack *) pkt; + struct ds_cap_state *cp = find_cap(dp, np->handle); + + if (!cp) { + printk(KERN_ERR "ds-%llu: REG NACK for " + "unknown handle %llx\n", + dp->id, np->handle); + return 0; + } + cp->state = CAP_STATE_UNKNOWN; + } + + return 0; + +conn_reset: + ds_conn_reset(dp); + return -ECONNRESET; +} + +static void __send_ds_nack(struct ds_info *dp, u64 handle) +{ + struct ds_data_nack nack = { + .tag = { + .type = DS_NACK, + .len = (sizeof(struct ds_data_nack) - + sizeof(struct ds_msg_tag)), + }, + .handle = handle, + .result = DS_INV_HDL, + }; + + __ds_send(dp->lp, &nack, sizeof(nack)); +} + +static LIST_HEAD(ds_work_list); +static DECLARE_WAIT_QUEUE_HEAD(ds_wait); + +struct ds_queue_entry { + struct list_head list; + struct ds_info *dp; + int req_len; + int __pad; + u64 req[]; +}; + +static void process_ds_work(void) +{ + struct ds_queue_entry *qp, *tmp; + unsigned long flags; + LIST_HEAD(todo); + + spin_lock_irqsave(&ds_lock, flags); + list_splice_init(&ds_work_list, &todo); + spin_unlock_irqrestore(&ds_lock, flags); + + list_for_each_entry_safe(qp, tmp, &todo, list) { + struct ds_data *dpkt = (struct ds_data *) qp->req; + struct ds_info *dp = qp->dp; + struct ds_cap_state *cp = find_cap(dp, dpkt->handle); + int req_len = qp->req_len; + + if (!cp) { + printk(KERN_ERR "ds-%llu: Data for unknown " + "handle %llu\n", + dp->id, dpkt->handle); + + spin_lock_irqsave(&ds_lock, flags); + __send_ds_nack(dp, dpkt->handle); + spin_unlock_irqrestore(&ds_lock, flags); + } else { + cp->data(dp, cp, dpkt, req_len); + } + + list_del(&qp->list); + kfree(qp); + } +} + +static int ds_thread(void *__unused) +{ + DEFINE_WAIT(wait); + + while (1) { + prepare_to_wait(&ds_wait, &wait, TASK_INTERRUPTIBLE); + if (list_empty(&ds_work_list)) + schedule(); + finish_wait(&ds_wait, &wait); + + if (kthread_should_stop()) + break; + + process_ds_work(); + } + + return 0; +} + +static int ds_data(struct ds_info *dp, struct ds_msg_tag *pkt, int len) +{ + struct ds_data *dpkt = (struct ds_data *) pkt; + struct ds_queue_entry *qp; + + qp = kmalloc(sizeof(struct ds_queue_entry) + len, GFP_ATOMIC); + if (!qp) { + __send_ds_nack(dp, dpkt->handle); + } else { + qp->dp = dp; + memcpy(&qp->req, pkt, len); + list_add_tail(&qp->list, &ds_work_list); + wake_up(&ds_wait); + } + return 0; +} + +static void ds_up(struct ds_info *dp) +{ + struct ldc_channel *lp = dp->lp; + struct ds_ver_req req; + int err; + + req.tag.type = DS_INIT_REQ; + req.tag.len = sizeof(req) - sizeof(struct ds_msg_tag); + req.ver.major = 1; + req.ver.minor = 0; + + err = __ds_send(lp, &req, sizeof(req)); + if (err > 0) + dp->hs_state = DS_HS_START; +} + +static void ds_reset(struct ds_info *dp) +{ + int i; + + dp->hs_state = 0; + + for (i = 0; i < dp->num_ds_states; i++) { + struct ds_cap_state *cp = &dp->ds_states[i]; + + cp->state = CAP_STATE_UNKNOWN; + } +} + +static void ds_event(void *arg, int event) +{ + struct ds_info *dp = arg; + struct ldc_channel *lp = dp->lp; + unsigned long flags; + int err; + + spin_lock_irqsave(&ds_lock, flags); + + if (event == LDC_EVENT_UP) { + ds_up(dp); + spin_unlock_irqrestore(&ds_lock, flags); + return; + } + + if (event == LDC_EVENT_RESET) { + ds_reset(dp); + spin_unlock_irqrestore(&ds_lock, flags); + return; + } + + if (event != LDC_EVENT_DATA_READY) { + printk(KERN_WARNING "ds-%llu: Unexpected LDC event %d\n", + dp->id, event); + spin_unlock_irqrestore(&ds_lock, flags); + return; + } + + err = 0; + while (1) { + struct ds_msg_tag *tag; + + err = ldc_read(lp, dp->rcv_buf, sizeof(*tag)); + + if (unlikely(err < 0)) { + if (err == -ECONNRESET) + ds_conn_reset(dp); + break; + } + if (err == 0) + break; + + tag = dp->rcv_buf; + err = ldc_read(lp, tag + 1, tag->len); + + if (unlikely(err < 0)) { + if (err == -ECONNRESET) + ds_conn_reset(dp); + break; + } + if (err < tag->len) + break; + + if (tag->type < DS_DATA) + err = ds_handshake(dp, dp->rcv_buf); + else + err = ds_data(dp, dp->rcv_buf, + sizeof(*tag) + err); + if (err == -ECONNRESET) + break; + } + + spin_unlock_irqrestore(&ds_lock, flags); +} + +static int ds_probe(struct vio_dev *vdev, const struct vio_device_id *id) +{ + static int ds_version_printed; + struct ldc_channel_config ds_cfg = { + .event = ds_event, + .mtu = 4096, + .mode = LDC_MODE_STREAM, + }; + struct mdesc_handle *hp; + struct ldc_channel *lp; + struct ds_info *dp; + const u64 *val; + int err, i; + + if (ds_version_printed++ == 0) + printk(KERN_INFO "%s", version); + + dp = kzalloc_obj(*dp); + err = -ENOMEM; + if (!dp) + goto out_err; + + hp = mdesc_grab(); + val = mdesc_get_property(hp, vdev->mp, "id", NULL); + if (val) + dp->id = *val; + mdesc_release(hp); + + dp->rcv_buf = kzalloc(4096, GFP_KERNEL); + if (!dp->rcv_buf) + goto out_free_dp; + + dp->rcv_buf_len = 4096; + + dp->ds_states = kmemdup(ds_states_template, + sizeof(ds_states_template), GFP_KERNEL); + if (!dp->ds_states) + goto out_free_rcv_buf; + + dp->num_ds_states = ARRAY_SIZE(ds_states_template); + + for (i = 0; i < dp->num_ds_states; i++) + dp->ds_states[i].handle = ((u64)i << 32); + + ds_cfg.tx_irq = vdev->tx_irq; + ds_cfg.rx_irq = vdev->rx_irq; + + lp = ldc_alloc(vdev->channel_id, &ds_cfg, dp, "DS"); + if (IS_ERR(lp)) { + err = PTR_ERR(lp); + goto out_free_ds_states; + } + dp->lp = lp; + + err = ldc_bind(lp); + if (err) + goto out_free_ldc; + + spin_lock_irq(&ds_lock); + dp->next = ds_info_list; + ds_info_list = dp; + spin_unlock_irq(&ds_lock); + + return err; + +out_free_ldc: + ldc_free(dp->lp); + +out_free_ds_states: + kfree(dp->ds_states); + +out_free_rcv_buf: + kfree(dp->rcv_buf); + +out_free_dp: + kfree(dp); + +out_err: + return err; +} + +static const struct vio_device_id ds_match[] = { + { + .type = "domain-services-port", + }, + {}, +}; + +static struct vio_driver ds_driver = { + .id_table = ds_match, + .probe = ds_probe, + .name = "ds", +}; + +static int __init ds_init(void) +{ + unsigned long hv_ret, major, minor; + + if (tlb_type == hypervisor) { + hv_ret = sun4v_get_version(HV_GRP_REBOOT_DATA, &major, &minor); + if (hv_ret == HV_EOK) { + pr_info("SUN4V: Reboot data supported (maj=%lu,min=%lu).\n", + major, minor); + reboot_data_supported = 1; + } + } + kthread_run(ds_thread, NULL, "kldomd"); + + return vio_register_driver(&ds_driver); +} + +fs_initcall(ds_init); diff --git a/arch/sparc/kernel/dtlb_miss.S b/arch/sparc/kernel/dtlb_miss.S new file mode 100644 index 000000000..fb9c78843 --- /dev/null +++ b/arch/sparc/kernel/dtlb_miss.S @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* DTLB ** ICACHE line 1: Context 0 check and TSB load */ + ldxa [%g0] ASI_DMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer + ldxa [%g0] ASI_DMMU, %g6 ! Get TAG TARGET + srlx %g6, 48, %g5 ! Get context + sllx %g6, 22, %g6 ! Zero out context + brz,pn %g5, kvmap_dtlb ! Context 0 processing + srlx %g6, 22, %g6 ! Delay slot + TSB_LOAD_QUAD(%g1, %g4) ! Load TSB entry + cmp %g4, %g6 ! Compare TAG + +/* DTLB ** ICACHE line 2: TSB compare and TLB load */ + bne,pn %xcc, tsb_miss_dtlb ! Miss + mov FAULT_CODE_DTLB, %g3 + stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Load TLB + retry ! Trap done + nop + nop + nop + nop + +/* DTLB ** ICACHE line 3: */ + nop + nop + nop + nop + nop + nop + nop + nop + +/* DTLB ** ICACHE line 4: */ + nop + nop + nop + nop + nop + nop + nop + nop diff --git a/arch/sparc/kernel/dtlb_prot.S b/arch/sparc/kernel/dtlb_prot.S new file mode 100644 index 000000000..9f945771b --- /dev/null +++ b/arch/sparc/kernel/dtlb_prot.S @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * dtlb_prot.S: DTLB protection trap strategy. + * This is included directly into the trap table. + * + * Copyright (C) 1996,1998 David S. Miller (davem@redhat.com) + * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz) + */ + +/* Ways we can get here: + * + * [TL == 0] 1) User stores to readonly pages. + * [TL == 0] 2) Nucleus stores to user readonly pages. + * [TL > 0] 3) Nucleus stores to user readonly stack frame. + */ + +/* PROT ** ICACHE line 1: User DTLB protection trap */ + mov TLB_SFSR, %g1 + stxa %g0, [%g1] ASI_DMMU ! Clear FaultValid bit + membar #Sync ! Synchronize stores + rdpr %pstate, %g5 ! Move into alt-globals + wrpr %g5, PSTATE_AG|PSTATE_MG, %pstate + rdpr %tl, %g1 ! Need a winfixup? + cmp %g1, 1 ! Trap level >1? + mov TLB_TAG_ACCESS, %g4 ! For reload of vaddr + +/* PROT ** ICACHE line 2: More real fault processing */ + ldxa [%g4] ASI_DMMU, %g5 ! Put tagaccess in %g5 + srlx %g5, PAGE_SHIFT, %g5 + sllx %g5, PAGE_SHIFT, %g5 ! Clear context ID bits + bgu,pn %xcc, winfix_trampoline ! Yes, perform winfixup + mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 + ba,pt %xcc, sparc64_realfault_common ! Nope, normal fault + nop + nop + +/* PROT ** ICACHE line 3: Unused... */ + nop + nop + nop + nop + nop + nop + nop + nop + +/* PROT ** ICACHE line 4: Unused... */ + nop + nop + nop + nop + nop + nop + nop + nop diff --git a/arch/sparc/kernel/ebus.c b/arch/sparc/kernel/ebus.c new file mode 100644 index 000000000..264b18647 --- /dev/null +++ b/arch/sparc/kernel/ebus.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: GPL-2.0 +/* ebus.c: EBUS DMA library code. + * + * Copyright (C) 1997 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1999 David S. Miller (davem@redhat.com) + */ + +#include +#include +#include +#include +#include + +#include +#include + +#define EBDMA_CSR 0x00UL /* Control/Status */ +#define EBDMA_ADDR 0x04UL /* DMA Address */ +#define EBDMA_COUNT 0x08UL /* DMA Count */ + +#define EBDMA_CSR_INT_PEND 0x00000001 +#define EBDMA_CSR_ERR_PEND 0x00000002 +#define EBDMA_CSR_DRAIN 0x00000004 +#define EBDMA_CSR_INT_EN 0x00000010 +#define EBDMA_CSR_RESET 0x00000080 +#define EBDMA_CSR_WRITE 0x00000100 +#define EBDMA_CSR_EN_DMA 0x00000200 +#define EBDMA_CSR_CYC_PEND 0x00000400 +#define EBDMA_CSR_DIAG_RD_DONE 0x00000800 +#define EBDMA_CSR_DIAG_WR_DONE 0x00001000 +#define EBDMA_CSR_EN_CNT 0x00002000 +#define EBDMA_CSR_TC 0x00004000 +#define EBDMA_CSR_DIS_CSR_DRN 0x00010000 +#define EBDMA_CSR_BURST_SZ_MASK 0x000c0000 +#define EBDMA_CSR_BURST_SZ_1 0x00080000 +#define EBDMA_CSR_BURST_SZ_4 0x00000000 +#define EBDMA_CSR_BURST_SZ_8 0x00040000 +#define EBDMA_CSR_BURST_SZ_16 0x000c0000 +#define EBDMA_CSR_DIAG_EN 0x00100000 +#define EBDMA_CSR_DIS_ERR_PEND 0x00400000 +#define EBDMA_CSR_TCI_DIS 0x00800000 +#define EBDMA_CSR_EN_NEXT 0x01000000 +#define EBDMA_CSR_DMA_ON 0x02000000 +#define EBDMA_CSR_A_LOADED 0x04000000 +#define EBDMA_CSR_NA_LOADED 0x08000000 +#define EBDMA_CSR_DEV_ID_MASK 0xf0000000 + +#define EBUS_DMA_RESET_TIMEOUT 10000 + +static void __ebus_dma_reset(struct ebus_dma_info *p, int no_drain) +{ + int i; + u32 val = 0; + + writel(EBDMA_CSR_RESET, p->regs + EBDMA_CSR); + udelay(1); + + if (no_drain) + return; + + for (i = EBUS_DMA_RESET_TIMEOUT; i > 0; i--) { + val = readl(p->regs + EBDMA_CSR); + + if (!(val & (EBDMA_CSR_DRAIN | EBDMA_CSR_CYC_PEND))) + break; + udelay(10); + } +} + +static irqreturn_t ebus_dma_irq(int irq, void *dev_id) +{ + struct ebus_dma_info *p = dev_id; + unsigned long flags; + u32 csr = 0; + + spin_lock_irqsave(&p->lock, flags); + csr = readl(p->regs + EBDMA_CSR); + writel(csr, p->regs + EBDMA_CSR); + spin_unlock_irqrestore(&p->lock, flags); + + if (csr & EBDMA_CSR_ERR_PEND) { + printk(KERN_CRIT "ebus_dma(%s): DMA error!\n", p->name); + p->callback(p, EBUS_DMA_EVENT_ERROR, p->client_cookie); + return IRQ_HANDLED; + } else if (csr & EBDMA_CSR_INT_PEND) { + p->callback(p, + (csr & EBDMA_CSR_TC) ? + EBUS_DMA_EVENT_DMA : EBUS_DMA_EVENT_DEVICE, + p->client_cookie); + return IRQ_HANDLED; + } + + return IRQ_NONE; + +} + +int ebus_dma_register(struct ebus_dma_info *p) +{ + u32 csr; + + if (!p->regs) + return -EINVAL; + if (p->flags & ~(EBUS_DMA_FLAG_USE_EBDMA_HANDLER | + EBUS_DMA_FLAG_TCI_DISABLE)) + return -EINVAL; + if ((p->flags & EBUS_DMA_FLAG_USE_EBDMA_HANDLER) && !p->callback) + return -EINVAL; + if (!strlen(p->name)) + return -EINVAL; + + __ebus_dma_reset(p, 1); + + csr = EBDMA_CSR_BURST_SZ_16 | EBDMA_CSR_EN_CNT; + + if (p->flags & EBUS_DMA_FLAG_TCI_DISABLE) + csr |= EBDMA_CSR_TCI_DIS; + + writel(csr, p->regs + EBDMA_CSR); + + return 0; +} +EXPORT_SYMBOL(ebus_dma_register); + +int ebus_dma_irq_enable(struct ebus_dma_info *p, int on) +{ + unsigned long flags; + u32 csr; + + if (on) { + if (p->flags & EBUS_DMA_FLAG_USE_EBDMA_HANDLER) { + if (request_irq(p->irq, ebus_dma_irq, IRQF_SHARED, p->name, p)) + return -EBUSY; + } + + spin_lock_irqsave(&p->lock, flags); + csr = readl(p->regs + EBDMA_CSR); + csr |= EBDMA_CSR_INT_EN; + writel(csr, p->regs + EBDMA_CSR); + spin_unlock_irqrestore(&p->lock, flags); + } else { + spin_lock_irqsave(&p->lock, flags); + csr = readl(p->regs + EBDMA_CSR); + csr &= ~EBDMA_CSR_INT_EN; + writel(csr, p->regs + EBDMA_CSR); + spin_unlock_irqrestore(&p->lock, flags); + + if (p->flags & EBUS_DMA_FLAG_USE_EBDMA_HANDLER) { + free_irq(p->irq, p); + } + } + + return 0; +} +EXPORT_SYMBOL(ebus_dma_irq_enable); + +void ebus_dma_unregister(struct ebus_dma_info *p) +{ + unsigned long flags; + u32 csr; + int irq_on = 0; + + spin_lock_irqsave(&p->lock, flags); + csr = readl(p->regs + EBDMA_CSR); + if (csr & EBDMA_CSR_INT_EN) { + csr &= ~EBDMA_CSR_INT_EN; + writel(csr, p->regs + EBDMA_CSR); + irq_on = 1; + } + spin_unlock_irqrestore(&p->lock, flags); + + if (irq_on) + free_irq(p->irq, p); +} +EXPORT_SYMBOL(ebus_dma_unregister); + +int ebus_dma_request(struct ebus_dma_info *p, dma_addr_t bus_addr, size_t len) +{ + unsigned long flags; + u32 csr; + int err; + + if (len >= (1 << 24)) + return -EINVAL; + + spin_lock_irqsave(&p->lock, flags); + csr = readl(p->regs + EBDMA_CSR); + err = -EINVAL; + if (!(csr & EBDMA_CSR_EN_DMA)) + goto out; + err = -EBUSY; + if (csr & EBDMA_CSR_NA_LOADED) + goto out; + + writel(len, p->regs + EBDMA_COUNT); + writel(bus_addr, p->regs + EBDMA_ADDR); + err = 0; + +out: + spin_unlock_irqrestore(&p->lock, flags); + + return err; +} +EXPORT_SYMBOL(ebus_dma_request); + +void ebus_dma_prepare(struct ebus_dma_info *p, int write) +{ + unsigned long flags; + u32 csr; + + spin_lock_irqsave(&p->lock, flags); + __ebus_dma_reset(p, 0); + + csr = (EBDMA_CSR_INT_EN | + EBDMA_CSR_EN_CNT | + EBDMA_CSR_BURST_SZ_16 | + EBDMA_CSR_EN_NEXT); + + if (write) + csr |= EBDMA_CSR_WRITE; + if (p->flags & EBUS_DMA_FLAG_TCI_DISABLE) + csr |= EBDMA_CSR_TCI_DIS; + + writel(csr, p->regs + EBDMA_CSR); + + spin_unlock_irqrestore(&p->lock, flags); +} +EXPORT_SYMBOL(ebus_dma_prepare); + +unsigned int ebus_dma_residue(struct ebus_dma_info *p) +{ + return readl(p->regs + EBDMA_COUNT); +} +EXPORT_SYMBOL(ebus_dma_residue); + +unsigned int ebus_dma_addr(struct ebus_dma_info *p) +{ + return readl(p->regs + EBDMA_ADDR); +} +EXPORT_SYMBOL(ebus_dma_addr); + +void ebus_dma_enable(struct ebus_dma_info *p, int on) +{ + unsigned long flags; + u32 orig_csr, csr; + + spin_lock_irqsave(&p->lock, flags); + orig_csr = csr = readl(p->regs + EBDMA_CSR); + if (on) + csr |= EBDMA_CSR_EN_DMA; + else + csr &= ~EBDMA_CSR_EN_DMA; + if ((orig_csr & EBDMA_CSR_EN_DMA) != + (csr & EBDMA_CSR_EN_DMA)) + writel(csr, p->regs + EBDMA_CSR); + spin_unlock_irqrestore(&p->lock, flags); +} +EXPORT_SYMBOL(ebus_dma_enable); diff --git a/arch/sparc/kernel/entry.S b/arch/sparc/kernel/entry.S new file mode 100644 index 000000000..ea51ef52c --- /dev/null +++ b/arch/sparc/kernel/entry.S @@ -0,0 +1,1370 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* arch/sparc/kernel/entry.S: Sparc trap low-level entry points. + * + * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) + * Copyright (C) 1996-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 1997 Anton Blanchard (anton@progsoc.uts.edu.au) + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define curptr g6 + +/* These are just handy. */ +#define _SV save %sp, -STACKFRAME_SZ, %sp +#define _RS restore + +#define FLUSH_ALL_KERNEL_WINDOWS \ + _SV; _SV; _SV; _SV; _SV; _SV; _SV; \ + _RS; _RS; _RS; _RS; _RS; _RS; _RS; + + .text + +#ifdef CONFIG_KGDB + .align 4 + .globl arch_kgdb_breakpoint + .type arch_kgdb_breakpoint,#function +arch_kgdb_breakpoint: + ta 0x7d + retl + nop + .size arch_kgdb_breakpoint,.-arch_kgdb_breakpoint +#endif + +#if defined(CONFIG_BLK_DEV_FD) || defined(CONFIG_BLK_DEV_FD_MODULE) + .align 4 + .globl floppy_hardint +floppy_hardint: + /* + * This code cannot touch registers %l0 %l1 and %l2 + * because SAVE_ALL depends on their values. It depends + * on %l3 also, but we regenerate it before a call. + * Other registers are: + * %l3 -- base address of fdc registers + * %l4 -- pdma_vaddr + * %l5 -- scratch for ld/st address + * %l6 -- pdma_size + * %l7 -- scratch [floppy byte, ld/st address, aux. data] + */ + + /* Do we have work to do? */ + sethi %hi(doing_pdma), %l7 + ld [%l7 + %lo(doing_pdma)], %l7 + cmp %l7, 0 + be floppy_dosoftint + nop + + /* Load fdc register base */ + sethi %hi(fdc_status), %l3 + ld [%l3 + %lo(fdc_status)], %l3 + + /* Setup register addresses */ + sethi %hi(pdma_vaddr), %l5 ! transfer buffer + ld [%l5 + %lo(pdma_vaddr)], %l4 + sethi %hi(pdma_size), %l5 ! bytes to go + ld [%l5 + %lo(pdma_size)], %l6 +next_byte: + ldub [%l3], %l7 + + andcc %l7, 0x80, %g0 ! Does fifo still have data + bz floppy_fifo_emptied ! fifo has been emptied... + andcc %l7, 0x20, %g0 ! in non-dma mode still? + bz floppy_overrun ! nope, overrun + andcc %l7, 0x40, %g0 ! 0=write 1=read + bz floppy_write + sub %l6, 0x1, %l6 + + /* Ok, actually read this byte */ + ldub [%l3 + 1], %l7 + orcc %g0, %l6, %g0 + stb %l7, [%l4] + bne next_byte + add %l4, 0x1, %l4 + + b floppy_tdone + nop + +floppy_write: + /* Ok, actually write this byte */ + ldub [%l4], %l7 + orcc %g0, %l6, %g0 + stb %l7, [%l3 + 1] + bne next_byte + add %l4, 0x1, %l4 + + /* fall through... */ +floppy_tdone: + sethi %hi(pdma_vaddr), %l5 + st %l4, [%l5 + %lo(pdma_vaddr)] + sethi %hi(pdma_size), %l5 + st %l6, [%l5 + %lo(pdma_size)] + /* Flip terminal count pin */ + set auxio_register, %l7 + ld [%l7], %l7 + + ldub [%l7], %l5 + + or %l5, 0xc2, %l5 + stb %l5, [%l7] + andn %l5, 0x02, %l5 + +2: + /* Kill some time so the bits set */ + WRITE_PAUSE + WRITE_PAUSE + + stb %l5, [%l7] + + /* Prevent recursion */ + sethi %hi(doing_pdma), %l7 + b floppy_dosoftint + st %g0, [%l7 + %lo(doing_pdma)] + + /* We emptied the FIFO, but we haven't read everything + * as of yet. Store the current transfer address and + * bytes left to read so we can continue when the next + * fast IRQ comes in. + */ +floppy_fifo_emptied: + sethi %hi(pdma_vaddr), %l5 + st %l4, [%l5 + %lo(pdma_vaddr)] + sethi %hi(pdma_size), %l7 + st %l6, [%l7 + %lo(pdma_size)] + + /* Restore condition codes */ + wr %l0, 0x0, %psr + WRITE_PAUSE + + jmp %l1 + rett %l2 + +floppy_overrun: + sethi %hi(pdma_vaddr), %l5 + st %l4, [%l5 + %lo(pdma_vaddr)] + sethi %hi(pdma_size), %l5 + st %l6, [%l5 + %lo(pdma_size)] + /* Prevent recursion */ + sethi %hi(doing_pdma), %l7 + st %g0, [%l7 + %lo(doing_pdma)] + + /* fall through... */ +floppy_dosoftint: + rd %wim, %l3 + SAVE_ALL + + /* Set all IRQs off. */ + or %l0, PSR_PIL, %l4 + wr %l4, 0x0, %psr + WRITE_PAUSE + wr %l4, PSR_ET, %psr + WRITE_PAUSE + + mov 11, %o0 ! floppy irq level (unused anyway) + mov %g0, %o1 ! devid is not used in fast interrupts + call sparc_floppy_irq + add %sp, STACKFRAME_SZ, %o2 ! struct pt_regs *regs + + RESTORE_ALL + +#endif /* (CONFIG_BLK_DEV_FD) */ + + /* Bad trap handler */ + .globl bad_trap_handler +bad_trap_handler: + SAVE_ALL + + wr %l0, PSR_ET, %psr + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 ! pt_regs + call do_hw_interrupt + mov %l7, %o1 ! trap number + + RESTORE_ALL + +/* For now all IRQ's not registered get sent here. handler_irq() will + * see if a routine is registered to handle this interrupt and if not + * it will say so on the console. + */ + + .align 4 + .globl real_irq_entry, patch_handler_irq +real_irq_entry: + SAVE_ALL + +#ifdef CONFIG_SMP + .globl patchme_maybe_smp_msg + + cmp %l7, 11 +patchme_maybe_smp_msg: + bgu maybe_smp4m_msg + nop +#endif + +real_irq_continue: + or %l0, PSR_PIL, %g2 + wr %g2, 0x0, %psr + WRITE_PAUSE + wr %g2, PSR_ET, %psr + WRITE_PAUSE + mov %l7, %o0 ! irq level +patch_handler_irq: + call handler_irq + add %sp, STACKFRAME_SZ, %o1 ! pt_regs ptr + or %l0, PSR_PIL, %g2 ! restore PIL after handler_irq + wr %g2, PSR_ET, %psr ! keep ET up + WRITE_PAUSE + + RESTORE_ALL + +#ifdef CONFIG_SMP + /* SMP per-cpu ticker interrupts are handled specially. */ +smp4m_ticker: + bne real_irq_continue+4 + or %l0, PSR_PIL, %g2 + wr %g2, 0x0, %psr + WRITE_PAUSE + wr %g2, PSR_ET, %psr + WRITE_PAUSE + call smp4m_percpu_timer_interrupt + add %sp, STACKFRAME_SZ, %o0 + wr %l0, PSR_ET, %psr + WRITE_PAUSE + RESTORE_ALL + +#define GET_PROCESSOR4M_ID(reg) \ + rd %tbr, %reg; \ + srl %reg, 12, %reg; \ + and %reg, 3, %reg; + + /* Here is where we check for possible SMP IPI passed to us + * on some level other than 15 which is the NMI and only used + * for cross calls. That has a separate entry point below. + * + * IPIs are sent on Level 12, 13 and 14. See IRQ_IPI_*. + */ +maybe_smp4m_msg: + GET_PROCESSOR4M_ID(o3) + sethi %hi(sun4m_irq_percpu), %l5 + sll %o3, 2, %o3 + or %l5, %lo(sun4m_irq_percpu), %o5 + sethi %hi(0x70000000), %o2 ! Check all soft-IRQs + ld [%o5 + %o3], %o1 + ld [%o1 + 0x00], %o3 ! sun4m_irq_percpu[cpu]->pending + andcc %o3, %o2, %g0 + be,a smp4m_ticker + cmp %l7, 14 + /* Soft-IRQ IPI */ + st %o2, [%o1 + 0x04] ! sun4m_irq_percpu[cpu]->clear=0x70000000 + WRITE_PAUSE + ld [%o1 + 0x00], %g0 ! sun4m_irq_percpu[cpu]->pending + WRITE_PAUSE + or %l0, PSR_PIL, %l4 + wr %l4, 0x0, %psr + WRITE_PAUSE + wr %l4, PSR_ET, %psr + WRITE_PAUSE + srl %o3, 28, %o2 ! shift for simpler checks below +maybe_smp4m_msg_check_single: + andcc %o2, 0x1, %g0 + beq,a maybe_smp4m_msg_check_mask + andcc %o2, 0x2, %g0 + call smp_call_function_single_interrupt + nop + andcc %o2, 0x2, %g0 +maybe_smp4m_msg_check_mask: + beq,a maybe_smp4m_msg_check_resched + andcc %o2, 0x4, %g0 + call smp_call_function_interrupt + nop + andcc %o2, 0x4, %g0 +maybe_smp4m_msg_check_resched: + /* rescheduling is done in RESTORE_ALL regardless, but incr stats */ + beq,a maybe_smp4m_msg_out + nop + call smp_resched_interrupt + nop +maybe_smp4m_msg_out: + RESTORE_ALL + + .align 4 + .globl linux_trap_ipi15_sun4m +linux_trap_ipi15_sun4m: + SAVE_ALL + sethi %hi(0x80000000), %o2 + GET_PROCESSOR4M_ID(o0) + sethi %hi(sun4m_irq_percpu), %l5 + or %l5, %lo(sun4m_irq_percpu), %o5 + sll %o0, 2, %o0 + ld [%o5 + %o0], %o5 + ld [%o5 + 0x00], %o3 ! sun4m_irq_percpu[cpu]->pending + andcc %o3, %o2, %g0 + be sun4m_nmi_error ! Must be an NMI async memory error + st %o2, [%o5 + 0x04] ! sun4m_irq_percpu[cpu]->clear=0x80000000 + WRITE_PAUSE + ld [%o5 + 0x00], %g0 ! sun4m_irq_percpu[cpu]->pending + WRITE_PAUSE + or %l0, PSR_PIL, %l4 + wr %l4, 0x0, %psr + WRITE_PAUSE + wr %l4, PSR_ET, %psr + WRITE_PAUSE + call smp4m_cross_call_irq + nop + b ret_trap_lockless_ipi + clr %l6 + + .globl smp4d_ticker + /* SMP per-cpu ticker interrupts are handled specially. */ +smp4d_ticker: + SAVE_ALL + or %l0, PSR_PIL, %g2 + sethi %hi(CC_ICLR), %o0 + sethi %hi(1 << 14), %o1 + or %o0, %lo(CC_ICLR), %o0 + stha %o1, [%o0] ASI_M_MXCC /* Clear PIL 14 in MXCC's ICLR */ + wr %g2, 0x0, %psr + WRITE_PAUSE + wr %g2, PSR_ET, %psr + WRITE_PAUSE + call smp4d_percpu_timer_interrupt + add %sp, STACKFRAME_SZ, %o0 + wr %l0, PSR_ET, %psr + WRITE_PAUSE + RESTORE_ALL + + .align 4 + .globl linux_trap_ipi15_sun4d +linux_trap_ipi15_sun4d: + SAVE_ALL + sethi %hi(CC_BASE), %o4 + sethi %hi(MXCC_ERR_ME|MXCC_ERR_PEW|MXCC_ERR_ASE|MXCC_ERR_PEE), %o2 + or %o4, (CC_EREG - CC_BASE), %o0 + ldda [%o0] ASI_M_MXCC, %o0 + andcc %o0, %o2, %g0 + bne 1f + sethi %hi(BB_STAT2), %o2 + lduba [%o2] ASI_M_CTL, %o2 + andcc %o2, BB_STAT2_MASK, %g0 + bne 2f + or %o4, (CC_ICLR - CC_BASE), %o0 + sethi %hi(1 << 15), %o1 + stha %o1, [%o0] ASI_M_MXCC /* Clear PIL 15 in MXCC's ICLR */ + or %l0, PSR_PIL, %l4 + wr %l4, 0x0, %psr + WRITE_PAUSE + wr %l4, PSR_ET, %psr + WRITE_PAUSE + call smp4d_cross_call_irq + nop + b ret_trap_lockless_ipi + clr %l6 + +1: /* MXCC error */ +2: /* BB error */ + /* Disable PIL 15 */ + set CC_IMSK, %l4 + lduha [%l4] ASI_M_MXCC, %l5 + sethi %hi(1 << 15), %l7 + or %l5, %l7, %l5 + stha %l5, [%l4] ASI_M_MXCC + /* FIXME */ +1: b,a 1b + + .globl smpleon_ipi + .extern leon_ipi_interrupt + /* SMP per-cpu IPI interrupts are handled specially. */ +smpleon_ipi: + SAVE_ALL + or %l0, PSR_PIL, %g2 + wr %g2, 0x0, %psr + WRITE_PAUSE + wr %g2, PSR_ET, %psr + WRITE_PAUSE + call leonsmp_ipi_interrupt + add %sp, STACKFRAME_SZ, %o1 ! pt_regs + wr %l0, PSR_ET, %psr + WRITE_PAUSE + RESTORE_ALL + + .align 4 + .globl linux_trap_ipi15_leon +linux_trap_ipi15_leon: + SAVE_ALL + or %l0, PSR_PIL, %l4 + wr %l4, 0x0, %psr + WRITE_PAUSE + wr %l4, PSR_ET, %psr + WRITE_PAUSE + call leon_cross_call_irq + nop + b ret_trap_lockless_ipi + clr %l6 + +#endif /* CONFIG_SMP */ + + /* This routine handles illegal instructions and privileged + * instruction attempts from user code. + */ + .align 4 + .globl bad_instruction +bad_instruction: + sethi %hi(0xc1f80000), %l4 + ld [%l1], %l5 + sethi %hi(0x81d80000), %l7 + and %l5, %l4, %l5 + cmp %l5, %l7 + be 1f + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call do_illegal_instruction + mov %l0, %o3 + + RESTORE_ALL + +1: /* unimplemented flush - just skip */ + jmpl %l2, %g0 + rett %l2 + 4 + + .align 4 + .globl priv_instruction +priv_instruction: + SAVE_ALL + + wr %l0, PSR_ET, %psr + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call do_priv_instruction + mov %l0, %o3 + + RESTORE_ALL + + /* This routine handles unaligned data accesses. */ + .align 4 + .globl mna_handler +mna_handler: + andcc %l0, PSR_PS, %g0 + be mna_fromuser + nop + + SAVE_ALL + + wr %l0, PSR_ET, %psr + WRITE_PAUSE + + ld [%l1], %o1 + call kernel_unaligned_trap + add %sp, STACKFRAME_SZ, %o0 + + RESTORE_ALL + +mna_fromuser: + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + ld [%l1], %o1 + call user_unaligned_trap + add %sp, STACKFRAME_SZ, %o0 + + RESTORE_ALL + + /* This routine handles floating point disabled traps. */ + .align 4 + .globl fpd_trap_handler +fpd_trap_handler: + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call do_fpd_trap + mov %l0, %o3 + + RESTORE_ALL + + /* This routine handles Floating Point Exceptions. */ + .align 4 + .globl fpe_trap_handler +fpe_trap_handler: + set fpsave_magic, %l5 + cmp %l1, %l5 + be 1f + sethi %hi(fpsave), %l5 + or %l5, %lo(fpsave), %l5 + cmp %l1, %l5 + bne 2f + sethi %hi(fpsave_catch2), %l5 + or %l5, %lo(fpsave_catch2), %l5 + wr %l0, 0x0, %psr + WRITE_PAUSE + jmp %l5 + rett %l5 + 4 +1: + sethi %hi(fpsave_catch), %l5 + or %l5, %lo(fpsave_catch), %l5 + wr %l0, 0x0, %psr + WRITE_PAUSE + jmp %l5 + rett %l5 + 4 + +2: + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call do_fpe_trap + mov %l0, %o3 + + RESTORE_ALL + + /* This routine handles Tag Overflow Exceptions. */ + .align 4 + .globl do_tag_overflow +do_tag_overflow: + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call handle_tag_overflow + mov %l0, %o3 + + RESTORE_ALL + + /* This routine handles Watchpoint Exceptions. */ + .align 4 + .globl do_watchpoint +do_watchpoint: + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call handle_watchpoint + mov %l0, %o3 + + RESTORE_ALL + + /* This routine handles Register Access Exceptions. */ + .align 4 + .globl do_reg_access +do_reg_access: + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call handle_reg_access + mov %l0, %o3 + + RESTORE_ALL + + /* This routine handles Co-Processor Disabled Exceptions. */ + .align 4 + .globl do_cp_disabled +do_cp_disabled: + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call handle_cp_disabled + mov %l0, %o3 + + RESTORE_ALL + + /* This routine handles Co-Processor Exceptions. */ + .align 4 + .globl do_cp_exception +do_cp_exception: + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call handle_cp_exception + mov %l0, %o3 + + RESTORE_ALL + + /* This routine handles Hardware Divide By Zero Exceptions. */ + .align 4 + .globl do_hw_divzero +do_hw_divzero: + SAVE_ALL + + wr %l0, PSR_ET, %psr ! re-enable traps + WRITE_PAUSE + + add %sp, STACKFRAME_SZ, %o0 + mov %l1, %o1 + mov %l2, %o2 + call handle_hw_divzero + mov %l0, %o3 + + RESTORE_ALL + + .align 4 + .globl do_flush_windows +do_flush_windows: + SAVE_ALL + + wr %l0, PSR_ET, %psr + WRITE_PAUSE + + andcc %l0, PSR_PS, %g0 + bne dfw_kernel + nop + + call flush_user_windows + nop + + /* Advance over the trap instruction. */ + ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 + add %l1, 0x4, %l2 + st %l1, [%sp + STACKFRAME_SZ + PT_PC] + st %l2, [%sp + STACKFRAME_SZ + PT_NPC] + + RESTORE_ALL + + .globl flush_patch_one + + /* We get these for debugging routines using __builtin_return_address() */ +dfw_kernel: +flush_patch_one: + FLUSH_ALL_KERNEL_WINDOWS + + /* Advance over the trap instruction. */ + ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 + add %l1, 0x4, %l2 + st %l1, [%sp + STACKFRAME_SZ + PT_PC] + st %l2, [%sp + STACKFRAME_SZ + PT_NPC] + + RESTORE_ALL + + /* The getcc software trap. The user wants the condition codes from + * the %psr in register %g1. + */ + + .align 4 + .globl getcc_trap_handler +getcc_trap_handler: + srl %l0, 20, %g1 ! give user + and %g1, 0xf, %g1 ! only ICC bits in %psr + jmp %l2 ! advance over trap instruction + rett %l2 + 0x4 ! like this... + + /* The setcc software trap. The user has condition codes in %g1 + * that it would like placed in the %psr. Be careful not to flip + * any unintentional bits! + */ + + .align 4 + .globl setcc_trap_handler +setcc_trap_handler: + sll %g1, 0x14, %l4 + set PSR_ICC, %l5 + andn %l0, %l5, %l0 ! clear ICC bits in %psr + and %l4, %l5, %l4 ! clear non-ICC bits in user value + or %l4, %l0, %l4 ! or them in... mix mix mix + + wr %l4, 0x0, %psr ! set new %psr + WRITE_PAUSE ! TI scumbags... + + jmp %l2 ! advance over trap instruction + rett %l2 + 0x4 ! like this... + +sun4m_nmi_error: + /* NMI async memory error handling. */ + sethi %hi(0x80000000), %l4 + sethi %hi(sun4m_irq_global), %o5 + ld [%o5 + %lo(sun4m_irq_global)], %l5 + st %l4, [%l5 + 0x0c] ! sun4m_irq_global->mask_set=0x80000000 + WRITE_PAUSE + ld [%l5 + 0x00], %g0 ! sun4m_irq_global->pending + WRITE_PAUSE + or %l0, PSR_PIL, %l4 + wr %l4, 0x0, %psr + WRITE_PAUSE + wr %l4, PSR_ET, %psr + WRITE_PAUSE + call sun4m_nmi + nop + st %l4, [%l5 + 0x08] ! sun4m_irq_global->mask_clear=0x80000000 + WRITE_PAUSE + ld [%l5 + 0x00], %g0 ! sun4m_irq_global->pending + WRITE_PAUSE + RESTORE_ALL + +#ifndef CONFIG_SMP + .align 4 + .globl linux_trap_ipi15_sun4m +linux_trap_ipi15_sun4m: + SAVE_ALL + + ba sun4m_nmi_error + nop +#endif /* CONFIG_SMP */ + + .align 4 + .globl srmmu_fault +srmmu_fault: + mov 0x400, %l5 + mov 0x300, %l4 + +LEON_PI(lda [%l5] ASI_LEON_MMUREGS, %l6) ! read sfar first +SUN_PI_(lda [%l5] ASI_M_MMUREGS, %l6) ! read sfar first + +LEON_PI(lda [%l4] ASI_LEON_MMUREGS, %l5) ! read sfsr last +SUN_PI_(lda [%l4] ASI_M_MMUREGS, %l5) ! read sfsr last + + andn %l6, 0xfff, %l6 + srl %l5, 6, %l5 ! and encode all info into l7 + + and %l5, 2, %l5 + or %l5, %l6, %l6 + + or %l6, %l7, %l7 ! l7 = [addr,write,txtfault] + + SAVE_ALL + + mov %l7, %o1 + mov %l7, %o2 + and %o1, 1, %o1 ! arg2 = text_faultp + mov %l7, %o3 + and %o2, 2, %o2 ! arg3 = writep + andn %o3, 0xfff, %o3 ! arg4 = faulting address + + wr %l0, PSR_ET, %psr + WRITE_PAUSE + + call do_sparc_fault + add %sp, STACKFRAME_SZ, %o0 ! arg1 = pt_regs ptr + + RESTORE_ALL + + .align 4 +sunos_execv: + .globl sunos_execv + b sys_execve + clr %i2 + + .align 4 + .globl sys_sigstack +sys_sigstack: + mov %o7, %l5 + mov %fp, %o2 + call do_sys_sigstack + mov %l5, %o7 + + .align 4 + .globl sys_sigreturn +sys_sigreturn: + call do_sigreturn + add %sp, STACKFRAME_SZ, %o0 + + ld [%curptr + TI_FLAGS], %l5 + andcc %l5, _TIF_SYSCALL_TRACE, %g0 + be 1f + nop + + call syscall_trace + mov 1, %o1 + +1: + /* We don't want to muck with user registers like a + * normal syscall, just return. + */ + RESTORE_ALL + + .align 4 + .globl sys_rt_sigreturn +sys_rt_sigreturn: + call do_rt_sigreturn + add %sp, STACKFRAME_SZ, %o0 + + ld [%curptr + TI_FLAGS], %l5 + andcc %l5, _TIF_SYSCALL_TRACE, %g0 + be 1f + nop + + add %sp, STACKFRAME_SZ, %o0 + call syscall_trace + mov 1, %o1 + +1: + /* We are returning to a signal handler. */ + RESTORE_ALL + + /* Now that we have a real sys_clone, sys_fork() is + * implemented in terms of it. Our _real_ implementation + * of SunOS vfork() will use sys_vfork(). + * + * XXX These three should be consolidated into mostly shared + * XXX code just like on sparc64... -DaveM + */ + .align 4 + .globl sys_fork, flush_patch_two +sys_fork: + mov %o7, %l5 +flush_patch_two: + FLUSH_ALL_KERNEL_WINDOWS; + ld [%curptr + TI_TASK], %o4 + rd %psr, %g4 + WRITE_PAUSE + rd %wim, %g5 + WRITE_PAUSE + std %g4, [%o4 + AOFF_task_thread + AOFF_thread_fork_kpsr] + add %sp, STACKFRAME_SZ, %o0 + call sparc_fork + mov %l5, %o7 + + /* Whee, kernel threads! */ + .globl sys_clone, flush_patch_three +sys_clone: + mov %o7, %l5 +flush_patch_three: + FLUSH_ALL_KERNEL_WINDOWS; + ld [%curptr + TI_TASK], %o4 + rd %psr, %g4 + WRITE_PAUSE + rd %wim, %g5 + WRITE_PAUSE + std %g4, [%o4 + AOFF_task_thread + AOFF_thread_fork_kpsr] + add %sp, STACKFRAME_SZ, %o0 + call sparc_clone + mov %l5, %o7 + + /* Whee, real vfork! */ + .globl sys_vfork, flush_patch_four +sys_vfork: +flush_patch_four: + FLUSH_ALL_KERNEL_WINDOWS; + ld [%curptr + TI_TASK], %o4 + rd %psr, %g4 + WRITE_PAUSE + rd %wim, %g5 + WRITE_PAUSE + std %g4, [%o4 + AOFF_task_thread + AOFF_thread_fork_kpsr] + sethi %hi(sparc_vfork), %l1 + jmpl %l1 + %lo(sparc_vfork), %g0 + add %sp, STACKFRAME_SZ, %o0 + + .globl __sys_clone3, flush_patch_five +__sys_clone3: + mov %o7, %l5 +flush_patch_five: + FLUSH_ALL_KERNEL_WINDOWS; + ld [%curptr + TI_TASK], %o4 + rd %psr, %g4 + WRITE_PAUSE + rd %wim, %g5 + WRITE_PAUSE + std %g4, [%o4 + AOFF_task_thread + AOFF_thread_fork_kpsr] + add %sp, STACKFRAME_SZ, %o0 + call sparc_clone3 + mov %l5, %o7 + + .align 4 +linux_sparc_ni_syscall: + sethi %hi(sys_ni_syscall), %l7 + b do_syscall + or %l7, %lo(sys_ni_syscall), %l7 + +linux_syscall_trace: + add %sp, STACKFRAME_SZ, %o0 + call syscall_trace + mov 0, %o1 + cmp %o0, 0 + bne 3f + mov -ENOSYS, %o0 + + /* Syscall tracing can modify the registers. */ + ld [%sp + STACKFRAME_SZ + PT_G1], %g1 + sethi %hi(sys_call_table), %l7 + ld [%sp + STACKFRAME_SZ + PT_I0], %i0 + or %l7, %lo(sys_call_table), %l7 + ld [%sp + STACKFRAME_SZ + PT_I1], %i1 + ld [%sp + STACKFRAME_SZ + PT_I2], %i2 + ld [%sp + STACKFRAME_SZ + PT_I3], %i3 + ld [%sp + STACKFRAME_SZ + PT_I4], %i4 + ld [%sp + STACKFRAME_SZ + PT_I5], %i5 + cmp %g1, NR_syscalls + bgeu 3f + mov -ENOSYS, %o0 + + sll %g1, 2, %l4 + mov %i0, %o0 + ld [%l7 + %l4], %l7 + mov %i1, %o1 + mov %i2, %o2 + mov %i3, %o3 + b 2f + mov %i4, %o4 + + .globl ret_from_fork +ret_from_fork: + call schedule_tail + ld [%g3 + TI_TASK], %o0 + b ret_sys_call + ld [%sp + STACKFRAME_SZ + PT_I0], %o0 + + .globl ret_from_kernel_thread +ret_from_kernel_thread: + call schedule_tail + ld [%g3 + TI_TASK], %o0 + ld [%sp + STACKFRAME_SZ + PT_G1], %l0 + call %l0 + ld [%sp + STACKFRAME_SZ + PT_G2], %o0 + rd %psr, %l1 + ld [%sp + STACKFRAME_SZ + PT_PSR], %l0 + andn %l0, PSR_CWP, %l0 + nop + and %l1, PSR_CWP, %l1 + or %l0, %l1, %l0 + st %l0, [%sp + STACKFRAME_SZ + PT_PSR] + b ret_sys_call + mov 0, %o0 + + /* Linux native system calls enter here... */ + .align 4 + .globl linux_sparc_syscall +linux_sparc_syscall: + sethi %hi(PSR_SYSCALL), %l4 + or %l0, %l4, %l0 + /* Direct access to user regs, must faster. */ + cmp %g1, NR_syscalls + bgeu linux_sparc_ni_syscall + sll %g1, 2, %l4 + ld [%l7 + %l4], %l7 + +do_syscall: + SAVE_ALL_HEAD + rd %wim, %l3 + + wr %l0, PSR_ET, %psr + mov %i0, %o0 + mov %i1, %o1 + mov %i2, %o2 + + ld [%curptr + TI_FLAGS], %l5 + mov %i3, %o3 + andcc %l5, _TIF_SYSCALL_TRACE, %g0 + mov %i4, %o4 + bne linux_syscall_trace + mov %i0, %l6 +2: + call %l7 + mov %i5, %o5 + +3: + st %o0, [%sp + STACKFRAME_SZ + PT_I0] + +ret_sys_call: + ld [%curptr + TI_FLAGS], %l5 + cmp %o0, -ERESTART_RESTARTBLOCK + ld [%sp + STACKFRAME_SZ + PT_PSR], %g3 + set PSR_C, %g2 + bgeu 1f + andcc %l5, _TIF_SYSCALL_TRACE, %g0 + + /* System call success, clear Carry condition code. */ + andn %g3, %g2, %g3 + st %g3, [%sp + STACKFRAME_SZ + PT_PSR] + bne linux_syscall_trace2 + ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 /* pc = npc */ + add %l1, 0x4, %l2 /* npc = npc+4 */ + st %l1, [%sp + STACKFRAME_SZ + PT_PC] + b ret_trap_entry + st %l2, [%sp + STACKFRAME_SZ + PT_NPC] +1: + /* System call failure, set Carry condition code. + * Also, get abs(errno) to return to the process. + */ + sub %g0, %o0, %o0 + or %g3, %g2, %g3 + st %o0, [%sp + STACKFRAME_SZ + PT_I0] + st %g3, [%sp + STACKFRAME_SZ + PT_PSR] + bne linux_syscall_trace2 + ld [%sp + STACKFRAME_SZ + PT_NPC], %l1 /* pc = npc */ + add %l1, 0x4, %l2 /* npc = npc+4 */ + st %l1, [%sp + STACKFRAME_SZ + PT_PC] + b ret_trap_entry + st %l2, [%sp + STACKFRAME_SZ + PT_NPC] + +linux_syscall_trace2: + add %sp, STACKFRAME_SZ, %o0 + mov 1, %o1 + call syscall_trace + add %l1, 0x4, %l2 /* npc = npc+4 */ + st %l1, [%sp + STACKFRAME_SZ + PT_PC] + b ret_trap_entry + st %l2, [%sp + STACKFRAME_SZ + PT_NPC] + + +/* Saving and restoring the FPU state is best done from lowlevel code. + * + * void fpsave(unsigned long *fpregs, unsigned long *fsr, + * void *fpqueue, unsigned long *fpqdepth) + */ + + .globl fpsave +fpsave: + st %fsr, [%o1] ! this can trap on us if fpu is in bogon state + ld [%o1], %g1 + set 0x2000, %g4 + andcc %g1, %g4, %g0 + be 2f + mov 0, %g2 + + /* We have an fpqueue to save. */ +1: + std %fq, [%o2] +fpsave_magic: + st %fsr, [%o1] + ld [%o1], %g3 + andcc %g3, %g4, %g0 + add %g2, 1, %g2 + bne 1b + add %o2, 8, %o2 + +2: + st %g2, [%o3] + + std %f0, [%o0 + 0x00] + std %f2, [%o0 + 0x08] + std %f4, [%o0 + 0x10] + std %f6, [%o0 + 0x18] + std %f8, [%o0 + 0x20] + std %f10, [%o0 + 0x28] + std %f12, [%o0 + 0x30] + std %f14, [%o0 + 0x38] + std %f16, [%o0 + 0x40] + std %f18, [%o0 + 0x48] + std %f20, [%o0 + 0x50] + std %f22, [%o0 + 0x58] + std %f24, [%o0 + 0x60] + std %f26, [%o0 + 0x68] + std %f28, [%o0 + 0x70] + retl + std %f30, [%o0 + 0x78] + + /* Thanks for Theo Deraadt and the authors of the Sprite/netbsd/openbsd + * code for pointing out this possible deadlock, while we save state + * above we could trap on the fsr store so our low level fpu trap + * code has to know how to deal with this. + */ +fpsave_catch: + b fpsave_magic + 4 + st %fsr, [%o1] + +fpsave_catch2: + b fpsave + 4 + st %fsr, [%o1] + + /* void fpload(unsigned long *fpregs, unsigned long *fsr); */ + + .globl fpload +fpload: + ldd [%o0 + 0x00], %f0 + ldd [%o0 + 0x08], %f2 + ldd [%o0 + 0x10], %f4 + ldd [%o0 + 0x18], %f6 + ldd [%o0 + 0x20], %f8 + ldd [%o0 + 0x28], %f10 + ldd [%o0 + 0x30], %f12 + ldd [%o0 + 0x38], %f14 + ldd [%o0 + 0x40], %f16 + ldd [%o0 + 0x48], %f18 + ldd [%o0 + 0x50], %f20 + ldd [%o0 + 0x58], %f22 + ldd [%o0 + 0x60], %f24 + ldd [%o0 + 0x68], %f26 + ldd [%o0 + 0x70], %f28 + ldd [%o0 + 0x78], %f30 + ld [%o1], %fsr + retl + nop + + /* __ndelay and __udelay take two arguments: + * 0 - nsecs or usecs to delay + * 1 - per_cpu udelay_val (loops per jiffy) + * + * Note that ndelay gives HZ times higher resolution but has a 10ms + * limit. udelay can handle up to 1s. + */ + .globl __ndelay +__ndelay: + save %sp, -STACKFRAME_SZ, %sp + mov %i0, %o0 ! round multiplier up so large ns ok + mov 0x1ae, %o1 ! 2**32 / (1 000 000 000 / HZ) + umul %o0, %o1, %o0 + rd %y, %o1 + mov %i1, %o1 ! udelay_val + umul %o0, %o1, %o0 + rd %y, %o1 + ba delay_continue + mov %o1, %o0 ! >>32 later for better resolution + + .globl __udelay +__udelay: + save %sp, -STACKFRAME_SZ, %sp + mov %i0, %o0 + sethi %hi(0x10c7), %o1 ! round multiplier up so large us ok + or %o1, %lo(0x10c7), %o1 ! 2**32 / 1 000 000 + umul %o0, %o1, %o0 + rd %y, %o1 + mov %i1, %o1 ! udelay_val + umul %o0, %o1, %o0 + rd %y, %o1 + sethi %hi(0x028f4b62), %l0 ! Add in rounding constant * 2**32, + or %g0, %lo(0x028f4b62), %l0 + addcc %o0, %l0, %o0 ! 2**32 * 0.009 999 + bcs,a 3f + add %o1, 0x01, %o1 +3: + mov HZ, %o0 ! >>32 earlier for wider range + umul %o0, %o1, %o0 + rd %y, %o1 + +delay_continue: + cmp %o0, 0x0 +1: + bne 1b + subcc %o0, 1, %o0 + + ret + restore +EXPORT_SYMBOL(__udelay) +EXPORT_SYMBOL(__ndelay) + + /* Handle a software breakpoint */ + /* We have to inform parent that child has stopped */ + .align 4 + .globl breakpoint_trap +breakpoint_trap: + rd %wim,%l3 + SAVE_ALL + wr %l0, PSR_ET, %psr + WRITE_PAUSE + + st %i0, [%sp + STACKFRAME_SZ + PT_G0] ! for restarting syscalls + call sparc_breakpoint + add %sp, STACKFRAME_SZ, %o0 + + RESTORE_ALL + +#ifdef CONFIG_KGDB + ENTRY(kgdb_trap_low) + rd %wim,%l3 + SAVE_ALL + wr %l0, PSR_ET, %psr + WRITE_PAUSE + + mov %l7, %o0 ! trap_level + call kgdb_trap + add %sp, STACKFRAME_SZ, %o1 ! struct pt_regs *regs + + RESTORE_ALL + ENDPROC(kgdb_trap_low) +#endif + + .align 4 + .globl flush_patch_exception +flush_patch_exception: + FLUSH_ALL_KERNEL_WINDOWS; + ldd [%o0], %o6 + jmpl %o7 + 0xc, %g0 ! see asm-sparc/processor.h + mov 1, %g1 ! signal EFAULT condition + + .align 4 + .globl kill_user_windows, kuw_patch1_7win + .globl kuw_patch1 +kuw_patch1_7win: sll %o3, 6, %o3 + + /* No matter how much overhead this routine has in the worst + * case scenario, it is several times better than taking the + * traps with the old method of just doing flush_user_windows(). + */ +kill_user_windows: + ld [%g6 + TI_UWINMASK], %o0 ! get current umask + orcc %g0, %o0, %g0 ! if no bits set, we are done + be 3f ! nothing to do + rd %psr, %o5 ! must clear interrupts + or %o5, PSR_PIL, %o4 ! or else that could change + wr %o4, 0x0, %psr ! the uwinmask state + WRITE_PAUSE ! burn them cycles +1: + ld [%g6 + TI_UWINMASK], %o0 ! get consistent state + orcc %g0, %o0, %g0 ! did an interrupt come in? + be 4f ! yep, we are done + rd %wim, %o3 ! get current wim + srl %o3, 1, %o4 ! simulate a save +kuw_patch1: + sll %o3, 7, %o3 ! compute next wim + or %o4, %o3, %o3 ! result + andncc %o0, %o3, %o0 ! clean this bit in umask + bne kuw_patch1 ! not done yet + srl %o3, 1, %o4 ! begin another save simulation + wr %o3, 0x0, %wim ! set the new wim + st %g0, [%g6 + TI_UWINMASK] ! clear uwinmask +4: + wr %o5, 0x0, %psr ! re-enable interrupts + WRITE_PAUSE ! burn baby burn +3: + retl ! return + st %g0, [%g6 + TI_W_SAVED] ! no windows saved + + .align 4 + .globl restore_current +restore_current: + LOAD_CURRENT(g6, o0) + retl + nop + +#ifdef CONFIG_PCIC_PCI +#include + + .align 4 + .globl linux_trap_ipi15_pcic +linux_trap_ipi15_pcic: + rd %wim, %l3 + SAVE_ALL + + /* + * First deactivate NMI + * or we cannot drop ET, cannot get window spill traps. + * The busy loop is necessary because the PIO error + * sometimes does not go away quickly and we trap again. + */ + sethi %hi(pcic_regs), %o1 + ld [%o1 + %lo(pcic_regs)], %o2 + + ! Get pending status for printouts later. + ld [%o2 + PCI_SYS_INT_PENDING], %o0 + + mov PCI_SYS_INT_PENDING_CLEAR_ALL, %o1 + stb %o1, [%o2 + PCI_SYS_INT_PENDING_CLEAR] +1: + ld [%o2 + PCI_SYS_INT_PENDING], %o1 + andcc %o1, ((PCI_SYS_INT_PENDING_PIO|PCI_SYS_INT_PENDING_PCI)>>24), %g0 + bne 1b + nop + + or %l0, PSR_PIL, %l4 + wr %l4, 0x0, %psr + WRITE_PAUSE + wr %l4, PSR_ET, %psr + WRITE_PAUSE + + call pcic_nmi + add %sp, STACKFRAME_SZ, %o1 ! struct pt_regs *regs + RESTORE_ALL + + .globl pcic_nmi_trap_patch +pcic_nmi_trap_patch: + sethi %hi(linux_trap_ipi15_pcic), %l3 + jmpl %l3 + %lo(linux_trap_ipi15_pcic), %g0 + rd %psr, %l0 + .word 0 + +#endif /* CONFIG_PCIC_PCI */ + + .globl flushw_all +flushw_all: + save %sp, -0x40, %sp + save %sp, -0x40, %sp + save %sp, -0x40, %sp + save %sp, -0x40, %sp + save %sp, -0x40, %sp + save %sp, -0x40, %sp + save %sp, -0x40, %sp + restore + restore + restore + restore + restore + restore + ret + restore + +#ifdef CONFIG_SMP +ENTRY(hard_smp_processor_id) +661: rd %tbr, %g1 + srl %g1, 12, %o0 + and %o0, 3, %o0 + .section .cpuid_patch, "ax" + /* Instruction location. */ + .word 661b + /* SUN4D implementation. */ + lda [%g0] ASI_M_VIKING_TMP1, %o0 + nop + nop + /* LEON implementation. */ + rd %asr17, %o0 + srl %o0, 0x1c, %o0 + nop + .previous + retl + nop +ENDPROC(hard_smp_processor_id) +#endif + +/* End of entry.S */ diff --git a/arch/sparc/kernel/entry.h b/arch/sparc/kernel/entry.h new file mode 100644 index 000000000..c746c0fd5 --- /dev/null +++ b/arch/sparc/kernel/entry.h @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ENTRY_H +#define _ENTRY_H + +#include +#include +#include + +/* irq */ +void handler_irq(int irq, struct pt_regs *regs); + +#ifdef CONFIG_SPARC32 +/* traps */ +void do_hw_interrupt(struct pt_regs *regs, unsigned long type); +void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); + +void do_priv_instruction(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void do_fpd_trap(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void do_fpe_trap(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_tag_overflow(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_watchpoint(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_reg_access(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_cp_disabled(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +void handle_cp_exception(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); + + + +/* entry.S */ +void fpsave(unsigned long *fpregs, unsigned long *fsr, + void *fpqueue, unsigned long *fpqdepth); +void fpload(unsigned long *fpregs, unsigned long *fsr); + +#else /* CONFIG_SPARC32 */ + +#include + +struct popc_3insn_patch_entry { + unsigned int addr; + unsigned int insns[3]; +}; +extern struct popc_3insn_patch_entry __popc_3insn_patch, + __popc_3insn_patch_end; + +struct popc_6insn_patch_entry { + unsigned int addr; + unsigned int insns[6]; +}; +extern struct popc_6insn_patch_entry __popc_6insn_patch, + __popc_6insn_patch_end; + +struct pause_patch_entry { + unsigned int addr; + unsigned int insns[3]; +}; +extern struct pause_patch_entry __pause_3insn_patch, + __pause_3insn_patch_end; + +void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *, + struct sun4v_1insn_patch_entry *); +void sun4v_patch_2insn_range(struct sun4v_2insn_patch_entry *, + struct sun4v_2insn_patch_entry *); +void sun_m7_patch_2insn_range(struct sun4v_2insn_patch_entry *, + struct sun4v_2insn_patch_entry *); +extern unsigned int dcache_parity_tl1_occurred; +extern unsigned int icache_parity_tl1_occurred; + +asmlinkage void sparc_breakpoint(struct pt_regs *regs); +void timer_interrupt(int irq, struct pt_regs *regs); + +void do_notify_resume(struct pt_regs *regs, + unsigned long orig_i0, + unsigned long thread_info_flags); + +asmlinkage int syscall_trace_enter(struct pt_regs *regs); +asmlinkage void syscall_trace_leave(struct pt_regs *regs); + +void bad_trap_tl1(struct pt_regs *regs, long lvl); + +void do_fpieee(struct pt_regs *regs); +void do_fpother(struct pt_regs *regs); +void do_tof(struct pt_regs *regs); +void do_div0(struct pt_regs *regs); +void do_illegal_instruction(struct pt_regs *regs); +void mem_address_unaligned(struct pt_regs *regs, + unsigned long sfar, + unsigned long sfsr); +void sun4v_do_mna(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void do_privop(struct pt_regs *regs); +void do_privact(struct pt_regs *regs); +void do_cee(struct pt_regs *regs); +void do_div0_tl1(struct pt_regs *regs); +void do_fpieee_tl1(struct pt_regs *regs); +void do_fpother_tl1(struct pt_regs *regs); +void do_ill_tl1(struct pt_regs *regs); +void do_irq_tl1(struct pt_regs *regs); +void do_lddfmna_tl1(struct pt_regs *regs); +void do_stdfmna_tl1(struct pt_regs *regs); +void do_paw(struct pt_regs *regs); +void do_paw_tl1(struct pt_regs *regs); +void do_vaw(struct pt_regs *regs); +void do_vaw_tl1(struct pt_regs *regs); +void do_tof_tl1(struct pt_regs *regs); +void do_getpsr(struct pt_regs *regs); + +void spitfire_insn_access_exception(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_insn_access_exception_tl1(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_data_access_exception(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_data_access_exception_tl1(struct pt_regs *regs, + unsigned long sfsr, + unsigned long sfar); +void spitfire_access_error(struct pt_regs *regs, + unsigned long status_encoded, + unsigned long afar); + +void cheetah_fecc_handler(struct pt_regs *regs, + unsigned long afsr, + unsigned long afar); +void cheetah_cee_handler(struct pt_regs *regs, + unsigned long afsr, + unsigned long afar); +void cheetah_deferred_handler(struct pt_regs *regs, + unsigned long afsr, + unsigned long afar); +void cheetah_plus_parity_error(int type, struct pt_regs *regs); + +void sun4v_insn_access_exception(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_insn_access_exception_tl1(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_data_access_exception(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_data_access_exception_tl1(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); +void sun4v_resum_error(struct pt_regs *regs, + unsigned long offset); +void sun4v_resum_overflow(struct pt_regs *regs); +void sun4v_nonresum_error(struct pt_regs *regs, + unsigned long offset); +void sun4v_nonresum_overflow(struct pt_regs *regs); +void sun4v_mem_corrupt_detect_precise(struct pt_regs *regs, + unsigned long addr, + unsigned long context); + +extern unsigned long sun4v_err_itlb_vaddr; +extern unsigned long sun4v_err_itlb_ctx; +extern unsigned long sun4v_err_itlb_pte; +extern unsigned long sun4v_err_itlb_error; + +void sun4v_itlb_error_report(struct pt_regs *regs, int tl); + +extern unsigned long sun4v_err_dtlb_vaddr; +extern unsigned long sun4v_err_dtlb_ctx; +extern unsigned long sun4v_err_dtlb_pte; +extern unsigned long sun4v_err_dtlb_error; + +void sun4v_dtlb_error_report(struct pt_regs *regs, int tl); +void hypervisor_tlbop_error(unsigned long err, + unsigned long op); +void hypervisor_tlbop_error_xcall(unsigned long err, + unsigned long op); + +/* WARNING: The error trap handlers in assembly know the precise + * layout of the following structure. + * + * C-level handlers in traps.c use this information to log the + * error and then determine how to recover (if possible). + */ +struct cheetah_err_info { +/*0x00*/u64 afsr; +/*0x08*/u64 afar; + + /* D-cache state */ +/*0x10*/u64 dcache_data[4]; /* The actual data */ +/*0x30*/u64 dcache_index; /* D-cache index */ +/*0x38*/u64 dcache_tag; /* D-cache tag/valid */ +/*0x40*/u64 dcache_utag; /* D-cache microtag */ +/*0x48*/u64 dcache_stag; /* D-cache snooptag */ + + /* I-cache state */ +/*0x50*/u64 icache_data[8]; /* The actual insns + predecode */ +/*0x90*/u64 icache_index; /* I-cache index */ +/*0x98*/u64 icache_tag; /* I-cache phys tag */ +/*0xa0*/u64 icache_utag; /* I-cache microtag */ +/*0xa8*/u64 icache_stag; /* I-cache snooptag */ +/*0xb0*/u64 icache_upper; /* I-cache upper-tag */ +/*0xb8*/u64 icache_lower; /* I-cache lower-tag */ + + /* E-cache state */ +/*0xc0*/u64 ecache_data[4]; /* 32 bytes from staging registers */ +/*0xe0*/u64 ecache_index; /* E-cache index */ +/*0xe8*/u64 ecache_tag; /* E-cache tag/state */ + +/*0xf0*/u64 __pad[32 - 30]; +}; +#define CHAFSR_INVALID ((u64)-1L) + +/* This is allocated at boot time based upon the largest hardware + * cpu ID in the system. We allocate two entries per cpu, one for + * TL==0 logging and one for TL >= 1 logging. + */ +extern struct cheetah_err_info *cheetah_error_log; + +/* UPA nodes send interrupt packet to UltraSparc with first data reg + * value low 5 (7 on Starfire) bits holding the IRQ identifier being + * delivered. We must translate this into a non-vector IRQ so we can + * set the softint on this cpu. + * + * To make processing these packets efficient and race free we use + * an array of irq buckets below. The interrupt vector handler in + * entry.S feeds incoming packets into per-cpu pil-indexed lists. + * + * If you make changes to ino_bucket, please update hand coded assembler + * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S + */ +struct ino_bucket { +/*0x00*/unsigned long __irq_chain_pa; + + /* Interrupt number assigned to this INO. */ +/*0x08*/unsigned int __irq; +/*0x0c*/unsigned int __pad; +}; + +extern struct ino_bucket *ivector_table; +extern unsigned long ivector_table_pa; + +void init_irqwork_curcpu(void); +void sun4v_register_mondo_queues(int this_cpu); + +#endif /* CONFIG_SPARC32 */ +#endif /* _ENTRY_H */ diff --git a/arch/sparc/kernel/etrap_32.S b/arch/sparc/kernel/etrap_32.S new file mode 100644 index 000000000..9f243f918 --- /dev/null +++ b/arch/sparc/kernel/etrap_32.S @@ -0,0 +1,281 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * etrap.S: Sparc trap window preparation for entry into the + * Linux kernel. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Registers to not touch at all. */ +#define t_psr l0 /* Set by caller */ +#define t_pc l1 /* Set by caller */ +#define t_npc l2 /* Set by caller */ +#define t_wim l3 /* Set by caller */ +#define t_twinmask l4 /* Set at beginning of this entry routine. */ +#define t_kstack l5 /* Set right before pt_regs frame is built */ +#define t_retpc l6 /* If you change this, change winmacro.h header file */ +#define t_systable l7 /* Never touch this, could be the syscall table ptr. */ +#define curptr g6 /* Set after pt_regs frame is built */ + + .text + .align 4 + + /* SEVEN WINDOW PATCH INSTRUCTIONS */ + .globl tsetup_7win_patch1, tsetup_7win_patch2 + .globl tsetup_7win_patch3, tsetup_7win_patch4 + .globl tsetup_7win_patch5, tsetup_7win_patch6 +tsetup_7win_patch1: sll %t_wim, 0x6, %t_wim +tsetup_7win_patch2: and %g2, 0x7f, %g2 +tsetup_7win_patch3: and %g2, 0x7f, %g2 +tsetup_7win_patch4: and %g1, 0x7f, %g1 +tsetup_7win_patch5: sll %t_wim, 0x6, %t_wim +tsetup_7win_patch6: and %g2, 0x7f, %g2 + /* END OF PATCH INSTRUCTIONS */ + + /* At trap time, interrupts and all generic traps do the + * following: + * + * rd %psr, %l0 + * b some_handler + * rd %wim, %l3 + * nop + * + * Then 'some_handler' if it needs a trap frame (ie. it has + * to call c-code and the trap cannot be handled in-window) + * then it does the SAVE_ALL macro in entry.S which does + * + * sethi %hi(trap_setup), %l4 + * jmpl %l4 + %lo(trap_setup), %l6 + * nop + */ + + /* 2 3 4 window number + * ----- + * O T S mnemonic + * + * O == Current window before trap + * T == Window entered when trap occurred + * S == Window we will need to save if (1<w_saved */ + st %g0, [%curptr + TI_W_SAVED] + + /* See if we are in the trap window. */ + andcc %t_twinmask, %t_wim, %g0 + bne trap_setup_user_spill ! yep we are + orn %g0, %t_twinmask, %g1 ! negate trap win mask into %g1 + + /* Trap from user, but not into the invalid window. + * Calculate new umask. The way this works is, + * any window from the %wim at trap time until + * the window right before the one we are in now, + * is a user window. A diagram: + * + * 7 6 5 4 3 2 1 0 window number + * --------------- + * I L T mnemonic + * + * Window 'I' is the invalid window in our example, + * window 'L' is the window the user was in when + * the trap occurred, window T is the trap window + * we are in now. So therefore, windows 5, 4 and + * 3 are user windows. The following sequence + * computes the user winmask to represent this. + */ + subcc %t_wim, %t_twinmask, %g2 + bneg,a 1f + sub %g2, 0x1, %g2 +1: + andn %g2, %t_twinmask, %g2 +tsetup_patch3: + and %g2, 0xff, %g2 ! patched on 7win Sparcs + st %g2, [%curptr + TI_UWINMASK] ! store new umask + + jmpl %t_retpc + 0x8, %g0 ! return to caller + mov %t_kstack, %sp ! and onto kernel stack + +trap_setup_user_spill: + /* A spill occurred from either kernel or user mode + * and there exist some user windows to deal with. + * A mask of the currently valid user windows + * is in %g1 upon entry to here. + */ + +tsetup_patch4: + and %g1, 0xff, %g1 ! patched on 7win Sparcs, mask + srl %t_wim, 0x1, %g2 ! compute new %wim +tsetup_patch5: + sll %t_wim, 0x7, %t_wim ! patched on 7win Sparcs + or %t_wim, %g2, %g2 ! %g2 is new %wim +tsetup_patch6: + and %g2, 0xff, %g2 ! patched on 7win Sparcs + andn %g1, %g2, %g1 ! clear this bit in %g1 + st %g1, [%curptr + TI_UWINMASK] + + save %g0, %g0, %g0 + + wr %g2, 0x0, %wim + + /* Call MMU-architecture dependent stack checking + * routine. + */ + b tsetup_srmmu_stackchk + andcc %sp, 0x7, %g0 + + /* Architecture specific stack checking routines. When either + * of these routines are called, the globals are free to use + * as they have been safely stashed on the new kernel stack + * pointer. Thus the definition below for simplicity. + */ +#define glob_tmp g1 + + .globl tsetup_srmmu_stackchk +tsetup_srmmu_stackchk: + /* Check results of callers andcc %sp, 0x7, %g0 */ + bne trap_setup_user_stack_is_bolixed + sethi %hi(PAGE_OFFSET), %glob_tmp + + cmp %glob_tmp, %sp + bleu,a 1f +LEON_PI( lda [%g0] ASI_LEON_MMUREGS, %glob_tmp) ! read MMU control +SUN_PI_( lda [%g0] ASI_M_MMUREGS, %glob_tmp) ! read MMU control + +trap_setup_user_stack_is_bolixed: + /* From user/kernel into invalid window w/bad user + * stack. Save bad user stack, and return to caller. + */ + SAVE_BOLIXED_USER_STACK(curptr, g3) + restore %g0, %g0, %g0 + + jmpl %t_retpc + 0x8, %g0 + mov %t_kstack, %sp + +1: + /* Clear the fault status and turn on the no_fault bit. */ + or %glob_tmp, 0x2, %glob_tmp ! or in no_fault bit +LEON_PI(sta %glob_tmp, [%g0] ASI_LEON_MMUREGS) ! set it +SUN_PI_(sta %glob_tmp, [%g0] ASI_M_MMUREGS) ! set it + + /* Dump the registers and cross fingers. */ + STORE_WINDOW(sp) + + /* Clear the no_fault bit and check the status. */ + andn %glob_tmp, 0x2, %glob_tmp +LEON_PI(sta %glob_tmp, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %glob_tmp, [%g0] ASI_M_MMUREGS) + + mov AC_M_SFAR, %glob_tmp +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %g0) +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %g0) + + mov AC_M_SFSR, %glob_tmp +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %glob_tmp)! save away status of winstore +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %glob_tmp) ! save away status of winstore + + andcc %glob_tmp, 0x2, %g0 ! did we fault? + bne trap_setup_user_stack_is_bolixed ! failure + nop + + restore %g0, %g0, %g0 + + jmpl %t_retpc + 0x8, %g0 + mov %t_kstack, %sp + diff --git a/arch/sparc/kernel/etrap_64.S b/arch/sparc/kernel/etrap_64.S new file mode 100644 index 000000000..08cc41f64 --- /dev/null +++ b/arch/sparc/kernel/etrap_64.S @@ -0,0 +1,284 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * etrap.S: Preparing for entry into the kernel on Sparc V9. + * + * Copyright (C) 1996, 1997 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz) + */ + + +#include +#include +#include +#include +#include +#include +#include +#include + +#define TASK_REGOFF (THREAD_SIZE-TRACEREG_SZ-STACKFRAME_SZ) +#define ETRAP_PSTATE1 (PSTATE_TSO | PSTATE_PRIV) +#define ETRAP_PSTATE2 \ + (PSTATE_TSO | PSTATE_PEF | PSTATE_PRIV | PSTATE_IE) + +/* + * On entry, %g7 is return address - 0x4. + * %g4 and %g5 will be preserved %l4 and %l5 respectively. + */ + + .text + .align 64 + .globl etrap_syscall, etrap, etrap_irq, etraptl1 +etrap: rdpr %pil, %g2 +etrap_irq: clr %g3 +etrap_syscall: TRAP_LOAD_THREAD_REG(%g6, %g1) + rdpr %tstate, %g1 + or %g1, %g3, %g1 + sllx %g2, 20, %g3 + andcc %g1, TSTATE_PRIV, %g0 + or %g1, %g3, %g1 + bne,pn %xcc, 1f + sub %sp, STACKFRAME_SZ+TRACEREG_SZ-STACK_BIAS, %g2 +661: wrpr %g0, 7, %cleanwin + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + .word 0x85880000 ! allclean + .previous + + sethi %hi(TASK_REGOFF), %g2 + sethi %hi(TSTATE_PEF), %g3 + or %g2, %lo(TASK_REGOFF), %g2 + and %g1, %g3, %g3 + brnz,pn %g3, 1f + add %g6, %g2, %g2 + wr %g0, 0, %fprs +1: rdpr %tpc, %g3 + + stx %g1, [%g2 + STACKFRAME_SZ + PT_V9_TSTATE] + rdpr %tnpc, %g1 + stx %g3, [%g2 + STACKFRAME_SZ + PT_V9_TPC] + rd %y, %g3 + stx %g1, [%g2 + STACKFRAME_SZ + PT_V9_TNPC] + rdpr %tt, %g1 + st %g3, [%g2 + STACKFRAME_SZ + PT_V9_Y] + sethi %hi(PT_REGS_MAGIC), %g3 + or %g3, %g1, %g1 + st %g1, [%g2 + STACKFRAME_SZ + PT_V9_MAGIC] + + rdpr %cansave, %g1 + brnz,pt %g1, etrap_save + nop + + rdpr %cwp, %g1 + add %g1, 2, %g1 + wrpr %g1, %cwp + be,pt %xcc, etrap_user_spill + mov ASI_AIUP, %g3 + + rdpr %otherwin, %g3 + brz %g3, etrap_kernel_spill + mov ASI_AIUS, %g3 + +etrap_user_spill: + + wr %g3, 0x0, %asi + ldx [%g6 + TI_FLAGS], %g3 + and %g3, _TIF_32BIT, %g3 + brnz,pt %g3, etrap_user_spill_32bit + nop + ba,a,pt %xcc, etrap_user_spill_64bit + +etrap_save: save %g2, -STACK_BIAS, %sp + mov %g6, %l6 + + bne,pn %xcc, 3f + mov PRIMARY_CONTEXT, %l4 +661: rdpr %canrestore, %g3 + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + nop + .previous + + rdpr %wstate, %g2 +661: wrpr %g0, 0, %canrestore + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + nop + .previous + sll %g2, 3, %g2 + + /* Set TI_SYS_FPDEPTH to 1 and clear TI_SYS_NOERROR. */ + mov 1, %l5 + sth %l5, [%l6 + TI_SYS_NOERROR] + +661: wrpr %g3, 0, %otherwin + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + .word 0x87880000 ! otherw + .previous + + wrpr %g2, 0, %wstate + sethi %hi(sparc64_kern_pri_context), %g2 + ldx [%g2 + %lo(sparc64_kern_pri_context)], %g3 + +661: stxa %g3, [%l4] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g3, [%l4] ASI_MMU + .previous + + sethi %hi(KERNBASE), %l4 + flush %l4 + mov ASI_AIUS, %l7 +2: mov %g4, %l4 + mov %g5, %l5 + add %g7, 4, %l2 + + /* Go to trap time globals so we can save them. */ +661: wrpr %g0, ETRAP_PSTATE1, %pstate + .section .sun4v_1insn_patch, "ax" + .word 661b + SET_GL(0) + .previous + + stx %g1, [%sp + PTREGS_OFF + PT_V9_G1] + stx %g2, [%sp + PTREGS_OFF + PT_V9_G2] + sllx %l7, 24, %l7 + stx %g3, [%sp + PTREGS_OFF + PT_V9_G3] + rdpr %cwp, %l0 + stx %g4, [%sp + PTREGS_OFF + PT_V9_G4] + stx %g5, [%sp + PTREGS_OFF + PT_V9_G5] + stx %g6, [%sp + PTREGS_OFF + PT_V9_G6] + stx %g7, [%sp + PTREGS_OFF + PT_V9_G7] + or %l7, %l0, %l7 +661: sethi %hi(TSTATE_TSO | TSTATE_PEF), %l0 + /* If userspace is using ADI, it could potentially pass + * a pointer with version tag embedded in it. To maintain + * the ADI security, we must enable PSTATE.mcde. Userspace + * would have already set TTE.mcd in an earlier call to + * kernel and set the version tag for the address being + * dereferenced. Setting PSTATE.mcde would ensure any + * access to userspace data through a system call honors + * ADI and does not allow a rogue app to bypass ADI by + * using system calls. Setting PSTATE.mcde only affects + * accesses to virtual addresses that have TTE.mcd set. + * Set PMCDPER to ensure any exceptions caused by ADI + * version tag mismatch are exposed before system call + * returns to userspace. Setting PMCDPER affects only + * writes to virtual addresses that have TTE.mcd set and + * have a version tag set as well. + */ + .section .sun_m7_1insn_patch, "ax" + .word 661b + sethi %hi(TSTATE_TSO | TSTATE_PEF | TSTATE_MCDE), %l0 + .previous +661: nop + .section .sun_m7_1insn_patch, "ax" + .word 661b + .word 0xaf902001 /* wrpr %g0, 1, %pmcdper */ + .previous + or %l7, %l0, %l7 + wrpr %l2, %tnpc + wrpr %l7, (TSTATE_PRIV | TSTATE_IE), %tstate + stx %i0, [%sp + PTREGS_OFF + PT_V9_I0] + stx %i1, [%sp + PTREGS_OFF + PT_V9_I1] + stx %i2, [%sp + PTREGS_OFF + PT_V9_I2] + stx %i3, [%sp + PTREGS_OFF + PT_V9_I3] + stx %i4, [%sp + PTREGS_OFF + PT_V9_I4] + stx %i5, [%sp + PTREGS_OFF + PT_V9_I5] + stx %i6, [%sp + PTREGS_OFF + PT_V9_I6] + mov %l6, %g6 + stx %i7, [%sp + PTREGS_OFF + PT_V9_I7] + LOAD_PER_CPU_BASE(%g5, %g6, %g4, %g3, %l1) + ldx [%g6 + TI_TASK], %g4 + done + +3: mov ASI_P, %l7 + ldub [%l6 + TI_FPDEPTH], %l5 + add %l6, TI_FPSAVED + 1, %l4 + srl %l5, 1, %l3 + add %l5, 2, %l5 + + /* Set TI_SYS_FPDEPTH to %l5 and clear TI_SYS_NOERROR. */ + sth %l5, [%l6 + TI_SYS_NOERROR] + ba,pt %xcc, 2b + stb %g0, [%l4 + %l3] + nop + +etraptl1: /* Save tstate/tpc/tnpc of TL 1-->4 and the tl register itself. + * We place this right after pt_regs on the trap stack. + * The layout is: + * 0x00 TL1's TSTATE + * 0x08 TL1's TPC + * 0x10 TL1's TNPC + * 0x18 TL1's TT + * ... + * 0x58 TL4's TT + * 0x60 TL + */ + TRAP_LOAD_THREAD_REG(%g6, %g1) + sub %sp, ((4 * 8) * 4) + 8, %g2 + rdpr %tl, %g1 + + wrpr %g0, 1, %tl + rdpr %tstate, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x00] + rdpr %tpc, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x08] + rdpr %tnpc, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x10] + rdpr %tt, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x18] + + wrpr %g0, 2, %tl + rdpr %tstate, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x20] + rdpr %tpc, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x28] + rdpr %tnpc, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x30] + rdpr %tt, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x38] + + sethi %hi(is_sun4v), %g3 + lduw [%g3 + %lo(is_sun4v)], %g3 + brnz,pn %g3, finish_tl1_capture + nop + + wrpr %g0, 3, %tl + rdpr %tstate, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x40] + rdpr %tpc, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x48] + rdpr %tnpc, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x50] + rdpr %tt, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x58] + + wrpr %g0, 4, %tl + rdpr %tstate, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x60] + rdpr %tpc, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x68] + rdpr %tnpc, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x70] + rdpr %tt, %g3 + stx %g3, [%g2 + STACK_BIAS + 0x78] + + stx %g1, [%g2 + STACK_BIAS + 0x80] + +finish_tl1_capture: + wrpr %g0, 1, %tl +661: nop + .section .sun4v_1insn_patch, "ax" + .word 661b + SET_GL(1) + .previous + + rdpr %tstate, %g1 + sub %g2, STACKFRAME_SZ + TRACEREG_SZ - STACK_BIAS, %g2 + ba,pt %xcc, 1b + andcc %g1, TSTATE_PRIV, %g0 + +#undef TASK_REGOFF +#undef ETRAP_PSTATE1 diff --git a/arch/sparc/kernel/fpu_traps.S b/arch/sparc/kernel/fpu_traps.S new file mode 100644 index 000000000..051659e29 --- /dev/null +++ b/arch/sparc/kernel/fpu_traps.S @@ -0,0 +1,384 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + /* This is trivial with the new code... */ + .globl do_fpdis + .type do_fpdis,#function +do_fpdis: + sethi %hi(TSTATE_PEF), %g4 + rdpr %tstate, %g5 + andcc %g5, %g4, %g0 + be,pt %xcc, 1f + nop + rd %fprs, %g5 + andcc %g5, FPRS_FEF, %g0 + be,pt %xcc, 1f + nop + + /* Legal state when DCR_IFPOE is set in Cheetah %dcr. */ + sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + add %g0, %g0, %g0 + ba,a,pt %xcc, rtrap + +1: TRAP_LOAD_THREAD_REG(%g6, %g1) + ldub [%g6 + TI_FPSAVED], %g5 + wr %g0, FPRS_FEF, %fprs + andcc %g5, FPRS_FEF, %g0 + be,a,pt %icc, 1f + clr %g7 + ldx [%g6 + TI_GSR], %g7 +1: andcc %g5, FPRS_DL, %g0 + bne,pn %icc, 2f + fzero %f0 + andcc %g5, FPRS_DU, %g0 + bne,pn %icc, 1f + fzero %f2 + faddd %f0, %f2, %f4 + fmuld %f0, %f2, %f6 + faddd %f0, %f2, %f8 + fmuld %f0, %f2, %f10 + faddd %f0, %f2, %f12 + fmuld %f0, %f2, %f14 + faddd %f0, %f2, %f16 + fmuld %f0, %f2, %f18 + faddd %f0, %f2, %f20 + fmuld %f0, %f2, %f22 + faddd %f0, %f2, %f24 + fmuld %f0, %f2, %f26 + faddd %f0, %f2, %f28 + fmuld %f0, %f2, %f30 + faddd %f0, %f2, %f32 + fmuld %f0, %f2, %f34 + faddd %f0, %f2, %f36 + fmuld %f0, %f2, %f38 + faddd %f0, %f2, %f40 + fmuld %f0, %f2, %f42 + faddd %f0, %f2, %f44 + fmuld %f0, %f2, %f46 + faddd %f0, %f2, %f48 + fmuld %f0, %f2, %f50 + faddd %f0, %f2, %f52 + fmuld %f0, %f2, %f54 + faddd %f0, %f2, %f56 + fmuld %f0, %f2, %f58 + b,pt %xcc, fpdis_exit2 + faddd %f0, %f2, %f60 +1: mov SECONDARY_CONTEXT, %g3 + add %g6, TI_FPREGS + 0x80, %g1 + faddd %f0, %f2, %f4 + fmuld %f0, %f2, %f6 + +661: ldxa [%g3] ASI_DMMU, %g5 + .section .sun4v_1insn_patch, "ax" + .word 661b + ldxa [%g3] ASI_MMU, %g5 + .previous + + sethi %hi(sparc64_kern_sec_context), %g2 + ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 + +661: stxa %g2, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g3] ASI_MMU + .previous + + membar #Sync + add %g6, TI_FPREGS + 0xc0, %g2 + faddd %f0, %f2, %f8 + fmuld %f0, %f2, %f10 + membar #Sync + ldda [%g1] ASI_BLK_S, %f32 + ldda [%g2] ASI_BLK_S, %f48 + membar #Sync + faddd %f0, %f2, %f12 + fmuld %f0, %f2, %f14 + faddd %f0, %f2, %f16 + fmuld %f0, %f2, %f18 + faddd %f0, %f2, %f20 + fmuld %f0, %f2, %f22 + faddd %f0, %f2, %f24 + fmuld %f0, %f2, %f26 + faddd %f0, %f2, %f28 + fmuld %f0, %f2, %f30 + ba,a,pt %xcc, fpdis_exit + +2: andcc %g5, FPRS_DU, %g0 + bne,pt %icc, 3f + fzero %f32 + mov SECONDARY_CONTEXT, %g3 + fzero %f34 + +661: ldxa [%g3] ASI_DMMU, %g5 + .section .sun4v_1insn_patch, "ax" + .word 661b + ldxa [%g3] ASI_MMU, %g5 + .previous + + add %g6, TI_FPREGS, %g1 + sethi %hi(sparc64_kern_sec_context), %g2 + ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 + +661: stxa %g2, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g3] ASI_MMU + .previous + + membar #Sync + add %g6, TI_FPREGS + 0x40, %g2 + faddd %f32, %f34, %f36 + fmuld %f32, %f34, %f38 + membar #Sync + ldda [%g1] ASI_BLK_S, %f0 + ldda [%g2] ASI_BLK_S, %f16 + membar #Sync + faddd %f32, %f34, %f40 + fmuld %f32, %f34, %f42 + faddd %f32, %f34, %f44 + fmuld %f32, %f34, %f46 + faddd %f32, %f34, %f48 + fmuld %f32, %f34, %f50 + faddd %f32, %f34, %f52 + fmuld %f32, %f34, %f54 + faddd %f32, %f34, %f56 + fmuld %f32, %f34, %f58 + faddd %f32, %f34, %f60 + fmuld %f32, %f34, %f62 + ba,a,pt %xcc, fpdis_exit + +3: mov SECONDARY_CONTEXT, %g3 + add %g6, TI_FPREGS, %g1 + +661: ldxa [%g3] ASI_DMMU, %g5 + .section .sun4v_1insn_patch, "ax" + .word 661b + ldxa [%g3] ASI_MMU, %g5 + .previous + + sethi %hi(sparc64_kern_sec_context), %g2 + ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 + +661: stxa %g2, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g3] ASI_MMU + .previous + + membar #Sync + mov 0x40, %g2 + membar #Sync + ldda [%g1] ASI_BLK_S, %f0 + ldda [%g1 + %g2] ASI_BLK_S, %f16 + add %g1, 0x80, %g1 + ldda [%g1] ASI_BLK_S, %f32 + ldda [%g1 + %g2] ASI_BLK_S, %f48 + membar #Sync +fpdis_exit: + +661: stxa %g5, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g5, [%g3] ASI_MMU + .previous + + membar #Sync +fpdis_exit2: + wr %g7, 0, %gsr + ldx [%g6 + TI_XFSR], %fsr + rdpr %tstate, %g3 + or %g3, %g4, %g3 ! anal... + wrpr %g3, %tstate + wr %g0, FPRS_FEF, %fprs ! clean DU/DL bits + retry + .size do_fpdis,.-do_fpdis + + .align 32 + .type fp_other_bounce,#function +fp_other_bounce: + call do_fpother + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + .size fp_other_bounce,.-fp_other_bounce + + .align 32 + .globl do_fpother_check_fitos + .type do_fpother_check_fitos,#function +do_fpother_check_fitos: + TRAP_LOAD_THREAD_REG(%g6, %g1) + sethi %hi(fp_other_bounce - 4), %g7 + or %g7, %lo(fp_other_bounce - 4), %g7 + + /* NOTE: Need to preserve %g7 until we fully commit + * to the fitos fixup. + */ + stx %fsr, [%g6 + TI_XFSR] + rdpr %tstate, %g3 + andcc %g3, TSTATE_PRIV, %g0 + bne,pn %xcc, do_fptrap_after_fsr + nop + ldx [%g6 + TI_XFSR], %g3 + srlx %g3, 14, %g1 + and %g1, 7, %g1 + cmp %g1, 2 ! Unfinished FP-OP + bne,pn %xcc, do_fptrap_after_fsr + sethi %hi(1 << 23), %g1 ! Inexact + andcc %g3, %g1, %g0 + bne,pn %xcc, do_fptrap_after_fsr + rdpr %tpc, %g1 + lduwa [%g1] ASI_AIUP, %g3 ! This cannot ever fail +#define FITOS_MASK 0xc1f83fe0 +#define FITOS_COMPARE 0x81a01880 + sethi %hi(FITOS_MASK), %g1 + or %g1, %lo(FITOS_MASK), %g1 + and %g3, %g1, %g1 + sethi %hi(FITOS_COMPARE), %g2 + or %g2, %lo(FITOS_COMPARE), %g2 + cmp %g1, %g2 + bne,pn %xcc, do_fptrap_after_fsr + nop + std %f62, [%g6 + TI_FPREGS + (62 * 4)] + sethi %hi(fitos_table_1), %g1 + and %g3, 0x1f, %g2 + or %g1, %lo(fitos_table_1), %g1 + sllx %g2, 2, %g2 + jmpl %g1 + %g2, %g0 + ba,pt %xcc, fitos_emul_continue + +fitos_table_1: + fitod %f0, %f62 + fitod %f1, %f62 + fitod %f2, %f62 + fitod %f3, %f62 + fitod %f4, %f62 + fitod %f5, %f62 + fitod %f6, %f62 + fitod %f7, %f62 + fitod %f8, %f62 + fitod %f9, %f62 + fitod %f10, %f62 + fitod %f11, %f62 + fitod %f12, %f62 + fitod %f13, %f62 + fitod %f14, %f62 + fitod %f15, %f62 + fitod %f16, %f62 + fitod %f17, %f62 + fitod %f18, %f62 + fitod %f19, %f62 + fitod %f20, %f62 + fitod %f21, %f62 + fitod %f22, %f62 + fitod %f23, %f62 + fitod %f24, %f62 + fitod %f25, %f62 + fitod %f26, %f62 + fitod %f27, %f62 + fitod %f28, %f62 + fitod %f29, %f62 + fitod %f30, %f62 + fitod %f31, %f62 + +fitos_emul_continue: + sethi %hi(fitos_table_2), %g1 + srl %g3, 25, %g2 + or %g1, %lo(fitos_table_2), %g1 + and %g2, 0x1f, %g2 + sllx %g2, 2, %g2 + jmpl %g1 + %g2, %g0 + ba,pt %xcc, fitos_emul_fini + +fitos_table_2: + fdtos %f62, %f0 + fdtos %f62, %f1 + fdtos %f62, %f2 + fdtos %f62, %f3 + fdtos %f62, %f4 + fdtos %f62, %f5 + fdtos %f62, %f6 + fdtos %f62, %f7 + fdtos %f62, %f8 + fdtos %f62, %f9 + fdtos %f62, %f10 + fdtos %f62, %f11 + fdtos %f62, %f12 + fdtos %f62, %f13 + fdtos %f62, %f14 + fdtos %f62, %f15 + fdtos %f62, %f16 + fdtos %f62, %f17 + fdtos %f62, %f18 + fdtos %f62, %f19 + fdtos %f62, %f20 + fdtos %f62, %f21 + fdtos %f62, %f22 + fdtos %f62, %f23 + fdtos %f62, %f24 + fdtos %f62, %f25 + fdtos %f62, %f26 + fdtos %f62, %f27 + fdtos %f62, %f28 + fdtos %f62, %f29 + fdtos %f62, %f30 + fdtos %f62, %f31 + +fitos_emul_fini: + ldd [%g6 + TI_FPREGS + (62 * 4)], %f62 + done + .size do_fpother_check_fitos,.-do_fpother_check_fitos + + .align 32 + .globl do_fptrap + .type do_fptrap,#function +do_fptrap: + TRAP_LOAD_THREAD_REG(%g6, %g1) + stx %fsr, [%g6 + TI_XFSR] +do_fptrap_after_fsr: + ldub [%g6 + TI_FPSAVED], %g3 + rd %fprs, %g1 + or %g3, %g1, %g3 + stb %g3, [%g6 + TI_FPSAVED] + rd %gsr, %g3 + stx %g3, [%g6 + TI_GSR] + mov SECONDARY_CONTEXT, %g3 + +661: ldxa [%g3] ASI_DMMU, %g5 + .section .sun4v_1insn_patch, "ax" + .word 661b + ldxa [%g3] ASI_MMU, %g5 + .previous + + sethi %hi(sparc64_kern_sec_context), %g2 + ldx [%g2 + %lo(sparc64_kern_sec_context)], %g2 + +661: stxa %g2, [%g3] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g3] ASI_MMU + .previous + + membar #Sync + add %g6, TI_FPREGS, %g2 + andcc %g1, FPRS_DL, %g0 + be,pn %icc, 4f + mov 0x40, %g3 + stda %f0, [%g2] ASI_BLK_S + stda %f16, [%g2 + %g3] ASI_BLK_S + andcc %g1, FPRS_DU, %g0 + be,pn %icc, 5f +4: add %g2, 128, %g2 + stda %f32, [%g2] ASI_BLK_S + stda %f48, [%g2 + %g3] ASI_BLK_S +5: mov SECONDARY_CONTEXT, %g1 + membar #Sync + +661: stxa %g5, [%g1] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g5, [%g1] ASI_MMU + .previous + + membar #Sync + ba,pt %xcc, etrap + wr %g0, 0, %fprs + .size do_fptrap,.-do_fptrap diff --git a/arch/sparc/kernel/ftrace.c b/arch/sparc/kernel/ftrace.c new file mode 100644 index 000000000..eaead3da8 --- /dev/null +++ b/arch/sparc/kernel/ftrace.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef CONFIG_DYNAMIC_FTRACE +static const u32 ftrace_nop = 0x01000000; + +static u32 ftrace_call_replace(unsigned long ip, unsigned long addr) +{ + u32 call; + s32 off; + + off = ((s32)addr - (s32)ip); + call = 0x40000000 | ((u32)off >> 2); + + return call; +} + +static int ftrace_modify_code(unsigned long ip, u32 old, u32 new) +{ + u32 replaced; + int faulted; + + __asm__ __volatile__( + "1: cas [%[ip]], %[old], %[new]\n" + " flush %[ip]\n" + " mov 0, %[faulted]\n" + "2:\n" + " .section .fixup,#alloc,#execinstr\n" + " .align 4\n" + "3: sethi %%hi(2b), %[faulted]\n" + " jmpl %[faulted] + %%lo(2b), %%g0\n" + " mov 1, %[faulted]\n" + " .previous\n" + " .section __ex_table,\"a\"\n" + " .align 4\n" + " .word 1b, 3b\n" + " .previous\n" + : "=r" (replaced), [faulted] "=r" (faulted) + : [new] "0" (new), [old] "r" (old), [ip] "r" (ip) + : "memory"); + + if (replaced != old && replaced != new) + faulted = 2; + + return faulted; +} + +int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long ip = rec->ip; + u32 old, new; + + old = ftrace_call_replace(ip, addr); + new = ftrace_nop; + return ftrace_modify_code(ip, old, new); +} + +int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr) +{ + unsigned long ip = rec->ip; + u32 old, new; + + old = ftrace_nop; + new = ftrace_call_replace(ip, addr); + return ftrace_modify_code(ip, old, new); +} + +int ftrace_update_ftrace_func(ftrace_func_t func) +{ + unsigned long ip = (unsigned long)(&ftrace_call); + u32 old, new; + + old = *(u32 *) &ftrace_call; + new = ftrace_call_replace(ip, (unsigned long)func); + return ftrace_modify_code(ip, old, new); +} +#endif + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + +#ifdef CONFIG_DYNAMIC_FTRACE +extern void ftrace_graph_call(void); + +int ftrace_enable_ftrace_graph_caller(void) +{ + unsigned long ip = (unsigned long)(&ftrace_graph_call); + u32 old, new; + + old = *(u32 *) &ftrace_graph_call; + new = ftrace_call_replace(ip, (unsigned long) &ftrace_graph_caller); + return ftrace_modify_code(ip, old, new); +} + +int ftrace_disable_ftrace_graph_caller(void) +{ + unsigned long ip = (unsigned long)(&ftrace_graph_call); + u32 old, new; + + old = *(u32 *) &ftrace_graph_call; + new = ftrace_call_replace(ip, (unsigned long) &ftrace_stub); + + return ftrace_modify_code(ip, old, new); +} + +#endif /* !CONFIG_DYNAMIC_FTRACE */ + +/* + * Hook the return address and push it in the stack of return addrs + * in current thread info. + */ +unsigned long prepare_ftrace_return(unsigned long parent, + unsigned long self_addr, + unsigned long frame_pointer) +{ + unsigned long return_hooker = (unsigned long) &return_to_handler; + + if (unlikely(atomic_read(¤t->tracing_graph_pause))) + return parent + 8UL; + + if (function_graph_enter(parent, self_addr, frame_pointer, NULL)) + return parent + 8UL; + + return return_hooker; +} +#endif /* CONFIG_FUNCTION_GRAPH_TRACER */ diff --git a/arch/sparc/kernel/getsetcc.S b/arch/sparc/kernel/getsetcc.S new file mode 100644 index 000000000..181e09fd1 --- /dev/null +++ b/arch/sparc/kernel/getsetcc.S @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + .globl getcc + .type getcc,#function +getcc: + ldx [%o0 + PT_V9_TSTATE], %o1 + srlx %o1, 32, %o1 + and %o1, 0xf, %o1 + retl + stx %o1, [%o0 + PT_V9_G1] + .size getcc,.-getcc + + .globl setcc + .type setcc,#function +setcc: + ldx [%o0 + PT_V9_TSTATE], %o1 + ldx [%o0 + PT_V9_G1], %o2 + or %g0, %ulo(TSTATE_ICC), %o3 + sllx %o3, 32, %o3 + andn %o1, %o3, %o1 + sllx %o2, 32, %o2 + and %o2, %o3, %o2 + or %o1, %o2, %o1 + retl + stx %o1, [%o0 + PT_V9_TSTATE] + .size setcc,.-setcc diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S new file mode 100644 index 000000000..8c320fa25 --- /dev/null +++ b/arch/sparc/kernel/head_32.S @@ -0,0 +1,812 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * head.S: The initial boot code for the Sparc port of Linux. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1995,1999 Pete Zaitcev (zaitcev@yahoo.com) + * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 1997 Michael A. Griffith (grif@acm.org) + * + * CompactPCI platform by Eric Brower, 1999. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include /* TI_UWINMASK */ +#include +#include /* PGDIR_SHIFT */ + + .data +/* The following are used with the prom_vector node-ops to figure out + * the cpu-type + */ + .align 4 + .globl cputypval +cputypval: + .asciz "sun4m" + .ascii " " + +/* Tested on SS-5, SS-10 */ + .align 4 +cputypvar: + .asciz "compatible" + + .align 4 + +notsup: + .asciz "Sparc-Linux sun4/sun4c or MMU-less not supported\n\n" + .align 4 + +sun4e_notsup: + .asciz "Sparc-Linux sun4e support does not exist\n\n" + .align 4 + +/* The trap-table - located in the __HEAD section */ +#include "ttable_32.S" + + .align PAGE_SIZE + + .global root_flags + .global ram_flags + .global root_dev + .global sparc_ramdisk_image + .global sparc_ramdisk_size + +/* This stuff has to be in sync with SILO and other potential boot loaders + * Fields should be kept upward compatible and whenever any change is made, + * HdrS version should be incremented. + */ + .ascii "HdrS" + .word LINUX_VERSION_CODE + .half 0x0203 /* HdrS version */ +root_flags: + .half 1 +root_dev: + .half 0 +ram_flags: + .half 0 +sparc_ramdisk_image: + .word 0 +sparc_ramdisk_size: + .word 0 + .word reboot_command + .word 0, 0, 0 + .word _end + +/* Cool, here we go. Pick up the romvec pointer in %o0 and stash it in + * %g7 and at prom_vector_p. And also quickly check whether we are on + * a v0, v2, or v3 prom. + */ +gokernel: + /* Ok, it's nice to know, as early as possible, if we + * are already mapped where we expect to be in virtual + * memory. The Solaris /boot elf format bootloader + * will peek into our elf header and load us where + * we want to be, otherwise we have to re-map. + * + * Some boot loaders don't place the jmp'rs address + * in %o7, so we do a pc-relative call to a local + * label, then see what %o7 has. + */ + + mov %o7, %g4 ! Save %o7 + + /* Jump to it, and pray... */ +current_pc: + call 1f + nop + +1: + mov %o7, %g3 + + tst %o0 + bne 2f + mov %g4, %o7 /* Previous %o7. */ + sethi %hi(no_sun4u_here), %l1 + jmpl %l1 + %lo(no_sun4u_here), %g0 + nop +2: + mov %o0, %l0 ! stash away romvec + mov %o0, %g7 ! put it here too + mov %o1, %l1 ! stash away debug_vec too + + /* Ok, let's check out our run time program counter. */ + set current_pc, %g5 + cmp %g3, %g5 + be already_mapped + nop + + /* %l6 will hold the offset we have to subtract + * from absolute symbols in order to access areas + * in our own image. If already mapped this is + * just plain zero, else it is KERNBASE. + */ + set KERNBASE, %l6 + b copy_prom_lvl14 + nop + +already_mapped: + mov 0, %l6 + + /* Copy over the Prom's level 14 clock handler. */ +copy_prom_lvl14: +#if 1 + /* DJHR + * preserve our linked/calculated instructions + */ + set lvl14_save, %g1 + set t_irq14, %g3 + sub %g1, %l6, %g1 ! translate to physical + sub %g3, %l6, %g3 ! translate to physical + ldd [%g3], %g4 + std %g4, [%g1] + ldd [%g3+8], %g4 + std %g4, [%g1+8] +#endif + rd %tbr, %g1 + andn %g1, 0xfff, %g1 ! proms trap table base + or %g0, (0x1e<<4), %g2 ! offset to lvl14 intr + or %g1, %g2, %g2 + set t_irq14, %g3 + sub %g3, %l6, %g3 + ldd [%g2], %g4 + std %g4, [%g3] + ldd [%g2 + 0x8], %g4 + std %g4, [%g3 + 0x8] ! Copy proms handler + +/* DON'T TOUCH %l0 thru %l5 in these remapping routines, + * we need their values afterwards! + */ + + /* Now check whether we are already mapped, if we + * are we can skip all this garbage coming up. + */ +copy_prom_done: + cmp %l6, 0 + be go_to_highmem ! this will be a nop then + nop + + /* Validate that we are in fact running on an + * SRMMU based cpu. + */ + set 0x4000, %g6 + cmp %g7, %g6 + bne not_a_sun4 + nop + +halt_notsup: + ld [%g7 + 0x68], %o1 + set notsup, %o0 + sub %o0, %l6, %o0 + call %o1 + nop + sethi %hi(halt_me), %o0 + jmpl %o0 + %lo(halt_me), %g0 + nop + +not_a_sun4: + /* It looks like this is a machine we support. + * Now find out what MMU we are dealing with + * LEON - identified by the psr.impl field + * Viking - identified by the psr.impl field + * In all other cases a sun4m srmmu. + * We check that the MMU is enabled in all cases. + */ + + /* Check if this is a LEON CPU */ + rd %psr, %g3 + srl %g3, PSR_IMPL_SHIFT, %g3 + and %g3, PSR_IMPL_SHIFTED_MASK, %g3 + cmp %g3, PSR_IMPL_LEON + be leon_remap /* It is a LEON - jump */ + nop + + /* Sanity-check, is MMU enabled */ + lda [%g0] ASI_M_MMUREGS, %g1 + andcc %g1, 1, %g0 + be halt_notsup + nop + + /* Check for a viking (TI) module. */ + cmp %g3, PSR_IMPL_TI + bne srmmu_not_viking + nop + + /* Figure out what kind of viking we are on. + * We need to know if we have to play with the + * AC bit and disable traps or not. + */ + + /* I've only seen MicroSparc's on SparcClassics with this + * bit set. + */ + set 0x800, %g2 + lda [%g0] ASI_M_MMUREGS, %g3 ! peek in the control reg + and %g2, %g3, %g3 + subcc %g3, 0x0, %g0 + bnz srmmu_not_viking ! is in mbus mode + nop + + rd %psr, %g3 ! DO NOT TOUCH %g3 + andn %g3, PSR_ET, %g2 + wr %g2, 0x0, %psr + WRITE_PAUSE + + /* Get context table pointer, then convert to + * a physical address, which is 36 bits. + */ + set AC_M_CTPR, %g4 + lda [%g4] ASI_M_MMUREGS, %g4 + sll %g4, 0x4, %g4 ! We use this below + ! DO NOT TOUCH %g4 + + /* Set the AC bit in the Viking's MMU control reg. */ + lda [%g0] ASI_M_MMUREGS, %g5 ! DO NOT TOUCH %g5 + set 0x8000, %g6 ! AC bit mask + or %g5, %g6, %g6 ! Or it in... + sta %g6, [%g0] ASI_M_MMUREGS ! Close your eyes... + + /* Grrr, why does it seem like every other load/store + * on the sun4m is in some ASI space... + * Fine with me, let's get the pointer to the level 1 + * page table directory and fetch its entry. + */ + lda [%g4] ASI_M_BYPASS, %o1 ! This is a level 1 ptr + srl %o1, 0x4, %o1 ! Clear low 4 bits + sll %o1, 0x8, %o1 ! Make physical + + /* Ok, pull in the PTD. */ + lda [%o1] ASI_M_BYPASS, %o2 ! This is the 0x0 16MB pgd + + /* Calculate to KERNBASE entry. */ + add %o1, KERNBASE >> (PGDIR_SHIFT - 2), %o3 + + /* Poke the entry into the calculated address. */ + sta %o2, [%o3] ASI_M_BYPASS + + /* I don't get it Sun, if you engineered all these + * boot loaders and the PROM (thank you for the debugging + * features btw) why did you not have them load kernel + * images up in high address space, since this is necessary + * for ABI compliance anyways? Does this low-mapping provide + * enhanced interoperability? + * + * "The PROM is the computer." + */ + + /* Ok, restore the MMU control register we saved in %g5 */ + sta %g5, [%g0] ASI_M_MMUREGS ! POW... ouch + + /* Turn traps back on. We saved it in %g3 earlier. */ + wr %g3, 0x0, %psr ! tick tock, tick tock + + /* Now we burn precious CPU cycles due to bad engineering. */ + WRITE_PAUSE + + /* Wow, all that just to move a 32-bit value from one + * place to another... Jump to high memory. + */ + b go_to_highmem + nop + +srmmu_not_viking: + /* This works on viking's in Mbus mode and all + * other MBUS modules. It is virtually the same as + * the above madness sans turning traps off and flipping + * the AC bit. + */ + set AC_M_CTPR, %g1 + lda [%g1] ASI_M_MMUREGS, %g1 ! get ctx table ptr + sll %g1, 0x4, %g1 ! make physical addr + lda [%g1] ASI_M_BYPASS, %g1 ! ptr to level 1 pg_table + srl %g1, 0x4, %g1 + sll %g1, 0x8, %g1 ! make phys addr for l1 tbl + + lda [%g1] ASI_M_BYPASS, %g2 ! get level1 entry for 0x0 + add %g1, KERNBASE >> (PGDIR_SHIFT - 2), %g3 + sta %g2, [%g3] ASI_M_BYPASS ! place at KERNBASE entry + b go_to_highmem + nop ! wheee.... + + +leon_remap: + /* Sanity-check, is MMU enabled */ + lda [%g0] ASI_LEON_MMUREGS, %g1 + andcc %g1, 1, %g0 + be halt_notsup + nop + + /* Same code as in the srmmu_not_viking case, + * with the LEON ASI for mmuregs + */ + set AC_M_CTPR, %g1 + lda [%g1] ASI_LEON_MMUREGS, %g1 ! get ctx table ptr + sll %g1, 0x4, %g1 ! make physical addr + lda [%g1] ASI_M_BYPASS, %g1 ! ptr to level 1 pg_table + srl %g1, 0x4, %g1 + sll %g1, 0x8, %g1 ! make phys addr for l1 tbl + + lda [%g1] ASI_M_BYPASS, %g2 ! get level1 entry for 0x0 + add %g1, KERNBASE >> (PGDIR_SHIFT - 2), %g3 + sta %g2, [%g3] ASI_M_BYPASS ! place at KERNBASE entry + b go_to_highmem + nop ! wheee.... + +/* Now do a non-relative jump so that PC is in high-memory */ +go_to_highmem: + set execute_in_high_mem, %g1 + jmpl %g1, %g0 + nop + +/* The code above should be at beginning and we have to take care about + * short jumps, as branching to .init.text section from .text is usually + * impossible */ + __INIT +/* Acquire boot time privileged register values, this will help debugging. + * I figure out and store nwindows and nwindowsm1 later on. + */ +execute_in_high_mem: + mov %l0, %o0 ! put back romvec + mov %l1, %o1 ! and debug_vec + + sethi %hi(prom_vector_p), %g1 + st %o0, [%g1 + %lo(prom_vector_p)] + + sethi %hi(linux_dbvec), %g1 + st %o1, [%g1 + %lo(linux_dbvec)] + + /* Get the machine type via the romvec + * getprops node operation + */ + add %g7, 0x1c, %l1 + ld [%l1], %l0 + ld [%l0], %l0 + call %l0 + or %g0, %g0, %o0 ! next_node(0) = first_node + or %o0, %g0, %g6 + + sethi %hi(cputypvar), %o1 ! First node has cpu-arch + or %o1, %lo(cputypvar), %o1 + sethi %hi(cputypval), %o2 ! information, the string + or %o2, %lo(cputypval), %o2 + ld [%l1], %l0 ! 'compatible' tells + ld [%l0 + 0xc], %l0 ! that we want 'sun4x' where + call %l0 ! x is one of 'm', 'd' or 'e'. + nop ! %o2 holds pointer + ! to a buf where above string + ! will get stored by the prom. + + + /* Check value of "compatible" property. + * "value" => "model" + * leon => sparc_leon + * sun4m => sun4m + * sun4s => sun4m + * sun4d => sun4d + * sun4e => "no_sun4e_here" + * '*' => "no_sun4u_here" + * Check single letters only + */ + + set cputypval, %o2 + /* If cputypval[0] == 'l' (lower case letter L) this is leon */ + ldub [%o2], %l1 + cmp %l1, 'l' + be leon_init + nop + + /* Check cputypval[4] to find the sun model */ + ldub [%o2 + 0x4], %l1 + + cmp %l1, 'm' + be sun4m_init + cmp %l1, 's' + be sun4m_init + cmp %l1, 'd' + be sun4d_init + cmp %l1, 'e' + be no_sun4e_here ! Could be a sun4e. + nop + b no_sun4u_here ! AIEEE, a V9 sun4u... Get our BIG BROTHER kernel :)) + nop + +leon_init: + /* LEON CPU - set boot_cpu_id */ + sethi %hi(boot_cpu_id), %g2 ! boot-cpu index + +#ifdef CONFIG_SMP + ldub [%g2 + %lo(boot_cpu_id)], %g1 + cmp %g1, 0xff ! unset means first CPU + be 1f + sethi %hi(leon_smp_cpu_startup), %g1 + jmpl %g1 + %lo(leon_smp_cpu_startup), %g0 + nop +1: +#endif + /* Get CPU-ID from most significant 4-bit of ASR17 */ + rd %asr17, %g1 + srl %g1, 28, %g1 + + /* Update boot_cpu_id only on boot cpu */ + stub %g1, [%g2 + %lo(boot_cpu_id)] + + ba continue_boot + nop + +/* CPUID in bootbus can be found at PA 0xff0140000 */ +#define SUN4D_BOOTBUS_CPUID 0xf0140000 + +sun4d_init: + /* Need to patch call to handler_irq */ + set patch_handler_irq, %g4 + set sun4d_handler_irq, %g5 + sethi %hi(0x40000000), %g3 ! call + sub %g5, %g4, %g5 + srl %g5, 2, %g5 + or %g5, %g3, %g5 + st %g5, [%g4] + +#ifdef CONFIG_SMP + /* Get our CPU id out of bootbus */ + set SUN4D_BOOTBUS_CPUID, %g3 + lduba [%g3] ASI_M_CTL, %g3 + and %g3, 0xf8, %g3 + srl %g3, 3, %g4 + sta %g4, [%g0] ASI_M_VIKING_TMP1 + sethi %hi(boot_cpu_id), %g5 + stb %g4, [%g5 + %lo(boot_cpu_id)] +#endif + + /* Fall through to sun4m_init */ + +sun4m_init: +/* Ok, the PROM could have done funny things and apple cider could still + * be sitting in the fault status/address registers. Read them all to + * clear them so we don't get magic faults later on. + */ +/* This sucks, apparently this makes Vikings call prom panic, will fix later */ +2: + rd %psr, %o1 + srl %o1, PSR_IMPL_SHIFT, %o1 ! Get a type of the CPU + + subcc %o1, PSR_IMPL_TI, %g0 ! TI: Viking or MicroSPARC + be continue_boot + nop + + set AC_M_SFSR, %o0 + lda [%o0] ASI_M_MMUREGS, %g0 + set AC_M_SFAR, %o0 + lda [%o0] ASI_M_MMUREGS, %g0 + + /* Fujitsu MicroSPARC-II has no asynchronous flavors of FARs */ + subcc %o1, 0, %g0 + be continue_boot + nop + + set AC_M_AFSR, %o0 + lda [%o0] ASI_M_MMUREGS, %g0 + set AC_M_AFAR, %o0 + lda [%o0] ASI_M_MMUREGS, %g0 + nop + + +continue_boot: + +/* Aieee, now set PC and nPC, enable traps, give ourselves a stack and it's + * show-time! + */ + /* Turn on Supervisor, EnableFloating, and all the PIL bits. + * Also puts us in register window zero with traps off. + */ + set (PSR_PS | PSR_S | PSR_PIL | PSR_EF), %g2 + wr %g2, 0x0, %psr + WRITE_PAUSE + + /* I want a kernel stack NOW! */ + set init_thread_union, %g1 + set (THREAD_SIZE - STACKFRAME_SZ - TRACEREG_SZ), %g2 + add %g1, %g2, %sp + mov 0, %fp /* And for good luck */ + + /* Zero out our BSS section. */ + set __bss_start , %o0 ! First address of BSS + set _end , %o1 ! Last address of BSS + add %o0, 0x1, %o0 +1: + stb %g0, [%o0] + subcc %o0, %o1, %g0 + bl 1b + add %o0, 0x1, %o0 + + /* If boot_cpu_id has not been setup by machine specific + * init-code above we default it to zero. + */ + sethi %hi(boot_cpu_id), %g2 + ldub [%g2 + %lo(boot_cpu_id)], %g3 + cmp %g3, 0xff + bne 1f + nop + mov %g0, %g3 + stub %g3, [%g2 + %lo(boot_cpu_id)] + +1: sll %g3, 2, %g3 + + /* Initialize the uwinmask value for init task just in case. + * But first make current_set[boot_cpu_id] point to something useful. + */ + set init_thread_union, %g6 + set current_set, %g2 +#ifdef CONFIG_SMP + st %g6, [%g2] + add %g2, %g3, %g2 +#endif + st %g6, [%g2] + + st %g0, [%g6 + TI_UWINMASK] + +/* Compute NWINDOWS and stash it away. Now uses %wim trick explained + * in the V8 manual. Ok, this method seems to work, Sparc is cool... + * No, it doesn't work, have to play the save/readCWP/restore trick. + */ + + wr %g0, 0x0, %wim ! so we do not get a trap + WRITE_PAUSE + + save + + rd %psr, %g3 + + restore + + and %g3, 0x1f, %g3 + add %g3, 0x1, %g3 + + mov 2, %g1 + wr %g1, 0x0, %wim ! make window 1 invalid + WRITE_PAUSE + + cmp %g3, 0x7 + bne 2f + nop + + /* Adjust our window handling routines to + * do things correctly on 7 window Sparcs. + */ + +#define PATCH_INSN(src, dest) \ + set src, %g5; \ + set dest, %g2; \ + ld [%g5], %g4; \ + st %g4, [%g2]; + + /* Patch for window spills... */ + PATCH_INSN(spnwin_patch1_7win, spnwin_patch1) + PATCH_INSN(spnwin_patch2_7win, spnwin_patch2) + PATCH_INSN(spnwin_patch3_7win, spnwin_patch3) + + /* Patch for window fills... */ + PATCH_INSN(fnwin_patch1_7win, fnwin_patch1) + PATCH_INSN(fnwin_patch2_7win, fnwin_patch2) + + /* Patch for trap entry setup... */ + PATCH_INSN(tsetup_7win_patch1, tsetup_patch1) + PATCH_INSN(tsetup_7win_patch2, tsetup_patch2) + PATCH_INSN(tsetup_7win_patch3, tsetup_patch3) + PATCH_INSN(tsetup_7win_patch4, tsetup_patch4) + PATCH_INSN(tsetup_7win_patch5, tsetup_patch5) + PATCH_INSN(tsetup_7win_patch6, tsetup_patch6) + + /* Patch for returning from traps... */ + PATCH_INSN(rtrap_7win_patch1, rtrap_patch1) + PATCH_INSN(rtrap_7win_patch2, rtrap_patch2) + PATCH_INSN(rtrap_7win_patch3, rtrap_patch3) + PATCH_INSN(rtrap_7win_patch4, rtrap_patch4) + PATCH_INSN(rtrap_7win_patch5, rtrap_patch5) + + /* Patch for killing user windows from the register file. */ + PATCH_INSN(kuw_patch1_7win, kuw_patch1) + + /* Now patch the kernel window flush sequences. + * This saves 2 traps on every switch and fork. + */ + set 0x01000000, %g4 + set flush_patch_one, %g5 + st %g4, [%g5 + 0x18] + st %g4, [%g5 + 0x1c] + set flush_patch_two, %g5 + st %g4, [%g5 + 0x18] + st %g4, [%g5 + 0x1c] + set flush_patch_three, %g5 + st %g4, [%g5 + 0x18] + st %g4, [%g5 + 0x1c] + set flush_patch_four, %g5 + st %g4, [%g5 + 0x18] + st %g4, [%g5 + 0x1c] + set flush_patch_exception, %g5 + st %g4, [%g5 + 0x18] + st %g4, [%g5 + 0x1c] + set flush_patch_switch, %g5 + st %g4, [%g5 + 0x18] + st %g4, [%g5 + 0x1c] + +2: + sethi %hi(nwindows), %g4 + st %g3, [%g4 + %lo(nwindows)] ! store final value + sub %g3, 0x1, %g3 + sethi %hi(nwindowsm1), %g4 + st %g3, [%g4 + %lo(nwindowsm1)] + + /* Here we go, start using Linux's trap table... */ + set trapbase, %g3 + wr %g3, 0x0, %tbr + WRITE_PAUSE + + /* Finally, turn on traps so that we can call c-code. */ + rd %psr, %g3 + wr %g3, 0x0, %psr + WRITE_PAUSE + + wr %g3, PSR_ET, %psr + WRITE_PAUSE + + /* Call sparc32_start_kernel(struct linux_romvec *rp) */ + sethi %hi(prom_vector_p), %g5 + ld [%g5 + %lo(prom_vector_p)], %o0 + call sparc32_start_kernel + nop + + /* We should not get here. */ + call halt_me + nop + +no_sun4e_here: + ld [%g7 + 0x68], %o1 + set sun4e_notsup, %o0 + call %o1 + nop + b halt_me + nop + + __INITDATA + +sun4u_1: + .asciz "finddevice" + .align 4 +sun4u_2: + .asciz "/chosen" + .align 4 +sun4u_3: + .asciz "getprop" + .align 4 +sun4u_4: + .asciz "stdout" + .align 4 +sun4u_5: + .asciz "write" + .align 4 +sun4u_6: + .asciz "\n\rOn sun4u you have to use sparc64 kernel\n\rand not a sparc32 version\n\r\n\r" +sun4u_6e: + .align 4 +sun4u_7: + .asciz "exit" + .align 8 +sun4u_a1: + .word 0, sun4u_1, 0, 1, 0, 1, 0, sun4u_2, 0 +sun4u_r1: + .word 0 +sun4u_a2: + .word 0, sun4u_3, 0, 4, 0, 1, 0 +sun4u_i2: + .word 0, 0, sun4u_4, 0, sun4u_1, 0, 8, 0 +sun4u_r2: + .word 0 +sun4u_a3: + .word 0, sun4u_5, 0, 3, 0, 1, 0 +sun4u_i3: + .word 0, 0, sun4u_6, 0, sun4u_6e - sun4u_6 - 1, 0 +sun4u_r3: + .word 0 +sun4u_a4: + .word 0, sun4u_7, 0, 0, 0, 0 +sun4u_r4: + + __INIT +no_sun4u_here: + set sun4u_a1, %o0 + set current_pc, %l2 + cmp %l2, %g3 + be 1f + mov %o4, %l0 + sub %g3, %l2, %l6 + add %o0, %l6, %o0 + mov %o0, %l4 + mov sun4u_r4 - sun4u_a1, %l3 + ld [%l4], %l5 +2: + add %l4, 4, %l4 + cmp %l5, %l2 + add %l5, %l6, %l5 + bgeu,a 3f + st %l5, [%l4 - 4] +3: + subcc %l3, 4, %l3 + bne 2b + ld [%l4], %l5 +1: + call %l0 + mov %o0, %l1 + + ld [%l1 + (sun4u_r1 - sun4u_a1)], %o1 + add %l1, (sun4u_a2 - sun4u_a1), %o0 + call %l0 + st %o1, [%o0 + (sun4u_i2 - sun4u_a2)] + + ld [%l1 + (sun4u_1 - sun4u_a1)], %o1 + add %l1, (sun4u_a3 - sun4u_a1), %o0 + call %l0 + st %o1, [%o0 + (sun4u_i3 - sun4u_a3)] + + call %l0 + add %l1, (sun4u_a4 - sun4u_a1), %o0 + + /* Not reached */ +halt_me: + ld [%g7 + 0x74], %o0 + call %o0 ! Get us out of here... + nop ! Apparently Solaris is better. + +/* Ok, now we continue in the .data/.text sections */ + + .data + .align 4 + +/* + * Fill up the prom vector, note in particular the kind first element, + * no joke. I don't need all of them in here as the entire prom vector + * gets initialized in c-code so all routines can use it. + */ + +prom_vector_p: + .word 0 + +/* We calculate the following at boot time, window fills/spills and trap entry + * code uses these to keep track of the register windows. + */ + + .align 4 + .globl nwindows + .globl nwindowsm1 +nwindows: + .word 8 +nwindowsm1: + .word 7 + +/* Boot time debugger vector value. We need this later on. */ + + .align 4 + .globl linux_dbvec +linux_dbvec: + .word 0 + .word 0 + + .align 8 + + .globl lvl14_save +lvl14_save: + .word 0 + .word 0 + .word 0 + .word 0 + .word t_irq14 diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S new file mode 100644 index 000000000..cf0549134 --- /dev/null +++ b/arch/sparc/kernel/head_64.S @@ -0,0 +1,969 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* head.S: Initial boot code for the Sparc64 port of Linux. + * + * Copyright (C) 1996, 1997, 2007 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996 David Sitsky (David.Sitsky@anu.edu.au) + * Copyright (C) 1997, 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 1997 Miguel de Icaza (miguel@nuclecu.unam.mx) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* This section from from _start to sparc64_boot_end should fit into + * 0x0000000000404000 to 0x0000000000408000. + */ + .text + .globl start, _start, stext, _stext +_start: +start: +_stext: +stext: +! 0x0000000000404000 + b sparc64_boot + flushw /* Flush register file. */ + +/* This stuff has to be in sync with SILO and other potential boot loaders + * Fields should be kept upward compatible and whenever any change is made, + * HdrS version should be incremented. + */ + .global root_flags, ram_flags, root_dev + .global sparc_ramdisk_image, sparc_ramdisk_size + .global sparc_ramdisk_image64 + + .ascii "HdrS" + .word LINUX_VERSION_CODE + + /* History: + * + * 0x0300 : Supports being located at other than 0x4000 + * 0x0202 : Supports kernel params string + * 0x0201 : Supports reboot_command + */ + .half 0x0301 /* HdrS version */ + +root_flags: + .half 1 +root_dev: + .half 0 +ram_flags: + .half 0 +sparc_ramdisk_image: + .word 0 +sparc_ramdisk_size: + .word 0 + .xword reboot_command + .xword bootstr_info +sparc_ramdisk_image64: + .xword 0 + .word _end + + /* PROM cif handler code address is in %o4. */ +sparc64_boot: + mov %o4, %l7 + + /* We need to remap the kernel. Use position independent + * code to remap us to KERNBASE. + * + * SILO can invoke us with 32-bit address masking enabled, + * so make sure that's clear. + */ + rdpr %pstate, %g1 + andn %g1, PSTATE_AM, %g1 + wrpr %g1, 0x0, %pstate + ba,a,pt %xcc, 1f + nop + + .globl prom_finddev_name, prom_chosen_path, prom_root_node + .globl prom_getprop_name, prom_mmu_name, prom_peer_name + .globl prom_callmethod_name, prom_translate_name, prom_root_compatible + .globl prom_map_name, prom_unmap_name, prom_mmu_ihandle_cache + .globl prom_boot_mapped_pc, prom_boot_mapping_mode + .globl prom_boot_mapping_phys_high, prom_boot_mapping_phys_low + .globl prom_compatible_name, prom_cpu_path, prom_cpu_compatible + .globl is_sun4v, sun4v_chip_type, prom_set_trap_table_name +prom_peer_name: + .asciz "peer" +prom_compatible_name: + .asciz "compatible" +prom_finddev_name: + .asciz "finddevice" +prom_chosen_path: + .asciz "/chosen" +prom_cpu_path: + .asciz "/cpu" +prom_getprop_name: + .asciz "getprop" +prom_mmu_name: + .asciz "mmu" +prom_callmethod_name: + .asciz "call-method" +prom_translate_name: + .asciz "translate" +prom_map_name: + .asciz "map" +prom_unmap_name: + .asciz "unmap" +prom_set_trap_table_name: + .asciz "SUNW,set-trap-table" +prom_sun4v_name: + .asciz "sun4v" +prom_niagara_prefix: + .asciz "SUNW,UltraSPARC-T" +prom_sparc_prefix: + .asciz "SPARC-" +prom_sparc64x_prefix: + .asciz "SPARC64-X" + .align 4 +prom_root_compatible: + .skip 64 +prom_cpu_compatible: + .skip 64 +prom_root_node: + .word 0 +EXPORT_SYMBOL(prom_root_node) +prom_mmu_ihandle_cache: + .word 0 +prom_boot_mapped_pc: + .word 0 +prom_boot_mapping_mode: + .word 0 + .align 8 +prom_boot_mapping_phys_high: + .xword 0 +prom_boot_mapping_phys_low: + .xword 0 +is_sun4v: + .word 0 +sun4v_chip_type: + .word SUN4V_CHIP_INVALID +EXPORT_SYMBOL(sun4v_chip_type) +1: + rd %pc, %l0 + + mov (1b - prom_peer_name), %l1 + sub %l0, %l1, %l1 + mov 0, %l2 + + /* prom_root_node = prom_peer(0) */ + stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "peer" + mov 1, %l3 + stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 1 + stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1 + stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1, 0 + stx %g0, [%sp + 2047 + 128 + 0x20] ! ret1 + call %l7 + add %sp, (2047 + 128), %o0 ! argument array + + ldx [%sp + 2047 + 128 + 0x20], %l4 ! prom root node + mov (1b - prom_root_node), %l1 + sub %l0, %l1, %l1 + stw %l4, [%l1] + + mov (1b - prom_getprop_name), %l1 + mov (1b - prom_compatible_name), %l2 + mov (1b - prom_root_compatible), %l5 + sub %l0, %l1, %l1 + sub %l0, %l2, %l2 + sub %l0, %l5, %l5 + + /* prom_getproperty(prom_root_node, "compatible", + * &prom_root_compatible, 64) + */ + stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "getprop" + mov 4, %l3 + stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 4 + mov 1, %l3 + stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1 + stx %l4, [%sp + 2047 + 128 + 0x18] ! arg1, prom_root_node + stx %l2, [%sp + 2047 + 128 + 0x20] ! arg2, "compatible" + stx %l5, [%sp + 2047 + 128 + 0x28] ! arg3, &prom_root_compatible + mov 64, %l3 + stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4, size + stx %g0, [%sp + 2047 + 128 + 0x38] ! ret1 + call %l7 + add %sp, (2047 + 128), %o0 ! argument array + + mov (1b - prom_finddev_name), %l1 + mov (1b - prom_chosen_path), %l2 + mov (1b - prom_boot_mapped_pc), %l3 + sub %l0, %l1, %l1 + sub %l0, %l2, %l2 + sub %l0, %l3, %l3 + stw %l0, [%l3] + sub %sp, (192 + 128), %sp + + /* chosen_node = prom_finddevice("/chosen") */ + stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "finddevice" + mov 1, %l3 + stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 1 + stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1 + stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1, "/chosen" + stx %g0, [%sp + 2047 + 128 + 0x20] ! ret1 + call %l7 + add %sp, (2047 + 128), %o0 ! argument array + + ldx [%sp + 2047 + 128 + 0x20], %l4 ! chosen device node + + mov (1b - prom_getprop_name), %l1 + mov (1b - prom_mmu_name), %l2 + mov (1b - prom_mmu_ihandle_cache), %l5 + sub %l0, %l1, %l1 + sub %l0, %l2, %l2 + sub %l0, %l5, %l5 + + /* prom_mmu_ihandle_cache = prom_getint(chosen_node, "mmu") */ + stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "getprop" + mov 4, %l3 + stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 4 + mov 1, %l3 + stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1 + stx %l4, [%sp + 2047 + 128 + 0x18] ! arg1, chosen_node + stx %l2, [%sp + 2047 + 128 + 0x20] ! arg2, "mmu" + stx %l5, [%sp + 2047 + 128 + 0x28] ! arg3, &prom_mmu_ihandle_cache + mov 4, %l3 + stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4, sizeof(arg3) + stx %g0, [%sp + 2047 + 128 + 0x38] ! ret1 + call %l7 + add %sp, (2047 + 128), %o0 ! argument array + + mov (1b - prom_callmethod_name), %l1 + mov (1b - prom_translate_name), %l2 + sub %l0, %l1, %l1 + sub %l0, %l2, %l2 + lduw [%l5], %l5 ! prom_mmu_ihandle_cache + + stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "call-method" + mov 3, %l3 + stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 3 + mov 5, %l3 + stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 5 + stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1: "translate" + stx %l5, [%sp + 2047 + 128 + 0x20] ! arg2: prom_mmu_ihandle_cache + /* PAGE align */ + srlx %l0, 13, %l3 + sllx %l3, 13, %l3 + stx %l3, [%sp + 2047 + 128 + 0x28] ! arg3: vaddr, our PC + stx %g0, [%sp + 2047 + 128 + 0x30] ! res1 + stx %g0, [%sp + 2047 + 128 + 0x38] ! res2 + stx %g0, [%sp + 2047 + 128 + 0x40] ! res3 + stx %g0, [%sp + 2047 + 128 + 0x48] ! res4 + stx %g0, [%sp + 2047 + 128 + 0x50] ! res5 + call %l7 + add %sp, (2047 + 128), %o0 ! argument array + + ldx [%sp + 2047 + 128 + 0x40], %l1 ! translation mode + mov (1b - prom_boot_mapping_mode), %l4 + sub %l0, %l4, %l4 + stw %l1, [%l4] + mov (1b - prom_boot_mapping_phys_high), %l4 + sub %l0, %l4, %l4 + ldx [%sp + 2047 + 128 + 0x48], %l2 ! physaddr high + stx %l2, [%l4 + 0x0] + ldx [%sp + 2047 + 128 + 0x50], %l3 ! physaddr low + /* 4MB align */ + srlx %l3, ILOG2_4MB, %l3 + sllx %l3, ILOG2_4MB, %l3 + stx %l3, [%l4 + 0x8] + + /* Leave service as-is, "call-method" */ + mov 7, %l3 + stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 7 + mov 1, %l3 + stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1 + mov (1b - prom_map_name), %l3 + sub %l0, %l3, %l3 + stx %l3, [%sp + 2047 + 128 + 0x18] ! arg1: "map" + /* Leave arg2 as-is, prom_mmu_ihandle_cache */ + mov -1, %l3 + stx %l3, [%sp + 2047 + 128 + 0x28] ! arg3: mode (-1 default) + /* 4MB align the kernel image size. */ + set (_end - KERNBASE), %l3 + set ((4 * 1024 * 1024) - 1), %l4 + add %l3, %l4, %l3 + andn %l3, %l4, %l3 + stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4: roundup(ksize, 4MB) + sethi %hi(KERNBASE), %l3 + stx %l3, [%sp + 2047 + 128 + 0x38] ! arg5: vaddr (KERNBASE) + stx %g0, [%sp + 2047 + 128 + 0x40] ! arg6: empty + mov (1b - prom_boot_mapping_phys_low), %l3 + sub %l0, %l3, %l3 + ldx [%l3], %l3 + stx %l3, [%sp + 2047 + 128 + 0x48] ! arg7: phys addr + call %l7 + add %sp, (2047 + 128), %o0 ! argument array + + add %sp, (192 + 128), %sp + + sethi %hi(prom_root_compatible), %g1 + or %g1, %lo(prom_root_compatible), %g1 + sethi %hi(prom_sun4v_name), %g7 + or %g7, %lo(prom_sun4v_name), %g7 + mov 5, %g3 +90: ldub [%g7], %g2 + ldub [%g1], %g4 + cmp %g2, %g4 + bne,pn %icc, 80f + add %g7, 1, %g7 + subcc %g3, 1, %g3 + bne,pt %xcc, 90b + add %g1, 1, %g1 + + sethi %hi(is_sun4v), %g1 + or %g1, %lo(is_sun4v), %g1 + mov 1, %g7 + stw %g7, [%g1] + + /* cpu_node = prom_finddevice("/cpu") */ + mov (1b - prom_finddev_name), %l1 + mov (1b - prom_cpu_path), %l2 + sub %l0, %l1, %l1 + sub %l0, %l2, %l2 + sub %sp, (192 + 128), %sp + + stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "finddevice" + mov 1, %l3 + stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 1 + stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1 + stx %l2, [%sp + 2047 + 128 + 0x18] ! arg1, "/cpu" + stx %g0, [%sp + 2047 + 128 + 0x20] ! ret1 + call %l7 + add %sp, (2047 + 128), %o0 ! argument array + + ldx [%sp + 2047 + 128 + 0x20], %l4 ! cpu device node + + mov (1b - prom_getprop_name), %l1 + mov (1b - prom_compatible_name), %l2 + mov (1b - prom_cpu_compatible), %l5 + sub %l0, %l1, %l1 + sub %l0, %l2, %l2 + sub %l0, %l5, %l5 + + /* prom_getproperty(cpu_node, "compatible", + * &prom_cpu_compatible, 64) + */ + stx %l1, [%sp + 2047 + 128 + 0x00] ! service, "getprop" + mov 4, %l3 + stx %l3, [%sp + 2047 + 128 + 0x08] ! num_args, 4 + mov 1, %l3 + stx %l3, [%sp + 2047 + 128 + 0x10] ! num_rets, 1 + stx %l4, [%sp + 2047 + 128 + 0x18] ! arg1, cpu_node + stx %l2, [%sp + 2047 + 128 + 0x20] ! arg2, "compatible" + stx %l5, [%sp + 2047 + 128 + 0x28] ! arg3, &prom_cpu_compatible + mov 64, %l3 + stx %l3, [%sp + 2047 + 128 + 0x30] ! arg4, size + stx %g0, [%sp + 2047 + 128 + 0x38] ! ret1 + call %l7 + add %sp, (2047 + 128), %o0 ! argument array + + add %sp, (192 + 128), %sp + + sethi %hi(prom_cpu_compatible), %g1 + or %g1, %lo(prom_cpu_compatible), %g1 + sethi %hi(prom_niagara_prefix), %g7 + or %g7, %lo(prom_niagara_prefix), %g7 + mov 17, %g3 +90: ldub [%g7], %g2 + ldub [%g1], %g4 + cmp %g2, %g4 + bne,pn %icc, 89f + add %g7, 1, %g7 + subcc %g3, 1, %g3 + bne,pt %xcc, 90b + add %g1, 1, %g1 + ba,pt %xcc, 91f + nop + +89: sethi %hi(prom_cpu_compatible), %g1 + or %g1, %lo(prom_cpu_compatible), %g1 + sethi %hi(prom_sparc_prefix), %g7 + or %g7, %lo(prom_sparc_prefix), %g7 + mov 6, %g3 +90: ldub [%g7], %g2 + ldub [%g1], %g4 + cmp %g2, %g4 + bne,pn %icc, 4f + add %g7, 1, %g7 + subcc %g3, 1, %g3 + bne,pt %xcc, 90b + add %g1, 1, %g1 + + sethi %hi(prom_cpu_compatible), %g1 + or %g1, %lo(prom_cpu_compatible), %g1 + ldub [%g1 + 6], %g2 + cmp %g2, 'T' + be,pt %xcc, 70f + cmp %g2, 'M' + be,pt %xcc, 70f + cmp %g2, 'S' + bne,pn %xcc, 49f + nop + +70: ldub [%g1 + 7], %g2 + cmp %g2, CPU_ID_NIAGARA3 + be,pt %xcc, 5f + mov SUN4V_CHIP_NIAGARA3, %g4 + cmp %g2, CPU_ID_NIAGARA4 + be,pt %xcc, 5f + mov SUN4V_CHIP_NIAGARA4, %g4 + cmp %g2, CPU_ID_NIAGARA5 + be,pt %xcc, 5f + mov SUN4V_CHIP_NIAGARA5, %g4 + cmp %g2, CPU_ID_M6 + be,pt %xcc, 5f + mov SUN4V_CHIP_SPARC_M6, %g4 + cmp %g2, CPU_ID_M7 + be,pt %xcc, 5f + mov SUN4V_CHIP_SPARC_M7, %g4 + cmp %g2, CPU_ID_M8 + be,pt %xcc, 5f + mov SUN4V_CHIP_SPARC_M8, %g4 + cmp %g2, CPU_ID_SONOMA1 + be,pt %xcc, 5f + mov SUN4V_CHIP_SPARC_SN, %g4 + ba,pt %xcc, 49f + nop + +91: sethi %hi(prom_cpu_compatible), %g1 + or %g1, %lo(prom_cpu_compatible), %g1 + ldub [%g1 + 17], %g2 + cmp %g2, CPU_ID_NIAGARA1 + be,pt %xcc, 5f + mov SUN4V_CHIP_NIAGARA1, %g4 + cmp %g2, CPU_ID_NIAGARA2 + be,pt %xcc, 5f + mov SUN4V_CHIP_NIAGARA2, %g4 + +4: + /* Athena */ + sethi %hi(prom_cpu_compatible), %g1 + or %g1, %lo(prom_cpu_compatible), %g1 + sethi %hi(prom_sparc64x_prefix), %g7 + or %g7, %lo(prom_sparc64x_prefix), %g7 + mov 9, %g3 +41: ldub [%g7], %g2 + ldub [%g1], %g4 + cmp %g2, %g4 + bne,pn %icc, 49f + add %g7, 1, %g7 + subcc %g3, 1, %g3 + bne,pt %xcc, 41b + add %g1, 1, %g1 + ba,pt %xcc, 5f + mov SUN4V_CHIP_SPARC64X, %g4 + +49: + mov SUN4V_CHIP_UNKNOWN, %g4 +5: sethi %hi(sun4v_chip_type), %g2 + or %g2, %lo(sun4v_chip_type), %g2 + stw %g4, [%g2] + +80: + BRANCH_IF_SUN4V(g1, jump_to_sun4u_init) + BRANCH_IF_CHEETAH_BASE(g1,g7,cheetah_boot) + BRANCH_IF_CHEETAH_PLUS_OR_FOLLOWON(g1,g7,cheetah_plus_boot) + ba,pt %xcc, spitfire_boot + nop + +cheetah_plus_boot: + /* Preserve OBP chosen DCU and DCR register settings. */ + ba,pt %xcc, cheetah_generic_boot + nop + +cheetah_boot: + mov DCR_BPE | DCR_RPE | DCR_SI | DCR_IFPOE | DCR_MS, %g1 + wr %g1, %asr18 + + sethi %uhi(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g7 + or %g7, %ulo(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g7 + sllx %g7, 32, %g7 + or %g7, DCU_DM | DCU_IM | DCU_DC | DCU_IC, %g7 + stxa %g7, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + +cheetah_generic_boot: + mov TSB_EXTENSION_P, %g3 + stxa %g0, [%g3] ASI_DMMU + stxa %g0, [%g3] ASI_IMMU + membar #Sync + + mov TSB_EXTENSION_S, %g3 + stxa %g0, [%g3] ASI_DMMU + membar #Sync + + mov TSB_EXTENSION_N, %g3 + stxa %g0, [%g3] ASI_DMMU + stxa %g0, [%g3] ASI_IMMU + membar #Sync + + ba,a,pt %xcc, jump_to_sun4u_init + +spitfire_boot: + /* Typically PROM has already enabled both MMU's and both on-chip + * caches, but we do it here anyway just to be paranoid. + */ + mov (LSU_CONTROL_IC|LSU_CONTROL_DC|LSU_CONTROL_IM|LSU_CONTROL_DM), %g1 + stxa %g1, [%g0] ASI_LSU_CONTROL + membar #Sync + +jump_to_sun4u_init: + /* + * Make sure we are in privileged mode, have address masking, + * using the ordinary globals and have enabled floating + * point. + * + * Again, typically PROM has left %pil at 13 or similar, and + * (PSTATE_PRIV | PSTATE_PEF | PSTATE_IE) in %pstate. + */ + wrpr %g0, (PSTATE_PRIV|PSTATE_PEF|PSTATE_IE), %pstate + wr %g0, 0, %fprs + + set sun4u_init, %g2 + jmpl %g2 + %g0, %g0 + nop + + __REF +sun4u_init: + BRANCH_IF_SUN4V(g1, sun4v_init) + + /* Set ctx 0 */ + mov PRIMARY_CONTEXT, %g7 + stxa %g0, [%g7] ASI_DMMU + membar #Sync + + mov SECONDARY_CONTEXT, %g7 + stxa %g0, [%g7] ASI_DMMU + membar #Sync + + ba,a,pt %xcc, sun4u_continue + +sun4v_init: + /* Set ctx 0 */ + mov PRIMARY_CONTEXT, %g7 + stxa %g0, [%g7] ASI_MMU + membar #Sync + + mov SECONDARY_CONTEXT, %g7 + stxa %g0, [%g7] ASI_MMU + membar #Sync + ba,a,pt %xcc, niagara_tlb_fixup + +sun4u_continue: + BRANCH_IF_ANY_CHEETAH(g1, g7, cheetah_tlb_fixup) + + ba,a,pt %xcc, spitfire_tlb_fixup + +niagara_tlb_fixup: + mov 3, %g2 /* Set TLB type to hypervisor. */ + sethi %hi(tlb_type), %g1 + stw %g2, [%g1 + %lo(tlb_type)] + + /* Patch copy/clear ops. */ + sethi %hi(sun4v_chip_type), %g1 + lduw [%g1 + %lo(sun4v_chip_type)], %g1 + cmp %g1, SUN4V_CHIP_NIAGARA1 + be,pt %xcc, niagara_patch + cmp %g1, SUN4V_CHIP_NIAGARA2 + be,pt %xcc, niagara2_patch + nop + cmp %g1, SUN4V_CHIP_NIAGARA3 + be,pt %xcc, niagara2_patch + nop + cmp %g1, SUN4V_CHIP_NIAGARA4 + be,pt %xcc, niagara4_patch + nop + cmp %g1, SUN4V_CHIP_NIAGARA5 + be,pt %xcc, niagara4_patch + nop + cmp %g1, SUN4V_CHIP_SPARC_M6 + be,pt %xcc, niagara4_patch + nop + cmp %g1, SUN4V_CHIP_SPARC_M7 + be,pt %xcc, sparc_m7_patch + nop + cmp %g1, SUN4V_CHIP_SPARC_M8 + be,pt %xcc, sparc_m7_patch + nop + cmp %g1, SUN4V_CHIP_SPARC_SN + be,pt %xcc, niagara4_patch + nop + + call generic_patch_copyops + nop + call generic_patch_bzero + nop + call generic_patch_pageops + nop + + ba,a,pt %xcc, 80f + nop + +sparc_m7_patch: + call m7_patch_copyops + nop + call m7_patch_bzero + nop + call m7_patch_pageops + nop + + ba,a,pt %xcc, 80f + nop + +niagara4_patch: + call niagara4_patch_copyops + nop + call niagara4_patch_bzero + nop + call niagara4_patch_pageops + nop + call niagara4_patch_fls + nop + + ba,a,pt %xcc, 80f + nop + +niagara2_patch: + call niagara2_patch_copyops + nop + call niagara_patch_bzero + nop + call niagara_patch_pageops + nop + + ba,a,pt %xcc, 80f + nop + +niagara_patch: + call niagara_patch_copyops + nop + call niagara_patch_bzero + nop + call niagara_patch_pageops + nop + +80: + /* Patch TLB/cache ops. */ + call hypervisor_patch_cachetlbops + nop + + ba,a,pt %xcc, tlb_fixup_done + +cheetah_tlb_fixup: + mov 2, %g2 /* Set TLB type to cheetah+. */ + BRANCH_IF_CHEETAH_PLUS_OR_FOLLOWON(g1,g7,1f) + + mov 1, %g2 /* Set TLB type to cheetah. */ + +1: sethi %hi(tlb_type), %g1 + stw %g2, [%g1 + %lo(tlb_type)] + + /* Patch copy/page operations to cheetah optimized versions. */ + call cheetah_patch_copyops + nop + call cheetah_patch_copy_page + nop + call cheetah_patch_cachetlbops + nop + + ba,a,pt %xcc, tlb_fixup_done + +spitfire_tlb_fixup: + /* Set TLB type to spitfire. */ + mov 0, %g2 + sethi %hi(tlb_type), %g1 + stw %g2, [%g1 + %lo(tlb_type)] + +tlb_fixup_done: + sethi %hi(init_thread_union), %g6 + or %g6, %lo(init_thread_union), %g6 + ldx [%g6 + TI_TASK], %g4 + + wr %g0, ASI_P, %asi + mov 1, %g1 + sllx %g1, THREAD_SHIFT, %g1 + sub %g1, (STACKFRAME_SZ + STACK_BIAS + TRACEREG_SZ), %g1 + add %g6, %g1, %sp + + /* Set per-cpu pointer initially to zero, this makes + * the boot-cpu use the in-kernel-image per-cpu areas + * before setup_per_cpu_area() is invoked. + */ + clr %g5 + + wrpr %g0, 0, %wstate + wrpr %g0, 0x0, %tl + + /* Clear the bss */ + sethi %hi(__bss_start), %o0 + or %o0, %lo(__bss_start), %o0 + sethi %hi(_end), %o1 + or %o1, %lo(_end), %o1 + call __bzero + sub %o1, %o0, %o1 + + call prom_init + mov %l7, %o0 ! OpenPROM cif handler + + /* To create a one-register-window buffer between the kernel's + * initial stack and the last stack frame we use from the firmware, + * do the rest of the boot from a C helper function. + */ + call start_early_boot + nop + /* Not reached... */ + + .previous + + /* This is meant to allow the sharing of this code between + * boot processor invocation (via setup_tba() below) and + * secondary processor startup (via trampoline.S). The + * former does use this code, the latter does not yet due + * to some complexities. That should be fixed up at some + * point. + * + * There used to be enormous complexity wrt. transferring + * over from the firmware's trap table to the Linux kernel's. + * For example, there was a chicken & egg problem wrt. building + * the OBP page tables, yet needing to be on the Linux kernel + * trap table (to translate PAGE_OFFSET addresses) in order to + * do that. + * + * We now handle OBP tlb misses differently, via linear lookups + * into the prom_trans[] array. So that specific problem no + * longer exists. Yet, unfortunately there are still some issues + * preventing trampoline.S from using this code... ho hum. + */ + .globl setup_trap_table +setup_trap_table: + save %sp, -192, %sp + + /* Force interrupts to be disabled. */ + rdpr %pstate, %l0 + andn %l0, PSTATE_IE, %o1 + wrpr %o1, 0x0, %pstate + rdpr %pil, %l1 + wrpr %g0, PIL_NORMAL_MAX, %pil + + /* Make the firmware call to jump over to the Linux trap table. */ + sethi %hi(is_sun4v), %o0 + lduw [%o0 + %lo(is_sun4v)], %o0 + brz,pt %o0, 1f + nop + + TRAP_LOAD_TRAP_BLOCK(%g2, %g3) + add %g2, TRAP_PER_CPU_FAULT_INFO, %g2 + stxa %g2, [%g0] ASI_SCRATCHPAD + + /* Compute physical address: + * + * paddr = kern_base + (mmfsa_vaddr - KERNBASE) + */ + sethi %hi(KERNBASE), %g3 + sub %g2, %g3, %g2 + sethi %hi(kern_base), %g3 + ldx [%g3 + %lo(kern_base)], %g3 + add %g2, %g3, %o1 + sethi %hi(sparc64_ttable_tl0), %o0 + + set prom_set_trap_table_name, %g2 + stx %g2, [%sp + 2047 + 128 + 0x00] + mov 2, %g2 + stx %g2, [%sp + 2047 + 128 + 0x08] + mov 0, %g2 + stx %g2, [%sp + 2047 + 128 + 0x10] + stx %o0, [%sp + 2047 + 128 + 0x18] + stx %o1, [%sp + 2047 + 128 + 0x20] + sethi %hi(p1275buf), %g2 + or %g2, %lo(p1275buf), %g2 + ldx [%g2 + 0x08], %o1 + call %o1 + add %sp, (2047 + 128), %o0 + + ba,a,pt %xcc, 2f + +1: sethi %hi(sparc64_ttable_tl0), %o0 + set prom_set_trap_table_name, %g2 + stx %g2, [%sp + 2047 + 128 + 0x00] + mov 1, %g2 + stx %g2, [%sp + 2047 + 128 + 0x08] + mov 0, %g2 + stx %g2, [%sp + 2047 + 128 + 0x10] + stx %o0, [%sp + 2047 + 128 + 0x18] + sethi %hi(p1275buf), %g2 + or %g2, %lo(p1275buf), %g2 + ldx [%g2 + 0x08], %o1 + call %o1 + add %sp, (2047 + 128), %o0 + + /* Start using proper page size encodings in ctx register. */ +2: sethi %hi(sparc64_kern_pri_context), %g3 + ldx [%g3 + %lo(sparc64_kern_pri_context)], %g2 + + mov PRIMARY_CONTEXT, %g1 + +661: stxa %g2, [%g1] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g1] ASI_MMU + .previous + + membar #Sync + + BRANCH_IF_SUN4V(o2, 1f) + + /* Kill PROM timer */ + sethi %hi(0x80000000), %o2 + sllx %o2, 32, %o2 + wr %o2, 0, %tick_cmpr + + BRANCH_IF_ANY_CHEETAH(o2, o3, 1f) + + ba,a,pt %xcc, 2f + + /* Disable STICK_INT interrupts. */ +1: + sethi %hi(0x80000000), %o2 + sllx %o2, 32, %o2 + wr %o2, %asr25 + +2: + wrpr %g0, %g0, %wstate + + call init_irqwork_curcpu + nop + + /* Now we can restore interrupt state. */ + wrpr %l0, 0, %pstate + wrpr %l1, 0x0, %pil + + ret + restore + + .globl setup_tba +setup_tba: + save %sp, -192, %sp + + /* The boot processor is the only cpu which invokes this + * routine, the other cpus set things up via trampoline.S. + * So save the OBP trap table address here. + */ + rdpr %tba, %g7 + sethi %hi(prom_tba), %o1 + or %o1, %lo(prom_tba), %o1 + stx %g7, [%o1] + + call setup_trap_table + nop + + ret + restore +sparc64_boot_end: + +#include "etrap_64.S" +#include "rtrap_64.S" +#include "winfixup.S" +#include "fpu_traps.S" +#include "ivec.S" +#include "getsetcc.S" +#include "utrap.S" +#include "spiterrs.S" +#include "cherrs.S" +#include "misctrap.S" +#include "syscalls.S" +#include "helpers.S" +#include "sun4v_tlb_miss.S" +#include "sun4v_mcd.S" +#include "sun4v_ivec.S" +#include "ktlb.S" +#include "tsb.S" + +/* + * The following skip makes sure the trap table in ttable.S is aligned + * on a 32K boundary as required by the v9 specs for TBA register. + * + * We align to a 32K boundary, then we have the 32K kernel TSB, + * the 64K kernel 4MB TSB, and then the 32K aligned trap table. + */ +1: + .skip 0x4000 + _start - 1b + +! 0x0000000000408000 + + .globl swapper_tsb +swapper_tsb: + .skip (32 * 1024) + + .globl swapper_4m_tsb +swapper_4m_tsb: + .skip (64 * 1024) + +! 0x0000000000420000 + + /* Some care needs to be exercised if you try to move the + * location of the trap table relative to other things. For + * one thing there are br* instructions in some of the + * trap table entires which branch back to code in ktlb.S + * Those instructions can only handle a signed 16-bit + * displacement. + * + * There is a binutils bug (bugzilla #4558) which causes + * the relocation overflow checks for such instructions to + * not be done correctly. So bintuils will not notice the + * error and will instead write junk into the relocation and + * you'll have an unbootable kernel. + */ +#include "ttable_64.S" + +! 0x0000000000428000 + +#include "hvcalls.S" +#include "systbls_64.S" + + .data + .align 8 + .globl prom_tba, tlb_type +prom_tba: .xword 0 +tlb_type: .word 0 /* Must NOT end up in BSS */ +EXPORT_SYMBOL(tlb_type) + .section ".fixup",#alloc,#execinstr + +ENTRY(__retl_efault) + retl + mov -EFAULT, %o0 +ENDPROC(__retl_efault) + +ENTRY(__retl_o1) + retl + mov %o1, %o0 +ENDPROC(__retl_o1) + +ENTRY(__retl_o1_asi) + wr %o5, 0x0, %asi + retl + mov %o1, %o0 +ENDPROC(__retl_o1_asi) diff --git a/arch/sparc/kernel/helpers.S b/arch/sparc/kernel/helpers.S new file mode 100644 index 000000000..9b3f74706 --- /dev/null +++ b/arch/sparc/kernel/helpers.S @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + .align 32 + .globl __flushw_user + .type __flushw_user,#function +__flushw_user: + rdpr %otherwin, %g1 + brz,pn %g1, 2f + clr %g2 +1: save %sp, -128, %sp + rdpr %otherwin, %g1 + brnz,pt %g1, 1b + add %g2, 1, %g2 +1: sub %g2, 1, %g2 + brnz,pt %g2, 1b + restore %g0, %g0, %g0 +2: retl + nop + .size __flushw_user,.-__flushw_user +EXPORT_SYMBOL(__flushw_user) + + /* Flush %fp and %i7 to the stack for all register + * windows active inside of the cpu. This allows + * show_stack_trace() to avoid using an expensive + * 'flushw'. + */ + .globl stack_trace_flush + .type stack_trace_flush,#function +stack_trace_flush: + rdpr %pstate, %o0 + wrpr %o0, PSTATE_IE, %pstate + + rdpr %cwp, %g1 + rdpr %canrestore, %g2 + sub %g1, 1, %g3 + +1: brz,pn %g2, 2f + sub %g2, 1, %g2 + wrpr %g3, %cwp + stx %fp, [%sp + STACK_BIAS + RW_V9_I6] + stx %i7, [%sp + STACK_BIAS + RW_V9_I7] + ba,pt %xcc, 1b + sub %g3, 1, %g3 + +2: wrpr %g1, %cwp + wrpr %o0, %pstate + + retl + nop + .size stack_trace_flush,.-stack_trace_flush + +#ifdef CONFIG_SMP + .globl hard_smp_processor_id + .type hard_smp_processor_id,#function +hard_smp_processor_id: +#endif + .globl real_hard_smp_processor_id + .type real_hard_smp_processor_id,#function +real_hard_smp_processor_id: + __GET_CPUID(%o0) + retl + nop +#ifdef CONFIG_SMP + .size hard_smp_processor_id,.-hard_smp_processor_id +#endif + .size real_hard_smp_processor_id,.-real_hard_smp_processor_id +EXPORT_SYMBOL_GPL(real_hard_smp_processor_id) diff --git a/arch/sparc/kernel/hvapi.c b/arch/sparc/kernel/hvapi.c new file mode 100644 index 000000000..717ec7ef0 --- /dev/null +++ b/arch/sparc/kernel/hvapi.c @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: GPL-2.0 +/* hvapi.c: Hypervisor API management. + * + * Copyright (C) 2007 David S. Miller + */ +#include +#include +#include + +#include +#include + +/* If the hypervisor indicates that the API setting + * calls are unsupported, by returning HV_EBADTRAP or + * HV_ENOTSUPPORTED, we assume that API groups with the + * PRE_API flag set are major 1 minor 0. + */ +struct api_info { + unsigned long group; + unsigned long major; + unsigned long minor; + unsigned int refcnt; + unsigned int flags; +#define FLAG_PRE_API 0x00000001 +}; + +static struct api_info api_table[] = { + { .group = HV_GRP_SUN4V, .flags = FLAG_PRE_API }, + { .group = HV_GRP_CORE, .flags = FLAG_PRE_API }, + { .group = HV_GRP_INTR, }, + { .group = HV_GRP_SOFT_STATE, }, + { .group = HV_GRP_TM, }, + { .group = HV_GRP_PCI, .flags = FLAG_PRE_API }, + { .group = HV_GRP_LDOM, }, + { .group = HV_GRP_SVC_CHAN, .flags = FLAG_PRE_API }, + { .group = HV_GRP_NCS, .flags = FLAG_PRE_API }, + { .group = HV_GRP_RNG, }, + { .group = HV_GRP_PBOOT, }, + { .group = HV_GRP_TPM, }, + { .group = HV_GRP_SDIO, }, + { .group = HV_GRP_SDIO_ERR, }, + { .group = HV_GRP_REBOOT_DATA, }, + { .group = HV_GRP_ATU, .flags = FLAG_PRE_API }, + { .group = HV_GRP_DAX, }, + { .group = HV_GRP_NIAG_PERF, .flags = FLAG_PRE_API }, + { .group = HV_GRP_FIRE_PERF, }, + { .group = HV_GRP_N2_CPU, }, + { .group = HV_GRP_NIU, }, + { .group = HV_GRP_VF_CPU, }, + { .group = HV_GRP_KT_CPU, }, + { .group = HV_GRP_VT_CPU, }, + { .group = HV_GRP_T5_CPU, }, + { .group = HV_GRP_DIAG, .flags = FLAG_PRE_API }, + { .group = HV_GRP_M7_PERF, }, +}; + +static DEFINE_SPINLOCK(hvapi_lock); + +static struct api_info *__get_info(unsigned long group) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(api_table); i++) { + if (api_table[i].group == group) + return &api_table[i]; + } + return NULL; +} + +static void __get_ref(struct api_info *p) +{ + p->refcnt++; +} + +static void __put_ref(struct api_info *p) +{ + if (--p->refcnt == 0) { + unsigned long ignore; + + sun4v_set_version(p->group, 0, 0, &ignore); + p->major = p->minor = 0; + } +} + +/* Register a hypervisor API specification. It indicates the + * API group and desired major+minor. + * + * If an existing API registration exists '0' (success) will + * be returned if it is compatible with the one being registered. + * Otherwise a negative error code will be returned. + * + * Otherwise an attempt will be made to negotiate the requested + * API group/major/minor with the hypervisor, and errors returned + * if that does not succeed. + */ +int sun4v_hvapi_register(unsigned long group, unsigned long major, + unsigned long *minor) +{ + struct api_info *p; + unsigned long flags; + int ret; + + spin_lock_irqsave(&hvapi_lock, flags); + p = __get_info(group); + ret = -EINVAL; + if (p) { + if (p->refcnt) { + ret = -EINVAL; + if (p->major == major) { + *minor = p->minor; + ret = 0; + } + } else { + unsigned long actual_minor; + unsigned long hv_ret; + + hv_ret = sun4v_set_version(group, major, *minor, + &actual_minor); + ret = -EINVAL; + if (hv_ret == HV_EOK) { + *minor = actual_minor; + p->major = major; + p->minor = actual_minor; + ret = 0; + } else if (hv_ret == HV_EBADTRAP || + hv_ret == HV_ENOTSUPPORTED) { + if (p->flags & FLAG_PRE_API) { + if (major == 1) { + p->major = 1; + p->minor = 0; + *minor = 0; + ret = 0; + } + } + } + } + + if (ret == 0) + __get_ref(p); + } + spin_unlock_irqrestore(&hvapi_lock, flags); + + return ret; +} +EXPORT_SYMBOL(sun4v_hvapi_register); + +void sun4v_hvapi_unregister(unsigned long group) +{ + struct api_info *p; + unsigned long flags; + + spin_lock_irqsave(&hvapi_lock, flags); + p = __get_info(group); + if (p) + __put_ref(p); + spin_unlock_irqrestore(&hvapi_lock, flags); +} +EXPORT_SYMBOL(sun4v_hvapi_unregister); + +int sun4v_hvapi_get(unsigned long group, + unsigned long *major, + unsigned long *minor) +{ + struct api_info *p; + unsigned long flags; + int ret; + + spin_lock_irqsave(&hvapi_lock, flags); + ret = -EINVAL; + p = __get_info(group); + if (p && p->refcnt) { + *major = p->major; + *minor = p->minor; + ret = 0; + } + spin_unlock_irqrestore(&hvapi_lock, flags); + + return ret; +} +EXPORT_SYMBOL(sun4v_hvapi_get); + +void __init sun4v_hvapi_init(void) +{ + unsigned long group, major, minor; + + group = HV_GRP_SUN4V; + major = 1; + minor = 0; + if (sun4v_hvapi_register(group, major, &minor)) + goto bad; + + group = HV_GRP_CORE; + major = 1; + minor = 6; + if (sun4v_hvapi_register(group, major, &minor)) + goto bad; + + return; + +bad: + prom_printf("HVAPI: Cannot register API group " + "%lx with major(%lu) minor(%lu)\n", + group, major, minor); + prom_halt(); +} diff --git a/arch/sparc/kernel/hvcalls.S b/arch/sparc/kernel/hvcalls.S new file mode 100644 index 000000000..2f865a464 --- /dev/null +++ b/arch/sparc/kernel/hvcalls.S @@ -0,0 +1,930 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + /* %o0: devhandle + * %o1: devino + * + * returns %o0: sysino + */ +ENTRY(sun4v_devino_to_sysino) + mov HV_FAST_INTR_DEVINO2SYSINO, %o5 + ta HV_FAST_TRAP + retl + mov %o1, %o0 +ENDPROC(sun4v_devino_to_sysino) + + /* %o0: sysino + * + * returns %o0: intr_enabled (HV_INTR_{DISABLED,ENABLED}) + */ +ENTRY(sun4v_intr_getenabled) + mov HV_FAST_INTR_GETENABLED, %o5 + ta HV_FAST_TRAP + retl + mov %o1, %o0 +ENDPROC(sun4v_intr_getenabled) + + /* %o0: sysino + * %o1: intr_enabled (HV_INTR_{DISABLED,ENABLED}) + */ +ENTRY(sun4v_intr_setenabled) + mov HV_FAST_INTR_SETENABLED, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_intr_setenabled) + + /* %o0: sysino + * + * returns %o0: intr_state (HV_INTR_STATE_*) + */ +ENTRY(sun4v_intr_getstate) + mov HV_FAST_INTR_GETSTATE, %o5 + ta HV_FAST_TRAP + retl + mov %o1, %o0 +ENDPROC(sun4v_intr_getstate) + + /* %o0: sysino + * %o1: intr_state (HV_INTR_STATE_*) + */ +ENTRY(sun4v_intr_setstate) + mov HV_FAST_INTR_SETSTATE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_intr_setstate) + + /* %o0: sysino + * + * returns %o0: cpuid + */ +ENTRY(sun4v_intr_gettarget) + mov HV_FAST_INTR_GETTARGET, %o5 + ta HV_FAST_TRAP + retl + mov %o1, %o0 +ENDPROC(sun4v_intr_gettarget) + + /* %o0: sysino + * %o1: cpuid + */ +ENTRY(sun4v_intr_settarget) + mov HV_FAST_INTR_SETTARGET, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_intr_settarget) + + /* %o0: cpuid + * %o1: pc + * %o2: rtba + * %o3: arg0 + * + * returns %o0: status + */ +ENTRY(sun4v_cpu_start) + mov HV_FAST_CPU_START, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_cpu_start) + + /* %o0: cpuid + * + * returns %o0: status + */ +ENTRY(sun4v_cpu_stop) + mov HV_FAST_CPU_STOP, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_cpu_stop) + + /* returns %o0: status */ +ENTRY(sun4v_cpu_yield) + mov HV_FAST_CPU_YIELD, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_cpu_yield) + + /* %o0: cpuid + * + * returns %o0: status + */ +ENTRY(sun4v_cpu_poke) + mov HV_FAST_CPU_POKE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_cpu_poke) + + /* %o0: type + * %o1: queue paddr + * %o2: num queue entries + * + * returns %o0: status + */ +ENTRY(sun4v_cpu_qconf) + mov HV_FAST_CPU_QCONF, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_cpu_qconf) + + /* %o0: num cpus in cpu list + * %o1: cpu list paddr + * %o2: mondo block paddr + * + * returns %o0: status + */ +ENTRY(sun4v_cpu_mondo_send) + mov HV_FAST_CPU_MONDO_SEND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_cpu_mondo_send) + + /* %o0: CPU ID + * + * returns %o0: -status if status non-zero, else + * %o0: cpu state as HV_CPU_STATE_* + */ +ENTRY(sun4v_cpu_state) + mov HV_FAST_CPU_STATE, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 1f + sub %g0, %o0, %o0 + mov %o1, %o0 +1: retl + nop +ENDPROC(sun4v_cpu_state) + + /* %o0: virtual address + * %o1: must be zero + * %o2: TTE + * %o3: HV_MMU_* flags + * + * returns %o0: status + */ +ENTRY(sun4v_mmu_map_perm_addr) + mov HV_FAST_MMU_MAP_PERM_ADDR, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_mmu_map_perm_addr) + + /* %o0: number of TSB descriptions + * %o1: TSB descriptions real address + * + * returns %o0: status + */ +ENTRY(sun4v_mmu_tsb_ctx0) + mov HV_FAST_MMU_TSB_CTX0, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_mmu_tsb_ctx0) + + /* %o0: API group number + * %o1: pointer to unsigned long major number storage + * %o2: pointer to unsigned long minor number storage + * + * returns %o0: status + */ +ENTRY(sun4v_get_version) + mov HV_CORE_GET_VER, %o5 + mov %o1, %o3 + mov %o2, %o4 + ta HV_CORE_TRAP + stx %o1, [%o3] + retl + stx %o2, [%o4] +ENDPROC(sun4v_get_version) + + /* %o0: API group number + * %o1: desired major number + * %o2: desired minor number + * %o3: pointer to unsigned long actual minor number storage + * + * returns %o0: status + */ +ENTRY(sun4v_set_version) + mov HV_CORE_SET_VER, %o5 + mov %o3, %o4 + ta HV_CORE_TRAP + retl + stx %o1, [%o4] +ENDPROC(sun4v_set_version) + + /* %o0: pointer to unsigned long time + * + * returns %o0: status + */ +ENTRY(sun4v_tod_get) + mov %o0, %o4 + mov HV_FAST_TOD_GET, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_tod_get) + + /* %o0: time + * + * returns %o0: status + */ +ENTRY(sun4v_tod_set) + mov HV_FAST_TOD_SET, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_tod_set) + + /* %o0: pointer to unsigned long status + * + * returns %o0: signed character + */ +ENTRY(sun4v_con_getchar) + mov %o0, %o4 + mov HV_FAST_CONS_GETCHAR, %o5 + clr %o0 + clr %o1 + ta HV_FAST_TRAP + stx %o0, [%o4] + retl + sra %o1, 0, %o0 +ENDPROC(sun4v_con_getchar) + + /* %o0: signed long character + * + * returns %o0: status + */ +ENTRY(sun4v_con_putchar) + mov HV_FAST_CONS_PUTCHAR, %o5 + ta HV_FAST_TRAP + retl + sra %o0, 0, %o0 +ENDPROC(sun4v_con_putchar) + + /* %o0: buffer real address + * %o1: buffer size + * %o2: pointer to unsigned long bytes_read + * + * returns %o0: status + */ +ENTRY(sun4v_con_read) + mov %o2, %o4 + mov HV_FAST_CONS_READ, %o5 + ta HV_FAST_TRAP + brnz %o0, 1f + cmp %o1, -1 /* break */ + be,a,pn %icc, 1f + mov %o1, %o0 + cmp %o1, -2 /* hup */ + be,a,pn %icc, 1f + mov %o1, %o0 + stx %o1, [%o4] +1: retl + nop +ENDPROC(sun4v_con_read) + + /* %o0: buffer real address + * %o1: buffer size + * %o2: pointer to unsigned long bytes_written + * + * returns %o0: status + */ +ENTRY(sun4v_con_write) + mov %o2, %o4 + mov HV_FAST_CONS_WRITE, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_con_write) + + /* %o0: soft state + * %o1: address of description string + * + * returns %o0: status + */ +ENTRY(sun4v_mach_set_soft_state) + mov HV_FAST_MACH_SET_SOFT_STATE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_mach_set_soft_state) + + /* %o0: exit code + * + * Does not return. + */ +ENTRY(sun4v_mach_exit) + mov HV_FAST_MACH_EXIT, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_mach_exit) + + /* %o0: buffer real address + * %o1: buffer length + * %o2: pointer to unsigned long real_buf_len + * + * returns %o0: status + */ +ENTRY(sun4v_mach_desc) + mov %o2, %o4 + mov HV_FAST_MACH_DESC, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_mach_desc) + + /* %o0: new timeout in milliseconds + * %o1: pointer to unsigned long orig_timeout + * + * returns %o0: status + */ +ENTRY(sun4v_mach_set_watchdog) + mov %o1, %o4 + mov HV_FAST_MACH_SET_WATCHDOG, %o5 + ta HV_FAST_TRAP + brnz,a,pn %o4, 0f + stx %o1, [%o4] +0: retl + nop +ENDPROC(sun4v_mach_set_watchdog) +EXPORT_SYMBOL(sun4v_mach_set_watchdog) + + /* No inputs and does not return. */ +ENTRY(sun4v_mach_sir) + mov %o1, %o4 + mov HV_FAST_MACH_SIR, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_mach_sir) + + /* %o0: channel + * %o1: ra + * %o2: num_entries + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_tx_qconf) + mov HV_FAST_LDC_TX_QCONF, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_ldc_tx_qconf) + + /* %o0: channel + * %o1: pointer to unsigned long ra + * %o2: pointer to unsigned long num_entries + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_tx_qinfo) + mov %o1, %g1 + mov %o2, %g2 + mov HV_FAST_LDC_TX_QINFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + retl + nop +ENDPROC(sun4v_ldc_tx_qinfo) + + /* %o0: channel + * %o1: pointer to unsigned long head_off + * %o2: pointer to unsigned long tail_off + * %o2: pointer to unsigned long chan_state + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_tx_get_state) + mov %o1, %g1 + mov %o2, %g2 + mov %o3, %g3 + mov HV_FAST_LDC_TX_GET_STATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + stx %o3, [%g3] + retl + nop +ENDPROC(sun4v_ldc_tx_get_state) + + /* %o0: channel + * %o1: tail_off + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_tx_set_qtail) + mov HV_FAST_LDC_TX_SET_QTAIL, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_ldc_tx_set_qtail) + + /* %o0: channel + * %o1: ra + * %o2: num_entries + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_rx_qconf) + mov HV_FAST_LDC_RX_QCONF, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_ldc_rx_qconf) + + /* %o0: channel + * %o1: pointer to unsigned long ra + * %o2: pointer to unsigned long num_entries + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_rx_qinfo) + mov %o1, %g1 + mov %o2, %g2 + mov HV_FAST_LDC_RX_QINFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + retl + nop +ENDPROC(sun4v_ldc_rx_qinfo) + + /* %o0: channel + * %o1: pointer to unsigned long head_off + * %o2: pointer to unsigned long tail_off + * %o2: pointer to unsigned long chan_state + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_rx_get_state) + mov %o1, %g1 + mov %o2, %g2 + mov %o3, %g3 + mov HV_FAST_LDC_RX_GET_STATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + stx %o3, [%g3] + retl + nop +ENDPROC(sun4v_ldc_rx_get_state) + + /* %o0: channel + * %o1: head_off + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_rx_set_qhead) + mov HV_FAST_LDC_RX_SET_QHEAD, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_ldc_rx_set_qhead) + + /* %o0: channel + * %o1: ra + * %o2: num_entries + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_set_map_table) + mov HV_FAST_LDC_SET_MAP_TABLE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_ldc_set_map_table) + + /* %o0: channel + * %o1: pointer to unsigned long ra + * %o2: pointer to unsigned long num_entries + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_get_map_table) + mov %o1, %g1 + mov %o2, %g2 + mov HV_FAST_LDC_GET_MAP_TABLE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + retl + nop +ENDPROC(sun4v_ldc_get_map_table) + + /* %o0: channel + * %o1: dir_code + * %o2: tgt_raddr + * %o3: lcl_raddr + * %o4: len + * %o5: pointer to unsigned long actual_len + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_copy) + mov %o5, %g1 + mov HV_FAST_LDC_COPY, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop +ENDPROC(sun4v_ldc_copy) + + /* %o0: channel + * %o1: cookie + * %o2: pointer to unsigned long ra + * %o3: pointer to unsigned long perm + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_mapin) + mov %o2, %g1 + mov %o3, %g2 + mov HV_FAST_LDC_MAPIN, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + stx %o2, [%g2] + retl + nop +ENDPROC(sun4v_ldc_mapin) + + /* %o0: ra + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_unmap) + mov HV_FAST_LDC_UNMAP, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_ldc_unmap) + + /* %o0: channel + * %o1: cookie + * %o2: mte_cookie + * + * returns %o0: status + */ +ENTRY(sun4v_ldc_revoke) + mov HV_FAST_LDC_REVOKE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_ldc_revoke) + + /* %o0: device handle + * %o1: device INO + * %o2: pointer to unsigned long cookie + * + * returns %o0: status + */ +ENTRY(sun4v_vintr_get_cookie) + mov %o2, %g1 + mov HV_FAST_VINTR_GET_COOKIE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop +ENDPROC(sun4v_vintr_get_cookie) + + /* %o0: device handle + * %o1: device INO + * %o2: cookie + * + * returns %o0: status + */ +ENTRY(sun4v_vintr_set_cookie) + mov HV_FAST_VINTR_SET_COOKIE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_vintr_set_cookie) + + /* %o0: device handle + * %o1: device INO + * %o2: pointer to unsigned long valid_state + * + * returns %o0: status + */ +ENTRY(sun4v_vintr_get_valid) + mov %o2, %g1 + mov HV_FAST_VINTR_GET_VALID, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop +ENDPROC(sun4v_vintr_get_valid) + + /* %o0: device handle + * %o1: device INO + * %o2: valid_state + * + * returns %o0: status + */ +ENTRY(sun4v_vintr_set_valid) + mov HV_FAST_VINTR_SET_VALID, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_vintr_set_valid) + + /* %o0: device handle + * %o1: device INO + * %o2: pointer to unsigned long state + * + * returns %o0: status + */ +ENTRY(sun4v_vintr_get_state) + mov %o2, %g1 + mov HV_FAST_VINTR_GET_STATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop +ENDPROC(sun4v_vintr_get_state) + + /* %o0: device handle + * %o1: device INO + * %o2: state + * + * returns %o0: status + */ +ENTRY(sun4v_vintr_set_state) + mov HV_FAST_VINTR_SET_STATE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_vintr_set_state) + + /* %o0: device handle + * %o1: device INO + * %o2: pointer to unsigned long cpuid + * + * returns %o0: status + */ +ENTRY(sun4v_vintr_get_target) + mov %o2, %g1 + mov HV_FAST_VINTR_GET_TARGET, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1] + retl + nop +ENDPROC(sun4v_vintr_get_target) + + /* %o0: device handle + * %o1: device INO + * %o2: cpuid + * + * returns %o0: status + */ +ENTRY(sun4v_vintr_set_target) + mov HV_FAST_VINTR_SET_TARGET, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_vintr_set_target) + + /* %o0: NCS sub-function + * %o1: sub-function arg real-address + * %o2: sub-function arg size + * + * returns %o0: status + */ +ENTRY(sun4v_ncs_request) + mov HV_FAST_NCS_REQUEST, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_ncs_request) + +ENTRY(sun4v_svc_send) + save %sp, -192, %sp + mov %i0, %o0 + mov %i1, %o1 + mov %i2, %o2 + mov HV_FAST_SVC_SEND, %o5 + ta HV_FAST_TRAP + stx %o1, [%i3] + ret + restore +ENDPROC(sun4v_svc_send) + +ENTRY(sun4v_svc_recv) + save %sp, -192, %sp + mov %i0, %o0 + mov %i1, %o1 + mov %i2, %o2 + mov HV_FAST_SVC_RECV, %o5 + ta HV_FAST_TRAP + stx %o1, [%i3] + ret + restore +ENDPROC(sun4v_svc_recv) + +ENTRY(sun4v_svc_getstatus) + mov HV_FAST_SVC_GETSTATUS, %o5 + mov %o1, %o4 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_svc_getstatus) + +ENTRY(sun4v_svc_setstatus) + mov HV_FAST_SVC_SETSTATUS, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_svc_setstatus) + +ENTRY(sun4v_svc_clrstatus) + mov HV_FAST_SVC_CLRSTATUS, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_svc_clrstatus) + +ENTRY(sun4v_mmustat_conf) + mov %o1, %o4 + mov HV_FAST_MMUSTAT_CONF, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_mmustat_conf) + +ENTRY(sun4v_mmustat_info) + mov %o0, %o4 + mov HV_FAST_MMUSTAT_INFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_mmustat_info) + +ENTRY(sun4v_mmu_demap_all) + clr %o0 + clr %o1 + mov HV_MMU_ALL, %o2 + mov HV_FAST_MMU_DEMAP_ALL, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_mmu_demap_all) + +ENTRY(sun4v_niagara_getperf) + mov %o0, %o4 + mov HV_FAST_GET_PERFREG, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_niagara_getperf) +EXPORT_SYMBOL(sun4v_niagara_getperf) + +ENTRY(sun4v_niagara_setperf) + mov HV_FAST_SET_PERFREG, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_niagara_setperf) +EXPORT_SYMBOL(sun4v_niagara_setperf) + +ENTRY(sun4v_niagara2_getperf) + mov %o0, %o4 + mov HV_FAST_N2_GET_PERFREG, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_niagara2_getperf) +EXPORT_SYMBOL(sun4v_niagara2_getperf) + +ENTRY(sun4v_niagara2_setperf) + mov HV_FAST_N2_SET_PERFREG, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_niagara2_setperf) +EXPORT_SYMBOL(sun4v_niagara2_setperf) + +ENTRY(sun4v_reboot_data_set) + mov HV_FAST_REBOOT_DATA_SET, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_reboot_data_set) + +ENTRY(sun4v_vt_get_perfreg) + mov %o1, %o4 + mov HV_FAST_VT_GET_PERFREG, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_vt_get_perfreg) + +ENTRY(sun4v_vt_set_perfreg) + mov HV_FAST_VT_SET_PERFREG, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_vt_set_perfreg) + +ENTRY(sun4v_t5_get_perfreg) + mov %o1, %o4 + mov HV_FAST_T5_GET_PERFREG, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_t5_get_perfreg) + +ENTRY(sun4v_t5_set_perfreg) + mov HV_FAST_T5_SET_PERFREG, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_t5_set_perfreg) + +ENTRY(sun4v_m7_get_perfreg) + mov %o1, %o4 + mov HV_FAST_M7_GET_PERFREG, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + nop +ENDPROC(sun4v_m7_get_perfreg) + +ENTRY(sun4v_m7_set_perfreg) + mov HV_FAST_M7_SET_PERFREG, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(sun4v_m7_set_perfreg) + + /* %o0: address of CCB array + * %o1: size (in bytes) of CCB array + * %o2: flags + * %o3: reserved + * + * returns: + * %o0: status + * %o1: size (in bytes) of the CCB array that was accepted + * %o2: status data + * %o3: reserved + */ +ENTRY(sun4v_ccb_submit) + mov %o5, %g1 + mov HV_CCB_SUBMIT, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + retl + stx %o2, [%g1] +ENDPROC(sun4v_ccb_submit) +EXPORT_SYMBOL(sun4v_ccb_submit) + + /* %o0: completion area ra for the ccb to get info + * + * returns: + * %o0: status + * %o1: CCB state + * %o2: position + * %o3: dax unit + * %o4: queue + */ +ENTRY(sun4v_ccb_info) + mov %o1, %g1 + mov HV_CCB_INFO, %o5 + ta HV_FAST_TRAP + sth %o1, [%g1 + CCB_INFO_OFFSET_CCB_STATE] + sth %o2, [%g1 + CCB_INFO_OFFSET_QUEUE_POS] + sth %o3, [%g1 + CCB_INFO_OFFSET_DAX_UNIT] + retl + sth %o4, [%g1 + CCB_INFO_OFFSET_QUEUE_NUM] +ENDPROC(sun4v_ccb_info) +EXPORT_SYMBOL(sun4v_ccb_info) + + /* %o0: completion area ra for the ccb to kill + * + * returns: + * %o0: status + * %o1: result of the kill + */ +ENTRY(sun4v_ccb_kill) + mov %o1, %g1 + mov HV_CCB_KILL, %o5 + ta HV_FAST_TRAP + retl + sth %o1, [%g1] +ENDPROC(sun4v_ccb_kill) +EXPORT_SYMBOL(sun4v_ccb_kill) diff --git a/arch/sparc/kernel/hvtramp.S b/arch/sparc/kernel/hvtramp.S new file mode 100644 index 000000000..f39220471 --- /dev/null +++ b/arch/sparc/kernel/hvtramp.S @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* hvtramp.S: Hypervisor start-cpu trampoline code. + * + * Copyright (C) 2007, 2008 David S. Miller + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + .align 8 + .globl hv_cpu_startup, hv_cpu_startup_end + + /* This code executes directly out of the hypervisor + * with physical addressing (va==pa). %o0 contains + * our client argument which for Linux points to + * a descriptor data structure which defines the + * MMU entries we need to load up. + * + * After we set things up we enable the MMU and call + * into the kernel. + * + * First setup basic privileged cpu state. + */ +hv_cpu_startup: + SET_GL(0) + wrpr %g0, PIL_NORMAL_MAX, %pil + wrpr %g0, 0, %canrestore + wrpr %g0, 0, %otherwin + wrpr %g0, 6, %cansave + wrpr %g0, 6, %cleanwin + wrpr %g0, 0, %cwp + wrpr %g0, 0, %wstate + wrpr %g0, 0, %tl + + sethi %hi(sparc64_ttable_tl0), %g1 + wrpr %g1, %tba + + mov %o0, %l0 + + lduw [%l0 + HVTRAMP_DESCR_CPU], %g1 + mov SCRATCHPAD_CPUID, %g2 + stxa %g1, [%g2] ASI_SCRATCHPAD + + ldx [%l0 + HVTRAMP_DESCR_FAULT_INFO_VA], %g2 + stxa %g2, [%g0] ASI_SCRATCHPAD + + mov 0, %l1 + lduw [%l0 + HVTRAMP_DESCR_NUM_MAPPINGS], %l2 + add %l0, HVTRAMP_DESCR_MAPS, %l3 + +1: ldx [%l3 + HVTRAMP_MAPPING_VADDR], %o0 + clr %o1 + ldx [%l3 + HVTRAMP_MAPPING_TTE], %o2 + mov HV_MMU_IMMU | HV_MMU_DMMU, %o3 + mov HV_FAST_MMU_MAP_PERM_ADDR, %o5 + ta HV_FAST_TRAP + + brnz,pn %o0, 80f + nop + + add %l1, 1, %l1 + cmp %l1, %l2 + blt,a,pt %xcc, 1b + add %l3, HVTRAMP_MAPPING_SIZE, %l3 + + ldx [%l0 + HVTRAMP_DESCR_FAULT_INFO_PA], %o0 + mov HV_FAST_MMU_FAULT_AREA_CONF, %o5 + ta HV_FAST_TRAP + + brnz,pn %o0, 80f + nop + + wrpr %g0, (PSTATE_PRIV | PSTATE_PEF), %pstate + + ldx [%l0 + HVTRAMP_DESCR_THREAD_REG], %l6 + + mov 1, %o0 + set 1f, %o1 + mov HV_FAST_MMU_ENABLE, %o5 + ta HV_FAST_TRAP + + ba,pt %xcc, 80f + nop + +1: + wr %g0, 0, %fprs + wr %g0, ASI_P, %asi + + mov PRIMARY_CONTEXT, %g7 + stxa %g0, [%g7] ASI_MMU + membar #Sync + + mov SECONDARY_CONTEXT, %g7 + stxa %g0, [%g7] ASI_MMU + membar #Sync + + mov %l6, %g6 + ldx [%g6 + TI_TASK], %g4 + + mov 1, %g5 + sllx %g5, THREAD_SHIFT, %g5 + sub %g5, (STACKFRAME_SZ + STACK_BIAS), %g5 + add %g6, %g5, %sp + + call init_irqwork_curcpu + nop + call hard_smp_processor_id + nop + + call sun4v_register_mondo_queues + nop + + call init_cur_cpu_trap + mov %g6, %o0 + + wrpr %g0, (PSTATE_PRIV | PSTATE_PEF | PSTATE_IE), %pstate + + call smp_callin + nop + + call cpu_panic + nop + +80: ba,pt %xcc, 80b + nop + + .align 8 +hv_cpu_startup_end: diff --git a/arch/sparc/kernel/idprom.c b/arch/sparc/kernel/idprom.c new file mode 100644 index 000000000..d6c46d512 --- /dev/null +++ b/arch/sparc/kernel/idprom.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * idprom.c: Routines to load the idprom into kernel addresses and + * interpret the data contained within. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include + +#include +#include + +struct idprom *idprom; +EXPORT_SYMBOL(idprom); + +static struct idprom idprom_buffer; + +#ifdef CONFIG_SPARC32 +#include /* Fun with Sun released architectures. */ + +/* Here is the master table of Sun machines which use some implementation + * of the Sparc CPU and have a meaningful IDPROM machtype value that we + * know about. See asm-sparc/machines.h for empirical constants. + */ +static struct Sun_Machine_Models Sun_Machines[] = { +/* First, Leon */ +{ .name = "Leon3 System-on-a-Chip", .id_machtype = (M_LEON | M_LEON3_SOC) }, +/* Finally, early Sun4m's */ +{ .name = "Sun4m SparcSystem600", .id_machtype = (SM_SUN4M | SM_4M_SS60) }, +{ .name = "Sun4m SparcStation10/20", .id_machtype = (SM_SUN4M | SM_4M_SS50) }, +{ .name = "Sun4m SparcStation5", .id_machtype = (SM_SUN4M | SM_4M_SS40) }, +/* One entry for the OBP arch's which are sun4d, sun4e, and newer sun4m's */ +{ .name = "Sun4M OBP based system", .id_machtype = (SM_SUN4M_OBP | 0x0) } }; + +static void __init display_system_type(unsigned char machtype) +{ + char sysname[128]; + register int i; + + for (i = 0; i < ARRAY_SIZE(Sun_Machines); i++) { + if (Sun_Machines[i].id_machtype == machtype) { + if (machtype != (SM_SUN4M_OBP | 0x00) || + prom_getproperty(prom_root_node, "banner-name", + sysname, sizeof(sysname)) <= 0) + printk(KERN_WARNING "TYPE: %s\n", + Sun_Machines[i].name); + else + printk(KERN_WARNING "TYPE: %s\n", sysname); + return; + } + } + + prom_printf("IDPROM: Warning, bogus id_machtype value, 0x%x\n", machtype); +} +#else +static void __init display_system_type(unsigned char machtype) +{ +} +#endif + +unsigned char *arch_get_platform_mac_address(void) +{ + return idprom->id_ethaddr; +} + +/* Calculate the IDPROM checksum (xor of the data bytes). */ +static unsigned char __init calc_idprom_cksum(struct idprom *idprom) +{ + unsigned char cksum, i, *ptr = (unsigned char *)idprom; + + for (i = cksum = 0; i <= 0x0E; i++) + cksum ^= *ptr++; + + return cksum; +} + +/* Create a local IDPROM copy, verify integrity, and display information. */ +void __init idprom_init(void) +{ + prom_get_idprom((char *) &idprom_buffer, sizeof(idprom_buffer)); + + idprom = &idprom_buffer; + + if (idprom->id_format != 0x01) + prom_printf("IDPROM: Warning, unknown format type!\n"); + + if (idprom->id_cksum != calc_idprom_cksum(idprom)) + prom_printf("IDPROM: Warning, checksum failure (nvram=%x, calc=%x)!\n", + idprom->id_cksum, calc_idprom_cksum(idprom)); + + display_system_type(idprom->id_machtype); + + printk(KERN_WARNING "Ethernet address: %pM\n", idprom->id_ethaddr); +} diff --git a/arch/sparc/kernel/iommu-common.c b/arch/sparc/kernel/iommu-common.c new file mode 100644 index 000000000..23ca75f09 --- /dev/null +++ b/arch/sparc/kernel/iommu-common.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * IOMMU mmap management and range allocation functions. + * Based almost entirely upon the powerpc iommu allocator. + */ + +#include +#include +#include +#include +#include +#include +#include + +static unsigned long iommu_large_alloc = 15; + +static DEFINE_PER_CPU(unsigned int, iommu_hash_common); + +static inline bool need_flush(struct iommu_map_table *iommu) +{ + return ((iommu->flags & IOMMU_NEED_FLUSH) != 0); +} + +static inline void set_flush(struct iommu_map_table *iommu) +{ + iommu->flags |= IOMMU_NEED_FLUSH; +} + +static inline void clear_flush(struct iommu_map_table *iommu) +{ + iommu->flags &= ~IOMMU_NEED_FLUSH; +} + +static void setup_iommu_pool_hash(void) +{ + unsigned int i; + static bool do_once; + + if (do_once) + return; + do_once = true; + for_each_possible_cpu(i) + per_cpu(iommu_hash_common, i) = hash_32(i, IOMMU_POOL_HASHBITS); +} + +/* + * Initialize iommu_pool entries for the iommu_map_table. `num_entries' + * is the number of table entries. If `large_pool' is set to true, + * the top 1/4 of the table will be set aside for pool allocations + * of more than iommu_large_alloc pages. + */ +void iommu_tbl_pool_init(struct iommu_map_table *iommu, + unsigned long num_entries, + u32 table_shift, + void (*lazy_flush)(struct iommu_map_table *), + bool large_pool, u32 npools, + bool skip_span_boundary_check) +{ + unsigned int start, i; + struct iommu_pool *p = &(iommu->large_pool); + + setup_iommu_pool_hash(); + if (npools == 0) + iommu->nr_pools = IOMMU_NR_POOLS; + else + iommu->nr_pools = npools; + BUG_ON(npools > IOMMU_NR_POOLS); + + iommu->table_shift = table_shift; + iommu->lazy_flush = lazy_flush; + start = 0; + if (skip_span_boundary_check) + iommu->flags |= IOMMU_NO_SPAN_BOUND; + if (large_pool) + iommu->flags |= IOMMU_HAS_LARGE_POOL; + + if (!large_pool) + iommu->poolsize = num_entries/iommu->nr_pools; + else + iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools; + for (i = 0; i < iommu->nr_pools; i++) { + spin_lock_init(&(iommu->pools[i].lock)); + iommu->pools[i].start = start; + iommu->pools[i].hint = start; + start += iommu->poolsize; /* start for next pool */ + iommu->pools[i].end = start - 1; + } + if (!large_pool) + return; + /* initialize large_pool */ + spin_lock_init(&(p->lock)); + p->start = start; + p->hint = p->start; + p->end = num_entries; +} + +unsigned long iommu_tbl_range_alloc(struct device *dev, + struct iommu_map_table *iommu, + unsigned long npages, + unsigned long *handle, + unsigned long mask, + unsigned int align_order) +{ + unsigned int pool_hash = __this_cpu_read(iommu_hash_common); + unsigned long n, end, start, limit, boundary_size; + struct iommu_pool *pool; + int pass = 0; + unsigned int pool_nr; + unsigned int npools = iommu->nr_pools; + unsigned long flags; + bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0); + bool largealloc = (large_pool && npages > iommu_large_alloc); + unsigned long shift; + unsigned long align_mask = 0; + + if (align_order > 0) + align_mask = ~0ul >> (BITS_PER_LONG - align_order); + + /* Sanity check */ + if (unlikely(npages == 0)) { + WARN_ON_ONCE(1); + return IOMMU_ERROR_CODE; + } + + if (largealloc) { + pool = &(iommu->large_pool); + pool_nr = 0; /* to keep compiler happy */ + } else { + /* pick out pool_nr */ + pool_nr = pool_hash & (npools - 1); + pool = &(iommu->pools[pool_nr]); + } + spin_lock_irqsave(&pool->lock, flags); + + again: + if (pass == 0 && handle && *handle && + (*handle >= pool->start) && (*handle < pool->end)) + start = *handle; + else + start = pool->hint; + + limit = pool->end; + + /* The case below can happen if we have a small segment appended + * to a large, or when the previous alloc was at the very end of + * the available space. If so, go back to the beginning. If a + * flush is needed, it will get done based on the return value + * from iommu_area_alloc() below. + */ + if (start >= limit) + start = pool->start; + shift = iommu->table_map_base >> iommu->table_shift; + if (limit + shift > mask) { + limit = mask - shift + 1; + /* If we're constrained on address range, first try + * at the masked hint to avoid O(n) search complexity, + * but on second pass, start at 0 in pool 0. + */ + if ((start & mask) >= limit || pass > 0) { + spin_unlock(&(pool->lock)); + pool = &(iommu->pools[0]); + spin_lock(&(pool->lock)); + start = pool->start; + } else { + start &= mask; + } + } + + /* + * if the skip_span_boundary_check had been set during init, we set + * things up so that iommu_is_span_boundary() merely checks if the + * (index + npages) < num_tsb_entries + */ + if ((iommu->flags & IOMMU_NO_SPAN_BOUND) != 0) { + shift = 0; + boundary_size = iommu->poolsize * iommu->nr_pools; + } else { + boundary_size = dma_get_seg_boundary_nr_pages(dev, + iommu->table_shift); + } + n = iommu_area_alloc(iommu->map, limit, start, npages, shift, + boundary_size, align_mask); + if (n == -1) { + if (likely(pass == 0)) { + /* First failure, rescan from the beginning. */ + pool->hint = pool->start; + set_flush(iommu); + pass++; + goto again; + } else if (!largealloc && pass <= iommu->nr_pools) { + spin_unlock(&(pool->lock)); + pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1); + pool = &(iommu->pools[pool_nr]); + spin_lock(&(pool->lock)); + pool->hint = pool->start; + set_flush(iommu); + pass++; + goto again; + } else { + /* give up */ + n = IOMMU_ERROR_CODE; + goto bail; + } + } + if (iommu->lazy_flush && + (n < pool->hint || need_flush(iommu))) { + clear_flush(iommu); + iommu->lazy_flush(iommu); + } + + end = n + npages; + pool->hint = end; + + /* Update handle for SG allocations */ + if (handle) + *handle = end; +bail: + spin_unlock_irqrestore(&(pool->lock), flags); + + return n; +} + +static struct iommu_pool *get_pool(struct iommu_map_table *tbl, + unsigned long entry) +{ + struct iommu_pool *p; + unsigned long largepool_start = tbl->large_pool.start; + bool large_pool = ((tbl->flags & IOMMU_HAS_LARGE_POOL) != 0); + + /* The large pool is the last pool at the top of the table */ + if (large_pool && entry >= largepool_start) { + p = &tbl->large_pool; + } else { + unsigned int pool_nr = entry / tbl->poolsize; + + BUG_ON(pool_nr >= tbl->nr_pools); + p = &tbl->pools[pool_nr]; + } + return p; +} + +/* Caller supplies the index of the entry into the iommu map table + * itself when the mapping from dma_addr to the entry is not the + * default addr->entry mapping below. + */ +void iommu_tbl_range_free(struct iommu_map_table *iommu, u64 dma_addr, + unsigned long npages, unsigned long entry) +{ + struct iommu_pool *pool; + unsigned long flags; + unsigned long shift = iommu->table_shift; + + if (entry == IOMMU_ERROR_CODE) /* use default addr->entry mapping */ + entry = (dma_addr - iommu->table_map_base) >> shift; + pool = get_pool(iommu, entry); + + spin_lock_irqsave(&(pool->lock), flags); + bitmap_clear(iommu->map, entry, npages); + spin_unlock_irqrestore(&(pool->lock), flags); +} diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c new file mode 100644 index 000000000..7613ab0ff --- /dev/null +++ b/arch/sparc/kernel/iommu.c @@ -0,0 +1,776 @@ +// SPDX-License-Identifier: GPL-2.0 +/* iommu.c: Generic sparc64 IOMMU support. + * + * Copyright (C) 1999, 2007, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1999, 2000 Jakub Jelinek (jakub@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_PCI +#include +#endif + +#include + +#include "iommu_common.h" +#include "kernel.h" + +#define STC_CTXMATCH_ADDR(STC, CTX) \ + ((STC)->strbuf_ctxmatch_base + ((CTX) << 3)) +#define STC_FLUSHFLAG_INIT(STC) \ + (*((STC)->strbuf_flushflag) = 0UL) +#define STC_FLUSHFLAG_SET(STC) \ + (*((STC)->strbuf_flushflag) != 0UL) + +#define iommu_read(__reg) \ +({ u64 __ret; \ + __asm__ __volatile__("ldxa [%1] %2, %0" \ + : "=r" (__ret) \ + : "r" (__reg), "i" (ASI_PHYS_BYPASS_EC_E) \ + : "memory"); \ + __ret; \ +}) +#define iommu_write(__reg, __val) \ + __asm__ __volatile__("stxa %0, [%1] %2" \ + : /* no outputs */ \ + : "r" (__val), "r" (__reg), \ + "i" (ASI_PHYS_BYPASS_EC_E)) + +/* Must be invoked under the IOMMU lock. */ +static void iommu_flushall(struct iommu_map_table *iommu_map_table) +{ + struct iommu *iommu = container_of(iommu_map_table, struct iommu, tbl); + if (iommu->iommu_flushinv) { + iommu_write(iommu->iommu_flushinv, ~(u64)0); + } else { + unsigned long tag; + int entry; + + tag = iommu->iommu_tags; + for (entry = 0; entry < 16; entry++) { + iommu_write(tag, 0); + tag += 8; + } + + /* Ensure completion of previous PIO writes. */ + (void) iommu_read(iommu->write_complete_reg); + } +} + +#define IOPTE_CONSISTENT(CTX) \ + (IOPTE_VALID | IOPTE_CACHE | \ + (((CTX) << 47) & IOPTE_CONTEXT)) + +#define IOPTE_STREAMING(CTX) \ + (IOPTE_CONSISTENT(CTX) | IOPTE_STBUF) + +/* Existing mappings are never marked invalid, instead they + * are pointed to a dummy page. + */ +#define IOPTE_IS_DUMMY(iommu, iopte) \ + ((iopte_val(*iopte) & IOPTE_PAGE) == (iommu)->dummy_page_pa) + +static inline void iopte_make_dummy(struct iommu *iommu, iopte_t *iopte) +{ + unsigned long val = iopte_val(*iopte); + + val &= ~IOPTE_PAGE; + val |= iommu->dummy_page_pa; + + iopte_val(*iopte) = val; +} + +int iommu_table_init(struct iommu *iommu, int tsbsize, + u32 dma_offset, u32 dma_addr_mask, + int numa_node) +{ + unsigned long i, order, sz, num_tsb_entries; + struct page *page; + + num_tsb_entries = tsbsize / sizeof(iopte_t); + + /* Setup initial software IOMMU state. */ + spin_lock_init(&iommu->lock); + iommu->ctx_lowest_free = 1; + iommu->tbl.table_map_base = dma_offset; + iommu->dma_addr_mask = dma_addr_mask; + + /* Allocate and initialize the free area map. */ + sz = num_tsb_entries / 8; + sz = (sz + 7UL) & ~7UL; + iommu->tbl.map = kzalloc_node(sz, GFP_KERNEL, numa_node); + if (!iommu->tbl.map) + return -ENOMEM; + + iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, + (tlb_type != hypervisor ? iommu_flushall : NULL), + false, 1, false); + + /* Allocate and initialize the dummy page which we + * set inactive IO PTEs to point to. + */ + page = alloc_pages_node(numa_node, GFP_KERNEL, 0); + if (!page) { + printk(KERN_ERR "IOMMU: Error, gfp(dummy_page) failed.\n"); + goto out_free_map; + } + iommu->dummy_page = (unsigned long) page_address(page); + memset((void *)iommu->dummy_page, 0, PAGE_SIZE); + iommu->dummy_page_pa = (unsigned long) __pa(iommu->dummy_page); + + /* Now allocate and setup the IOMMU page table itself. */ + order = get_order(tsbsize); + page = alloc_pages_node(numa_node, GFP_KERNEL, order); + if (!page) { + printk(KERN_ERR "IOMMU: Error, gfp(tsb) failed.\n"); + goto out_free_dummy_page; + } + iommu->page_table = (iopte_t *)page_address(page); + + for (i = 0; i < num_tsb_entries; i++) + iopte_make_dummy(iommu, &iommu->page_table[i]); + + return 0; + +out_free_dummy_page: + free_page(iommu->dummy_page); + iommu->dummy_page = 0UL; + +out_free_map: + kfree(iommu->tbl.map); + iommu->tbl.map = NULL; + + return -ENOMEM; +} + +static inline iopte_t *alloc_npages(struct device *dev, + struct iommu *iommu, + unsigned long npages) +{ + unsigned long entry; + + entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, + (unsigned long)(-1), 0); + if (unlikely(entry == IOMMU_ERROR_CODE)) + return NULL; + + return iommu->page_table + entry; +} + +static int iommu_alloc_ctx(struct iommu *iommu) +{ + int lowest = iommu->ctx_lowest_free; + int n = find_next_zero_bit(iommu->ctx_bitmap, IOMMU_NUM_CTXS, lowest); + + if (unlikely(n == IOMMU_NUM_CTXS)) { + n = find_next_zero_bit(iommu->ctx_bitmap, lowest, 1); + if (unlikely(n == lowest)) { + printk(KERN_WARNING "IOMMU: Ran out of contexts.\n"); + n = 0; + } + } + if (n) + __set_bit(n, iommu->ctx_bitmap); + + return n; +} + +static inline void iommu_free_ctx(struct iommu *iommu, int ctx) +{ + if (likely(ctx)) { + __clear_bit(ctx, iommu->ctx_bitmap); + if (ctx < iommu->ctx_lowest_free) + iommu->ctx_lowest_free = ctx; + } +} + +static void *dma_4u_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_addrp, gfp_t gfp, + unsigned long attrs) +{ + unsigned long order, first_page; + struct iommu *iommu; + struct page *page; + int npages, nid; + iopte_t *iopte; + void *ret; + + size = IO_PAGE_ALIGN(size); + order = get_order(size); + if (order >= 10) + return NULL; + + nid = dev->archdata.numa_node; + page = alloc_pages_node(nid, gfp, order); + if (unlikely(!page)) + return NULL; + + first_page = (unsigned long) page_address(page); + memset((char *)first_page, 0, PAGE_SIZE << order); + + iommu = dev->archdata.iommu; + + iopte = alloc_npages(dev, iommu, size >> IO_PAGE_SHIFT); + + if (unlikely(iopte == NULL)) { + free_pages(first_page, order); + return NULL; + } + + *dma_addrp = (iommu->tbl.table_map_base + + ((iopte - iommu->page_table) << IO_PAGE_SHIFT)); + ret = (void *) first_page; + npages = size >> IO_PAGE_SHIFT; + first_page = __pa(first_page); + while (npages--) { + iopte_val(*iopte) = (IOPTE_CONSISTENT(0UL) | + IOPTE_WRITE | + (first_page & IOPTE_PAGE)); + iopte++; + first_page += IO_PAGE_SIZE; + } + + return ret; +} + +static void dma_4u_free_coherent(struct device *dev, size_t size, + void *cpu, dma_addr_t dvma, + unsigned long attrs) +{ + struct iommu *iommu; + unsigned long order, npages; + + npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; + iommu = dev->archdata.iommu; + + iommu_tbl_range_free(&iommu->tbl, dvma, npages, IOMMU_ERROR_CODE); + + order = get_order(size); + if (order < 10) + free_pages((unsigned long)cpu, order); +} + +static dma_addr_t dma_4u_map_phys(struct device *dev, phys_addr_t phys, + size_t sz, enum dma_data_direction direction, + unsigned long attrs) +{ + struct iommu *iommu; + struct strbuf *strbuf; + iopte_t *base; + unsigned long flags, npages, oaddr; + unsigned long i, ctx; + u32 bus_addr, ret; + unsigned long iopte_protection; + + if (unlikely(attrs & DMA_ATTR_MMIO)) + /* + * This check is included because older versions of the code + * lacked MMIO path support, and my ability to test this path + * is limited. However, from a software technical standpoint, + * there is no restriction, as the following code operates + * solely on physical addresses. + */ + goto bad_no_ctx; + + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; + + if (unlikely(direction == DMA_NONE)) + goto bad_no_ctx; + + oaddr = (unsigned long)(phys_to_virt(phys)); + npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); + npages >>= IO_PAGE_SHIFT; + + base = alloc_npages(dev, iommu, npages); + spin_lock_irqsave(&iommu->lock, flags); + ctx = 0; + if (iommu->iommu_ctxflush) + ctx = iommu_alloc_ctx(iommu); + spin_unlock_irqrestore(&iommu->lock, flags); + + if (unlikely(!base)) + goto bad; + + bus_addr = (iommu->tbl.table_map_base + + ((base - iommu->page_table) << IO_PAGE_SHIFT)); + ret = bus_addr | (oaddr & ~IO_PAGE_MASK); + if (strbuf->strbuf_enabled) + iopte_protection = IOPTE_STREAMING(ctx); + else + iopte_protection = IOPTE_CONSISTENT(ctx); + if (direction != DMA_TO_DEVICE) + iopte_protection |= IOPTE_WRITE; + + phys &= IO_PAGE_MASK; + + for (i = 0; i < npages; i++, base++, phys += IO_PAGE_SIZE) + iopte_val(*base) = iopte_protection | phys; + + return ret; + +bad: + iommu_free_ctx(iommu, ctx); +bad_no_ctx: + if (printk_ratelimit()) + WARN_ON(1); + return DMA_MAPPING_ERROR; +} + +static void strbuf_flush(struct strbuf *strbuf, struct iommu *iommu, + u32 vaddr, unsigned long ctx, unsigned long npages, + enum dma_data_direction direction) +{ + int limit; + + if (strbuf->strbuf_ctxflush && + iommu->iommu_ctxflush) { + unsigned long matchreg, flushreg; + u64 val; + + flushreg = strbuf->strbuf_ctxflush; + matchreg = STC_CTXMATCH_ADDR(strbuf, ctx); + + iommu_write(flushreg, ctx); + val = iommu_read(matchreg); + val &= 0xffff; + if (!val) + goto do_flush_sync; + + while (val) { + if (val & 0x1) + iommu_write(flushreg, ctx); + val >>= 1; + } + val = iommu_read(matchreg); + if (unlikely(val)) { + printk(KERN_WARNING "strbuf_flush: ctx flush " + "timeout matchreg[%llx] ctx[%lx]\n", + val, ctx); + goto do_page_flush; + } + } else { + unsigned long i; + + do_page_flush: + for (i = 0; i < npages; i++, vaddr += IO_PAGE_SIZE) + iommu_write(strbuf->strbuf_pflush, vaddr); + } + +do_flush_sync: + /* If the device could not have possibly put dirty data into + * the streaming cache, no flush-flag synchronization needs + * to be performed. + */ + if (direction == DMA_TO_DEVICE) + return; + + STC_FLUSHFLAG_INIT(strbuf); + iommu_write(strbuf->strbuf_fsync, strbuf->strbuf_flushflag_pa); + (void) iommu_read(iommu->write_complete_reg); + + limit = 100000; + while (!STC_FLUSHFLAG_SET(strbuf)) { + limit--; + if (!limit) + break; + udelay(1); + rmb(); + } + if (!limit) + printk(KERN_WARNING "strbuf_flush: flushflag timeout " + "vaddr[%08x] ctx[%lx] npages[%ld]\n", + vaddr, ctx, npages); +} + +static void dma_4u_unmap_phys(struct device *dev, dma_addr_t bus_addr, + size_t sz, enum dma_data_direction direction, + unsigned long attrs) +{ + struct iommu *iommu; + struct strbuf *strbuf; + iopte_t *base; + unsigned long flags, npages, ctx, i; + + if (unlikely(direction == DMA_NONE)) { + if (printk_ratelimit()) + WARN_ON(1); + return; + } + + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; + + npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); + npages >>= IO_PAGE_SHIFT; + base = iommu->page_table + + ((bus_addr - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT); + bus_addr &= IO_PAGE_MASK; + + spin_lock_irqsave(&iommu->lock, flags); + + /* Record the context, if any. */ + ctx = 0; + if (iommu->iommu_ctxflush) + ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; + + /* Step 1: Kick data out of streaming buffers if necessary. */ + if (strbuf->strbuf_enabled && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + strbuf_flush(strbuf, iommu, bus_addr, ctx, + npages, direction); + + /* Step 2: Clear out TSB entries. */ + for (i = 0; i < npages; i++) + iopte_make_dummy(iommu, base + i); + + iommu_free_ctx(iommu, ctx); + spin_unlock_irqrestore(&iommu->lock, flags); + + iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, IOMMU_ERROR_CODE); +} + +static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction direction, + unsigned long attrs) +{ + struct scatterlist *s, *outs, *segstart; + unsigned long flags, handle, prot, ctx; + dma_addr_t dma_next = 0, dma_addr; + unsigned int max_seg_size; + unsigned long seg_boundary_size; + int outcount, incount, i; + struct strbuf *strbuf; + struct iommu *iommu; + unsigned long base_shift; + + BUG_ON(direction == DMA_NONE); + + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; + if (nelems == 0 || !iommu) + return -EINVAL; + + spin_lock_irqsave(&iommu->lock, flags); + + ctx = 0; + if (iommu->iommu_ctxflush) + ctx = iommu_alloc_ctx(iommu); + + if (strbuf->strbuf_enabled) + prot = IOPTE_STREAMING(ctx); + else + prot = IOPTE_CONSISTENT(ctx); + if (direction != DMA_TO_DEVICE) + prot |= IOPTE_WRITE; + + outs = s = segstart = &sglist[0]; + outcount = 1; + incount = nelems; + handle = 0; + + /* Init first segment length for backout at failure */ + outs->dma_length = 0; + + max_seg_size = dma_get_max_seg_size(dev); + seg_boundary_size = dma_get_seg_boundary_nr_pages(dev, IO_PAGE_SHIFT); + base_shift = iommu->tbl.table_map_base >> IO_PAGE_SHIFT; + for_each_sg(sglist, s, nelems, i) { + unsigned long paddr, npages, entry, out_entry = 0, slen; + iopte_t *base; + + slen = s->length; + /* Sanity check */ + if (slen == 0) { + dma_next = 0; + continue; + } + /* Allocate iommu entries for that segment */ + paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); + npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); + entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, + &handle, (unsigned long)(-1), 0); + + /* Handle failure */ + if (unlikely(entry == IOMMU_ERROR_CODE)) { + if (printk_ratelimit()) + printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" + " npages %lx\n", iommu, paddr, npages); + goto iommu_map_failed; + } + + base = iommu->page_table + entry; + + /* Convert entry to a dma_addr_t */ + dma_addr = iommu->tbl.table_map_base + + (entry << IO_PAGE_SHIFT); + dma_addr |= (s->offset & ~IO_PAGE_MASK); + + /* Insert into HW table */ + paddr &= IO_PAGE_MASK; + while (npages--) { + iopte_val(*base) = prot | paddr; + base++; + paddr += IO_PAGE_SIZE; + } + + /* If we are in an open segment, try merging */ + if (segstart != s) { + /* We cannot merge if: + * - allocated dma_addr isn't contiguous to previous allocation + */ + if ((dma_addr != dma_next) || + (outs->dma_length + s->length > max_seg_size) || + (is_span_boundary(out_entry, base_shift, + seg_boundary_size, outs, s))) { + /* Can't merge: create a new segment */ + segstart = s; + outcount++; + outs = sg_next(outs); + } else { + outs->dma_length += s->length; + } + } + + if (segstart == s) { + /* This is a new segment, fill entries */ + outs->dma_address = dma_addr; + outs->dma_length = slen; + out_entry = entry; + } + + /* Calculate next page pointer for contiguous check */ + dma_next = dma_addr + slen; + } + + spin_unlock_irqrestore(&iommu->lock, flags); + + if (outcount < incount) { + outs = sg_next(outs); + outs->dma_length = 0; + } + + return outcount; + +iommu_map_failed: + for_each_sg(sglist, s, nelems, i) { + if (s->dma_length != 0) { + unsigned long vaddr, npages, entry, j; + iopte_t *base; + + vaddr = s->dma_address & IO_PAGE_MASK; + npages = iommu_num_pages(s->dma_address, s->dma_length, + IO_PAGE_SIZE); + + entry = (vaddr - iommu->tbl.table_map_base) + >> IO_PAGE_SHIFT; + base = iommu->page_table + entry; + + for (j = 0; j < npages; j++) + iopte_make_dummy(iommu, base + j); + + iommu_tbl_range_free(&iommu->tbl, vaddr, npages, + IOMMU_ERROR_CODE); + + s->dma_length = 0; + } + if (s == outs) + break; + } + spin_unlock_irqrestore(&iommu->lock, flags); + + return -EINVAL; +} + +/* If contexts are being used, they are the same in all of the mappings + * we make for a particular SG. + */ +static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg) +{ + unsigned long ctx = 0; + + if (iommu->iommu_ctxflush) { + iopte_t *base; + u32 bus_addr; + struct iommu_map_table *tbl = &iommu->tbl; + + bus_addr = sg->dma_address & IO_PAGE_MASK; + base = iommu->page_table + + ((bus_addr - tbl->table_map_base) >> IO_PAGE_SHIFT); + + ctx = (iopte_val(*base) & IOPTE_CONTEXT) >> 47UL; + } + return ctx; +} + +static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction direction, + unsigned long attrs) +{ + unsigned long flags, ctx; + struct scatterlist *sg; + struct strbuf *strbuf; + struct iommu *iommu; + + BUG_ON(direction == DMA_NONE); + + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; + + ctx = fetch_sg_ctx(iommu, sglist); + + spin_lock_irqsave(&iommu->lock, flags); + + sg = sglist; + while (nelems--) { + dma_addr_t dma_handle = sg->dma_address; + unsigned int len = sg->dma_length; + unsigned long npages, entry; + iopte_t *base; + int i; + + if (!len) + break; + npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); + + entry = ((dma_handle - iommu->tbl.table_map_base) + >> IO_PAGE_SHIFT); + base = iommu->page_table + entry; + + dma_handle &= IO_PAGE_MASK; + if (strbuf->strbuf_enabled && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + strbuf_flush(strbuf, iommu, dma_handle, ctx, + npages, direction); + + for (i = 0; i < npages; i++) + iopte_make_dummy(iommu, base + i); + + iommu_tbl_range_free(&iommu->tbl, dma_handle, npages, + IOMMU_ERROR_CODE); + sg = sg_next(sg); + } + + iommu_free_ctx(iommu, ctx); + + spin_unlock_irqrestore(&iommu->lock, flags); +} + +static void dma_4u_sync_single_for_cpu(struct device *dev, + dma_addr_t bus_addr, size_t sz, + enum dma_data_direction direction) +{ + struct iommu *iommu; + struct strbuf *strbuf; + unsigned long flags, ctx, npages; + + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; + + if (!strbuf->strbuf_enabled) + return; + + spin_lock_irqsave(&iommu->lock, flags); + + npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); + npages >>= IO_PAGE_SHIFT; + bus_addr &= IO_PAGE_MASK; + + /* Step 1: Record the context, if any. */ + ctx = 0; + if (iommu->iommu_ctxflush && + strbuf->strbuf_ctxflush) { + iopte_t *iopte; + struct iommu_map_table *tbl = &iommu->tbl; + + iopte = iommu->page_table + + ((bus_addr - tbl->table_map_base)>>IO_PAGE_SHIFT); + ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; + } + + /* Step 2: Kick data out of streaming buffers. */ + strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); + + spin_unlock_irqrestore(&iommu->lock, flags); +} + +static void dma_4u_sync_sg_for_cpu(struct device *dev, + struct scatterlist *sglist, int nelems, + enum dma_data_direction direction) +{ + struct iommu *iommu; + struct strbuf *strbuf; + unsigned long flags, ctx, npages, i; + struct scatterlist *sg, *sgprv; + u32 bus_addr; + + iommu = dev->archdata.iommu; + strbuf = dev->archdata.stc; + + if (!strbuf->strbuf_enabled) + return; + + spin_lock_irqsave(&iommu->lock, flags); + + /* Step 1: Record the context, if any. */ + ctx = 0; + if (iommu->iommu_ctxflush && + strbuf->strbuf_ctxflush) { + iopte_t *iopte; + struct iommu_map_table *tbl = &iommu->tbl; + + iopte = iommu->page_table + ((sglist[0].dma_address - + tbl->table_map_base) >> IO_PAGE_SHIFT); + ctx = (iopte_val(*iopte) & IOPTE_CONTEXT) >> 47UL; + } + + /* Step 2: Kick data out of streaming buffers. */ + bus_addr = sglist[0].dma_address & IO_PAGE_MASK; + sgprv = NULL; + for_each_sg(sglist, sg, nelems, i) { + if (sg->dma_length == 0) + break; + sgprv = sg; + } + + npages = (IO_PAGE_ALIGN(sgprv->dma_address + sgprv->dma_length) + - bus_addr) >> IO_PAGE_SHIFT; + strbuf_flush(strbuf, iommu, bus_addr, ctx, npages, direction); + + spin_unlock_irqrestore(&iommu->lock, flags); +} + +static int dma_4u_supported(struct device *dev, u64 device_mask) +{ + struct iommu *iommu = dev->archdata.iommu; + + if (ali_sound_dma_hack(dev, device_mask)) + return 1; + + if (device_mask < iommu->dma_addr_mask) + return 0; + return 1; +} + +static const struct dma_map_ops sun4u_dma_ops = { + .alloc = dma_4u_alloc_coherent, + .free = dma_4u_free_coherent, + .map_phys = dma_4u_map_phys, + .unmap_phys = dma_4u_unmap_phys, + .map_sg = dma_4u_map_sg, + .unmap_sg = dma_4u_unmap_sg, + .sync_single_for_cpu = dma_4u_sync_single_for_cpu, + .sync_sg_for_cpu = dma_4u_sync_sg_for_cpu, + .dma_supported = dma_4u_supported, +}; + +const struct dma_map_ops *dma_ops = &sun4u_dma_ops; +EXPORT_SYMBOL(dma_ops); diff --git a/arch/sparc/kernel/iommu_common.h b/arch/sparc/kernel/iommu_common.h new file mode 100644 index 000000000..d62ed9c56 --- /dev/null +++ b/arch/sparc/kernel/iommu_common.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* iommu_common.h: UltraSparc SBUS/PCI common iommu declarations. + * + * Copyright (C) 1999, 2008 David S. Miller (davem@davemloft.net) + */ + +#ifndef _IOMMU_COMMON_H +#define _IOMMU_COMMON_H + +#include +#include +#include +#include +#include +#include +#include + +#include + +/* + * These give mapping size of each iommu pte/tlb. + */ +#define IO_PAGE_SHIFT 13 +#define IO_PAGE_SIZE (1UL << IO_PAGE_SHIFT) +#define IO_PAGE_MASK (~(IO_PAGE_SIZE-1)) +#define IO_PAGE_ALIGN(addr) ALIGN(addr, IO_PAGE_SIZE) + +#define IO_TSB_ENTRIES (128*1024) +#define IO_TSB_SIZE (IO_TSB_ENTRIES * 8) + +/* + * This is the hardwired shift in the iotlb tag/data parts. + */ +#define IOMMU_PAGE_SHIFT 13 + +#define SG_ENT_PHYS_ADDRESS(SG) (__pa(sg_virt((SG)))) + +static inline int is_span_boundary(unsigned long entry, + unsigned long shift, + unsigned long boundary_size, + struct scatterlist *outs, + struct scatterlist *sg) +{ + unsigned long paddr = SG_ENT_PHYS_ADDRESS(outs); + int nr = iommu_num_pages(paddr, outs->dma_length + sg->length, + IO_PAGE_SIZE); + + return iommu_is_span_boundary(entry, nr, shift, boundary_size); +} + +#endif /* _IOMMU_COMMON_H */ diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c new file mode 100644 index 000000000..cbc0680a4 --- /dev/null +++ b/arch/sparc/kernel/ioport.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ioport.c: Simple io mapping allocator. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1995 Miguel de Icaza (miguel@nuclecu.unam.mx) + * + * 1996: sparc_free_io, 1999: ioremap()/iounmap() by Pete Zaitcev. + * + * 2000/01/29 + * zait: as long as pci_alloc_consistent produces something addressable, + * things are ok. + * rth: no, it is relevant, because get_free_pages returns you a + * pointer into the big page mapping + * zait: so what? + * zait: remap_it_my_way(virt_to_phys(get_free_page())) + * Hmm + * Suppose I did this remap_it_my_way(virt_to_phys(get_free_page())). + * So far so good. + * Now, driver calls pci_free_consistent(with result of + * remap_it_my_way()). + * How do you find the address to pass to free_pages()? + * zait: walk the page tables? It's only two or three level after all. + * zait: you have to walk them anyway to remove the mapping. + * Hmm + * Sounds reasonable + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include /* struct pci_dev */ +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void __iomem *_sparc_ioremap(struct resource *res, u32 bus, u32 pa, int sz); +static void __iomem *_sparc_alloc_io(unsigned int busno, unsigned long phys, + unsigned long size, char *name); +static void _sparc_free_io(struct resource *res); + +static void register_proc_sparc_ioport(void); + +/* This points to the next to use virtual memory for DVMA mappings */ +static struct resource _sparc_dvma = { + .name = "sparc_dvma", .start = DVMA_VADDR, .end = DVMA_END - 1 +}; +/* This points to the start of I/O mappings, cluable from outside. */ +/*ext*/ struct resource sparc_iomap = { + .name = "sparc_iomap", .start = IOBASE_VADDR, .end = IOBASE_END - 1 +}; + +/* + * Our mini-allocator... + * Boy this is gross! We need it because we must map I/O for + * timers and interrupt controller before the kmalloc is available. + */ + +#define XNMLN 15 +#define XNRES 10 /* SS-10 uses 8 */ + +struct xresource { + struct resource xres; /* Must be first */ + int xflag; /* 1 == used */ + char xname[XNMLN+1]; +}; + +static struct xresource xresv[XNRES]; + +static struct xresource *xres_alloc(void) { + struct xresource *xrp; + int n; + + xrp = xresv; + for (n = 0; n < XNRES; n++) { + if (xrp->xflag == 0) { + xrp->xflag = 1; + return xrp; + } + xrp++; + } + return NULL; +} + +static void xres_free(struct xresource *xrp) { + xrp->xflag = 0; +} + +/* + * These are typically used in PCI drivers + * which are trying to be cross-platform. + * + * Bus type is always zero on IIep. + */ +void __iomem *ioremap(phys_addr_t offset, size_t size) +{ + char name[14]; + + sprintf(name, "phys_%08x", (u32)offset); + return _sparc_alloc_io(0, (unsigned long)offset, size, name); +} +EXPORT_SYMBOL(ioremap); + +/* + * Complementary to ioremap(). + */ +void iounmap(volatile void __iomem *virtual) +{ + unsigned long vaddr = (unsigned long) virtual & PAGE_MASK; + struct resource *res; + + /* + * XXX Too slow. Can have 8192 DVMA pages on sun4m in the worst case. + * This probably warrants some sort of hashing. + */ + if ((res = lookup_resource(&sparc_iomap, vaddr)) == NULL) { + printk("free_io/iounmap: cannot free %lx\n", vaddr); + return; + } + _sparc_free_io(res); + + if ((char *)res >= (char*)xresv && (char *)res < (char *)&xresv[XNRES]) { + xres_free((struct xresource *)res); + } else { + kfree(res); + } +} +EXPORT_SYMBOL(iounmap); + +void __iomem *of_ioremap(struct resource *res, unsigned long offset, + unsigned long size, char *name) +{ + return _sparc_alloc_io(res->flags & 0xF, + res->start + offset, + size, name); +} +EXPORT_SYMBOL(of_ioremap); + +void of_iounmap(struct resource *res, void __iomem *base, unsigned long size) +{ + iounmap(base); +} +EXPORT_SYMBOL(of_iounmap); + +/* + * Meat of mapping + */ +static void __iomem *_sparc_alloc_io(unsigned int busno, unsigned long phys, + unsigned long size, char *name) +{ + static int printed_full; + struct xresource *xres; + struct resource *res; + char *tack; + int tlen; + void __iomem *va; /* P3 diag */ + + if (name == NULL) name = "???"; + + if ((xres = xres_alloc()) != NULL) { + tack = xres->xname; + res = &xres->xres; + } else { + if (!printed_full) { + printk("ioremap: done with statics, switching to malloc\n"); + printed_full = 1; + } + tlen = strlen(name); + tack = kmalloc(sizeof (struct resource) + tlen + 1, GFP_KERNEL); + if (tack == NULL) return NULL; + memset(tack, 0, sizeof(struct resource)); + res = (struct resource *) tack; + tack += sizeof (struct resource); + } + + strscpy(tack, name, XNMLN+1); + res->name = tack; + + va = _sparc_ioremap(res, busno, phys, size); + /* printk("ioremap(0x%x:%08lx[0x%lx])=%p\n", busno, phys, size, va); */ /* P3 diag */ + return va; +} + +/* + */ +static void __iomem * +_sparc_ioremap(struct resource *res, u32 bus, u32 pa, int sz) +{ + unsigned long offset = ((unsigned long) pa) & (~PAGE_MASK); + + if (allocate_resource(&sparc_iomap, res, + (offset + sz + PAGE_SIZE-1) & PAGE_MASK, + sparc_iomap.start, sparc_iomap.end, PAGE_SIZE, NULL, NULL) != 0) { + /* Usually we cannot see printks in this case. */ + prom_printf("alloc_io_res(%s): cannot occupy\n", + (res->name != NULL)? res->name: "???"); + prom_halt(); + } + + pa &= PAGE_MASK; + srmmu_mapiorange(bus, pa, res->start, resource_size(res)); + + return (void __iomem *)(unsigned long)(res->start + offset); +} + +/* + * Complementary to _sparc_ioremap(). + */ +static void _sparc_free_io(struct resource *res) +{ + unsigned long plen; + + plen = resource_size(res); + BUG_ON((plen & (PAGE_SIZE-1)) != 0); + srmmu_unmapiorange(res->start, plen); + release_resource(res); +} + +unsigned long sparc_dma_alloc_resource(struct device *dev, size_t len) +{ + struct resource *res; + + res = kzalloc_obj(*res); + if (!res) + return 0; + res->name = dev->of_node->full_name; + + if (allocate_resource(&_sparc_dvma, res, len, _sparc_dvma.start, + _sparc_dvma.end, PAGE_SIZE, NULL, NULL) != 0) { + printk("%s: cannot occupy 0x%zx", __func__, len); + kfree(res); + return 0; + } + + return res->start; +} + +bool sparc_dma_free_resource(void *cpu_addr, size_t size) +{ + unsigned long addr = (unsigned long)cpu_addr; + struct resource *res; + + res = lookup_resource(&_sparc_dvma, addr); + if (!res) { + printk("%s: cannot free %p\n", __func__, cpu_addr); + return false; + } + + if ((addr & (PAGE_SIZE - 1)) != 0) { + printk("%s: unaligned va %p\n", __func__, cpu_addr); + return false; + } + + size = PAGE_ALIGN(size); + if (resource_size(res) != size) { + printk("%s: region 0x%lx asked 0x%zx\n", + __func__, (long)resource_size(res), size); + return false; + } + + release_resource(res); + kfree(res); + return true; +} + +#ifdef CONFIG_SBUS + +void sbus_set_sbus64(struct device *dev, int x) +{ + printk("sbus_set_sbus64: unsupported\n"); +} +EXPORT_SYMBOL(sbus_set_sbus64); + +static int __init sparc_register_ioport(void) +{ + register_proc_sparc_ioport(); + + return 0; +} + +arch_initcall(sparc_register_ioport); + +#endif /* CONFIG_SBUS */ + +/* + * IIep is write-through, not flushing on cpu to device transfer. + * + * On LEON systems without cache snooping, the entire D-CACHE must be flushed to + * make DMA to cacheable memory coherent. + */ +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) +{ + if (dir != DMA_TO_DEVICE && + sparc_cpu_model == sparc_leon && + !sparc_leon3_snooping_enabled()) + leon_flush_dcache_all(); +} + +#ifdef CONFIG_PROC_FS + +static int sparc_io_proc_show(struct seq_file *m, void *v) +{ + struct resource *root = m->private, *r; + const char *nm; + + for (r = root->child; r != NULL; r = r->sibling) { + if ((nm = r->name) == NULL) nm = "???"; + seq_printf(m, "%016llx-%016llx: %s\n", + (unsigned long long)r->start, + (unsigned long long)r->end, nm); + } + + return 0; +} +#endif /* CONFIG_PROC_FS */ + +static void register_proc_sparc_ioport(void) +{ +#ifdef CONFIG_PROC_FS + proc_create_single_data("io_map", 0, NULL, sparc_io_proc_show, + &sparc_iomap); + proc_create_single_data("dvma_map", 0, NULL, sparc_io_proc_show, + &_sparc_dvma); +#endif +} diff --git a/arch/sparc/kernel/irq.h b/arch/sparc/kernel/irq.h new file mode 100644 index 000000000..b02026ad6 --- /dev/null +++ b/arch/sparc/kernel/irq.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include + +#include + +struct irq_bucket { + struct irq_bucket *next; + unsigned int real_irq; + unsigned int irq; + unsigned int pil; +}; + +#define SUN4M_HARD_INT(x) (0x000000001 << (x)) +#define SUN4M_SOFT_INT(x) (0x000010000 << (x)) + +#define SUN4D_MAX_BOARD 10 +#define SUN4D_MAX_IRQ ((SUN4D_MAX_BOARD + 2) << 5) + +/* Map between the irq identifier used in hw to the + * irq_bucket. The map is sufficient large to hold + * the sun4d hw identifiers. + */ +extern struct irq_bucket *irq_map[SUN4D_MAX_IRQ]; + + +/* sun4m specific type definitions */ + +/* This maps direct to CPU specific interrupt registers */ +struct sun4m_irq_percpu { + u32 pending; + u32 clear; + u32 set; +}; + +/* This maps direct to global interrupt registers */ +struct sun4m_irq_global { + u32 pending; + u32 mask; + u32 mask_clear; + u32 mask_set; + u32 interrupt_target; +}; + +extern struct sun4m_irq_percpu __iomem *sun4m_irq_percpu[SUN4M_NCPUS]; +extern struct sun4m_irq_global __iomem *sun4m_irq_global; + +/* The following definitions describe the individual platform features: */ +#define FEAT_L10_CLOCKSOURCE (1 << 0) /* L10 timer is used as a clocksource */ +#define FEAT_L10_CLOCKEVENT (1 << 1) /* L10 timer is used as a clockevent */ +#define FEAT_L14_ONESHOT (1 << 2) /* L14 timer clockevent can oneshot */ + +/* + * Platform specific configuration + * The individual platforms assign their platform + * specifics in their init functions. + */ +struct sparc_config { + void (*init_timers)(void); + unsigned int (*build_device_irq)(struct platform_device *op, + unsigned int real_irq); + + /* generic clockevent features - see FEAT_* above */ + int features; + + /* clock rate used for clock event timer */ + int clock_rate; + + /* one period for clock source timer */ + unsigned int cs_period; + + /* function to obtain offsett for cs period */ + unsigned int (*get_cycles_offset)(void); + + void (*clear_clock_irq)(void); + void (*load_profile_irq)(int cpu, unsigned int limit); +}; +extern struct sparc_config sparc_config; + +unsigned int irq_alloc(unsigned int real_irq, unsigned int pil); +void irq_link(unsigned int irq); +void irq_unlink(unsigned int irq); +void handler_irq(unsigned int pil, struct pt_regs *regs); + +unsigned long leon_get_irqmask(unsigned int irq); + +/* irq_32.c */ +void sparc_floppy_irq(int irq, void *dev_id, struct pt_regs *regs); + +/* sun4m_irq.c */ +void sun4m_nmi(struct pt_regs *regs); + +/* sun4d_irq.c */ +void sun4d_handler_irq(unsigned int pil, struct pt_regs *regs); + +#ifdef CONFIG_SMP + +/* All SUN4D IPIs are sent on this IRQ, may be shared with hard IRQs */ +#define SUN4D_IPI_IRQ 13 + +void sun4d_ipi_interrupt(void); + +#endif diff --git a/arch/sparc/kernel/irq_32.c b/arch/sparc/kernel/irq_32.c new file mode 100644 index 000000000..22db72765 --- /dev/null +++ b/arch/sparc/kernel/irq_32.c @@ -0,0 +1,363 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Interrupt request handling routines. On the + * Sparc the IRQs are basically 'cast in stone' + * and you are supposed to probe the prom's device + * node trees to find out who's got which IRQ. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1995 Miguel de Icaza (miguel@nuclecu.unam.mx) + * Copyright (C) 1995,2002 Pete A. Zaitcev (zaitcev@yahoo.com) + * Copyright (C) 1996 Dave Redman (djhr@tadpole.co.uk) + * Copyright (C) 1998-2000 Anton Blanchard (anton@samba.org) + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "kernel.h" +#include "irq.h" + +/* platform specific irq setup */ +struct sparc_config sparc_config; + +unsigned long arch_local_irq_save(void) +{ + unsigned long retval; + unsigned long tmp; + + __asm__ __volatile__( + "rd %%psr, %0\n\t" + "or %0, %2, %1\n\t" + "wr %1, 0, %%psr\n\t" + "nop; nop; nop\n" + : "=&r" (retval), "=r" (tmp) + : "i" (PSR_PIL) + : "memory"); + + return retval; +} +EXPORT_SYMBOL(arch_local_irq_save); + +void arch_local_irq_enable(void) +{ + unsigned long tmp; + + __asm__ __volatile__( + "rd %%psr, %0\n\t" + "andn %0, %1, %0\n\t" + "wr %0, 0, %%psr\n\t" + "nop; nop; nop\n" + : "=&r" (tmp) + : "i" (PSR_PIL) + : "memory"); +} +EXPORT_SYMBOL(arch_local_irq_enable); + +void arch_local_irq_restore(unsigned long old_psr) +{ + unsigned long tmp; + + __asm__ __volatile__( + "rd %%psr, %0\n\t" + "and %2, %1, %2\n\t" + "andn %0, %1, %0\n\t" + "wr %0, %2, %%psr\n\t" + "nop; nop; nop\n" + : "=&r" (tmp) + : "i" (PSR_PIL), "r" (old_psr) + : "memory"); +} +EXPORT_SYMBOL(arch_local_irq_restore); + +/* + * Dave Redman (djhr@tadpole.co.uk) + * + * IRQ numbers.. These are no longer restricted to 15.. + * + * this is done to enable SBUS cards and onboard IO to be masked + * correctly. using the interrupt level isn't good enough. + * + * For example: + * A device interrupting at sbus level6 and the Floppy both come in + * at IRQ11, but enabling and disabling them requires writing to + * different bits in the SLAVIO/SEC. + * + * As a result of these changes sun4m machines could now support + * directed CPU interrupts using the existing enable/disable irq code + * with tweaks. + * + * Sun4d complicates things even further. IRQ numbers are arbitrary + * 32-bit values in that case. Since this is similar to sparc64, + * we adopt a virtual IRQ numbering scheme as is done there. + * Virutal interrupt numbers are allocated by build_irq(). So NR_IRQS + * just becomes a limit of how many interrupt sources we can handle in + * a single system. Even fully loaded SS2000 machines top off at + * about 32 interrupt sources or so, therefore a NR_IRQS value of 64 + * is more than enough. + * + * We keep a map of per-PIL enable interrupts. These get wired + * up via the irq_chip->startup() method which gets invoked by + * the generic IRQ layer during request_irq(). + */ + + +/* Table of allocated irqs. Unused entries has irq == 0 */ +static struct irq_bucket irq_table[NR_IRQS]; +/* Protect access to irq_table */ +static DEFINE_SPINLOCK(irq_table_lock); + +/* Map between the irq identifier used in hw to the irq_bucket. */ +struct irq_bucket *irq_map[SUN4D_MAX_IRQ]; +/* Protect access to irq_map */ +static DEFINE_SPINLOCK(irq_map_lock); + +/* Allocate a new irq from the irq_table */ +unsigned int irq_alloc(unsigned int real_irq, unsigned int pil) +{ + unsigned long flags; + unsigned int i; + + spin_lock_irqsave(&irq_table_lock, flags); + for (i = 1; i < NR_IRQS; i++) { + if (irq_table[i].real_irq == real_irq && irq_table[i].pil == pil) + goto found; + } + + for (i = 1; i < NR_IRQS; i++) { + if (!irq_table[i].irq) + break; + } + + if (i < NR_IRQS) { + irq_table[i].real_irq = real_irq; + irq_table[i].irq = i; + irq_table[i].pil = pil; + } else { + printk(KERN_ERR "IRQ: Out of virtual IRQs.\n"); + i = 0; + } +found: + spin_unlock_irqrestore(&irq_table_lock, flags); + + return i; +} + +/* Based on a single pil handler_irq may need to call several + * interrupt handlers. Use irq_map as entry to irq_table, + * and let each entry in irq_table point to the next entry. + */ +void irq_link(unsigned int irq) +{ + struct irq_bucket *p; + unsigned long flags; + unsigned int pil; + + BUG_ON(irq >= NR_IRQS); + + spin_lock_irqsave(&irq_map_lock, flags); + + p = &irq_table[irq]; + pil = p->pil; + BUG_ON(pil >= SUN4D_MAX_IRQ); + p->next = irq_map[pil]; + irq_map[pil] = p; + + spin_unlock_irqrestore(&irq_map_lock, flags); +} + +void irq_unlink(unsigned int irq) +{ + struct irq_bucket *p, **pnext; + unsigned long flags; + + BUG_ON(irq >= NR_IRQS); + + spin_lock_irqsave(&irq_map_lock, flags); + + p = &irq_table[irq]; + BUG_ON(p->pil >= SUN4D_MAX_IRQ); + pnext = &irq_map[p->pil]; + while (*pnext != p) + pnext = &(*pnext)->next; + *pnext = p->next; + + spin_unlock_irqrestore(&irq_map_lock, flags); +} + + +/* /proc/interrupts printing */ +int arch_show_interrupts(struct seq_file *p, int prec) +{ + int j; + +#ifdef CONFIG_SMP + seq_printf(p, "%*s:", prec, "RES"); + for_each_online_cpu(j) + seq_put_decimal_ull_width(p, " ", cpu_data(j).irq_resched_count, 10); + seq_printf(p, " IPI rescheduling interrupts\n"); + seq_printf(p, "%*s:", prec, "CAL"); + for_each_online_cpu(j) + seq_put_decimal_ull_width(p, " ", cpu_data(j).irq_call_count, 10); + seq_printf(p, " IPI function call interrupts\n"); +#endif + seq_printf(p, "%*s:", prec, "NMI"); + for_each_online_cpu(j) + seq_put_decimal_ull_width(p, " ", cpu_data(j).counter, 10); + seq_printf(p, " Non-maskable interrupts\n"); + return 0; +} + +void handler_irq(unsigned int pil, struct pt_regs *regs) +{ + struct pt_regs *old_regs; + struct irq_bucket *p; + + BUG_ON(pil > 15); + old_regs = set_irq_regs(regs); + irq_enter(); + + p = irq_map[pil]; + while (p) { + struct irq_bucket *next = p->next; + + generic_handle_irq(p->irq); + p = next; + } + irq_exit(); + set_irq_regs(old_regs); +} + +#if defined(CONFIG_BLK_DEV_FD) || defined(CONFIG_BLK_DEV_FD_MODULE) +static unsigned int floppy_irq; + +int sparc_floppy_request_irq(unsigned int irq, irq_handler_t irq_handler) +{ + unsigned int cpu_irq; + int err; + + + err = request_irq(irq, irq_handler, 0, "floppy", NULL); + if (err) + return -1; + + /* Save for later use in floppy interrupt handler */ + floppy_irq = irq; + + cpu_irq = (irq & (NR_IRQS - 1)); + + /* Dork with trap table if we get this far. */ +#define INSTANTIATE(table) \ + table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_one = SPARC_RD_PSR_L0; \ + table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_two = \ + SPARC_BRANCH((unsigned long) floppy_hardint, \ + (unsigned long) &table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_two);\ + table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_three = SPARC_RD_WIM_L3; \ + table[SP_TRAP_IRQ1+(cpu_irq-1)].inst_four = SPARC_NOP; + + INSTANTIATE(sparc_ttable) + +#if defined CONFIG_SMP + if (sparc_cpu_model != sparc_leon) { + struct tt_entry *trap_table; + + trap_table = &trapbase_cpu1[0]; + INSTANTIATE(trap_table) + trap_table = &trapbase_cpu2[0]; + INSTANTIATE(trap_table) + trap_table = &trapbase_cpu3[0]; + INSTANTIATE(trap_table) + } +#endif +#undef INSTANTIATE + /* + * XXX Correct thing whould be to flush only I- and D-cache lines + * which contain the handler in question. But as of time of the + * writing we have no CPU-neutral interface to fine-grained flushes. + */ + flush_cache_all(); + return 0; +} +EXPORT_SYMBOL(sparc_floppy_request_irq); + +/* + * These variables are used to access state from the assembler + * interrupt handler, floppy_hardint, so we cannot put these in + * the floppy driver image because that would not work in the + * modular case. + */ +volatile unsigned char *fdc_status; +EXPORT_SYMBOL(fdc_status); + +char *pdma_vaddr; +EXPORT_SYMBOL(pdma_vaddr); + +unsigned long pdma_size; +EXPORT_SYMBOL(pdma_size); + +volatile int doing_pdma; +EXPORT_SYMBOL(doing_pdma); + +char *pdma_base; +EXPORT_SYMBOL(pdma_base); + +unsigned long pdma_areasize; +EXPORT_SYMBOL(pdma_areasize); + +/* Use the generic irq support to call floppy_interrupt + * which was setup using request_irq() in sparc_floppy_request_irq(). + * We only have one floppy interrupt so we do not need to check + * for additional handlers being wired up by irq_link() + */ +void sparc_floppy_irq(int irq, void *dev_id, struct pt_regs *regs) +{ + struct pt_regs *old_regs; + + old_regs = set_irq_regs(regs); + irq_enter(); + generic_handle_irq(floppy_irq); + irq_exit(); + set_irq_regs(old_regs); +} +#endif + +/* djhr + * This could probably be made indirect too and assigned in the CPU + * bits of the code. That would be much nicer I think and would also + * fit in with the idea of being able to tune your kernel for your machine + * by removing unrequired machine and device support. + * + */ + +void __init init_IRQ(void) +{ + switch (sparc_cpu_model) { + case sun4m: + pcic_probe(); + if (pcic_present()) + sun4m_pci_init_IRQ(); + else + sun4m_init_IRQ(); + break; + + case sun4d: + sun4d_init_IRQ(); + break; + + case sparc_leon: + leon_init_IRQ(); + break; + + default: + prom_printf("Cannot initialize IRQs on this Sun machine..."); + break; + } +} + diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c new file mode 100644 index 000000000..3f55c69d5 --- /dev/null +++ b/arch/sparc/kernel/irq_64.c @@ -0,0 +1,1154 @@ +// SPDX-License-Identifier: GPL-2.0 +/* irq.c: UltraSparc IRQ handling/init/registry. + * + * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "entry.h" +#include "cpumap.h" +#include "kstack.h" + +struct ino_bucket *ivector_table; +unsigned long ivector_table_pa; + +/* On several sun4u processors, it is illegal to mix bypass and + * non-bypass accesses. Therefore we access all INO buckets + * using bypass accesses only. + */ +static unsigned long bucket_get_chain_pa(unsigned long bucket_pa) +{ + unsigned long ret; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=&r" (ret) + : "r" (bucket_pa + + offsetof(struct ino_bucket, + __irq_chain_pa)), + "i" (ASI_PHYS_USE_EC)); + + return ret; +} + +static void bucket_clear_chain_pa(unsigned long bucket_pa) +{ + __asm__ __volatile__("stxa %%g0, [%0] %1" + : /* no outputs */ + : "r" (bucket_pa + + offsetof(struct ino_bucket, + __irq_chain_pa)), + "i" (ASI_PHYS_USE_EC)); +} + +static unsigned int bucket_get_irq(unsigned long bucket_pa) +{ + unsigned int ret; + + __asm__ __volatile__("lduwa [%1] %2, %0" + : "=&r" (ret) + : "r" (bucket_pa + + offsetof(struct ino_bucket, + __irq)), + "i" (ASI_PHYS_USE_EC)); + + return ret; +} + +static void bucket_set_irq(unsigned long bucket_pa, unsigned int irq) +{ + __asm__ __volatile__("stwa %0, [%1] %2" + : /* no outputs */ + : "r" (irq), + "r" (bucket_pa + + offsetof(struct ino_bucket, + __irq)), + "i" (ASI_PHYS_USE_EC)); +} + +#define irq_work_pa(__cpu) &(trap_block[(__cpu)].irq_worklist_pa) + +static unsigned long hvirq_major __initdata; +static int __init early_hvirq_major(char *p) +{ + int rc = kstrtoul(p, 10, &hvirq_major); + + return rc; +} +early_param("hvirq", early_hvirq_major); + +static int hv_irq_version; + +/* Major version 2.0 of HV_GRP_INTR added support for the VIRQ cookie + * based interfaces, but: + * + * 1) Several OSs, Solaris and Linux included, use them even when only + * negotiating version 1.0 (or failing to negotiate at all). So the + * hypervisor has a workaround that provides the VIRQ interfaces even + * when only verion 1.0 of the API is in use. + * + * 2) Second, and more importantly, with major version 2.0 these VIRQ + * interfaces only were actually hooked up for LDC interrupts, even + * though the Hypervisor specification clearly stated: + * + * The new interrupt API functions will be available to a guest + * when it negotiates version 2.0 in the interrupt API group 0x2. When + * a guest negotiates version 2.0, all interrupt sources will only + * support using the cookie interface, and any attempt to use the + * version 1.0 interrupt APIs numbered 0xa0 to 0xa6 will result in the + * ENOTSUPPORTED error being returned. + * + * with an emphasis on "all interrupt sources". + * + * To correct this, major version 3.0 was created which does actually + * support VIRQs for all interrupt sources (not just LDC devices). So + * if we want to move completely over the cookie based VIRQs we must + * negotiate major version 3.0 or later of HV_GRP_INTR. + */ +static bool sun4v_cookie_only_virqs(void) +{ + return hv_irq_version >= 3; +} + +static void __init irq_init_hv(void) +{ + unsigned long hv_error, major, minor = 0; + + if (tlb_type != hypervisor) + return; + + if (hvirq_major) + major = hvirq_major; + else + major = 3; + + hv_error = sun4v_hvapi_register(HV_GRP_INTR, major, &minor); + if (!hv_error) + hv_irq_version = major; + else + hv_irq_version = 1; + + pr_info("SUN4V: Using IRQ API major %d, cookie only virqs %s\n", + hv_irq_version, + str_enabled_disabled(sun4v_cookie_only_virqs())); +} + +/* This function is for the timer interrupt.*/ +int __init arch_probe_nr_irqs(void) +{ + return 1; +} + +#define DEFAULT_NUM_IVECS (0xfffU) +static unsigned int nr_ivec = DEFAULT_NUM_IVECS; +#define NUM_IVECS (nr_ivec) + +static unsigned int __init size_nr_ivec(void) +{ + if (tlb_type == hypervisor) { + switch (sun4v_chip_type) { + /* Athena's devhandle|devino is large.*/ + case SUN4V_CHIP_SPARC64X: + nr_ivec = 0xffff; + break; + } + } + return nr_ivec; +} + +struct irq_handler_data { + union { + struct { + unsigned int dev_handle; + unsigned int dev_ino; + }; + unsigned long sysino; + }; + struct ino_bucket bucket; + unsigned long iclr; + unsigned long imap; +}; + +static inline unsigned int irq_data_to_handle(struct irq_data *data) +{ + struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); + + return ihd->dev_handle; +} + +static inline unsigned int irq_data_to_ino(struct irq_data *data) +{ + struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); + + return ihd->dev_ino; +} + +static inline unsigned long irq_data_to_sysino(struct irq_data *data) +{ + struct irq_handler_data *ihd = irq_data_get_irq_handler_data(data); + + return ihd->sysino; +} + +void irq_free(unsigned int irq) +{ + void *data = irq_get_handler_data(irq); + + kfree(data); + irq_set_handler_data(irq, NULL); + irq_free_descs(irq, 1); +} + +unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino) +{ + int irq; + + irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL, NULL); + if (irq <= 0) + goto out; + + return irq; +out: + return 0; +} + +static unsigned int cookie_exists(u32 devhandle, unsigned int devino) +{ + unsigned long hv_err, cookie; + struct ino_bucket *bucket; + unsigned int irq = 0U; + + hv_err = sun4v_vintr_get_cookie(devhandle, devino, &cookie); + if (hv_err) { + pr_err("HV get cookie failed hv_err = %ld\n", hv_err); + goto out; + } + + if (cookie & ((1UL << 63UL))) { + cookie = ~cookie; + bucket = (struct ino_bucket *) __va(cookie); + irq = bucket->__irq; + } +out: + return irq; +} + +static unsigned int sysino_exists(u32 devhandle, unsigned int devino) +{ + unsigned long sysino = sun4v_devino_to_sysino(devhandle, devino); + struct ino_bucket *bucket; + unsigned int irq; + + bucket = &ivector_table[sysino]; + irq = bucket_get_irq(__pa(bucket)); + + return irq; +} + +void ack_bad_irq(unsigned int irq) +{ + pr_crit("BAD IRQ ack %d\n", irq); +} + +void irq_install_pre_handler(int irq, + void (*func)(unsigned int, void *, void *), + void *arg1, void *arg2) +{ + pr_warn("IRQ pre handler NOT supported.\n"); +} + +/* + * /proc/interrupts printing: + */ +int arch_show_interrupts(struct seq_file *p, int prec) +{ + int j; + + seq_printf(p, "%*s:", prec, "NMI"); + for_each_online_cpu(j) + seq_put_decimal_ull_width(p, " ", cpu_data(j).__nmi_count, 10); + seq_printf(p, " Non-maskable interrupts\n"); + return 0; +} + +static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid) +{ + unsigned int tid; + + if (this_is_starfire) { + tid = starfire_translate(imap, cpuid); + tid <<= IMAP_TID_SHIFT; + tid &= IMAP_TID_UPA; + } else { + if (tlb_type == cheetah || tlb_type == cheetah_plus) { + unsigned long ver; + + __asm__ ("rdpr %%ver, %0" : "=r" (ver)); + if ((ver >> 32UL) == __JALAPENO_ID || + (ver >> 32UL) == __SERRANO_ID) { + tid = cpuid << IMAP_TID_SHIFT; + tid &= IMAP_TID_JBUS; + } else { + unsigned int a = cpuid & 0x1f; + unsigned int n = (cpuid >> 5) & 0x1f; + + tid = ((a << IMAP_AID_SHIFT) | + (n << IMAP_NID_SHIFT)); + tid &= (IMAP_AID_SAFARI | + IMAP_NID_SAFARI); + } + } else { + tid = cpuid << IMAP_TID_SHIFT; + tid &= IMAP_TID_UPA; + } + } + + return tid; +} + +#ifdef CONFIG_SMP +static int irq_choose_cpu(unsigned int irq, const struct cpumask *affinity) +{ + int cpuid; + + if (cpumask_equal(affinity, cpu_online_mask)) { + cpuid = map_to_cpu(irq); + } else { + cpuid = cpumask_first_and(affinity, cpu_online_mask); + cpuid = cpuid < nr_cpu_ids ? cpuid : map_to_cpu(irq); + } + + return cpuid; +} +#else +#define irq_choose_cpu(irq, affinity) \ + real_hard_smp_processor_id() +#endif + +static void sun4u_irq_enable(struct irq_data *data) +{ + struct irq_handler_data *handler_data; + + handler_data = irq_data_get_irq_handler_data(data); + if (likely(handler_data)) { + unsigned long cpuid, imap, val; + unsigned int tid; + + cpuid = irq_choose_cpu(data->irq, + irq_data_get_affinity_mask(data)); + imap = handler_data->imap; + + tid = sun4u_compute_tid(imap, cpuid); + + val = upa_readq(imap); + val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS | + IMAP_AID_SAFARI | IMAP_NID_SAFARI); + val |= tid | IMAP_VALID; + upa_writeq(val, imap); + upa_writeq(ICLR_IDLE, handler_data->iclr); + } +} + +static int sun4u_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) +{ + struct irq_handler_data *handler_data; + + handler_data = irq_data_get_irq_handler_data(data); + if (likely(handler_data)) { + unsigned long cpuid, imap, val; + unsigned int tid; + + cpuid = irq_choose_cpu(data->irq, mask); + imap = handler_data->imap; + + tid = sun4u_compute_tid(imap, cpuid); + + val = upa_readq(imap); + val &= ~(IMAP_TID_UPA | IMAP_TID_JBUS | + IMAP_AID_SAFARI | IMAP_NID_SAFARI); + val |= tid | IMAP_VALID; + upa_writeq(val, imap); + upa_writeq(ICLR_IDLE, handler_data->iclr); + } + + return 0; +} + +/* Don't do anything. The desc->status check for IRQ_DISABLED in + * handler_irq() will skip the handler call and that will leave the + * interrupt in the sent state. The next ->enable() call will hit the + * ICLR register to reset the state machine. + * + * This scheme is necessary, instead of clearing the Valid bit in the + * IMAP register, to handle the case of IMAP registers being shared by + * multiple INOs (and thus ICLR registers). Since we use a different + * virtual IRQ for each shared IMAP instance, the generic code thinks + * there is only one user so it prematurely calls ->disable() on + * free_irq(). + * + * We have to provide an explicit ->disable() method instead of using + * NULL to get the default. The reason is that if the generic code + * sees that, it also hooks up a default ->shutdown method which + * invokes ->mask() which we do not want. See irq_chip_set_defaults(). + */ +static void sun4u_irq_disable(struct irq_data *data) +{ +} + +static void sun4u_irq_eoi(struct irq_data *data) +{ + struct irq_handler_data *handler_data; + + handler_data = irq_data_get_irq_handler_data(data); + if (likely(handler_data)) + upa_writeq(ICLR_IDLE, handler_data->iclr); +} + +static void sun4v_irq_enable(struct irq_data *data) +{ + unsigned long cpuid = irq_choose_cpu(data->irq, + irq_data_get_affinity_mask(data)); + unsigned int ino = irq_data_to_sysino(data); + int err; + + err = sun4v_intr_settarget(ino, cpuid); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): " + "err(%d)\n", ino, cpuid, err); + err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_intr_setstate(%x): " + "err(%d)\n", ino, err); + err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_intr_setenabled(%x): err(%d)\n", + ino, err); +} + +static int sun4v_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) +{ + unsigned long cpuid = irq_choose_cpu(data->irq, mask); + unsigned int ino = irq_data_to_sysino(data); + int err; + + err = sun4v_intr_settarget(ino, cpuid); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_intr_settarget(%x,%lu): " + "err(%d)\n", ino, cpuid, err); + + return 0; +} + +static void sun4v_irq_disable(struct irq_data *data) +{ + unsigned int ino = irq_data_to_sysino(data); + int err; + + err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_intr_setenabled(%x): " + "err(%d)\n", ino, err); +} + +static void sun4v_irq_eoi(struct irq_data *data) +{ + unsigned int ino = irq_data_to_sysino(data); + int err; + + err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_intr_setstate(%x): " + "err(%d)\n", ino, err); +} + +static void sun4v_virq_enable(struct irq_data *data) +{ + unsigned long dev_handle = irq_data_to_handle(data); + unsigned long dev_ino = irq_data_to_ino(data); + unsigned long cpuid; + int err; + + cpuid = irq_choose_cpu(data->irq, irq_data_get_affinity_mask(data)); + + err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): " + "err(%d)\n", + dev_handle, dev_ino, cpuid, err); + err = sun4v_vintr_set_state(dev_handle, dev_ino, + HV_INTR_STATE_IDLE); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx," + "HV_INTR_STATE_IDLE): err(%d)\n", + dev_handle, dev_ino, err); + err = sun4v_vintr_set_valid(dev_handle, dev_ino, + HV_INTR_ENABLED); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx," + "HV_INTR_ENABLED): err(%d)\n", + dev_handle, dev_ino, err); +} + +static int sun4v_virt_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) +{ + unsigned long dev_handle = irq_data_to_handle(data); + unsigned long dev_ino = irq_data_to_ino(data); + unsigned long cpuid; + int err; + + cpuid = irq_choose_cpu(data->irq, mask); + + err = sun4v_vintr_set_target(dev_handle, dev_ino, cpuid); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_vintr_set_target(%lx,%lx,%lu): " + "err(%d)\n", + dev_handle, dev_ino, cpuid, err); + + return 0; +} + +static void sun4v_virq_disable(struct irq_data *data) +{ + unsigned long dev_handle = irq_data_to_handle(data); + unsigned long dev_ino = irq_data_to_ino(data); + int err; + + + err = sun4v_vintr_set_valid(dev_handle, dev_ino, + HV_INTR_DISABLED); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx," + "HV_INTR_DISABLED): err(%d)\n", + dev_handle, dev_ino, err); +} + +static void sun4v_virq_eoi(struct irq_data *data) +{ + unsigned long dev_handle = irq_data_to_handle(data); + unsigned long dev_ino = irq_data_to_ino(data); + int err; + + err = sun4v_vintr_set_state(dev_handle, dev_ino, + HV_INTR_STATE_IDLE); + if (err != HV_EOK) + printk(KERN_ERR "sun4v_vintr_set_state(%lx,%lx," + "HV_INTR_STATE_IDLE): err(%d)\n", + dev_handle, dev_ino, err); +} + +static struct irq_chip sun4u_irq = { + .name = "sun4u", + .irq_enable = sun4u_irq_enable, + .irq_disable = sun4u_irq_disable, + .irq_eoi = sun4u_irq_eoi, + .irq_set_affinity = sun4u_set_affinity, + .flags = IRQCHIP_EOI_IF_HANDLED, +}; + +static struct irq_chip sun4v_irq = { + .name = "sun4v", + .irq_enable = sun4v_irq_enable, + .irq_disable = sun4v_irq_disable, + .irq_eoi = sun4v_irq_eoi, + .irq_set_affinity = sun4v_set_affinity, + .flags = IRQCHIP_EOI_IF_HANDLED, +}; + +static struct irq_chip sun4v_virq = { + .name = "vsun4v", + .irq_enable = sun4v_virq_enable, + .irq_disable = sun4v_virq_disable, + .irq_eoi = sun4v_virq_eoi, + .irq_set_affinity = sun4v_virt_set_affinity, + .flags = IRQCHIP_EOI_IF_HANDLED, +}; + +unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap) +{ + struct irq_handler_data *handler_data; + struct ino_bucket *bucket; + unsigned int irq; + int ino; + + BUG_ON(tlb_type == hypervisor); + + ino = (upa_readq(imap) & (IMAP_IGN | IMAP_INO)) + inofixup; + bucket = &ivector_table[ino]; + irq = bucket_get_irq(__pa(bucket)); + if (!irq) { + irq = irq_alloc(0, ino); + bucket_set_irq(__pa(bucket), irq); + irq_set_chip_and_handler_name(irq, &sun4u_irq, + handle_fasteoi_irq, "IVEC"); + } + + handler_data = irq_get_handler_data(irq); + if (unlikely(handler_data)) + goto out; + + handler_data = kzalloc_obj(struct irq_handler_data, GFP_ATOMIC); + if (unlikely(!handler_data)) { + prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n"); + prom_halt(); + } + irq_set_handler_data(irq, handler_data); + + handler_data->imap = imap; + handler_data->iclr = iclr; + +out: + return irq; +} + +static unsigned int sun4v_build_common(u32 devhandle, unsigned int devino, + void (*handler_data_init)(struct irq_handler_data *data, + u32 devhandle, unsigned int devino), + struct irq_chip *chip) +{ + struct irq_handler_data *data; + unsigned int irq; + + irq = irq_alloc(devhandle, devino); + if (!irq) + goto out; + + data = kzalloc_obj(struct irq_handler_data, GFP_ATOMIC); + if (unlikely(!data)) { + pr_err("IRQ handler data allocation failed.\n"); + irq_free(irq); + irq = 0; + goto out; + } + + irq_set_handler_data(irq, data); + handler_data_init(data, devhandle, devino); + irq_set_chip_and_handler_name(irq, chip, handle_fasteoi_irq, "IVEC"); + data->imap = ~0UL; + data->iclr = ~0UL; +out: + return irq; +} + +static unsigned long cookie_assign(unsigned int irq, u32 devhandle, + unsigned int devino) +{ + struct irq_handler_data *ihd = irq_get_handler_data(irq); + unsigned long hv_error, cookie; + + /* handler_irq needs to find the irq. cookie is seen signed in + * sun4v_dev_mondo and treated as a non ivector_table delivery. + */ + ihd->bucket.__irq = irq; + cookie = ~__pa(&ihd->bucket); + + hv_error = sun4v_vintr_set_cookie(devhandle, devino, cookie); + if (hv_error) + pr_err("HV vintr set cookie failed = %ld\n", hv_error); + + return hv_error; +} + +static void cookie_handler_data(struct irq_handler_data *data, + u32 devhandle, unsigned int devino) +{ + data->dev_handle = devhandle; + data->dev_ino = devino; +} + +static unsigned int cookie_build_irq(u32 devhandle, unsigned int devino, + struct irq_chip *chip) +{ + unsigned long hv_error; + unsigned int irq; + + irq = sun4v_build_common(devhandle, devino, cookie_handler_data, chip); + + hv_error = cookie_assign(irq, devhandle, devino); + if (hv_error) { + irq_free(irq); + irq = 0; + } + + return irq; +} + +static unsigned int sun4v_build_cookie(u32 devhandle, unsigned int devino) +{ + unsigned int irq; + + irq = cookie_exists(devhandle, devino); + if (irq) + goto out; + + irq = cookie_build_irq(devhandle, devino, &sun4v_virq); + +out: + return irq; +} + +static void sysino_set_bucket(unsigned int irq) +{ + struct irq_handler_data *ihd = irq_get_handler_data(irq); + struct ino_bucket *bucket; + unsigned long sysino; + + sysino = sun4v_devino_to_sysino(ihd->dev_handle, ihd->dev_ino); + BUG_ON(sysino >= nr_ivec); + bucket = &ivector_table[sysino]; + bucket_set_irq(__pa(bucket), irq); +} + +static void sysino_handler_data(struct irq_handler_data *data, + u32 devhandle, unsigned int devino) +{ + unsigned long sysino; + + sysino = sun4v_devino_to_sysino(devhandle, devino); + data->sysino = sysino; +} + +static unsigned int sysino_build_irq(u32 devhandle, unsigned int devino, + struct irq_chip *chip) +{ + unsigned int irq; + + irq = sun4v_build_common(devhandle, devino, sysino_handler_data, chip); + if (!irq) + goto out; + + sysino_set_bucket(irq); +out: + return irq; +} + +static int sun4v_build_sysino(u32 devhandle, unsigned int devino) +{ + int irq; + + irq = sysino_exists(devhandle, devino); + if (irq) + goto out; + + irq = sysino_build_irq(devhandle, devino, &sun4v_irq); +out: + return irq; +} + +unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino) +{ + unsigned int irq; + + if (sun4v_cookie_only_virqs()) + irq = sun4v_build_cookie(devhandle, devino); + else + irq = sun4v_build_sysino(devhandle, devino); + + return irq; +} + +unsigned int sun4v_build_virq(u32 devhandle, unsigned int devino) +{ + int irq; + + irq = cookie_build_irq(devhandle, devino, &sun4v_virq); + if (!irq) + goto out; + + /* This is borrowed from the original function. + */ + irq_set_status_flags(irq, IRQ_NOAUTOEN); + +out: + return irq; +} + +void *hardirq_stack[NR_CPUS]; +void *softirq_stack[NR_CPUS]; + +void __irq_entry handler_irq(int pil, struct pt_regs *regs) +{ + unsigned long pstate, bucket_pa; + struct pt_regs *old_regs; + void *orig_sp; + + clear_softint(1 << pil); + + old_regs = set_irq_regs(regs); + irq_enter(); + + /* Grab an atomic snapshot of the pending IVECs. */ + __asm__ __volatile__("rdpr %%pstate, %0\n\t" + "wrpr %0, %3, %%pstate\n\t" + "ldx [%2], %1\n\t" + "stx %%g0, [%2]\n\t" + "wrpr %0, 0x0, %%pstate\n\t" + : "=&r" (pstate), "=&r" (bucket_pa) + : "r" (irq_work_pa(smp_processor_id())), + "i" (PSTATE_IE) + : "memory"); + + orig_sp = set_hardirq_stack(); + + while (bucket_pa) { + unsigned long next_pa; + unsigned int irq; + + next_pa = bucket_get_chain_pa(bucket_pa); + irq = bucket_get_irq(bucket_pa); + bucket_clear_chain_pa(bucket_pa); + + generic_handle_irq(irq); + + bucket_pa = next_pa; + } + + restore_hardirq_stack(orig_sp); + + irq_exit(); + set_irq_regs(old_regs); +} + +#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK +void do_softirq_own_stack(void) +{ + void *orig_sp, *sp = softirq_stack[smp_processor_id()]; + + sp += THREAD_SIZE - 192 - STACK_BIAS; + + __asm__ __volatile__("mov %%sp, %0\n\t" + "mov %1, %%sp" + : "=&r" (orig_sp) + : "r" (sp)); + __do_softirq(); + __asm__ __volatile__("mov %0, %%sp" + : : "r" (orig_sp)); +} +#endif + +#ifdef CONFIG_HOTPLUG_CPU +void fixup_irqs(void) +{ + unsigned int irq; + + for (irq = 0; irq < NR_IRQS; irq++) { + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *data; + unsigned long flags; + + if (!desc) + continue; + data = irq_desc_get_irq_data(desc); + raw_spin_lock_irqsave(&desc->lock, flags); + if (desc->action && !irqd_is_per_cpu(data)) { + if (data->chip->irq_set_affinity) + data->chip->irq_set_affinity(data, + irq_data_get_affinity_mask(data), + false); + } + raw_spin_unlock_irqrestore(&desc->lock, flags); + } + + tick_ops->disable_irq(); +} +#endif + +struct sun5_timer { + u64 count0; + u64 limit0; + u64 count1; + u64 limit1; +}; + +static struct sun5_timer *prom_timers; +static u64 prom_limit0, prom_limit1; + +static void map_prom_timers(void) +{ + struct device_node *dp; + const unsigned int *addr; + + /* PROM timer node hangs out in the top level of device siblings... */ + dp = of_find_node_by_path("/"); + dp = dp->child; + while (dp) { + if (of_node_name_eq(dp, "counter-timer")) + break; + dp = dp->sibling; + } + + /* Assume if node is not present, PROM uses different tick mechanism + * which we should not care about. + */ + if (!dp) { + prom_timers = (struct sun5_timer *) 0; + return; + } + + /* If PROM is really using this, it must be mapped by him. */ + addr = of_get_property(dp, "address", NULL); + if (!addr) { + prom_printf("PROM does not have timer mapped, trying to continue.\n"); + prom_timers = (struct sun5_timer *) 0; + return; + } + prom_timers = (struct sun5_timer *) ((unsigned long)addr[0]); +} + +static void kill_prom_timer(void) +{ + if (!prom_timers) + return; + + /* Save them away for later. */ + prom_limit0 = prom_timers->limit0; + prom_limit1 = prom_timers->limit1; + + /* Just as in sun4c PROM uses timer which ticks at IRQ 14. + * We turn both off here just to be paranoid. + */ + prom_timers->limit0 = 0; + prom_timers->limit1 = 0; + + /* Wheee, eat the interrupt packet too... */ + __asm__ __volatile__( +" mov 0x40, %%g2\n" +" ldxa [%%g0] %0, %%g1\n" +" ldxa [%%g2] %1, %%g1\n" +" stxa %%g0, [%%g0] %0\n" +" membar #Sync\n" + : /* no outputs */ + : "i" (ASI_INTR_RECEIVE), "i" (ASI_INTR_R) + : "g1", "g2"); +} + +void notrace init_irqwork_curcpu(void) +{ + int cpu = hard_smp_processor_id(); + + trap_block[cpu].irq_worklist_pa = 0UL; +} + +/* Please be very careful with register_one_mondo() and + * sun4v_register_mondo_queues(). + * + * On SMP this gets invoked from the CPU trampoline before + * the cpu has fully taken over the trap table from OBP, + * and its kernel stack + %g6 thread register state is + * not fully cooked yet. + * + * Therefore you cannot make any OBP calls, not even prom_printf, + * from these two routines. + */ +static void notrace register_one_mondo(unsigned long paddr, unsigned long type, + unsigned long qmask) +{ + unsigned long num_entries = (qmask + 1) / 64; + unsigned long status; + + status = sun4v_cpu_qconf(type, paddr, num_entries); + if (status != HV_EOK) { + prom_printf("SUN4V: sun4v_cpu_qconf(%lu:%lx:%lu) failed, " + "err %lu\n", type, paddr, num_entries, status); + prom_halt(); + } +} + +void notrace sun4v_register_mondo_queues(int this_cpu) +{ + struct trap_per_cpu *tb = &trap_block[this_cpu]; + + register_one_mondo(tb->cpu_mondo_pa, HV_CPU_QUEUE_CPU_MONDO, + tb->cpu_mondo_qmask); + register_one_mondo(tb->dev_mondo_pa, HV_CPU_QUEUE_DEVICE_MONDO, + tb->dev_mondo_qmask); + register_one_mondo(tb->resum_mondo_pa, HV_CPU_QUEUE_RES_ERROR, + tb->resum_qmask); + register_one_mondo(tb->nonresum_mondo_pa, HV_CPU_QUEUE_NONRES_ERROR, + tb->nonresum_qmask); +} + +/* Each queue region must be a power of 2 multiple of 64 bytes in + * size. The base real address must be aligned to the size of the + * region. Thus, an 8KB queue must be 8KB aligned, for example. + */ +static void __init alloc_one_queue(unsigned long *pa_ptr, unsigned long qmask) +{ + unsigned long size = PAGE_ALIGN(qmask + 1); + unsigned long order = get_order(size); + unsigned long p; + + p = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!p) { + prom_printf("SUN4V: Error, cannot allocate queue.\n"); + prom_halt(); + } + + *pa_ptr = __pa(p); +} + +static void __init init_cpu_send_mondo_info(struct trap_per_cpu *tb) +{ +#ifdef CONFIG_SMP + unsigned long page; + void *mondo, *p; + + BUILD_BUG_ON((NR_CPUS * sizeof(u16)) > PAGE_SIZE); + + /* Make sure mondo block is 64byte aligned */ + p = kzalloc(127, GFP_KERNEL); + if (!p) { + prom_printf("SUN4V: Error, cannot allocate mondo block.\n"); + prom_halt(); + } + mondo = (void *)(((unsigned long)p + 63) & ~0x3f); + tb->cpu_mondo_block_pa = __pa(mondo); + + page = get_zeroed_page(GFP_KERNEL); + if (!page) { + prom_printf("SUN4V: Error, cannot allocate cpu list page.\n"); + prom_halt(); + } + + tb->cpu_list_pa = __pa(page); +#endif +} + +/* Allocate mondo and error queues for all possible cpus. */ +static void __init sun4v_init_mondo_queues(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct trap_per_cpu *tb = &trap_block[cpu]; + + alloc_one_queue(&tb->cpu_mondo_pa, tb->cpu_mondo_qmask); + alloc_one_queue(&tb->dev_mondo_pa, tb->dev_mondo_qmask); + alloc_one_queue(&tb->resum_mondo_pa, tb->resum_qmask); + alloc_one_queue(&tb->resum_kernel_buf_pa, tb->resum_qmask); + alloc_one_queue(&tb->nonresum_mondo_pa, tb->nonresum_qmask); + alloc_one_queue(&tb->nonresum_kernel_buf_pa, + tb->nonresum_qmask); + } +} + +static void __init init_send_mondo_info(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + struct trap_per_cpu *tb = &trap_block[cpu]; + + init_cpu_send_mondo_info(tb); + } +} + +static struct irqaction timer_irq_action = { + .name = "timer", +}; + +static void __init irq_ivector_init(void) +{ + unsigned long size, order; + unsigned int ivecs; + + /* If we are doing cookie only VIRQs then we do not need the ivector + * table to process interrupts. + */ + if (sun4v_cookie_only_virqs()) + return; + + ivecs = size_nr_ivec(); + size = sizeof(struct ino_bucket) * ivecs; + order = get_order(size); + ivector_table = (struct ino_bucket *) + __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!ivector_table) { + prom_printf("Fatal error, cannot allocate ivector_table\n"); + prom_halt(); + } + __flush_dcache_range((unsigned long) ivector_table, + ((unsigned long) ivector_table) + size); + + ivector_table_pa = __pa(ivector_table); +} + +/* Only invoked on boot processor.*/ +void __init init_IRQ(void) +{ + irq_init_hv(); + irq_ivector_init(); + map_prom_timers(); + kill_prom_timer(); + + if (tlb_type == hypervisor) + sun4v_init_mondo_queues(); + + init_send_mondo_info(); + + if (tlb_type == hypervisor) { + /* Load up the boot cpu's entries. */ + sun4v_register_mondo_queues(hard_smp_processor_id()); + } + + /* We need to clear any IRQ's pending in the soft interrupt + * registers, a spurious one could be left around from the + * PROM timer which we just disabled. + */ + clear_softint(get_softint()); + + /* Now that ivector table is initialized, it is safe + * to receive IRQ vector traps. We will normally take + * one or two right now, in case some device PROM used + * to boot us wants to speak to us. We just ignore them. + */ + __asm__ __volatile__("rdpr %%pstate, %%g1\n\t" + "or %%g1, %0, %%g1\n\t" + "wrpr %%g1, 0x0, %%pstate" + : /* No outputs */ + : "i" (PSTATE_IE) + : "g1"); + + irq_to_desc(0)->action = &timer_irq_action; +} diff --git a/arch/sparc/kernel/itlb_miss.S b/arch/sparc/kernel/itlb_miss.S new file mode 100644 index 000000000..5a5d92482 --- /dev/null +++ b/arch/sparc/kernel/itlb_miss.S @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* ITLB ** ICACHE line 1: Context 0 check and TSB load */ + ldxa [%g0] ASI_IMMU_TSB_8KB_PTR, %g1 ! Get TSB 8K pointer + ldxa [%g0] ASI_IMMU, %g6 ! Get TAG TARGET + srlx %g6, 48, %g5 ! Get context + sllx %g6, 22, %g6 ! Zero out context + brz,pn %g5, kvmap_itlb ! Context 0 processing + srlx %g6, 22, %g6 ! Delay slot + TSB_LOAD_QUAD(%g1, %g4) ! Load TSB entry + cmp %g4, %g6 ! Compare TAG + +/* ITLB ** ICACHE line 2: TSB compare and TLB load */ + bne,pn %xcc, tsb_miss_itlb ! Miss + mov FAULT_CODE_ITLB, %g3 + sethi %hi(_PAGE_EXEC_4U), %g4 + andcc %g5, %g4, %g0 ! Executable? + be,pn %xcc, tsb_do_fault + nop ! Delay slot, fill me + stxa %g5, [%g0] ASI_ITLB_DATA_IN ! Load TLB + retry ! Trap done + +/* ITLB ** ICACHE line 3: */ + nop + nop + nop + nop + nop + nop + nop + nop + +/* ITLB ** ICACHE line 4: */ + nop + nop + nop + nop + nop + nop + nop + nop diff --git a/arch/sparc/kernel/ivec.S b/arch/sparc/kernel/ivec.S new file mode 100644 index 000000000..94ba2c3a2 --- /dev/null +++ b/arch/sparc/kernel/ivec.S @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + /* The registers for cross calls will be: + * + * DATA 0: [low 32-bits] Address of function to call, jmp to this + * [high 32-bits] MMU Context Argument 0, place in %g5 + * DATA 1: Address Argument 1, place in %g1 + * DATA 2: Address Argument 2, place in %g7 + * + * With this method we can do most of the cross-call tlb/cache + * flushing very quickly. + */ + .align 32 + .globl do_ivec + .type do_ivec,#function +do_ivec: + mov 0x40, %g3 + ldxa [%g3 + %g0] ASI_INTR_R, %g3 + sethi %hi(KERNBASE), %g4 + cmp %g3, %g4 + bgeu,pn %xcc, do_ivec_xcall + srlx %g3, 32, %g5 + stxa %g0, [%g0] ASI_INTR_RECEIVE + membar #Sync + + sethi %hi(ivector_table_pa), %g2 + ldx [%g2 + %lo(ivector_table_pa)], %g2 + sllx %g3, 4, %g3 + add %g2, %g3, %g3 + + TRAP_LOAD_IRQ_WORK_PA(%g6, %g1) + + ldx [%g6], %g5 + stxa %g5, [%g3] ASI_PHYS_USE_EC + stx %g3, [%g6] + wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint + retry +do_ivec_xcall: + mov 0x50, %g1 + ldxa [%g1 + %g0] ASI_INTR_R, %g1 + srl %g3, 0, %g3 + + mov 0x60, %g7 + ldxa [%g7 + %g0] ASI_INTR_R, %g7 + stxa %g0, [%g0] ASI_INTR_RECEIVE + membar #Sync + ba,pt %xcc, 1f + nop + + .align 32 +1: jmpl %g3, %g0 + nop + .size do_ivec,.-do_ivec diff --git a/arch/sparc/kernel/jump_label.c b/arch/sparc/kernel/jump_label.c new file mode 100644 index 000000000..a4cfaeeca --- /dev/null +++ b/arch/sparc/kernel/jump_label.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +#include +#include + +#include + +void arch_jump_label_transform(struct jump_entry *entry, + enum jump_label_type type) +{ + u32 *insn = (u32 *) (unsigned long) entry->code; + u32 val; + + if (type == JUMP_LABEL_JMP) { + s32 off = (s32)entry->target - (s32)entry->code; + bool use_v9_branch = false; + + BUG_ON(off & 3); + +#ifdef CONFIG_SPARC64 + if (off <= 0xfffff && off >= -0x100000) + use_v9_branch = true; +#endif + if (use_v9_branch) { + /* WDISP19 - target is . + immed << 2 */ + /* ba,pt %xcc, . + off */ + val = 0x10680000 | (((u32) off >> 2) & 0x7ffff); + } else { + /* WDISP22 - target is . + immed << 2 */ + BUG_ON(off > 0x7fffff); + BUG_ON(off < -0x800000); + /* ba . + off */ + val = 0x10800000 | (((u32) off >> 2) & 0x3fffff); + } + } else { + val = 0x01000000; + } + + mutex_lock(&text_mutex); + *insn = val; + flushi(insn); + mutex_unlock(&text_mutex); +} diff --git a/arch/sparc/kernel/kernel.h b/arch/sparc/kernel/kernel.h new file mode 100644 index 000000000..4ee850515 --- /dev/null +++ b/arch/sparc/kernel/kernel.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SPARC_KERNEL_H +#define __SPARC_KERNEL_H + +#include +#include + +#include +#include +#include + +/* cpu.c */ +extern const char *sparc_pmu_type; +extern unsigned int fsr_storage; +extern int ncpus_probed; + +/* process{_32,_64}.c */ +asmlinkage long sparc_clone(struct pt_regs *regs); +asmlinkage long sparc_fork(struct pt_regs *regs); +asmlinkage long sparc_vfork(struct pt_regs *regs); +asmlinkage long sparc_clone3(struct pt_regs *regs); + +#ifdef CONFIG_SPARC64 +/* setup_64.c */ +struct seq_file; +void cpucap_info(struct seq_file *); + +static inline unsigned long kimage_addr_to_ra(const void *p) +{ + unsigned long val = (unsigned long) p; + + return kern_base + (val - KERNBASE); +} + +/* sys_sparc_64.c */ +asmlinkage long sys_kern_features(void); + +/* unaligned_64.c */ +asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); +int handle_popc(u32 insn, struct pt_regs *regs); +void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr); +void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr); + +/* uprobes.c */ +asmlinkage void uprobe_trap(struct pt_regs *regs, + unsigned long trap_level); + +/* smp_64.c */ +void __irq_entry smp_call_function_client(int irq, struct pt_regs *regs); +void __irq_entry smp_call_function_single_client(int irq, struct pt_regs *regs); +void __irq_entry smp_penguin_jailcell(int irq, struct pt_regs *regs); +void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs); + +/* kgdb_64.c */ +void __irq_entry smp_kgdb_capture_client(int irq, struct pt_regs *regs); + +/* pci.c */ +#ifdef CONFIG_PCI +int ali_sound_dma_hack(struct device *dev, u64 device_mask); +#else +#define ali_sound_dma_hack(dev, mask) (0) +#endif + +/* signal32.c */ +void do_sigreturn32(struct pt_regs *regs); +asmlinkage void do_rt_sigreturn32(struct pt_regs *regs); +void do_signal32(struct pt_regs * regs); +asmlinkage int do_sys32_sigstack(u32 u_ssptr, u32 u_ossptr, unsigned long sp); + +/* time_64.c */ +void __init time_init_early(void); + +/* compat_audit.c */ +extern unsigned int sparc32_dir_class[]; +extern unsigned int sparc32_chattr_class[]; +extern unsigned int sparc32_write_class[]; +extern unsigned int sparc32_read_class[]; +extern unsigned int sparc32_signal_class[]; +int sparc32_classify_syscall(unsigned int syscall); +#endif + +#ifdef CONFIG_SPARC32 +/* setup_32.c */ +struct linux_romvec; +void sparc32_start_kernel(struct linux_romvec *rp); + +/* cpu.c */ +void cpu_probe(void); + +/* traps_32.c */ +void handle_hw_divzero(struct pt_regs *regs, unsigned long pc, + unsigned long npc, unsigned long psr); +/* irq_32.c */ +extern struct irqaction static_irqaction[]; +extern int static_irq_count; +extern spinlock_t irq_action_lock; + +void unexpected_irq(int irq, void *dev_id, struct pt_regs * regs); + +/* sun4m_irq.c */ +void sun4m_init_IRQ(void); +void sun4m_unmask_profile_irq(void); +void sun4m_clear_profile_irq(int cpu); + +/* sun4m_smp.c */ +void sun4m_cpu_pre_starting(void *arg); +void sun4m_cpu_pre_online(void *arg); +void __init smp4m_boot_cpus(void); +int smp4m_boot_one_cpu(int i, struct task_struct *idle); +void __init smp4m_smp_done(void); +void smp4m_cross_call_irq(void); +void smp4m_percpu_timer_interrupt(struct pt_regs *regs); + +/* sun4d_irq.c */ +extern spinlock_t sun4d_imsk_lock; + +void sun4d_init_IRQ(void); +int sun4d_request_irq(unsigned int irq, + irq_handler_t handler, + unsigned long irqflags, + const char *devname, void *dev_id); +int show_sun4d_interrupts(struct seq_file *, void *); +void sun4d_distribute_irqs(void); +void sun4d_free_irq(unsigned int irq, void *dev_id); + +/* sun4d_smp.c */ +void sun4d_cpu_pre_starting(void *arg); +void sun4d_cpu_pre_online(void *arg); +void __init smp4d_boot_cpus(void); +int smp4d_boot_one_cpu(int i, struct task_struct *idle); +void __init smp4d_smp_done(void); +void smp4d_cross_call_irq(void); +void smp4d_percpu_timer_interrupt(struct pt_regs *regs); + +/* leon_smp.c */ +void leon_cpu_pre_starting(void *arg); +void leon_cpu_pre_online(void *arg); +void leonsmp_ipi_interrupt(void); +void leon_cross_call_irq(void); + +/* head_32.S */ +extern unsigned int t_nmi[]; +extern unsigned int linux_trap_ipi15_sun4d[]; +extern unsigned int linux_trap_ipi15_sun4m[]; + +extern struct tt_entry trapbase[]; +extern struct tt_entry trapbase_cpu1[]; +extern struct tt_entry trapbase_cpu2[]; +extern struct tt_entry trapbase_cpu3[]; + +extern char cputypval[]; + +/* entry.S */ +extern unsigned long lvl14_save[4]; +extern unsigned int real_irq_entry[]; +extern unsigned int smp4d_ticker[]; +extern unsigned int patchme_maybe_smp_msg[]; + +void floppy_hardint(void); + +/* trampoline_32.S */ +extern unsigned long sun4m_cpu_startup; +extern unsigned long sun4d_cpu_startup; + +/* signal_32.c */ +asmlinkage void do_sigreturn(struct pt_regs *regs); +asmlinkage void do_rt_sigreturn(struct pt_regs *regs); +void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, + unsigned long thread_info_flags); +asmlinkage int do_sys_sigstack(struct sigstack __user *ssptr, + struct sigstack __user *ossptr, + unsigned long sp); + +/* ptrace_32.c */ +asmlinkage int syscall_trace(struct pt_regs *regs, int syscall_exit_p); + +/* unaligned_32.c */ +asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn); +asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn); + +/* windows.c */ +void try_to_clear_window_buffer(struct pt_regs *regs, int who); + +/* auxio_32.c */ +void __init auxio_probe(void); +void __init auxio_power_probe(void); + +/* pcic.c */ +extern void __iomem *pcic_regs; +void pcic_nmi(unsigned int pend, struct pt_regs *regs); + +/* time_32.c */ +void __init time_init(void); + +#else /* CONFIG_SPARC32 */ +#endif /* CONFIG_SPARC32 */ +#endif /* !(__SPARC_KERNEL_H) */ diff --git a/arch/sparc/kernel/kgdb_32.c b/arch/sparc/kernel/kgdb_32.c new file mode 100644 index 000000000..3b2c673ec --- /dev/null +++ b/arch/sparc/kernel/kgdb_32.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0 +/* kgdb.c: KGDB support for 32-bit sparc. + * + * Copyright (C) 2008 David S. Miller + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "kernel.h" +#include "entry.h" + +void pt_regs_to_gdb_regs(unsigned long *gdb_regs, struct pt_regs *regs) +{ + struct reg_window32 *win; + int i; + + gdb_regs[GDB_G0] = 0; + for (i = 0; i < 15; i++) + gdb_regs[GDB_G1 + i] = regs->u_regs[UREG_G1 + i]; + + win = (struct reg_window32 *) regs->u_regs[UREG_FP]; + for (i = 0; i < 8; i++) + gdb_regs[GDB_L0 + i] = win->locals[i]; + for (i = 0; i < 8; i++) + gdb_regs[GDB_I0 + i] = win->ins[i]; + + for (i = GDB_F0; i <= GDB_F31; i++) + gdb_regs[i] = 0; + + gdb_regs[GDB_Y] = regs->y; + gdb_regs[GDB_PSR] = regs->psr; + gdb_regs[GDB_WIM] = 0; + gdb_regs[GDB_TBR] = (unsigned long) &trapbase[0]; + gdb_regs[GDB_PC] = regs->pc; + gdb_regs[GDB_NPC] = regs->npc; + gdb_regs[GDB_FSR] = 0; + gdb_regs[GDB_CSR] = 0; +} + +void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) +{ + struct thread_info *t = task_thread_info(p); + struct reg_window32 *win; + int i; + + for (i = GDB_G0; i < GDB_G6; i++) + gdb_regs[i] = 0; + gdb_regs[GDB_G6] = (unsigned long) t; + gdb_regs[GDB_G7] = 0; + for (i = GDB_O0; i < GDB_SP; i++) + gdb_regs[i] = 0; + gdb_regs[GDB_SP] = t->ksp; + gdb_regs[GDB_O7] = 0; + + win = (struct reg_window32 *) t->ksp; + for (i = 0; i < 8; i++) + gdb_regs[GDB_L0 + i] = win->locals[i]; + for (i = 0; i < 8; i++) + gdb_regs[GDB_I0 + i] = win->ins[i]; + + for (i = GDB_F0; i <= GDB_F31; i++) + gdb_regs[i] = 0; + + gdb_regs[GDB_Y] = 0; + + gdb_regs[GDB_PSR] = t->kpsr; + gdb_regs[GDB_WIM] = t->kwim; + gdb_regs[GDB_TBR] = (unsigned long) &trapbase[0]; + gdb_regs[GDB_PC] = t->kpc; + gdb_regs[GDB_NPC] = t->kpc + 4; + gdb_regs[GDB_FSR] = 0; + gdb_regs[GDB_CSR] = 0; +} + +void gdb_regs_to_pt_regs(unsigned long *gdb_regs, struct pt_regs *regs) +{ + struct reg_window32 *win; + int i; + + for (i = 0; i < 15; i++) + regs->u_regs[UREG_G1 + i] = gdb_regs[GDB_G1 + i]; + + /* If the PSR register is changing, we have to preserve + * the CWP field, otherwise window save/restore explodes. + */ + if (regs->psr != gdb_regs[GDB_PSR]) { + unsigned long cwp = regs->psr & PSR_CWP; + + regs->psr = (gdb_regs[GDB_PSR] & ~PSR_CWP) | cwp; + } + + regs->pc = gdb_regs[GDB_PC]; + regs->npc = gdb_regs[GDB_NPC]; + regs->y = gdb_regs[GDB_Y]; + + win = (struct reg_window32 *) regs->u_regs[UREG_FP]; + for (i = 0; i < 8; i++) + win->locals[i] = gdb_regs[GDB_L0 + i]; + for (i = 0; i < 8; i++) + win->ins[i] = gdb_regs[GDB_I0 + i]; +} + +int kgdb_arch_handle_exception(int e_vector, int signo, int err_code, + char *remcomInBuffer, char *remcomOutBuffer, + struct pt_regs *linux_regs) +{ + unsigned long addr; + char *ptr; + + switch (remcomInBuffer[0]) { + case 'c': + /* try to read optional parameter, pc unchanged if no parm */ + ptr = &remcomInBuffer[1]; + if (kgdb_hex2long(&ptr, &addr)) { + linux_regs->pc = addr; + linux_regs->npc = addr + 4; + } + fallthrough; + + case 'D': + case 'k': + if (linux_regs->pc == (unsigned long) arch_kgdb_breakpoint) { + linux_regs->pc = linux_regs->npc; + linux_regs->npc += 4; + } + return 0; + } + return -1; +} + +asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) +{ + unsigned long flags; + + if (user_mode(regs)) { + do_hw_interrupt(regs, trap_level); + return; + } + + flushw_all(); + + local_irq_save(flags); + kgdb_handle_exception(trap_level, SIGTRAP, 0, regs); + local_irq_restore(flags); +} + +int kgdb_arch_init(void) +{ + return 0; +} + +void kgdb_arch_exit(void) +{ +} + +void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long ip) +{ + regs->pc = ip; + regs->npc = regs->pc + 4; +} + +const struct kgdb_arch arch_kgdb_ops = { + /* Breakpoint instruction: ta 0x7d */ + .gdb_bpt_instr = { 0x91, 0xd0, 0x20, 0x7d }, +}; diff --git a/arch/sparc/kernel/kgdb_64.c b/arch/sparc/kernel/kgdb_64.c new file mode 100644 index 000000000..177746ae2 --- /dev/null +++ b/arch/sparc/kernel/kgdb_64.c @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: GPL-2.0 +/* kgdb.c: KGDB support for 64-bit sparc. + * + * Copyright (C) 2008 David S. Miller + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "kernel.h" + +void pt_regs_to_gdb_regs(unsigned long *gdb_regs, struct pt_regs *regs) +{ + struct reg_window *win; + int i; + + gdb_regs[GDB_G0] = 0; + for (i = 0; i < 15; i++) + gdb_regs[GDB_G1 + i] = regs->u_regs[UREG_G1 + i]; + + win = (struct reg_window *) (regs->u_regs[UREG_FP] + STACK_BIAS); + for (i = 0; i < 8; i++) + gdb_regs[GDB_L0 + i] = win->locals[i]; + for (i = 0; i < 8; i++) + gdb_regs[GDB_I0 + i] = win->ins[i]; + + for (i = GDB_F0; i <= GDB_F62; i++) + gdb_regs[i] = 0; + + gdb_regs[GDB_PC] = regs->tpc; + gdb_regs[GDB_NPC] = regs->tnpc; + gdb_regs[GDB_STATE] = regs->tstate; + gdb_regs[GDB_FSR] = 0; + gdb_regs[GDB_FPRS] = 0; + gdb_regs[GDB_Y] = regs->y; +} + +void sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *p) +{ + struct thread_info *t = task_thread_info(p); + extern unsigned int switch_to_pc; + extern unsigned int ret_from_fork; + struct reg_window *win; + unsigned long pc, cwp; + int i; + + for (i = GDB_G0; i < GDB_G6; i++) + gdb_regs[i] = 0; + gdb_regs[GDB_G6] = (unsigned long) t; + gdb_regs[GDB_G7] = (unsigned long) p; + for (i = GDB_O0; i < GDB_SP; i++) + gdb_regs[i] = 0; + gdb_regs[GDB_SP] = t->ksp; + gdb_regs[GDB_O7] = 0; + + win = (struct reg_window *) (t->ksp + STACK_BIAS); + for (i = 0; i < 8; i++) + gdb_regs[GDB_L0 + i] = win->locals[i]; + for (i = 0; i < 8; i++) + gdb_regs[GDB_I0 + i] = win->ins[i]; + + for (i = GDB_F0; i <= GDB_F62; i++) + gdb_regs[i] = 0; + + if (t->new_child) + pc = (unsigned long) &ret_from_fork; + else + pc = (unsigned long) &switch_to_pc; + + gdb_regs[GDB_PC] = pc; + gdb_regs[GDB_NPC] = pc + 4; + + cwp = __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP]; + + gdb_regs[GDB_STATE] = (TSTATE_PRIV | TSTATE_IE | cwp); + gdb_regs[GDB_FSR] = 0; + gdb_regs[GDB_FPRS] = 0; + gdb_regs[GDB_Y] = 0; +} + +void gdb_regs_to_pt_regs(unsigned long *gdb_regs, struct pt_regs *regs) +{ + struct reg_window *win; + int i; + + for (i = 0; i < 15; i++) + regs->u_regs[UREG_G1 + i] = gdb_regs[GDB_G1 + i]; + + /* If the TSTATE register is changing, we have to preserve + * the CWP field, otherwise window save/restore explodes. + */ + if (regs->tstate != gdb_regs[GDB_STATE]) { + unsigned long cwp = regs->tstate & TSTATE_CWP; + + regs->tstate = (gdb_regs[GDB_STATE] & ~TSTATE_CWP) | cwp; + } + + regs->tpc = gdb_regs[GDB_PC]; + regs->tnpc = gdb_regs[GDB_NPC]; + regs->y = gdb_regs[GDB_Y]; + + win = (struct reg_window *) (regs->u_regs[UREG_FP] + STACK_BIAS); + for (i = 0; i < 8; i++) + win->locals[i] = gdb_regs[GDB_L0 + i]; + for (i = 0; i < 8; i++) + win->ins[i] = gdb_regs[GDB_I0 + i]; +} + +#ifdef CONFIG_SMP +void __irq_entry smp_kgdb_capture_client(int irq, struct pt_regs *regs) +{ + unsigned long flags; + + __asm__ __volatile__("rdpr %%pstate, %0\n\t" + "wrpr %0, %1, %%pstate" + : "=r" (flags) + : "i" (PSTATE_IE)); + + flushw_all(); + + if (atomic_read(&kgdb_active) != -1) + kgdb_nmicallback(raw_smp_processor_id(), regs); + + __asm__ __volatile__("wrpr %0, 0, %%pstate" + : : "r" (flags)); +} +#endif + +int kgdb_arch_handle_exception(int e_vector, int signo, int err_code, + char *remcomInBuffer, char *remcomOutBuffer, + struct pt_regs *linux_regs) +{ + unsigned long addr; + char *ptr; + + switch (remcomInBuffer[0]) { + case 'c': + /* try to read optional parameter, pc unchanged if no parm */ + ptr = &remcomInBuffer[1]; + if (kgdb_hex2long(&ptr, &addr)) { + linux_regs->tpc = addr; + linux_regs->tnpc = addr + 4; + } + fallthrough; + + case 'D': + case 'k': + if (linux_regs->tpc == (unsigned long) arch_kgdb_breakpoint) { + linux_regs->tpc = linux_regs->tnpc; + linux_regs->tnpc += 4; + } + return 0; + } + return -1; +} + +asmlinkage void kgdb_trap(unsigned long trap_level, struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + unsigned long flags; + + if (user_mode(regs)) { + bad_trap(regs, trap_level); + goto out; + } + + flushw_all(); + + local_irq_save(flags); + kgdb_handle_exception(0x172, SIGTRAP, 0, regs); + local_irq_restore(flags); +out: + exception_exit(prev_state); +} + +int kgdb_arch_init(void) +{ + return 0; +} + +void kgdb_arch_exit(void) +{ +} + +void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long ip) +{ + regs->tpc = ip; + regs->tnpc = regs->tpc + 4; +} + +const struct kgdb_arch arch_kgdb_ops = { + /* Breakpoint instruction: ta 0x72 */ + .gdb_bpt_instr = { 0x91, 0xd0, 0x20, 0x72 }, +}; diff --git a/arch/sparc/kernel/kprobes.c b/arch/sparc/kernel/kprobes.c new file mode 100644 index 000000000..191bbaca9 --- /dev/null +++ b/arch/sparc/kernel/kprobes.c @@ -0,0 +1,489 @@ +// SPDX-License-Identifier: GPL-2.0 +/* arch/sparc64/kernel/kprobes.c + * + * Copyright (C) 2004 David S. Miller + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* We do not have hardware single-stepping on sparc64. + * So we implement software single-stepping with breakpoint + * traps. The top-level scheme is similar to that used + * in the x86 kprobes implementation. + * + * In the kprobe->ainsn.insn[] array we store the original + * instruction at index zero and a break instruction at + * index one. + * + * When we hit a kprobe we: + * - Run the pre-handler + * - Remember "regs->tnpc" and interrupt level stored in + * "regs->tstate" so we can restore them later + * - Disable PIL interrupts + * - Set regs->tpc to point to kprobe->ainsn.insn[0] + * - Set regs->tnpc to point to kprobe->ainsn.insn[1] + * - Mark that we are actively in a kprobe + * + * At this point we wait for the second breakpoint at + * kprobe->ainsn.insn[1] to hit. When it does we: + * - Run the post-handler + * - Set regs->tpc to "remembered" regs->tnpc stored above, + * restore the PIL interrupt level in "regs->tstate" as well + * - Make any adjustments necessary to regs->tnpc in order + * to handle relative branches correctly. See below. + * - Mark that we are no longer actively in a kprobe. + */ + +DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; +DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk); + +struct kretprobe_blackpoint kretprobe_blacklist[] = {{NULL, NULL}}; + +int __kprobes arch_prepare_kprobe(struct kprobe *p) +{ + if ((unsigned long) p->addr & 0x3UL) + return -EILSEQ; + + p->ainsn.insn[0] = *p->addr; + flushi(&p->ainsn.insn[0]); + + p->ainsn.insn[1] = BREAKPOINT_INSTRUCTION_2; + flushi(&p->ainsn.insn[1]); + + p->opcode = *p->addr; + return 0; +} + +void __kprobes arch_arm_kprobe(struct kprobe *p) +{ + *p->addr = BREAKPOINT_INSTRUCTION; + flushi(p->addr); +} + +void __kprobes arch_disarm_kprobe(struct kprobe *p) +{ + *p->addr = p->opcode; + flushi(p->addr); +} + +static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + kcb->prev_kprobe.kp = kprobe_running(); + kcb->prev_kprobe.status = kcb->kprobe_status; + kcb->prev_kprobe.orig_tnpc = kcb->kprobe_orig_tnpc; + kcb->prev_kprobe.orig_tstate_pil = kcb->kprobe_orig_tstate_pil; +} + +static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); + kcb->kprobe_status = kcb->prev_kprobe.status; + kcb->kprobe_orig_tnpc = kcb->prev_kprobe.orig_tnpc; + kcb->kprobe_orig_tstate_pil = kcb->prev_kprobe.orig_tstate_pil; +} + +static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + __this_cpu_write(current_kprobe, p); + kcb->kprobe_orig_tnpc = regs->tnpc; + kcb->kprobe_orig_tstate_pil = (regs->tstate & TSTATE_PIL); +} + +static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs, + struct kprobe_ctlblk *kcb) +{ + regs->tstate |= TSTATE_PIL; + + /*single step inline, if it a breakpoint instruction*/ + if (p->opcode == BREAKPOINT_INSTRUCTION) { + regs->tpc = (unsigned long) p->addr; + regs->tnpc = kcb->kprobe_orig_tnpc; + } else { + regs->tpc = (unsigned long) &p->ainsn.insn[0]; + regs->tnpc = (unsigned long) &p->ainsn.insn[1]; + } +} + +static int __kprobes kprobe_handler(struct pt_regs *regs) +{ + struct kprobe *p; + void *addr = (void *) regs->tpc; + int ret = 0; + struct kprobe_ctlblk *kcb; + + /* + * We don't want to be preempted for the entire + * duration of kprobe processing + */ + preempt_disable(); + kcb = get_kprobe_ctlblk(); + + if (kprobe_running()) { + p = get_kprobe(addr); + if (p) { + if (kcb->kprobe_status == KPROBE_HIT_SS) { + regs->tstate = ((regs->tstate & ~TSTATE_PIL) | + kcb->kprobe_orig_tstate_pil); + goto no_kprobe; + } + /* We have reentered the kprobe_handler(), since + * another probe was hit while within the handler. + * We here save the original kprobes variables and + * just single step on the instruction of the new probe + * without calling any user handlers. + */ + save_previous_kprobe(kcb); + set_current_kprobe(p, regs, kcb); + kprobes_inc_nmissed_count(p); + kcb->kprobe_status = KPROBE_REENTER; + prepare_singlestep(p, regs, kcb); + return 1; + } else if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { + /* The breakpoint instruction was removed by + * another cpu right after we hit, no further + * handling of this interrupt is appropriate + */ + ret = 1; + } + goto no_kprobe; + } + + p = get_kprobe(addr); + if (!p) { + if (*(u32 *)addr != BREAKPOINT_INSTRUCTION) { + /* + * The breakpoint instruction was removed right + * after we hit it. Another cpu has removed + * either a probepoint or a debugger breakpoint + * at this address. In either case, no further + * handling of this interrupt is appropriate. + */ + ret = 1; + } + /* Not one of ours: let kernel handle it */ + goto no_kprobe; + } + + set_current_kprobe(p, regs, kcb); + kcb->kprobe_status = KPROBE_HIT_ACTIVE; + if (p->pre_handler && p->pre_handler(p, regs)) { + reset_current_kprobe(); + preempt_enable_no_resched(); + return 1; + } + + prepare_singlestep(p, regs, kcb); + kcb->kprobe_status = KPROBE_HIT_SS; + return 1; + +no_kprobe: + preempt_enable_no_resched(); + return ret; +} + +/* If INSN is a relative control transfer instruction, + * return the corrected branch destination value. + * + * regs->tpc and regs->tnpc still hold the values of the + * program counters at the time of trap due to the execution + * of the BREAKPOINT_INSTRUCTION_2 at p->ainsn.insn[1] + * + */ +static unsigned long __kprobes relbranch_fixup(u32 insn, struct kprobe *p, + struct pt_regs *regs) +{ + unsigned long real_pc = (unsigned long) p->addr; + + /* Branch not taken, no mods necessary. */ + if (regs->tnpc == regs->tpc + 0x4UL) + return real_pc + 0x8UL; + + /* The three cases are call, branch w/prediction, + * and traditional branch. + */ + if ((insn & 0xc0000000) == 0x40000000 || + (insn & 0xc1c00000) == 0x00400000 || + (insn & 0xc1c00000) == 0x00800000) { + unsigned long ainsn_addr; + + ainsn_addr = (unsigned long) &p->ainsn.insn[0]; + + /* The instruction did all the work for us + * already, just apply the offset to the correct + * instruction location. + */ + return (real_pc + (regs->tnpc - ainsn_addr)); + } + + /* It is jmpl or some other absolute PC modification instruction, + * leave NPC as-is. + */ + return regs->tnpc; +} + +/* If INSN is an instruction which writes its PC location + * into a destination register, fix that up. + */ +static void __kprobes retpc_fixup(struct pt_regs *regs, u32 insn, + unsigned long real_pc) +{ + unsigned long *slot = NULL; + + /* Simplest case is 'call', which always uses %o7 */ + if ((insn & 0xc0000000) == 0x40000000) { + slot = ®s->u_regs[UREG_I7]; + } + + /* 'jmpl' encodes the register inside of the opcode */ + if ((insn & 0xc1f80000) == 0x81c00000) { + unsigned long rd = ((insn >> 25) & 0x1f); + + if (rd <= 15) { + slot = ®s->u_regs[rd]; + } else { + /* Hard case, it goes onto the stack. */ + flushw_all(); + + rd -= 16; + slot = (unsigned long *) + (regs->u_regs[UREG_FP] + STACK_BIAS); + slot += rd; + } + } + if (slot != NULL) + *slot = real_pc; +} + +/* + * Called after single-stepping. p->addr is the address of the + * instruction which has been replaced by the breakpoint + * instruction. To avoid the SMP problems that can occur when we + * temporarily put back the original opcode to single-step, we + * single-stepped a copy of the instruction. The address of this + * copy is &p->ainsn.insn[0]. + * + * This function prepares to return from the post-single-step + * breakpoint trap. + */ +static void __kprobes resume_execution(struct kprobe *p, + struct pt_regs *regs, struct kprobe_ctlblk *kcb) +{ + u32 insn = p->ainsn.insn[0]; + + regs->tnpc = relbranch_fixup(insn, p, regs); + + /* This assignment must occur after relbranch_fixup() */ + regs->tpc = kcb->kprobe_orig_tnpc; + + retpc_fixup(regs, insn, (unsigned long) p->addr); + + regs->tstate = ((regs->tstate & ~TSTATE_PIL) | + kcb->kprobe_orig_tstate_pil); +} + +static int __kprobes post_kprobe_handler(struct pt_regs *regs) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + + if (!cur) + return 0; + + if ((kcb->kprobe_status != KPROBE_REENTER) && cur->post_handler) { + kcb->kprobe_status = KPROBE_HIT_SSDONE; + cur->post_handler(cur, regs, 0); + } + + resume_execution(cur, regs, kcb); + + /*Restore back the original saved kprobes variables and continue. */ + if (kcb->kprobe_status == KPROBE_REENTER) { + restore_previous_kprobe(kcb); + goto out; + } + reset_current_kprobe(); +out: + preempt_enable_no_resched(); + + return 1; +} + +int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) +{ + struct kprobe *cur = kprobe_running(); + struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); + const struct exception_table_entry *entry; + + switch(kcb->kprobe_status) { + case KPROBE_HIT_SS: + case KPROBE_REENTER: + /* + * We are here because the instruction being single + * stepped caused a page fault. We reset the current + * kprobe and the tpc points back to the probe address + * and allow the page fault handler to continue as a + * normal page fault. + */ + regs->tpc = (unsigned long)cur->addr; + regs->tnpc = kcb->kprobe_orig_tnpc; + regs->tstate = ((regs->tstate & ~TSTATE_PIL) | + kcb->kprobe_orig_tstate_pil); + if (kcb->kprobe_status == KPROBE_REENTER) + restore_previous_kprobe(kcb); + else + reset_current_kprobe(); + preempt_enable_no_resched(); + break; + case KPROBE_HIT_ACTIVE: + case KPROBE_HIT_SSDONE: + /* + * In case the user-specified fault handler returned + * zero, try to fix up. + */ + + entry = search_exception_tables(regs->tpc); + if (entry) { + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return 1; + } + + /* + * fixup_exception() could not handle it, + * Let do_page_fault() fix it. + */ + break; + default: + break; + } + + return 0; +} + +/* + * Wrapper routine to for handling exceptions. + */ +int __kprobes kprobe_exceptions_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + struct die_args *args = (struct die_args *)data; + int ret = NOTIFY_DONE; + + if (args->regs && user_mode(args->regs)) + return ret; + + switch (val) { + case DIE_DEBUG: + if (kprobe_handler(args->regs)) + ret = NOTIFY_STOP; + break; + case DIE_DEBUG_2: + if (post_kprobe_handler(args->regs)) + ret = NOTIFY_STOP; + break; + default: + break; + } + return ret; +} + +asmlinkage void __kprobes kprobe_trap(unsigned long trap_level, + struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + + BUG_ON(trap_level != 0x170 && trap_level != 0x171); + + if (user_mode(regs)) { + local_irq_enable(); + bad_trap(regs, trap_level); + goto out; + } + + /* trap_level == 0x170 --> ta 0x70 + * trap_level == 0x171 --> ta 0x71 + */ + if (notify_die((trap_level == 0x170) ? DIE_DEBUG : DIE_DEBUG_2, + (trap_level == 0x170) ? "debug" : "debug_2", + regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) + bad_trap(regs, trap_level); +out: + exception_exit(prev_state); +} + +/* The value stored in the return address register is actually 2 + * instructions before where the callee will return to. + * Sequences usually look something like this + * + * call some_function <--- return register points here + * nop <--- call delay slot + * whatever <--- where callee returns to + * + * To keep trampoline_probe_handler logic simpler, we normalize the + * value kept in ri->ret_addr so we don't need to keep adjusting it + * back and forth. + */ +void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri, + struct pt_regs *regs) +{ + ri->ret_addr = (kprobe_opcode_t *)(regs->u_regs[UREG_RETPC] + 8); + ri->fp = NULL; + + /* Replace the return addr with trampoline addr */ + regs->u_regs[UREG_RETPC] = + ((unsigned long)__kretprobe_trampoline) - 8; +} + +/* + * Called when the probe at kretprobe trampoline is hit + */ +static int __kprobes trampoline_probe_handler(struct kprobe *p, + struct pt_regs *regs) +{ + unsigned long orig_ret_address = 0; + + orig_ret_address = __kretprobe_trampoline_handler(regs, NULL); + regs->tpc = orig_ret_address; + regs->tnpc = orig_ret_address + 4; + + /* + * By returning a non-zero value, we are telling + * kprobe_handler() that we don't want the post_handler + * to run (and have re-enabled preemption) + */ + return 1; +} + +static void __used kretprobe_trampoline_holder(void) +{ + asm volatile(".global __kretprobe_trampoline\n" + "__kretprobe_trampoline:\n" + "\tnop\n" + "\tnop\n"); +} +static struct kprobe trampoline_p = { + .addr = (kprobe_opcode_t *) &__kretprobe_trampoline, + .pre_handler = trampoline_probe_handler +}; + +int __init arch_init_kprobes(void) +{ + return register_kprobe(&trampoline_p); +} + +int __kprobes arch_trampoline_kprobe(struct kprobe *p) +{ + if (p->addr == (kprobe_opcode_t *)&__kretprobe_trampoline) + return 1; + + return 0; +} diff --git a/arch/sparc/kernel/kstack.h b/arch/sparc/kernel/kstack.h new file mode 100644 index 000000000..b3c5e8f24 --- /dev/null +++ b/arch/sparc/kernel/kstack.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _KSTACK_H +#define _KSTACK_H + +#include +#include +#include +#include + +/* SP must be STACK_BIAS adjusted already. */ +static inline bool kstack_valid(struct thread_info *tp, unsigned long sp) +{ + unsigned long base = (unsigned long) tp; + + /* Stack pointer must be 16-byte aligned. */ + if (sp & (16UL - 1)) + return false; + + if (sp >= (base + sizeof(struct thread_info)) && + sp <= (base + THREAD_SIZE - sizeof(struct sparc_stackf))) + return true; + + if (hardirq_stack[tp->cpu]) { + base = (unsigned long) hardirq_stack[tp->cpu]; + if (sp >= base && + sp <= (base + THREAD_SIZE - sizeof(struct sparc_stackf))) + return true; + base = (unsigned long) softirq_stack[tp->cpu]; + if (sp >= base && + sp <= (base + THREAD_SIZE - sizeof(struct sparc_stackf))) + return true; + } + return false; +} + +/* Does "regs" point to a valid pt_regs trap frame? */ +static inline bool kstack_is_trap_frame(struct thread_info *tp, struct pt_regs *regs) +{ + unsigned long base = (unsigned long) tp; + unsigned long addr = (unsigned long) regs; + + if (addr >= base && + addr <= (base + THREAD_SIZE - sizeof(*regs))) + goto check_magic; + + if (hardirq_stack[tp->cpu]) { + base = (unsigned long) hardirq_stack[tp->cpu]; + if (addr >= base && + addr <= (base + THREAD_SIZE - sizeof(*regs))) + goto check_magic; + base = (unsigned long) softirq_stack[tp->cpu]; + if (addr >= base && + addr <= (base + THREAD_SIZE - sizeof(*regs))) + goto check_magic; + } + return false; + +check_magic: + if ((regs->magic & ~0x1ff) == PT_REGS_MAGIC) + return true; + return false; + +} + +static inline __attribute__((always_inline)) void *set_hardirq_stack(void) +{ + void *orig_sp, *sp = hardirq_stack[smp_processor_id()]; + + __asm__ __volatile__("mov %%sp, %0" : "=r" (orig_sp)); + if (orig_sp < sp || + orig_sp > (sp + THREAD_SIZE)) { + sp += THREAD_SIZE - 192 - STACK_BIAS; + __asm__ __volatile__("mov %0, %%sp" : : "r" (sp)); + } + + return orig_sp; +} + +static inline __attribute__((always_inline)) void restore_hardirq_stack(void *orig_sp) +{ + __asm__ __volatile__("mov %0, %%sp" : : "r" (orig_sp)); +} + +#endif /* _KSTACK_H */ diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S new file mode 100644 index 000000000..6bfaf73ce --- /dev/null +++ b/arch/sparc/kernel/ktlb.S @@ -0,0 +1,270 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling. + * + * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller + * Copyright (C) 1996 Eddie C. Dost (ecd@brainaid.de) + * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) + * Copyright (C) 1996,98,99 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include + + .text + .align 32 + +kvmap_itlb: + /* g6: TAG TARGET */ + mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_IMMU, %g4 + + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g4 here. + */ + + /* sun4v_itlb_miss branches here with the missing virtual + * address already loaded into %g4 + */ +kvmap_itlb_4v: + + /* Catch kernel NULL pointer calls. */ + sethi %hi(PAGE_SIZE), %g5 + cmp %g4, %g5 + blu,pn %xcc, kvmap_itlb_longpath + nop + + KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load) + +kvmap_itlb_tsb_miss: + sethi %hi(LOW_OBP_ADDRESS), %g5 + cmp %g4, %g5 + blu,pn %xcc, kvmap_itlb_vmalloc_addr + mov 0x1, %g5 + sllx %g5, 32, %g5 + cmp %g4, %g5 + blu,pn %xcc, kvmap_itlb_obp + nop + +kvmap_itlb_vmalloc_addr: + KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath) + + TSB_LOCK_TAG(%g1, %g2, %g7) + TSB_WRITE(%g1, %g5, %g6) + + /* fallthrough to TLB load */ + +kvmap_itlb_load: + +661: stxa %g5, [%g0] ASI_ITLB_DATA_IN + retry + .section .sun4v_2insn_patch, "ax" + .word 661b + nop + nop + .previous + + /* For sun4v the ASI_ITLB_DATA_IN store and the retry + * instruction get nop'd out and we get here to branch + * to the sun4v tlb load code. The registers are setup + * as follows: + * + * %g4: vaddr + * %g5: PTE + * %g6: TAG + * + * The sun4v TLB load wants the PTE in %g3 so we fix that + * up here. + */ + ba,pt %xcc, sun4v_itlb_load + mov %g5, %g3 + +kvmap_itlb_longpath: + +661: rdpr %pstate, %g5 + wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate + .section .sun4v_2insn_patch, "ax" + .word 661b + SET_GL(1) + nop + .previous + + rdpr %tpc, %g5 + ba,pt %xcc, sparc64_realfault_common + mov FAULT_CODE_ITLB, %g4 + +kvmap_itlb_obp: + OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath) + + TSB_LOCK_TAG(%g1, %g2, %g7) + + TSB_WRITE(%g1, %g5, %g6) + + ba,pt %xcc, kvmap_itlb_load + nop + +kvmap_dtlb_obp: + OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath) + + TSB_LOCK_TAG(%g1, %g2, %g7) + + TSB_WRITE(%g1, %g5, %g6) + + ba,pt %xcc, kvmap_dtlb_load + nop + +kvmap_linear_early: + sethi %hi(kern_linear_pte_xor), %g7 + ldx [%g7 + %lo(kern_linear_pte_xor)], %g2 + ba,pt %xcc, kvmap_dtlb_tsb4m_load + xor %g2, %g4, %g5 + + .align 32 +kvmap_dtlb_tsb4m_load: + TSB_LOCK_TAG(%g1, %g2, %g7) + TSB_WRITE(%g1, %g5, %g6) + ba,pt %xcc, kvmap_dtlb_load + nop + +kvmap_dtlb: + /* %g6: TAG TARGET */ + mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_DMMU, %g4 + + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g4 here. + */ + + /* sun4v_dtlb_miss branches here with the missing virtual + * address already loaded into %g4 + */ +kvmap_dtlb_4v: + brgez,pn %g4, kvmap_dtlb_nonlinear + nop + +#ifdef CONFIG_DEBUG_PAGEALLOC + /* Index through the base page size TSB even for linear + * mappings when using page allocation debugging. + */ + KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) +#else + /* Correct TAG_TARGET is already in %g6, check 4mb TSB. */ + KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) +#endif + /* Linear mapping TSB lookup failed. Fallthrough to kernel + * page table based lookup. + */ + .globl kvmap_linear_patch +kvmap_linear_patch: + ba,a,pt %xcc, kvmap_linear_early + +kvmap_dtlb_vmalloc_addr: + KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) + + TSB_LOCK_TAG(%g1, %g2, %g7) + TSB_WRITE(%g1, %g5, %g6) + + /* fallthrough to TLB load */ + +kvmap_dtlb_load: + +661: stxa %g5, [%g0] ASI_DTLB_DATA_IN ! Reload TLB + retry + .section .sun4v_2insn_patch, "ax" + .word 661b + nop + nop + .previous + + /* For sun4v the ASI_DTLB_DATA_IN store and the retry + * instruction get nop'd out and we get here to branch + * to the sun4v tlb load code. The registers are setup + * as follows: + * + * %g4: vaddr + * %g5: PTE + * %g6: TAG + * + * The sun4v TLB load wants the PTE in %g3 so we fix that + * up here. + */ + ba,pt %xcc, sun4v_dtlb_load + mov %g5, %g3 + +#ifdef CONFIG_SPARSEMEM_VMEMMAP +kvmap_vmemmap: + KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath) + ba,a,pt %xcc, kvmap_dtlb_load +#endif + +kvmap_dtlb_nonlinear: + /* Catch kernel NULL pointer derefs. */ + sethi %hi(PAGE_SIZE), %g5 + cmp %g4, %g5 + bleu,pn %xcc, kvmap_dtlb_longpath + nop + +#ifdef CONFIG_SPARSEMEM_VMEMMAP + /* Do not use the TSB for vmemmap. */ + sethi %hi(VMEMMAP_BASE), %g5 + ldx [%g5 + %lo(VMEMMAP_BASE)], %g5 + cmp %g4,%g5 + bgeu,pn %xcc, kvmap_vmemmap + nop +#endif + + KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load) + +kvmap_dtlb_tsbmiss: + sethi %hi(MODULES_VADDR), %g5 + cmp %g4, %g5 + blu,pn %xcc, kvmap_dtlb_longpath + sethi %hi(VMALLOC_END), %g5 + ldx [%g5 + %lo(VMALLOC_END)], %g5 + cmp %g4, %g5 + bgeu,pn %xcc, kvmap_dtlb_longpath + nop + +kvmap_check_obp: + sethi %hi(LOW_OBP_ADDRESS), %g5 + cmp %g4, %g5 + blu,pn %xcc, kvmap_dtlb_vmalloc_addr + mov 0x1, %g5 + sllx %g5, 32, %g5 + cmp %g4, %g5 + blu,pn %xcc, kvmap_dtlb_obp + nop + ba,pt %xcc, kvmap_dtlb_vmalloc_addr + nop + +kvmap_dtlb_longpath: + +661: rdpr %pstate, %g5 + wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate + .section .sun4v_2insn_patch, "ax" + .word 661b + SET_GL(1) + ldxa [%g0] ASI_SCRATCHPAD, %g5 + .previous + + rdpr %tl, %g3 + cmp %g3, 1 + +661: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_DMMU, %g5 + .section .sun4v_2insn_patch, "ax" + .word 661b + ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 + nop + .previous + + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g5 here. + */ + + be,pt %xcc, sparc64_realfault_common + mov FAULT_CODE_DTLB, %g4 + ba,pt %xcc, winfix_trampoline + nop diff --git a/arch/sparc/kernel/ldc.c b/arch/sparc/kernel/ldc.c new file mode 100644 index 000000000..826fe8ed0 --- /dev/null +++ b/arch/sparc/kernel/ldc.c @@ -0,0 +1,2428 @@ +// SPDX-License-Identifier: GPL-2.0 +/* ldc.c: Logical Domain Channel link-layer protocol driver. + * + * Copyright (C) 2007, 2008 David S. Miller + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define DRV_MODULE_NAME "ldc" +#define PFX DRV_MODULE_NAME ": " +#define DRV_MODULE_VERSION "1.1" +#define DRV_MODULE_RELDATE "July 22, 2008" + +#define COOKIE_PGSZ_CODE 0xf000000000000000ULL +#define COOKIE_PGSZ_CODE_SHIFT 60ULL + + +static char version[] = + DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; + +/* Packet header layout for unreliable and reliable mode frames. + * When in RAW mode, packets are simply straight 64-byte payloads + * with no headers. + */ +struct ldc_packet { + u8 type; +#define LDC_CTRL 0x01 +#define LDC_DATA 0x02 +#define LDC_ERR 0x10 + + u8 stype; +#define LDC_INFO 0x01 +#define LDC_ACK 0x02 +#define LDC_NACK 0x04 + + u8 ctrl; +#define LDC_VERS 0x01 /* Link Version */ +#define LDC_RTS 0x02 /* Request To Send */ +#define LDC_RTR 0x03 /* Ready To Receive */ +#define LDC_RDX 0x04 /* Ready for Data eXchange */ +#define LDC_CTRL_MSK 0x0f + + u8 env; +#define LDC_LEN 0x3f +#define LDC_FRAG_MASK 0xc0 +#define LDC_START 0x40 +#define LDC_STOP 0x80 + + u32 seqid; + + union { + u8 u_data[LDC_PACKET_SIZE - 8]; + struct { + u32 pad; + u32 ackid; + u8 r_data[LDC_PACKET_SIZE - 8 - 8]; + } r; + } u; +}; + +struct ldc_version { + u16 major; + u16 minor; +}; + +/* Ordered from largest major to lowest. */ +static struct ldc_version ver_arr[] = { + { .major = 1, .minor = 0 }, +}; + +#define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE) +#define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE) + +struct ldc_channel; + +struct ldc_mode_ops { + int (*write)(struct ldc_channel *, const void *, unsigned int); + int (*read)(struct ldc_channel *, void *, unsigned int); +}; + +static const struct ldc_mode_ops raw_ops; +static const struct ldc_mode_ops nonraw_ops; +static const struct ldc_mode_ops stream_ops; + +int ldom_domaining_enabled; + +struct ldc_iommu { + /* Protects ldc_unmap. */ + spinlock_t lock; + struct ldc_mtable_entry *page_table; + struct iommu_map_table iommu_map_table; +}; + +struct ldc_channel { + /* Protects all operations that depend upon channel state. */ + spinlock_t lock; + + unsigned long id; + + u8 *mssbuf; + u32 mssbuf_len; + u32 mssbuf_off; + + struct ldc_packet *tx_base; + unsigned long tx_head; + unsigned long tx_tail; + unsigned long tx_num_entries; + unsigned long tx_ra; + + unsigned long tx_acked; + + struct ldc_packet *rx_base; + unsigned long rx_head; + unsigned long rx_tail; + unsigned long rx_num_entries; + unsigned long rx_ra; + + u32 rcv_nxt; + u32 snd_nxt; + + unsigned long chan_state; + + struct ldc_channel_config cfg; + void *event_arg; + + const struct ldc_mode_ops *mops; + + struct ldc_iommu iommu; + + struct ldc_version ver; + + u8 hs_state; +#define LDC_HS_CLOSED 0x00 +#define LDC_HS_OPEN 0x01 +#define LDC_HS_GOTVERS 0x02 +#define LDC_HS_SENTRTR 0x03 +#define LDC_HS_GOTRTR 0x04 +#define LDC_HS_COMPLETE 0x10 + + u8 flags; +#define LDC_FLAG_ALLOCED_QUEUES 0x01 +#define LDC_FLAG_REGISTERED_QUEUES 0x02 +#define LDC_FLAG_REGISTERED_IRQS 0x04 +#define LDC_FLAG_RESET 0x10 + + u8 mss; + u8 state; + +#define LDC_IRQ_NAME_MAX 32 + char rx_irq_name[LDC_IRQ_NAME_MAX]; + char tx_irq_name[LDC_IRQ_NAME_MAX]; + + struct hlist_head mh_list; + + struct hlist_node list; +}; + +#define ldcdbg(TYPE, f, a...) \ +do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \ + printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \ +} while (0) + +#define LDC_ABORT(lp) ldc_abort((lp), __func__) + +static const char *state_to_str(u8 state) +{ + switch (state) { + case LDC_STATE_INVALID: + return "INVALID"; + case LDC_STATE_INIT: + return "INIT"; + case LDC_STATE_BOUND: + return "BOUND"; + case LDC_STATE_READY: + return "READY"; + case LDC_STATE_CONNECTED: + return "CONNECTED"; + default: + return ""; + } +} + +static unsigned long __advance(unsigned long off, unsigned long num_entries) +{ + off += LDC_PACKET_SIZE; + if (off == (num_entries * LDC_PACKET_SIZE)) + off = 0; + + return off; +} + +static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off) +{ + return __advance(off, lp->rx_num_entries); +} + +static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off) +{ + return __advance(off, lp->tx_num_entries); +} + +static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp, + unsigned long *new_tail) +{ + struct ldc_packet *p; + unsigned long t; + + t = tx_advance(lp, lp->tx_tail); + if (t == lp->tx_head) + return NULL; + + *new_tail = t; + + p = lp->tx_base; + return p + (lp->tx_tail / LDC_PACKET_SIZE); +} + +/* When we are in reliable or stream mode, have to track the next packet + * we haven't gotten an ACK for in the TX queue using tx_acked. We have + * to be careful not to stomp over the queue past that point. During + * the handshake, we don't have TX data packets pending in the queue + * and that's why handshake_get_tx_packet() need not be mindful of + * lp->tx_acked. + */ +static unsigned long head_for_data(struct ldc_channel *lp) +{ + if (lp->cfg.mode == LDC_MODE_STREAM) + return lp->tx_acked; + return lp->tx_head; +} + +static int tx_has_space_for(struct ldc_channel *lp, unsigned int size) +{ + unsigned long limit, tail, new_tail, diff; + unsigned int mss; + + limit = head_for_data(lp); + tail = lp->tx_tail; + new_tail = tx_advance(lp, tail); + if (new_tail == limit) + return 0; + + if (limit > new_tail) + diff = limit - new_tail; + else + diff = (limit + + ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail)); + diff /= LDC_PACKET_SIZE; + mss = lp->mss; + + if (diff * mss < size) + return 0; + + return 1; +} + +static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp, + unsigned long *new_tail) +{ + struct ldc_packet *p; + unsigned long h, t; + + h = head_for_data(lp); + t = tx_advance(lp, lp->tx_tail); + if (t == h) + return NULL; + + *new_tail = t; + + p = lp->tx_base; + return p + (lp->tx_tail / LDC_PACKET_SIZE); +} + +static int set_tx_tail(struct ldc_channel *lp, unsigned long tail) +{ + unsigned long orig_tail = lp->tx_tail; + int limit = 1000; + + lp->tx_tail = tail; + while (limit-- > 0) { + unsigned long err; + + err = sun4v_ldc_tx_set_qtail(lp->id, tail); + if (!err) + return 0; + + if (err != HV_EWOULDBLOCK) { + lp->tx_tail = orig_tail; + return -EINVAL; + } + udelay(1); + } + + lp->tx_tail = orig_tail; + return -EBUSY; +} + +/* This just updates the head value in the hypervisor using + * a polling loop with a timeout. The caller takes care of + * upating software state representing the head change, if any. + */ +static int __set_rx_head(struct ldc_channel *lp, unsigned long head) +{ + int limit = 1000; + + while (limit-- > 0) { + unsigned long err; + + err = sun4v_ldc_rx_set_qhead(lp->id, head); + if (!err) + return 0; + + if (err != HV_EWOULDBLOCK) + return -EINVAL; + + udelay(1); + } + + return -EBUSY; +} + +static int send_tx_packet(struct ldc_channel *lp, + struct ldc_packet *p, + unsigned long new_tail) +{ + BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE))); + + return set_tx_tail(lp, new_tail); +} + +static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp, + u8 stype, u8 ctrl, + void *data, int dlen, + unsigned long *new_tail) +{ + struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail); + + if (p) { + memset(p, 0, sizeof(*p)); + p->type = LDC_CTRL; + p->stype = stype; + p->ctrl = ctrl; + if (data) + memcpy(p->u.u_data, data, dlen); + } + return p; +} + +static int start_handshake(struct ldc_channel *lp) +{ + struct ldc_packet *p; + struct ldc_version *ver; + unsigned long new_tail; + + ver = &ver_arr[0]; + + ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n", + ver->major, ver->minor); + + p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, + ver, sizeof(*ver), &new_tail); + if (p) { + int err = send_tx_packet(lp, p, new_tail); + if (!err) + lp->flags &= ~LDC_FLAG_RESET; + return err; + } + return -EBUSY; +} + +static int send_version_nack(struct ldc_channel *lp, + u16 major, u16 minor) +{ + struct ldc_packet *p; + struct ldc_version ver; + unsigned long new_tail; + + ver.major = major; + ver.minor = minor; + + p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS, + &ver, sizeof(ver), &new_tail); + if (p) { + ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n", + ver.major, ver.minor); + + return send_tx_packet(lp, p, new_tail); + } + return -EBUSY; +} + +static int send_version_ack(struct ldc_channel *lp, + struct ldc_version *vp) +{ + struct ldc_packet *p; + unsigned long new_tail; + + p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS, + vp, sizeof(*vp), &new_tail); + if (p) { + ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n", + vp->major, vp->minor); + + return send_tx_packet(lp, p, new_tail); + } + return -EBUSY; +} + +static int send_rts(struct ldc_channel *lp) +{ + struct ldc_packet *p; + unsigned long new_tail; + + p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0, + &new_tail); + if (p) { + p->env = lp->cfg.mode; + p->seqid = 0; + lp->rcv_nxt = 0; + + ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n", + p->env, p->seqid); + + return send_tx_packet(lp, p, new_tail); + } + return -EBUSY; +} + +static int send_rtr(struct ldc_channel *lp) +{ + struct ldc_packet *p; + unsigned long new_tail; + + p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0, + &new_tail); + if (p) { + p->env = lp->cfg.mode; + p->seqid = 0; + + ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n", + p->env, p->seqid); + + return send_tx_packet(lp, p, new_tail); + } + return -EBUSY; +} + +static int send_rdx(struct ldc_channel *lp) +{ + struct ldc_packet *p; + unsigned long new_tail; + + p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0, + &new_tail); + if (p) { + p->env = 0; + p->seqid = ++lp->snd_nxt; + p->u.r.ackid = lp->rcv_nxt; + + ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n", + p->env, p->seqid, p->u.r.ackid); + + return send_tx_packet(lp, p, new_tail); + } + return -EBUSY; +} + +static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt) +{ + struct ldc_packet *p; + unsigned long new_tail; + int err; + + p = data_get_tx_packet(lp, &new_tail); + if (!p) + return -EBUSY; + memset(p, 0, sizeof(*p)); + p->type = data_pkt->type; + p->stype = LDC_NACK; + p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK; + p->seqid = lp->snd_nxt + 1; + p->u.r.ackid = lp->rcv_nxt; + + ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n", + p->type, p->ctrl, p->seqid, p->u.r.ackid); + + err = send_tx_packet(lp, p, new_tail); + if (!err) + lp->snd_nxt++; + + return err; +} + +static int ldc_abort(struct ldc_channel *lp, const char *msg) +{ + unsigned long hv_err; + + ldcdbg(STATE, "ABORT[%s]\n", msg); + ldc_print(lp); + + /* We report but do not act upon the hypervisor errors because + * there really isn't much we can do if they fail at this point. + */ + hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); + if (hv_err) + printk(KERN_ERR PFX "ldc_abort: " + "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n", + lp->id, lp->tx_ra, lp->tx_num_entries, hv_err); + + hv_err = sun4v_ldc_tx_get_state(lp->id, + &lp->tx_head, + &lp->tx_tail, + &lp->chan_state); + if (hv_err) + printk(KERN_ERR PFX "ldc_abort: " + "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n", + lp->id, hv_err); + + hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); + if (hv_err) + printk(KERN_ERR PFX "ldc_abort: " + "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n", + lp->id, lp->rx_ra, lp->rx_num_entries, hv_err); + + /* Refetch the RX queue state as well, because we could be invoked + * here in the queue processing context. + */ + hv_err = sun4v_ldc_rx_get_state(lp->id, + &lp->rx_head, + &lp->rx_tail, + &lp->chan_state); + if (hv_err) + printk(KERN_ERR PFX "ldc_abort: " + "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n", + lp->id, hv_err); + + return -ECONNRESET; +} + +static struct ldc_version *find_by_major(u16 major) +{ + struct ldc_version *ret = NULL; + int i; + + for (i = 0; i < ARRAY_SIZE(ver_arr); i++) { + struct ldc_version *v = &ver_arr[i]; + if (v->major <= major) { + ret = v; + break; + } + } + return ret; +} + +static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp) +{ + struct ldc_version *vap; + int err; + + ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n", + vp->major, vp->minor); + + if (lp->hs_state == LDC_HS_GOTVERS) { + lp->hs_state = LDC_HS_OPEN; + memset(&lp->ver, 0, sizeof(lp->ver)); + } + + vap = find_by_major(vp->major); + if (!vap) { + err = send_version_nack(lp, 0, 0); + } else if (vap->major != vp->major) { + err = send_version_nack(lp, vap->major, vap->minor); + } else { + struct ldc_version ver = *vp; + if (ver.minor > vap->minor) + ver.minor = vap->minor; + err = send_version_ack(lp, &ver); + if (!err) { + lp->ver = ver; + lp->hs_state = LDC_HS_GOTVERS; + } + } + if (err) + return LDC_ABORT(lp); + + return 0; +} + +static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp) +{ + ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n", + vp->major, vp->minor); + + if (lp->hs_state == LDC_HS_GOTVERS) { + if (lp->ver.major != vp->major || + lp->ver.minor != vp->minor) + return LDC_ABORT(lp); + } else { + lp->ver = *vp; + lp->hs_state = LDC_HS_GOTVERS; + } + if (send_rts(lp)) + return LDC_ABORT(lp); + return 0; +} + +static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp) +{ + struct ldc_version *vap; + struct ldc_packet *p; + unsigned long new_tail; + + if (vp->major == 0 && vp->minor == 0) + return LDC_ABORT(lp); + + vap = find_by_major(vp->major); + if (!vap) + return LDC_ABORT(lp); + + p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, + vap, sizeof(*vap), + &new_tail); + if (!p) + return LDC_ABORT(lp); + + return send_tx_packet(lp, p, new_tail); +} + +static int process_version(struct ldc_channel *lp, + struct ldc_packet *p) +{ + struct ldc_version *vp; + + vp = (struct ldc_version *) p->u.u_data; + + switch (p->stype) { + case LDC_INFO: + return process_ver_info(lp, vp); + + case LDC_ACK: + return process_ver_ack(lp, vp); + + case LDC_NACK: + return process_ver_nack(lp, vp); + + default: + return LDC_ABORT(lp); + } +} + +static int process_rts(struct ldc_channel *lp, + struct ldc_packet *p) +{ + ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n", + p->stype, p->seqid, p->env); + + if (p->stype != LDC_INFO || + lp->hs_state != LDC_HS_GOTVERS || + p->env != lp->cfg.mode) + return LDC_ABORT(lp); + + lp->snd_nxt = p->seqid; + lp->rcv_nxt = p->seqid; + lp->hs_state = LDC_HS_SENTRTR; + if (send_rtr(lp)) + return LDC_ABORT(lp); + + return 0; +} + +static int process_rtr(struct ldc_channel *lp, + struct ldc_packet *p) +{ + ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n", + p->stype, p->seqid, p->env); + + if (p->stype != LDC_INFO || + p->env != lp->cfg.mode) + return LDC_ABORT(lp); + + lp->snd_nxt = p->seqid; + lp->hs_state = LDC_HS_COMPLETE; + ldc_set_state(lp, LDC_STATE_CONNECTED); + send_rdx(lp); + + return LDC_EVENT_UP; +} + +static int rx_seq_ok(struct ldc_channel *lp, u32 seqid) +{ + return lp->rcv_nxt + 1 == seqid; +} + +static int process_rdx(struct ldc_channel *lp, + struct ldc_packet *p) +{ + ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n", + p->stype, p->seqid, p->env, p->u.r.ackid); + + if (p->stype != LDC_INFO || + !(rx_seq_ok(lp, p->seqid))) + return LDC_ABORT(lp); + + lp->rcv_nxt = p->seqid; + + lp->hs_state = LDC_HS_COMPLETE; + ldc_set_state(lp, LDC_STATE_CONNECTED); + + return LDC_EVENT_UP; +} + +static int process_control_frame(struct ldc_channel *lp, + struct ldc_packet *p) +{ + switch (p->ctrl) { + case LDC_VERS: + return process_version(lp, p); + + case LDC_RTS: + return process_rts(lp, p); + + case LDC_RTR: + return process_rtr(lp, p); + + case LDC_RDX: + return process_rdx(lp, p); + + default: + return LDC_ABORT(lp); + } +} + +static int process_error_frame(struct ldc_channel *lp, + struct ldc_packet *p) +{ + return LDC_ABORT(lp); +} + +static int process_data_ack(struct ldc_channel *lp, + struct ldc_packet *ack) +{ + unsigned long head = lp->tx_acked; + u32 ackid = ack->u.r.ackid; + + while (1) { + struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE); + + head = tx_advance(lp, head); + + if (p->seqid == ackid) { + lp->tx_acked = head; + return 0; + } + if (head == lp->tx_tail) + return LDC_ABORT(lp); + } + + return 0; +} + +static void send_events(struct ldc_channel *lp, unsigned int event_mask) +{ + if (event_mask & LDC_EVENT_RESET) + lp->cfg.event(lp->event_arg, LDC_EVENT_RESET); + if (event_mask & LDC_EVENT_UP) + lp->cfg.event(lp->event_arg, LDC_EVENT_UP); + if (event_mask & LDC_EVENT_DATA_READY) + lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY); +} + +static irqreturn_t ldc_rx(int irq, void *dev_id) +{ + struct ldc_channel *lp = dev_id; + unsigned long orig_state, flags; + unsigned int event_mask; + + spin_lock_irqsave(&lp->lock, flags); + + orig_state = lp->chan_state; + + /* We should probably check for hypervisor errors here and + * reset the LDC channel if we get one. + */ + sun4v_ldc_rx_get_state(lp->id, + &lp->rx_head, + &lp->rx_tail, + &lp->chan_state); + + ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", + orig_state, lp->chan_state, lp->rx_head, lp->rx_tail); + + event_mask = 0; + + if (lp->cfg.mode == LDC_MODE_RAW && + lp->chan_state == LDC_CHANNEL_UP) { + lp->hs_state = LDC_HS_COMPLETE; + ldc_set_state(lp, LDC_STATE_CONNECTED); + + /* + * Generate an LDC_EVENT_UP event if the channel + * was not already up. + */ + if (orig_state != LDC_CHANNEL_UP) { + event_mask |= LDC_EVENT_UP; + orig_state = lp->chan_state; + } + } + + /* If we are in reset state, flush the RX queue and ignore + * everything. + */ + if (lp->flags & LDC_FLAG_RESET) { + (void) ldc_rx_reset(lp); + goto out; + } + + /* Once we finish the handshake, we let the ldc_read() + * paths do all of the control frame and state management. + * Just trigger the callback. + */ + if (lp->hs_state == LDC_HS_COMPLETE) { +handshake_complete: + if (lp->chan_state != orig_state) { + unsigned int event = LDC_EVENT_RESET; + + if (lp->chan_state == LDC_CHANNEL_UP) + event = LDC_EVENT_UP; + + event_mask |= event; + } + if (lp->rx_head != lp->rx_tail) + event_mask |= LDC_EVENT_DATA_READY; + + goto out; + } + + if (lp->chan_state != orig_state) + goto out; + + while (lp->rx_head != lp->rx_tail) { + struct ldc_packet *p; + unsigned long new; + int err; + + p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); + + switch (p->type) { + case LDC_CTRL: + err = process_control_frame(lp, p); + if (err > 0) + event_mask |= err; + break; + + case LDC_DATA: + event_mask |= LDC_EVENT_DATA_READY; + err = 0; + break; + + case LDC_ERR: + err = process_error_frame(lp, p); + break; + + default: + err = LDC_ABORT(lp); + break; + } + + if (err < 0) + break; + + new = lp->rx_head; + new += LDC_PACKET_SIZE; + if (new == (lp->rx_num_entries * LDC_PACKET_SIZE)) + new = 0; + lp->rx_head = new; + + err = __set_rx_head(lp, new); + if (err < 0) { + (void) LDC_ABORT(lp); + break; + } + if (lp->hs_state == LDC_HS_COMPLETE) + goto handshake_complete; + } + +out: + spin_unlock_irqrestore(&lp->lock, flags); + + send_events(lp, event_mask); + + return IRQ_HANDLED; +} + +static irqreturn_t ldc_tx(int irq, void *dev_id) +{ + struct ldc_channel *lp = dev_id; + unsigned long flags, orig_state; + unsigned int event_mask = 0; + + spin_lock_irqsave(&lp->lock, flags); + + orig_state = lp->chan_state; + + /* We should probably check for hypervisor errors here and + * reset the LDC channel if we get one. + */ + sun4v_ldc_tx_get_state(lp->id, + &lp->tx_head, + &lp->tx_tail, + &lp->chan_state); + + ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", + orig_state, lp->chan_state, lp->tx_head, lp->tx_tail); + + if (lp->cfg.mode == LDC_MODE_RAW && + lp->chan_state == LDC_CHANNEL_UP) { + lp->hs_state = LDC_HS_COMPLETE; + ldc_set_state(lp, LDC_STATE_CONNECTED); + + /* + * Generate an LDC_EVENT_UP event if the channel + * was not already up. + */ + if (orig_state != LDC_CHANNEL_UP) { + event_mask |= LDC_EVENT_UP; + orig_state = lp->chan_state; + } + } + + spin_unlock_irqrestore(&lp->lock, flags); + + send_events(lp, event_mask); + + return IRQ_HANDLED; +} + +/* XXX ldc_alloc() and ldc_free() needs to run under a mutex so + * XXX that addition and removal from the ldc_channel_list has + * XXX atomicity, otherwise the __ldc_channel_exists() check is + * XXX totally pointless as another thread can slip into ldc_alloc() + * XXX and add a channel with the same ID. There also needs to be + * XXX a spinlock for ldc_channel_list. + */ +static HLIST_HEAD(ldc_channel_list); + +static int __ldc_channel_exists(unsigned long id) +{ + struct ldc_channel *lp; + + hlist_for_each_entry(lp, &ldc_channel_list, list) { + if (lp->id == id) + return 1; + } + return 0; +} + +static int alloc_queue(const char *name, unsigned long num_entries, + struct ldc_packet **base, unsigned long *ra) +{ + unsigned long size, order; + void *q; + + size = num_entries * LDC_PACKET_SIZE; + order = get_order(size); + + q = (void *) __get_free_pages(GFP_KERNEL, order); + if (!q) { + printk(KERN_ERR PFX "Alloc of %s queue failed with " + "size=%lu order=%lu\n", name, size, order); + return -ENOMEM; + } + + memset(q, 0, PAGE_SIZE << order); + + *base = q; + *ra = __pa(q); + + return 0; +} + +static void free_queue(unsigned long num_entries, struct ldc_packet *q) +{ + unsigned long size, order; + + if (!q) + return; + + size = num_entries * LDC_PACKET_SIZE; + order = get_order(size); + + free_pages((unsigned long)q, order); +} + +static unsigned long ldc_cookie_to_index(u64 cookie, void *arg) +{ + u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; + /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */ + + cookie &= ~COOKIE_PGSZ_CODE; + + return (cookie >> (13ULL + (szcode * 3ULL))); +} + +static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie, + unsigned long entry, unsigned long npages) +{ + struct ldc_mtable_entry *base; + unsigned long i, shift; + + shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3; + base = iommu->page_table + entry; + for (i = 0; i < npages; i++) { + if (base->cookie) + sun4v_ldc_revoke(id, cookie + (i << shift), + base->cookie); + base->mte = 0; + } +} + +/* XXX Make this configurable... XXX */ +#define LDC_IOTABLE_SIZE (8 * 1024) + +static int ldc_iommu_init(const char *name, struct ldc_channel *lp) +{ + unsigned long sz, num_tsb_entries, tsbsize, order; + struct ldc_iommu *ldc_iommu = &lp->iommu; + struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; + struct ldc_mtable_entry *table; + unsigned long hv_err; + int err; + + num_tsb_entries = LDC_IOTABLE_SIZE; + tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); + spin_lock_init(&ldc_iommu->lock); + + sz = num_tsb_entries / 8; + sz = (sz + 7UL) & ~7UL; + iommu->map = kzalloc(sz, GFP_KERNEL); + if (!iommu->map) { + printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); + return -ENOMEM; + } + iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT, + NULL, false /* no large pool */, + 1 /* npools */, + true /* skip span boundary check */); + + order = get_order(tsbsize); + + table = (struct ldc_mtable_entry *) + __get_free_pages(GFP_KERNEL, order); + err = -ENOMEM; + if (!table) { + printk(KERN_ERR PFX "Alloc of MTE table failed, " + "size=%lu order=%lu\n", tsbsize, order); + goto out_free_map; + } + + memset(table, 0, PAGE_SIZE << order); + + ldc_iommu->page_table = table; + + hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), + num_tsb_entries); + err = -EINVAL; + if (hv_err) + goto out_free_table; + + return 0; + +out_free_table: + free_pages((unsigned long) table, order); + ldc_iommu->page_table = NULL; + +out_free_map: + kfree(iommu->map); + iommu->map = NULL; + + return err; +} + +static void ldc_iommu_release(struct ldc_channel *lp) +{ + struct ldc_iommu *ldc_iommu = &lp->iommu; + struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; + unsigned long num_tsb_entries, tsbsize, order; + + (void) sun4v_ldc_set_map_table(lp->id, 0, 0); + + num_tsb_entries = iommu->poolsize * iommu->nr_pools; + tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); + order = get_order(tsbsize); + + free_pages((unsigned long) ldc_iommu->page_table, order); + ldc_iommu->page_table = NULL; + + kfree(iommu->map); + iommu->map = NULL; +} + +struct ldc_channel *ldc_alloc(unsigned long id, + const struct ldc_channel_config *cfgp, + void *event_arg, + const char *name) +{ + struct ldc_channel *lp; + const struct ldc_mode_ops *mops; + unsigned long dummy1, dummy2, hv_err; + u8 mss, *mssbuf; + int err; + + err = -ENODEV; + if (!ldom_domaining_enabled) + goto out_err; + + err = -EINVAL; + if (!cfgp) + goto out_err; + if (!name) + goto out_err; + + switch (cfgp->mode) { + case LDC_MODE_RAW: + mops = &raw_ops; + mss = LDC_PACKET_SIZE; + break; + + case LDC_MODE_UNRELIABLE: + mops = &nonraw_ops; + mss = LDC_PACKET_SIZE - 8; + break; + + case LDC_MODE_STREAM: + mops = &stream_ops; + mss = LDC_PACKET_SIZE - 8 - 8; + break; + + default: + goto out_err; + } + + if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq) + goto out_err; + + hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2); + err = -ENODEV; + if (hv_err == HV_ECHANNEL) + goto out_err; + + err = -EEXIST; + if (__ldc_channel_exists(id)) + goto out_err; + + mssbuf = NULL; + + lp = kzalloc_obj(*lp); + err = -ENOMEM; + if (!lp) + goto out_err; + + spin_lock_init(&lp->lock); + + lp->id = id; + + err = ldc_iommu_init(name, lp); + if (err) + goto out_free_ldc; + + lp->mops = mops; + lp->mss = mss; + + lp->cfg = *cfgp; + if (!lp->cfg.mtu) + lp->cfg.mtu = LDC_DEFAULT_MTU; + + if (lp->cfg.mode == LDC_MODE_STREAM) { + mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL); + if (!mssbuf) { + err = -ENOMEM; + goto out_free_iommu; + } + lp->mssbuf = mssbuf; + } + + lp->event_arg = event_arg; + + /* XXX allow setting via ldc_channel_config to override defaults + * XXX or use some formula based upon mtu + */ + lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES; + lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES; + + err = alloc_queue("TX", lp->tx_num_entries, + &lp->tx_base, &lp->tx_ra); + if (err) + goto out_free_mssbuf; + + err = alloc_queue("RX", lp->rx_num_entries, + &lp->rx_base, &lp->rx_ra); + if (err) + goto out_free_txq; + + lp->flags |= LDC_FLAG_ALLOCED_QUEUES; + + lp->hs_state = LDC_HS_CLOSED; + ldc_set_state(lp, LDC_STATE_INIT); + + INIT_HLIST_NODE(&lp->list); + hlist_add_head(&lp->list, &ldc_channel_list); + + INIT_HLIST_HEAD(&lp->mh_list); + + snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); + snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); + + err = request_irq(lp->cfg.rx_irq, ldc_rx, 0, + lp->rx_irq_name, lp); + if (err) + goto out_free_txq; + + err = request_irq(lp->cfg.tx_irq, ldc_tx, 0, + lp->tx_irq_name, lp); + if (err) { + free_irq(lp->cfg.rx_irq, lp); + goto out_free_txq; + } + + return lp; + +out_free_txq: + free_queue(lp->tx_num_entries, lp->tx_base); + +out_free_mssbuf: + kfree(mssbuf); + +out_free_iommu: + ldc_iommu_release(lp); + +out_free_ldc: + kfree(lp); + +out_err: + return ERR_PTR(err); +} +EXPORT_SYMBOL(ldc_alloc); + +void ldc_unbind(struct ldc_channel *lp) +{ + if (lp->flags & LDC_FLAG_REGISTERED_IRQS) { + free_irq(lp->cfg.rx_irq, lp); + free_irq(lp->cfg.tx_irq, lp); + lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; + } + + if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) { + sun4v_ldc_tx_qconf(lp->id, 0, 0); + sun4v_ldc_rx_qconf(lp->id, 0, 0); + lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; + } + if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) { + free_queue(lp->tx_num_entries, lp->tx_base); + free_queue(lp->rx_num_entries, lp->rx_base); + lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES; + } + + ldc_set_state(lp, LDC_STATE_INIT); +} +EXPORT_SYMBOL(ldc_unbind); + +void ldc_free(struct ldc_channel *lp) +{ + ldc_unbind(lp); + hlist_del(&lp->list); + kfree(lp->mssbuf); + ldc_iommu_release(lp); + + kfree(lp); +} +EXPORT_SYMBOL(ldc_free); + +/* Bind the channel. This registers the LDC queues with + * the hypervisor and puts the channel into a pseudo-listening + * state. This does not initiate a handshake, ldc_connect() does + * that. + */ +int ldc_bind(struct ldc_channel *lp) +{ + unsigned long hv_err, flags; + int err = -EINVAL; + + if (lp->state != LDC_STATE_INIT) + return -EINVAL; + + spin_lock_irqsave(&lp->lock, flags); + + enable_irq(lp->cfg.rx_irq); + enable_irq(lp->cfg.tx_irq); + + lp->flags |= LDC_FLAG_REGISTERED_IRQS; + + err = -ENODEV; + hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); + if (hv_err) + goto out_free_irqs; + + hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); + if (hv_err) + goto out_free_irqs; + + hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); + if (hv_err) + goto out_unmap_tx; + + hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); + if (hv_err) + goto out_unmap_tx; + + lp->flags |= LDC_FLAG_REGISTERED_QUEUES; + + hv_err = sun4v_ldc_tx_get_state(lp->id, + &lp->tx_head, + &lp->tx_tail, + &lp->chan_state); + err = -EBUSY; + if (hv_err) + goto out_unmap_rx; + + lp->tx_acked = lp->tx_head; + + lp->hs_state = LDC_HS_OPEN; + ldc_set_state(lp, LDC_STATE_BOUND); + + if (lp->cfg.mode == LDC_MODE_RAW) { + /* + * There is no handshake in RAW mode, so handshake + * is completed. + */ + lp->hs_state = LDC_HS_COMPLETE; + } + + spin_unlock_irqrestore(&lp->lock, flags); + + return 0; + +out_unmap_rx: + lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; + sun4v_ldc_rx_qconf(lp->id, 0, 0); + +out_unmap_tx: + sun4v_ldc_tx_qconf(lp->id, 0, 0); + +out_free_irqs: + lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; + free_irq(lp->cfg.tx_irq, lp); + free_irq(lp->cfg.rx_irq, lp); + + spin_unlock_irqrestore(&lp->lock, flags); + + return err; +} +EXPORT_SYMBOL(ldc_bind); + +int ldc_connect(struct ldc_channel *lp) +{ + unsigned long flags; + int err; + + if (lp->cfg.mode == LDC_MODE_RAW) + return -EINVAL; + + spin_lock_irqsave(&lp->lock, flags); + + if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || + !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) || + lp->hs_state != LDC_HS_OPEN) + err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL); + else + err = start_handshake(lp); + + spin_unlock_irqrestore(&lp->lock, flags); + + return err; +} +EXPORT_SYMBOL(ldc_connect); + +int ldc_disconnect(struct ldc_channel *lp) +{ + unsigned long hv_err, flags; + int err; + + if (lp->cfg.mode == LDC_MODE_RAW) + return -EINVAL; + + if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || + !(lp->flags & LDC_FLAG_REGISTERED_QUEUES)) + return -EINVAL; + + spin_lock_irqsave(&lp->lock, flags); + + err = -ENODEV; + hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); + if (hv_err) + goto out_err; + + hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); + if (hv_err) + goto out_err; + + hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); + if (hv_err) + goto out_err; + + hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); + if (hv_err) + goto out_err; + + ldc_set_state(lp, LDC_STATE_BOUND); + lp->hs_state = LDC_HS_OPEN; + lp->flags |= LDC_FLAG_RESET; + + spin_unlock_irqrestore(&lp->lock, flags); + + return 0; + +out_err: + sun4v_ldc_tx_qconf(lp->id, 0, 0); + sun4v_ldc_rx_qconf(lp->id, 0, 0); + free_irq(lp->cfg.tx_irq, lp); + free_irq(lp->cfg.rx_irq, lp); + lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS | + LDC_FLAG_REGISTERED_QUEUES); + ldc_set_state(lp, LDC_STATE_INIT); + + spin_unlock_irqrestore(&lp->lock, flags); + + return err; +} +EXPORT_SYMBOL(ldc_disconnect); + +int ldc_state(struct ldc_channel *lp) +{ + return lp->state; +} +EXPORT_SYMBOL(ldc_state); + +void ldc_set_state(struct ldc_channel *lp, u8 state) +{ + ldcdbg(STATE, "STATE (%s) --> (%s)\n", + state_to_str(lp->state), + state_to_str(state)); + + lp->state = state; +} +EXPORT_SYMBOL(ldc_set_state); + +int ldc_mode(struct ldc_channel *lp) +{ + return lp->cfg.mode; +} +EXPORT_SYMBOL(ldc_mode); + +int ldc_rx_reset(struct ldc_channel *lp) +{ + return __set_rx_head(lp, lp->rx_tail); +} +EXPORT_SYMBOL(ldc_rx_reset); + +void __ldc_print(struct ldc_channel *lp, const char *caller) +{ + pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n" + "\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n" + "\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n" + "\trcv_nxt=%u snd_nxt=%u\n", + caller, lp->id, lp->flags, state_to_str(lp->state), + lp->chan_state, lp->hs_state, + lp->rx_head, lp->rx_tail, lp->rx_num_entries, + lp->tx_head, lp->tx_tail, lp->tx_num_entries, + lp->rcv_nxt, lp->snd_nxt); +} +EXPORT_SYMBOL(__ldc_print); + +static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size) +{ + struct ldc_packet *p; + unsigned long new_tail, hv_err; + int err; + + hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, + &lp->chan_state); + if (unlikely(hv_err)) + return -EBUSY; + + if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) + return LDC_ABORT(lp); + + if (size > LDC_PACKET_SIZE) + return -EMSGSIZE; + + p = data_get_tx_packet(lp, &new_tail); + if (!p) + return -EAGAIN; + + memcpy(p, buf, size); + + err = send_tx_packet(lp, p, new_tail); + if (!err) + err = size; + + return err; +} + +static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size) +{ + struct ldc_packet *p; + unsigned long hv_err, new; + int err; + + if (size < LDC_PACKET_SIZE) + return -EINVAL; + + hv_err = sun4v_ldc_rx_get_state(lp->id, + &lp->rx_head, + &lp->rx_tail, + &lp->chan_state); + if (hv_err) + return LDC_ABORT(lp); + + if (lp->chan_state == LDC_CHANNEL_DOWN || + lp->chan_state == LDC_CHANNEL_RESETTING) + return -ECONNRESET; + + if (lp->rx_head == lp->rx_tail) + return 0; + + p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); + memcpy(buf, p, LDC_PACKET_SIZE); + + new = rx_advance(lp, lp->rx_head); + lp->rx_head = new; + + err = __set_rx_head(lp, new); + if (err < 0) + err = -ECONNRESET; + else + err = LDC_PACKET_SIZE; + + return err; +} + +static const struct ldc_mode_ops raw_ops = { + .write = write_raw, + .read = read_raw, +}; + +static int write_nonraw(struct ldc_channel *lp, const void *buf, + unsigned int size) +{ + unsigned long hv_err, tail; + unsigned int copied; + u32 seq; + int err; + + hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, + &lp->chan_state); + if (unlikely(hv_err)) + return -EBUSY; + + if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) + return LDC_ABORT(lp); + + if (!tx_has_space_for(lp, size)) + return -EAGAIN; + + seq = lp->snd_nxt; + copied = 0; + tail = lp->tx_tail; + while (copied < size) { + struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE); + u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ? + p->u.u_data : + p->u.r.r_data); + int data_len; + + p->type = LDC_DATA; + p->stype = LDC_INFO; + p->ctrl = 0; + + data_len = size - copied; + if (data_len > lp->mss) + data_len = lp->mss; + + BUG_ON(data_len > LDC_LEN); + + p->env = (data_len | + (copied == 0 ? LDC_START : 0) | + (data_len == size - copied ? LDC_STOP : 0)); + + p->seqid = ++seq; + + ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n", + p->type, + p->stype, + p->ctrl, + p->env, + p->seqid); + + memcpy(data, buf, data_len); + buf += data_len; + copied += data_len; + + tail = tx_advance(lp, tail); + } + + err = set_tx_tail(lp, tail); + if (!err) { + lp->snd_nxt = seq; + err = size; + } + + return err; +} + +static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p, + struct ldc_packet *first_frag) +{ + int err; + + if (first_frag) + lp->rcv_nxt = first_frag->seqid - 1; + + err = send_data_nack(lp, p); + if (err) + return err; + + err = ldc_rx_reset(lp); + if (err < 0) + return LDC_ABORT(lp); + + return 0; +} + +static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p) +{ + if (p->stype & LDC_ACK) { + int err = process_data_ack(lp, p); + if (err) + return err; + } + if (p->stype & LDC_NACK) + return LDC_ABORT(lp); + + return 0; +} + +static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head) +{ + unsigned long dummy; + int limit = 1000; + + ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n", + cur_head, lp->rx_head, lp->rx_tail); + while (limit-- > 0) { + unsigned long hv_err; + + hv_err = sun4v_ldc_rx_get_state(lp->id, + &dummy, + &lp->rx_tail, + &lp->chan_state); + if (hv_err) + return LDC_ABORT(lp); + + if (lp->chan_state == LDC_CHANNEL_DOWN || + lp->chan_state == LDC_CHANNEL_RESETTING) + return -ECONNRESET; + + if (cur_head != lp->rx_tail) { + ldcdbg(DATA, "DATA WAIT DONE " + "head[%lx] tail[%lx] chan_state[%lx]\n", + dummy, lp->rx_tail, lp->chan_state); + return 0; + } + + udelay(1); + } + return -EAGAIN; +} + +static int rx_set_head(struct ldc_channel *lp, unsigned long head) +{ + int err = __set_rx_head(lp, head); + + if (err < 0) + return LDC_ABORT(lp); + + lp->rx_head = head; + return 0; +} + +static void send_data_ack(struct ldc_channel *lp) +{ + unsigned long new_tail; + struct ldc_packet *p; + + p = data_get_tx_packet(lp, &new_tail); + if (likely(p)) { + int err; + + memset(p, 0, sizeof(*p)); + p->type = LDC_DATA; + p->stype = LDC_ACK; + p->ctrl = 0; + p->seqid = lp->snd_nxt + 1; + p->u.r.ackid = lp->rcv_nxt; + + err = send_tx_packet(lp, p, new_tail); + if (!err) + lp->snd_nxt++; + } +} + +static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) +{ + struct ldc_packet *first_frag; + unsigned long hv_err, new; + int err, copied; + + hv_err = sun4v_ldc_rx_get_state(lp->id, + &lp->rx_head, + &lp->rx_tail, + &lp->chan_state); + if (hv_err) + return LDC_ABORT(lp); + + if (lp->chan_state == LDC_CHANNEL_DOWN || + lp->chan_state == LDC_CHANNEL_RESETTING) + return -ECONNRESET; + + if (lp->rx_head == lp->rx_tail) + return 0; + + first_frag = NULL; + copied = err = 0; + new = lp->rx_head; + while (1) { + struct ldc_packet *p; + int pkt_len; + + BUG_ON(new == lp->rx_tail); + p = lp->rx_base + (new / LDC_PACKET_SIZE); + + ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] " + "rcv_nxt[%08x]\n", + p->type, + p->stype, + p->ctrl, + p->env, + p->seqid, + p->u.r.ackid, + lp->rcv_nxt); + + if (unlikely(!rx_seq_ok(lp, p->seqid))) { + err = rx_bad_seq(lp, p, first_frag); + copied = 0; + break; + } + + if (p->type & LDC_CTRL) { + err = process_control_frame(lp, p); + if (err < 0) + break; + err = 0; + } + + lp->rcv_nxt = p->seqid; + + /* + * If this is a control-only packet, there is nothing + * else to do but advance the rx queue since the packet + * was already processed above. + */ + if (!(p->type & LDC_DATA)) { + new = rx_advance(lp, new); + break; + } + if (p->stype & (LDC_ACK | LDC_NACK)) { + err = data_ack_nack(lp, p); + if (err) + break; + } + if (!(p->stype & LDC_INFO)) { + new = rx_advance(lp, new); + err = rx_set_head(lp, new); + if (err) + break; + goto no_data; + } + + pkt_len = p->env & LDC_LEN; + + /* Every initial packet starts with the START bit set. + * + * Singleton packets will have both START+STOP set. + * + * Fragments will have START set in the first frame, STOP + * set in the last frame, and neither bit set in middle + * frames of the packet. + * + * Therefore if we are at the beginning of a packet and + * we don't see START, or we are in the middle of a fragmented + * packet and do see START, we are unsynchronized and should + * flush the RX queue. + */ + if ((first_frag == NULL && !(p->env & LDC_START)) || + (first_frag != NULL && (p->env & LDC_START))) { + if (!first_frag) + new = rx_advance(lp, new); + + err = rx_set_head(lp, new); + if (err) + break; + + if (!first_frag) + goto no_data; + } + if (!first_frag) + first_frag = p; + + if (pkt_len > size - copied) { + /* User didn't give us a big enough buffer, + * what to do? This is a pretty serious error. + * + * Since we haven't updated the RX ring head to + * consume any of the packets, signal the error + * to the user and just leave the RX ring alone. + * + * This seems the best behavior because this allows + * a user of the LDC layer to start with a small + * RX buffer for ldc_read() calls and use -EMSGSIZE + * as a cue to enlarge its read buffer. + */ + err = -EMSGSIZE; + break; + } + + /* Ok, we are gonna eat this one. */ + new = rx_advance(lp, new); + + memcpy(buf, + (lp->cfg.mode == LDC_MODE_UNRELIABLE ? + p->u.u_data : p->u.r.r_data), pkt_len); + buf += pkt_len; + copied += pkt_len; + + if (p->env & LDC_STOP) + break; + +no_data: + if (new == lp->rx_tail) { + err = rx_data_wait(lp, new); + if (err) + break; + } + } + + if (!err) + err = rx_set_head(lp, new); + + if (err && first_frag) + lp->rcv_nxt = first_frag->seqid - 1; + + if (!err) { + err = copied; + if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE) + send_data_ack(lp); + } + + return err; +} + +static const struct ldc_mode_ops nonraw_ops = { + .write = write_nonraw, + .read = read_nonraw, +}; + +static int write_stream(struct ldc_channel *lp, const void *buf, + unsigned int size) +{ + if (size > lp->cfg.mtu) + size = lp->cfg.mtu; + return write_nonraw(lp, buf, size); +} + +static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size) +{ + if (!lp->mssbuf_len) { + int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu); + if (err < 0) + return err; + + lp->mssbuf_len = err; + lp->mssbuf_off = 0; + } + + if (size > lp->mssbuf_len) + size = lp->mssbuf_len; + memcpy(buf, lp->mssbuf + lp->mssbuf_off, size); + + lp->mssbuf_off += size; + lp->mssbuf_len -= size; + + return size; +} + +static const struct ldc_mode_ops stream_ops = { + .write = write_stream, + .read = read_stream, +}; + +int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size) +{ + unsigned long flags; + int err; + + if (!buf) + return -EINVAL; + + if (!size) + return 0; + + spin_lock_irqsave(&lp->lock, flags); + + if (lp->hs_state != LDC_HS_COMPLETE) + err = -ENOTCONN; + else + err = lp->mops->write(lp, buf, size); + + spin_unlock_irqrestore(&lp->lock, flags); + + return err; +} +EXPORT_SYMBOL(ldc_write); + +int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) +{ + unsigned long flags; + int err; + + ldcdbg(RX, "%s: entered size=%d\n", __func__, size); + + if (!buf) + return -EINVAL; + + if (!size) + return 0; + + spin_lock_irqsave(&lp->lock, flags); + + if (lp->hs_state != LDC_HS_COMPLETE) + err = -ENOTCONN; + else + err = lp->mops->read(lp, buf, size); + + spin_unlock_irqrestore(&lp->lock, flags); + + ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__, + lp->cfg.mode, lp->rx_head, lp->rx_tail, err); + + return err; +} +EXPORT_SYMBOL(ldc_read); + +static u64 pagesize_code(void) +{ + switch (PAGE_SIZE) { + default: + case (8ULL * 1024ULL): + return 0; + case (64ULL * 1024ULL): + return 1; + case (512ULL * 1024ULL): + return 2; + case (4ULL * 1024ULL * 1024ULL): + return 3; + case (32ULL * 1024ULL * 1024ULL): + return 4; + case (256ULL * 1024ULL * 1024ULL): + return 5; + } +} + +static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) +{ + return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) | + (index << PAGE_SHIFT) | + page_offset); +} + + +static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, + unsigned long npages) +{ + long entry; + + entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table, + npages, NULL, (unsigned long)-1, 0); + if (unlikely(entry == IOMMU_ERROR_CODE)) + return NULL; + + return iommu->page_table + entry; +} + +static u64 perm_to_mte(unsigned int map_perm) +{ + u64 mte_base; + + mte_base = pagesize_code(); + + if (map_perm & LDC_MAP_SHADOW) { + if (map_perm & LDC_MAP_R) + mte_base |= LDC_MTE_COPY_R; + if (map_perm & LDC_MAP_W) + mte_base |= LDC_MTE_COPY_W; + } + if (map_perm & LDC_MAP_DIRECT) { + if (map_perm & LDC_MAP_R) + mte_base |= LDC_MTE_READ; + if (map_perm & LDC_MAP_W) + mte_base |= LDC_MTE_WRITE; + if (map_perm & LDC_MAP_X) + mte_base |= LDC_MTE_EXEC; + } + if (map_perm & LDC_MAP_IO) { + if (map_perm & LDC_MAP_R) + mte_base |= LDC_MTE_IOMMU_R; + if (map_perm & LDC_MAP_W) + mte_base |= LDC_MTE_IOMMU_W; + } + + return mte_base; +} + +static int pages_in_region(unsigned long base, long len) +{ + int count = 0; + + do { + unsigned long new = (base + PAGE_SIZE) & PAGE_MASK; + + len -= (new - base); + base = new; + count++; + } while (len > 0); + + return count; +} + +struct cookie_state { + struct ldc_mtable_entry *page_table; + struct ldc_trans_cookie *cookies; + u64 mte_base; + u64 prev_cookie; + u32 pte_idx; + u32 nc; +}; + +static void fill_cookies(struct cookie_state *sp, unsigned long pa, + unsigned long off, unsigned long len) +{ + do { + unsigned long tlen, new = pa + PAGE_SIZE; + u64 this_cookie; + + sp->page_table[sp->pte_idx].mte = sp->mte_base | pa; + + tlen = PAGE_SIZE; + if (off) + tlen = PAGE_SIZE - off; + if (tlen > len) + tlen = len; + + this_cookie = make_cookie(sp->pte_idx, + pagesize_code(), off); + + off = 0; + + if (this_cookie == sp->prev_cookie) { + sp->cookies[sp->nc - 1].cookie_size += tlen; + } else { + sp->cookies[sp->nc].cookie_addr = this_cookie; + sp->cookies[sp->nc].cookie_size = tlen; + sp->nc++; + } + sp->prev_cookie = this_cookie + tlen; + + sp->pte_idx++; + + len -= tlen; + pa = new; + } while (len > 0); +} + +static int sg_count_one(struct scatterlist *sg) +{ + unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT; + long len = sg->length; + + if ((sg->offset | len) & (8UL - 1)) + return -EFAULT; + + return pages_in_region(base + sg->offset, len); +} + +static int sg_count_pages(struct scatterlist *sg, int num_sg) +{ + int count; + int i; + + count = 0; + for (i = 0; i < num_sg; i++) { + int err = sg_count_one(sg + i); + if (err < 0) + return err; + count += err; + } + + return count; +} + +int ldc_map_sg(struct ldc_channel *lp, + struct scatterlist *sg, int num_sg, + struct ldc_trans_cookie *cookies, int ncookies, + unsigned int map_perm) +{ + unsigned long i, npages; + struct ldc_mtable_entry *base; + struct cookie_state state; + struct ldc_iommu *iommu; + int err; + struct scatterlist *s; + + if (map_perm & ~LDC_MAP_ALL) + return -EINVAL; + + err = sg_count_pages(sg, num_sg); + if (err < 0) + return err; + + npages = err; + if (err > ncookies) + return -EMSGSIZE; + + iommu = &lp->iommu; + + base = alloc_npages(iommu, npages); + + if (!base) + return -ENOMEM; + + state.page_table = iommu->page_table; + state.cookies = cookies; + state.mte_base = perm_to_mte(map_perm); + state.prev_cookie = ~(u64)0; + state.pte_idx = (base - iommu->page_table); + state.nc = 0; + + for_each_sg(sg, s, num_sg, i) { + fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT, + s->offset, s->length); + } + + return state.nc; +} +EXPORT_SYMBOL(ldc_map_sg); + +int ldc_map_single(struct ldc_channel *lp, + void *buf, unsigned int len, + struct ldc_trans_cookie *cookies, int ncookies, + unsigned int map_perm) +{ + unsigned long npages, pa; + struct ldc_mtable_entry *base; + struct cookie_state state; + struct ldc_iommu *iommu; + + if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1)) + return -EINVAL; + + pa = __pa(buf); + if ((pa | len) & (8UL - 1)) + return -EFAULT; + + npages = pages_in_region(pa, len); + + iommu = &lp->iommu; + + base = alloc_npages(iommu, npages); + + if (!base) + return -ENOMEM; + + state.page_table = iommu->page_table; + state.cookies = cookies; + state.mte_base = perm_to_mte(map_perm); + state.prev_cookie = ~(u64)0; + state.pte_idx = (base - iommu->page_table); + state.nc = 0; + fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len); + BUG_ON(state.nc > ncookies); + + return state.nc; +} +EXPORT_SYMBOL(ldc_map_single); + + +static void free_npages(unsigned long id, struct ldc_iommu *iommu, + u64 cookie, u64 size) +{ + unsigned long npages, entry; + + npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; + + entry = ldc_cookie_to_index(cookie, iommu); + ldc_demap(iommu, id, cookie, entry, npages); + iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry); +} + +void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, + int ncookies) +{ + struct ldc_iommu *iommu = &lp->iommu; + int i; + unsigned long flags; + + spin_lock_irqsave(&iommu->lock, flags); + for (i = 0; i < ncookies; i++) { + u64 addr = cookies[i].cookie_addr; + u64 size = cookies[i].cookie_size; + + free_npages(lp->id, iommu, addr, size); + } + spin_unlock_irqrestore(&iommu->lock, flags); +} +EXPORT_SYMBOL(ldc_unmap); + +int ldc_copy(struct ldc_channel *lp, int copy_dir, + void *buf, unsigned int len, unsigned long offset, + struct ldc_trans_cookie *cookies, int ncookies) +{ + unsigned int orig_len; + unsigned long ra; + int i; + + if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) { + printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n", + lp->id, copy_dir); + return -EINVAL; + } + + ra = __pa(buf); + if ((ra | len | offset) & (8UL - 1)) { + printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer " + "ra[%lx] len[%x] offset[%lx]\n", + lp->id, ra, len, offset); + return -EFAULT; + } + + if (lp->hs_state != LDC_HS_COMPLETE || + (lp->flags & LDC_FLAG_RESET)) { + printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] " + "flags[%x]\n", lp->id, lp->hs_state, lp->flags); + return -ECONNRESET; + } + + orig_len = len; + for (i = 0; i < ncookies; i++) { + unsigned long cookie_raddr = cookies[i].cookie_addr; + unsigned long this_len = cookies[i].cookie_size; + unsigned long actual_len; + + if (unlikely(offset)) { + unsigned long this_off = offset; + + if (this_off > this_len) + this_off = this_len; + + offset -= this_off; + this_len -= this_off; + if (!this_len) + continue; + cookie_raddr += this_off; + } + + if (this_len > len) + this_len = len; + + while (1) { + unsigned long hv_err; + + hv_err = sun4v_ldc_copy(lp->id, copy_dir, + cookie_raddr, ra, + this_len, &actual_len); + if (unlikely(hv_err)) { + printk(KERN_ERR PFX "ldc_copy: ID[%lu] " + "HV error %lu\n", + lp->id, hv_err); + if (lp->hs_state != LDC_HS_COMPLETE || + (lp->flags & LDC_FLAG_RESET)) + return -ECONNRESET; + else + return -EFAULT; + } + + cookie_raddr += actual_len; + ra += actual_len; + len -= actual_len; + if (actual_len == this_len) + break; + + this_len -= actual_len; + } + + if (!len) + break; + } + + /* It is caller policy what to do about short copies. + * For example, a networking driver can declare the + * packet a runt and drop it. + */ + + return orig_len - len; +} +EXPORT_SYMBOL(ldc_copy); + +void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, + struct ldc_trans_cookie *cookies, int *ncookies, + unsigned int map_perm) +{ + void *buf; + int err; + + if (len & (8UL - 1)) + return ERR_PTR(-EINVAL); + + buf = kzalloc(len, GFP_ATOMIC); + if (!buf) + return ERR_PTR(-ENOMEM); + + err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm); + if (err < 0) { + kfree(buf); + return ERR_PTR(err); + } + *ncookies = err; + + return buf; +} +EXPORT_SYMBOL(ldc_alloc_exp_dring); + +void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len, + struct ldc_trans_cookie *cookies, int ncookies) +{ + ldc_unmap(lp, cookies, ncookies); + kfree(buf); +} +EXPORT_SYMBOL(ldc_free_exp_dring); + +static int __init ldc_init(void) +{ + unsigned long major, minor; + struct mdesc_handle *hp; + const u64 *v; + int err; + u64 mp; + + hp = mdesc_grab(); + if (!hp) + return -ENODEV; + + mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); + err = -ENODEV; + if (mp == MDESC_NODE_NULL) + goto out; + + v = mdesc_get_property(hp, mp, "domaining-enabled", NULL); + if (!v) + goto out; + + major = 1; + minor = 0; + if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) { + printk(KERN_INFO PFX "Could not register LDOM hvapi.\n"); + goto out; + } + + printk(KERN_INFO "%s", version); + + if (!*v) { + printk(KERN_INFO PFX "Domaining disabled.\n"); + goto out; + } + ldom_domaining_enabled = 1; + err = 0; + +out: + mdesc_release(hp); + return err; +} + +core_initcall(ldc_init); diff --git a/arch/sparc/kernel/led.c b/arch/sparc/kernel/led.c new file mode 100644 index 000000000..9b53ac1fe --- /dev/null +++ b/arch/sparc/kernel/led.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define LED_MAX_LENGTH 8 /* maximum chars written to proc file */ + +static inline void led_toggle(void) +{ + unsigned char val = get_auxio(); + unsigned char on, off; + + if (val & AUXIO_LED) { + on = 0; + off = AUXIO_LED; + } else { + on = AUXIO_LED; + off = 0; + } + + set_auxio(on, off); +} + +static struct timer_list led_blink_timer; +static unsigned long led_blink_timer_timeout; + +static void led_blink(struct timer_list *unused) +{ + unsigned long timeout = led_blink_timer_timeout; + + led_toggle(); + + /* reschedule */ + if (!timeout) { /* blink according to load */ + led_blink_timer.expires = jiffies + + ((1 + (avenrun[0] >> FSHIFT)) * HZ); + } else { /* blink at user specified interval */ + led_blink_timer.expires = jiffies + (timeout * HZ); + } + add_timer(&led_blink_timer); +} + +#ifdef CONFIG_PROC_FS +static int led_proc_show(struct seq_file *m, void *v) +{ + if (get_auxio() & AUXIO_LED) + seq_puts(m, "on\n"); + else + seq_puts(m, "off\n"); + return 0; +} + +static int led_proc_open(struct inode *inode, struct file *file) +{ + return single_open(file, led_proc_show, NULL); +} + +static ssize_t led_proc_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + char *buf = NULL; + + if (count > LED_MAX_LENGTH) + count = LED_MAX_LENGTH; + + buf = memdup_user_nul(buffer, count); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + /* work around \n when echo'ing into proc */ + if (count > 0 && buf[count - 1] == '\n') + buf[count - 1] = '\0'; + + /* before we change anything we want to stop any running timers, + * otherwise calls such as on will have no persistent effect + */ + timer_delete_sync(&led_blink_timer); + + if (!strcmp(buf, "on")) { + auxio_set_led(AUXIO_LED_ON); + } else if (!strcmp(buf, "toggle")) { + led_toggle(); + } else if ((*buf > '0') && (*buf <= '9')) { + led_blink_timer_timeout = simple_strtoul(buf, NULL, 10); + led_blink(&led_blink_timer); + } else if (!strcmp(buf, "load")) { + led_blink_timer_timeout = 0; + led_blink(&led_blink_timer); + } else { + auxio_set_led(AUXIO_LED_OFF); + } + + kfree(buf); + + return count; +} + +static const struct proc_ops led_proc_ops = { + .proc_open = led_proc_open, + .proc_read = seq_read, + .proc_lseek = seq_lseek, + .proc_release = single_release, + .proc_write = led_proc_write, +}; +#endif + +#define LED_VERSION "0.1" + +static int __init led_init(void) +{ + timer_setup(&led_blink_timer, led_blink, 0); + +#ifdef CONFIG_PROC_FS + if (!proc_create("led", 0, NULL, &led_proc_ops)) + return -ENOMEM; +#endif + printk(KERN_INFO + "led: version %s, Lars Kotthoff \n", + LED_VERSION); + + return 0; +} + +static void __exit led_exit(void) +{ + remove_proc_entry("led", NULL); + timer_delete_sync(&led_blink_timer); +} + +module_init(led_init); +module_exit(led_exit); + +MODULE_AUTHOR("Lars Kotthoff "); +MODULE_DESCRIPTION("Provides control of the front LED on SPARC systems."); +MODULE_LICENSE("GPL"); +MODULE_VERSION(LED_VERSION); diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c new file mode 100644 index 000000000..a43cf794b --- /dev/null +++ b/arch/sparc/kernel/leon_kernel.c @@ -0,0 +1,508 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2009 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB + * Copyright (C) 2009 Konrad Eisele (konrad@gaisler.com) Aeroflex Gaisler AB + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" +#include "prom.h" +#include "irq.h" + +struct leon3_irqctrl_regs_map *leon3_irqctrl_regs; /* interrupt controller base address */ +struct leon3_gptimer_regs_map *leon3_gptimer_regs; /* timer controller base address */ + +int leondebug_irq_disable; +int leon_debug_irqout; +static volatile u32 dummy_master_l10_counter; +unsigned long amba_system_id; +static DEFINE_SPINLOCK(leon_irq_lock); + +static unsigned long leon3_gptimer_idx; /* Timer Index (0..6) within Timer Core */ +static unsigned long leon3_gptimer_ackmask; /* For clearing pending bit */ +unsigned long leon3_gptimer_irq; /* interrupt controller irq number */ +unsigned int sparc_leon_eirq; +#define LEON_IMASK(cpu) (&leon3_irqctrl_regs->mask[cpu]) +#define LEON_IACK (&leon3_irqctrl_regs->iclear) +#define LEON_DO_ACK_HW 1 + +/* Return the last ACKed IRQ by the Extended IRQ controller. It has already + * been (automatically) ACKed when the CPU takes the trap. + */ +static inline unsigned int leon_eirq_get(int cpu) +{ + return LEON3_BYPASS_LOAD_PA(&leon3_irqctrl_regs->intid[cpu]) & 0x1f; +} + +/* Handle one or multiple IRQs from the extended interrupt controller */ +static void leon_handle_ext_irq(struct irq_desc *desc) +{ + unsigned int eirq; + struct irq_bucket *p; + int cpu = sparc_leon3_cpuid(); + + eirq = leon_eirq_get(cpu); + p = irq_map[eirq]; + if ((eirq & 0x10) && p && p->irq) /* bit4 tells if IRQ happened */ + generic_handle_irq(p->irq); +} + +/* The extended IRQ controller has been found, this function registers it */ +static void leon_eirq_setup(unsigned int eirq) +{ + unsigned long mask, oldmask; + unsigned int veirq; + + if (eirq < 1 || eirq > 0xf) { + printk(KERN_ERR "LEON EXT IRQ NUMBER BAD: %d\n", eirq); + return; + } + + veirq = leon_build_device_irq(eirq, leon_handle_ext_irq, "extirq", 0); + + /* + * Unmask the Extended IRQ, the IRQs routed through the Ext-IRQ + * controller have a mask-bit of their own, so this is safe. + */ + irq_link(veirq); + mask = 1 << eirq; + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(boot_cpu_id)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(boot_cpu_id), (oldmask | mask)); + sparc_leon_eirq = eirq; +} + +unsigned long leon_get_irqmask(unsigned int irq) +{ + unsigned long mask; + + if (!irq || ((irq > 0xf) && !sparc_leon_eirq) + || ((irq > 0x1f) && sparc_leon_eirq)) { + printk(KERN_ERR + "leon_get_irqmask: false irq number: %d\n", irq); + mask = 0; + } else { + mask = LEON_HARD_INT(irq); + } + return mask; +} + +#ifdef CONFIG_SMP +static int irq_choose_cpu(const struct cpumask *affinity) +{ + unsigned int cpu = cpumask_first_and(affinity, cpu_online_mask); + + if (cpumask_subset(cpu_online_mask, affinity) || cpu >= nr_cpu_ids) + return boot_cpu_id; + else + return cpu; +} +#else +#define irq_choose_cpu(affinity) boot_cpu_id +#endif + +static int leon_set_affinity(struct irq_data *data, const struct cpumask *dest, + bool force) +{ + unsigned long mask, oldmask, flags; + int oldcpu, newcpu; + + mask = (unsigned long)data->chip_data; + oldcpu = irq_choose_cpu(irq_data_get_affinity_mask(data)); + newcpu = irq_choose_cpu(dest); + + if (oldcpu == newcpu) + goto out; + + /* unmask on old CPU first before enabling on the selected CPU */ + spin_lock_irqsave(&leon_irq_lock, flags); + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(oldcpu)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(oldcpu), (oldmask & ~mask)); + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(newcpu)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(newcpu), (oldmask | mask)); + spin_unlock_irqrestore(&leon_irq_lock, flags); +out: + return IRQ_SET_MASK_OK; +} + +static void leon_unmask_irq(struct irq_data *data) +{ + unsigned long mask, oldmask, flags; + int cpu; + + mask = (unsigned long)data->chip_data; + cpu = irq_choose_cpu(irq_data_get_affinity_mask(data)); + spin_lock_irqsave(&leon_irq_lock, flags); + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(cpu)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(cpu), (oldmask | mask)); + spin_unlock_irqrestore(&leon_irq_lock, flags); +} + +static void leon_mask_irq(struct irq_data *data) +{ + unsigned long mask, oldmask, flags; + int cpu; + + mask = (unsigned long)data->chip_data; + cpu = irq_choose_cpu(irq_data_get_affinity_mask(data)); + spin_lock_irqsave(&leon_irq_lock, flags); + oldmask = LEON3_BYPASS_LOAD_PA(LEON_IMASK(cpu)); + LEON3_BYPASS_STORE_PA(LEON_IMASK(cpu), (oldmask & ~mask)); + spin_unlock_irqrestore(&leon_irq_lock, flags); +} + +static unsigned int leon_startup_irq(struct irq_data *data) +{ + irq_link(data->irq); + leon_unmask_irq(data); + return 0; +} + +static void leon_shutdown_irq(struct irq_data *data) +{ + leon_mask_irq(data); + irq_unlink(data->irq); +} + +/* Used by external level sensitive IRQ handlers on the LEON: ACK IRQ ctrl */ +static void leon_eoi_irq(struct irq_data *data) +{ + unsigned long mask = (unsigned long)data->chip_data; + + if (mask & LEON_DO_ACK_HW) + LEON3_BYPASS_STORE_PA(LEON_IACK, mask & ~LEON_DO_ACK_HW); +} + +static struct irq_chip leon_irq = { + .name = "leon", + .irq_startup = leon_startup_irq, + .irq_shutdown = leon_shutdown_irq, + .irq_mask = leon_mask_irq, + .irq_unmask = leon_unmask_irq, + .irq_eoi = leon_eoi_irq, + .irq_set_affinity = leon_set_affinity, +}; + +/* + * Build a LEON IRQ for the edge triggered LEON IRQ controller: + * Edge (normal) IRQ - handle_simple_irq, ack=DON'T-CARE, never ack + * Level IRQ (PCI|Level-GPIO) - handle_fasteoi_irq, ack=1, ack after ISR + * Per-CPU Edge - handle_percpu_irq, ack=0 + */ +unsigned int leon_build_device_irq(unsigned int real_irq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack) +{ + unsigned int irq; + unsigned long mask; + struct irq_desc *desc; + + irq = 0; + mask = leon_get_irqmask(real_irq); + if (mask == 0) + goto out; + + irq = irq_alloc(real_irq, real_irq); + if (irq == 0) + goto out; + + if (do_ack) + mask |= LEON_DO_ACK_HW; + + desc = irq_to_desc(irq); + if (!desc || !desc->handle_irq || desc->handle_irq == handle_bad_irq) { + irq_set_chip_and_handler_name(irq, &leon_irq, + flow_handler, name); + irq_set_chip_data(irq, (void *)mask); + } + +out: + return irq; +} + +static unsigned int _leon_build_device_irq(struct platform_device *op, + unsigned int real_irq) +{ + return leon_build_device_irq(real_irq, handle_simple_irq, "edge", 0); +} + +void leon_update_virq_handling(unsigned int virq, + irq_flow_handler_t flow_handler, + const char *name, int do_ack) +{ + unsigned long mask = (unsigned long)irq_get_chip_data(virq); + + mask &= ~LEON_DO_ACK_HW; + if (do_ack) + mask |= LEON_DO_ACK_HW; + + irq_set_chip_and_handler_name(virq, &leon_irq, + flow_handler, name); + irq_set_chip_data(virq, (void *)mask); +} + +static u32 leon_cycles_offset(void) +{ + u32 rld, val, ctrl, off; + + rld = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld); + val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val); + ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl); + if (LEON3_GPTIMER_CTRL_ISPENDING(ctrl)) { + val = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val); + off = 2 * rld - val; + } else { + off = rld - val; + } + + return off; +} + +#ifdef CONFIG_SMP + +/* smp clockevent irq */ +static irqreturn_t leon_percpu_timer_ce_interrupt(int irq, void *unused) +{ + struct clock_event_device *ce; + int cpu = smp_processor_id(); + + leon_clear_profile_irq(cpu); + + if (cpu == boot_cpu_id) + timer_interrupt(irq, NULL); + + ce = &per_cpu(sparc32_clockevent, cpu); + + irq_enter(); + if (ce->event_handler) + ce->event_handler(ce); + irq_exit(); + + return IRQ_HANDLED; +} + +#endif /* CONFIG_SMP */ + +void __init leon_init_timers(void) +{ + int irq, eirq; + struct device_node *rootnp, *np, *nnp; + struct property *pp; + int len; + int icsel; + int ampopts; + int err; + u32 config; + u32 ctrl; + + sparc_config.get_cycles_offset = leon_cycles_offset; + sparc_config.cs_period = 1000000 / HZ; + sparc_config.features |= FEAT_L10_CLOCKSOURCE; + +#ifndef CONFIG_SMP + sparc_config.features |= FEAT_L10_CLOCKEVENT; +#endif + + leondebug_irq_disable = 0; + leon_debug_irqout = 0; + master_l10_counter = (u32 __iomem *)&dummy_master_l10_counter; + dummy_master_l10_counter = 0; + + rootnp = of_find_node_by_path("/ambapp0"); + if (!rootnp) + goto bad; + + /* Find System ID: GRLIB build ID and optional CHIP ID */ + pp = of_find_property(rootnp, "systemid", &len); + if (pp) + amba_system_id = *(unsigned long *)pp->value; + + /* Find IRQMP IRQ Controller Registers base adr otherwise bail out */ + np = of_find_node_by_name(rootnp, "GAISLER_IRQMP"); + if (!np) { + np = of_find_node_by_name(rootnp, "01_00d"); + if (!np) + goto bad; + } + pp = of_find_property(np, "reg", &len); + if (!pp) + goto bad; + leon3_irqctrl_regs = *(struct leon3_irqctrl_regs_map **)pp->value; + + /* Find GPTIMER Timer Registers base address otherwise bail out. */ + nnp = rootnp; + +retry: + np = of_find_node_by_name(nnp, "GAISLER_GPTIMER"); + if (!np) { + np = of_find_node_by_name(nnp, "01_011"); + if (!np) + goto bad; + } + + ampopts = 0; + pp = of_find_property(np, "ampopts", &len); + if (pp) { + ampopts = *(int *)pp->value; + if (ampopts == 0) { + /* Skip this instance, resource already + * allocated by other OS */ + nnp = np; + goto retry; + } + } + + /* Select Timer-Instance on Timer Core. Default is zero */ + leon3_gptimer_idx = ampopts & 0x7; + + pp = of_find_property(np, "reg", &len); + if (pp) + leon3_gptimer_regs = *(struct leon3_gptimer_regs_map **) + pp->value; + pp = of_find_property(np, "interrupts", &len); + if (pp) + leon3_gptimer_irq = *(unsigned int *)pp->value; + + if (!(leon3_gptimer_regs && leon3_irqctrl_regs && leon3_gptimer_irq)) + goto bad; + + ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl); + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, + ctrl | LEON3_GPTIMER_CTRL_PENDING); + ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl); + + if ((ctrl & LEON3_GPTIMER_CTRL_PENDING) != 0) + leon3_gptimer_ackmask = ~LEON3_GPTIMER_CTRL_PENDING; + else + leon3_gptimer_ackmask = ~0; + + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].val, 0); + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].rld, + (((1000000 / HZ) - 1))); + LEON3_BYPASS_STORE_PA( + &leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, 0); + + /* + * The IRQ controller may (if implemented) consist of multiple + * IRQ controllers, each mapped on a 4Kb boundary. + * Each CPU may be routed to different IRQCTRLs, however + * we assume that all CPUs (in SMP system) is routed to the + * same IRQ Controller, and for non-SMP only one IRQCTRL is + * accessed anyway. + * In AMP systems, Linux must run on CPU0 for the time being. + */ + icsel = LEON3_BYPASS_LOAD_PA(&leon3_irqctrl_regs->icsel[boot_cpu_id/8]); + icsel = (icsel >> ((7 - (boot_cpu_id&0x7)) * 4)) & 0xf; + leon3_irqctrl_regs += icsel; + + /* Mask all IRQs on boot-cpu IRQ controller */ + LEON3_BYPASS_STORE_PA(&leon3_irqctrl_regs->mask[boot_cpu_id], 0); + + /* Probe extended IRQ controller */ + eirq = (LEON3_BYPASS_LOAD_PA(&leon3_irqctrl_regs->mpstatus) + >> 16) & 0xf; + if (eirq != 0) + leon_eirq_setup(eirq); + +#ifdef CONFIG_SMP + { + unsigned long flags; + + /* + * In SMP, sun4m adds a IPI handler to IRQ trap handler that + * LEON never must take, sun4d and LEON overwrites the branch + * with a NOP. + */ + local_irq_save(flags); + patchme_maybe_smp_msg[0] = 0x01000000; /* NOP out the branch */ + local_ops->cache_all(); + local_irq_restore(flags); + } +#endif + + config = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->config); + if (config & (1 << LEON3_GPTIMER_SEPIRQ)) + leon3_gptimer_irq += leon3_gptimer_idx; + else if ((config & LEON3_GPTIMER_TIMERS) > 1) + pr_warn("GPTIMER uses shared irqs, using other timers of the same core will fail.\n"); + +#ifdef CONFIG_SMP + /* Install per-cpu IRQ handler for broadcasted ticker */ + irq = leon_build_device_irq(leon3_gptimer_irq, handle_percpu_irq, + "per-cpu", 0); + err = request_irq(irq, leon_percpu_timer_ce_interrupt, + IRQF_PERCPU | IRQF_TIMER, "timer", NULL); +#else + irq = _leon_build_device_irq(NULL, leon3_gptimer_irq); + err = request_irq(irq, timer_interrupt, IRQF_TIMER, "timer", NULL); +#endif + if (err) { + pr_err("Unable to attach timer IRQ%d\n", irq); + prom_halt(); + } + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, + LEON3_GPTIMER_EN | + LEON3_GPTIMER_RL | + LEON3_GPTIMER_LD | + LEON3_GPTIMER_IRQEN); + return; +bad: + printk(KERN_ERR "No Timer/irqctrl found\n"); + BUG(); + return; +} + +static void leon_clear_clock_irq(void) +{ + u32 ctrl; + + ctrl = LEON3_BYPASS_LOAD_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl); + LEON3_BYPASS_STORE_PA(&leon3_gptimer_regs->e[leon3_gptimer_idx].ctrl, + ctrl & leon3_gptimer_ackmask); +} + +static void leon_load_profile_irq(int cpu, unsigned int limit) +{ +} + +#ifdef CONFIG_SMP +void leon_clear_profile_irq(int cpu) +{ +} + +void leon_enable_irq_cpu(unsigned int irq_nr, unsigned int cpu) +{ + unsigned long mask, flags, *addr; + mask = leon_get_irqmask(irq_nr); + spin_lock_irqsave(&leon_irq_lock, flags); + addr = (unsigned long *)LEON_IMASK(cpu); + LEON3_BYPASS_STORE_PA(addr, (LEON3_BYPASS_LOAD_PA(addr) | mask)); + spin_unlock_irqrestore(&leon_irq_lock, flags); +} + +#endif + +void __init leon_init_IRQ(void) +{ + sparc_config.init_timers = leon_init_timers; + sparc_config.build_device_irq = _leon_build_device_irq; + sparc_config.clock_rate = 1000000; + sparc_config.clear_clock_irq = leon_clear_clock_irq; + sparc_config.load_profile_irq = leon_load_profile_irq; +} diff --git a/arch/sparc/kernel/leon_pci.c b/arch/sparc/kernel/leon_pci.c new file mode 100644 index 000000000..10934dfa9 --- /dev/null +++ b/arch/sparc/kernel/leon_pci.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * leon_pci.c: LEON Host PCI support + * + * Copyright (C) 2011 Aeroflex Gaisler AB, Daniel Hellstrom + * + * Code is partially derived from pcic.c + */ + +#include +#include +#include +#include +#include +#include +#include + +/* The LEON architecture does not rely on a BIOS or bootloader to setup + * PCI for us. The Linux generic routines are used to setup resources, + * reset values of configuration-space register settings are preserved. + * + * PCI Memory and Prefetchable Memory is direct-mapped. However I/O Space is + * accessed through a Window which is translated to low 64KB in PCI space, the + * first 4KB is not used so 60KB is available. + */ +void leon_pci_init(struct platform_device *ofdev, struct leon_pci_info *info) +{ + LIST_HEAD(resources); + struct pci_bus *root_bus; + struct pci_host_bridge *bridge; + int ret; + + bridge = pci_alloc_host_bridge(0); + if (!bridge) + return; + + pci_add_resource_offset(&resources, &info->io_space, + info->io_space.start - 0x1000); + pci_add_resource(&resources, &info->mem_space); + info->busn.flags = IORESOURCE_BUS; + pci_add_resource(&resources, &info->busn); + + list_splice_init(&resources, &bridge->windows); + bridge->dev.parent = &ofdev->dev; + bridge->sysdata = info; + bridge->busnr = 0; + bridge->ops = info->ops; + bridge->swizzle_irq = pci_common_swizzle; + bridge->map_irq = info->map_irq; + + ret = pci_scan_root_bus_bridge(bridge); + if (ret) { + pci_free_host_bridge(bridge); + return; + } + + root_bus = bridge->bus; + + /* Assign devices with resources */ + pci_assign_unassigned_resources(); + pci_bus_add_devices(root_bus); +} diff --git a/arch/sparc/kernel/leon_pci_grpci1.c b/arch/sparc/kernel/leon_pci_grpci1.c new file mode 100644 index 000000000..b2b639bee --- /dev/null +++ b/arch/sparc/kernel/leon_pci_grpci1.c @@ -0,0 +1,723 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * leon_pci_grpci1.c: GRPCI1 Host PCI driver + * + * Copyright (C) 2013 Aeroflex Gaisler AB + * + * This GRPCI1 driver does not support PCI interrupts taken from + * GPIO pins. Interrupt generation at PCI parity and system error + * detection is by default turned off since some GRPCI1 cores does + * not support detection. It can be turned on from the bootloader + * using the all_pci_errors property. + * + * Contributors: Daniel Hellstrom + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "irq.h" + +/* Enable/Disable Debugging Configuration Space Access */ +#undef GRPCI1_DEBUG_CFGACCESS + +/* + * GRPCI1 APB Register MAP + */ +struct grpci1_regs { + unsigned int cfg_stat; /* 0x00 Configuration / Status */ + unsigned int bar0; /* 0x04 BAR0 (RO) */ + unsigned int page0; /* 0x08 PAGE0 (RO) */ + unsigned int bar1; /* 0x0C BAR1 (RO) */ + unsigned int page1; /* 0x10 PAGE1 */ + unsigned int iomap; /* 0x14 IO Map */ + unsigned int stat_cmd; /* 0x18 PCI Status & Command (RO) */ + unsigned int irq; /* 0x1C Interrupt register */ +}; + +#define REGLOAD(a) (be32_to_cpu(__raw_readl(&(a)))) +#define REGSTORE(a, v) (__raw_writel(cpu_to_be32(v), &(a))) + +#define PAGE0_BTEN_BIT 0 +#define PAGE0_BTEN (1 << PAGE0_BTEN_BIT) + +#define CFGSTAT_HOST_BIT 13 +#define CFGSTAT_CTO_BIT 8 +#define CFGSTAT_HOST (1 << CFGSTAT_HOST_BIT) +#define CFGSTAT_CTO (1 << CFGSTAT_CTO_BIT) + +#define IRQ_DPE (1 << 9) +#define IRQ_SSE (1 << 8) +#define IRQ_RMA (1 << 7) +#define IRQ_RTA (1 << 6) +#define IRQ_STA (1 << 5) +#define IRQ_DPED (1 << 4) +#define IRQ_INTD (1 << 3) +#define IRQ_INTC (1 << 2) +#define IRQ_INTB (1 << 1) +#define IRQ_INTA (1 << 0) +#define IRQ_DEF_ERRORS (IRQ_RMA | IRQ_RTA | IRQ_STA) +#define IRQ_ALL_ERRORS (IRQ_DPED | IRQ_DEF_ERRORS | IRQ_SSE | IRQ_DPE) +#define IRQ_INTX (IRQ_INTA | IRQ_INTB | IRQ_INTC | IRQ_INTD) +#define IRQ_MASK_BIT 16 + +#define DEF_PCI_ERRORS (PCI_STATUS_SIG_TARGET_ABORT | \ + PCI_STATUS_REC_TARGET_ABORT | \ + PCI_STATUS_REC_MASTER_ABORT) +#define ALL_PCI_ERRORS (PCI_STATUS_PARITY | PCI_STATUS_DETECTED_PARITY | \ + PCI_STATUS_SIG_SYSTEM_ERROR | DEF_PCI_ERRORS) + +#define TGT 256 + +struct grpci1_priv { + struct leon_pci_info info; /* must be on top of this structure */ + struct grpci1_regs __iomem *regs; /* GRPCI register map */ + struct device *dev; + int pci_err_mask; /* STATUS register error mask */ + int irq; /* LEON irqctrl GRPCI IRQ */ + unsigned char irq_map[4]; /* GRPCI nexus PCI INTX# IRQs */ + unsigned int irq_err; /* GRPCI nexus Virt Error IRQ */ + + /* AHB PCI Windows */ + unsigned long pci_area; /* MEMORY */ + unsigned long pci_area_end; + unsigned long pci_io; /* I/O */ + unsigned long pci_conf; /* CONFIGURATION */ + unsigned long pci_conf_end; + unsigned long pci_io_va; +}; + +static struct grpci1_priv *grpci1priv; + +static int grpci1_cfg_w32(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val); + +static int grpci1_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + struct grpci1_priv *priv = dev->bus->sysdata; + int irq_group; + + /* Use default IRQ decoding on PCI BUS0 according slot numbering */ + irq_group = slot & 0x3; + pin = ((pin - 1) + irq_group) & 0x3; + + return priv->irq_map[pin]; +} + +static int grpci1_cfg_r32(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 *pci_conf, tmp, cfg; + + if (where & 0x3) + return -EINVAL; + + if (bus == 0) { + devfn += (0x8 * 6); /* start at AD16=Device0 */ + } else if (bus == TGT) { + bus = 0; + devfn = 0; /* special case: bridge controller itself */ + } + + /* Select bus */ + cfg = REGLOAD(priv->regs->cfg_stat); + REGSTORE(priv->regs->cfg_stat, (cfg & ~(0xf << 23)) | (bus << 23)); + + /* do read access */ + pci_conf = (u32 *) (priv->pci_conf | (devfn << 8) | (where & 0xfc)); + tmp = LEON3_BYPASS_LOAD_PA(pci_conf); + + /* check if master abort was received */ + if (REGLOAD(priv->regs->cfg_stat) & CFGSTAT_CTO) { + *val = 0xffffffff; + /* Clear Master abort bit in PCI cfg space (is set) */ + tmp = REGLOAD(priv->regs->stat_cmd); + grpci1_cfg_w32(priv, TGT, 0, PCI_COMMAND, tmp); + } else { + /* Bus always little endian (unaffected by byte-swapping) */ + *val = swab32(tmp); + } + + return 0; +} + +static int grpci1_cfg_r16(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 v; + int ret; + + if (where & 0x1) + return -EINVAL; + ret = grpci1_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + *val = 0xffff & (v >> (8 * (where & 0x3))); + return ret; +} + +static int grpci1_cfg_r8(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 v; + int ret; + + ret = grpci1_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + *val = 0xff & (v >> (8 * (where & 3))); + + return ret; +} + +static int grpci1_cfg_w32(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + unsigned int *pci_conf; + u32 cfg; + + if (where & 0x3) + return -EINVAL; + + if (bus == 0) { + devfn += (0x8 * 6); /* start at AD16=Device0 */ + } else if (bus == TGT) { + bus = 0; + devfn = 0; /* special case: bridge controller itself */ + } + + /* Select bus */ + cfg = REGLOAD(priv->regs->cfg_stat); + REGSTORE(priv->regs->cfg_stat, (cfg & ~(0xf << 23)) | (bus << 23)); + + pci_conf = (unsigned int *) (priv->pci_conf | + (devfn << 8) | (where & 0xfc)); + LEON3_BYPASS_STORE_PA(pci_conf, swab32(val)); + + return 0; +} + +static int grpci1_cfg_w16(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + int ret; + u32 v; + + if (where & 0x1) + return -EINVAL; + ret = grpci1_cfg_r32(priv, bus, devfn, where&~3, &v); + if (ret) + return ret; + v = (v & ~(0xffff << (8 * (where & 0x3)))) | + ((0xffff & val) << (8 * (where & 0x3))); + return grpci1_cfg_w32(priv, bus, devfn, where & ~0x3, v); +} + +static int grpci1_cfg_w8(struct grpci1_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + int ret; + u32 v; + + ret = grpci1_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + if (ret != 0) + return ret; + v = (v & ~(0xff << (8 * (where & 0x3)))) | + ((0xff & val) << (8 * (where & 0x3))); + return grpci1_cfg_w32(priv, bus, devfn, where & ~0x3, v); +} + +/* Read from Configuration Space. When entering here the PCI layer has taken + * the pci_lock spinlock and IRQ is off. + */ +static int grpci1_read_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct grpci1_priv *priv = grpci1priv; + unsigned int busno = bus->number; + int ret; + + if (PCI_SLOT(devfn) > 15 || busno > 15) { + *val = ~0; + return 0; + } + + switch (size) { + case 1: + ret = grpci1_cfg_r8(priv, busno, devfn, where, val); + break; + case 2: + ret = grpci1_cfg_r16(priv, busno, devfn, where, val); + break; + case 4: + ret = grpci1_cfg_r32(priv, busno, devfn, where, val); + break; + default: + ret = -EINVAL; + break; + } + +#ifdef GRPCI1_DEBUG_CFGACCESS + printk(KERN_INFO + "grpci1_read_config: [%02x:%02x:%x] ofs=%d val=%x size=%d\n", + busno, PCI_SLOT(devfn), PCI_FUNC(devfn), where, *val, size); +#endif + + return ret; +} + +/* Write to Configuration Space. When entering here the PCI layer has taken + * the pci_lock spinlock and IRQ is off. + */ +static int grpci1_write_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct grpci1_priv *priv = grpci1priv; + unsigned int busno = bus->number; + + if (PCI_SLOT(devfn) > 15 || busno > 15) + return 0; + +#ifdef GRPCI1_DEBUG_CFGACCESS + printk(KERN_INFO + "grpci1_write_config: [%02x:%02x:%x] ofs=%d size=%d val=%x\n", + busno, PCI_SLOT(devfn), PCI_FUNC(devfn), where, size, val); +#endif + + switch (size) { + default: + return -EINVAL; + case 1: + return grpci1_cfg_w8(priv, busno, devfn, where, val); + case 2: + return grpci1_cfg_w16(priv, busno, devfn, where, val); + case 4: + return grpci1_cfg_w32(priv, busno, devfn, where, val); + } +} + +static struct pci_ops grpci1_ops = { + .read = grpci1_read_config, + .write = grpci1_write_config, +}; + +/* GENIRQ IRQ chip implementation for grpci1 irqmode=0..2. In configuration + * 3 where all PCI Interrupts has a separate IRQ on the system IRQ controller + * this is not needed and the standard IRQ controller can be used. + */ + +static void grpci1_mask_irq(struct irq_data *data) +{ + u32 irqidx; + struct grpci1_priv *priv = grpci1priv; + + irqidx = (u32)data->chip_data - 1; + if (irqidx > 3) /* only mask PCI interrupts here */ + return; + irqidx += IRQ_MASK_BIT; + + REGSTORE(priv->regs->irq, REGLOAD(priv->regs->irq) & ~(1 << irqidx)); +} + +static void grpci1_unmask_irq(struct irq_data *data) +{ + u32 irqidx; + struct grpci1_priv *priv = grpci1priv; + + irqidx = (u32)data->chip_data - 1; + if (irqidx > 3) /* only unmask PCI interrupts here */ + return; + irqidx += IRQ_MASK_BIT; + + REGSTORE(priv->regs->irq, REGLOAD(priv->regs->irq) | (1 << irqidx)); +} + +static unsigned int grpci1_startup_irq(struct irq_data *data) +{ + grpci1_unmask_irq(data); + return 0; +} + +static void grpci1_shutdown_irq(struct irq_data *data) +{ + grpci1_mask_irq(data); +} + +static struct irq_chip grpci1_irq = { + .name = "grpci1", + .irq_startup = grpci1_startup_irq, + .irq_shutdown = grpci1_shutdown_irq, + .irq_mask = grpci1_mask_irq, + .irq_unmask = grpci1_unmask_irq, +}; + +/* Handle one or multiple IRQs from the PCI core */ +static void grpci1_pci_flow_irq(struct irq_desc *desc) +{ + struct grpci1_priv *priv = grpci1priv; + int i, ack = 0; + unsigned int irqreg; + + irqreg = REGLOAD(priv->regs->irq); + irqreg = (irqreg >> IRQ_MASK_BIT) & irqreg; + + /* Error Interrupt? */ + if (irqreg & IRQ_ALL_ERRORS) { + generic_handle_irq(priv->irq_err); + ack = 1; + } + + /* PCI Interrupt? */ + if (irqreg & IRQ_INTX) { + /* Call respective PCI Interrupt handler */ + for (i = 0; i < 4; i++) { + if (irqreg & (1 << i)) + generic_handle_irq(priv->irq_map[i]); + } + ack = 1; + } + + /* + * Call "first level" IRQ chip end-of-irq handler. It will ACK LEON IRQ + * Controller, this must be done after IRQ sources have been handled to + * avoid double IRQ generation + */ + if (ack) + desc->irq_data.chip->irq_eoi(&desc->irq_data); +} + +/* Create a virtual IRQ */ +static unsigned int grpci1_build_device_irq(unsigned int irq) +{ + unsigned int virq = 0, pil; + + pil = 1 << 8; + virq = irq_alloc(irq, pil); + if (virq == 0) + goto out; + + irq_set_chip_and_handler_name(virq, &grpci1_irq, handle_simple_irq, + "pcilvl"); + irq_set_chip_data(virq, (void *)irq); + +out: + return virq; +} + +/* + * Initialize mappings AMBA<->PCI, clear IRQ state, setup PCI interface + * + * Target BARs: + * BAR0: unused in this implementation + * BAR1: peripheral DMA to host's memory (size at least 256MByte) + * BAR2..BAR5: not implemented in hardware + */ +static void grpci1_hw_init(struct grpci1_priv *priv) +{ + u32 ahbadr, bar_sz, data, pciadr; + struct grpci1_regs __iomem *regs = priv->regs; + + /* set 1:1 mapping between AHB -> PCI memory space */ + REGSTORE(regs->cfg_stat, priv->pci_area & 0xf0000000); + + /* map PCI accesses to target BAR1 to Linux kernel memory 1:1 */ + ahbadr = 0xf0000000 & (u32)__pa(PAGE_ALIGN((unsigned long) &_end)); + REGSTORE(regs->page1, ahbadr); + + /* translate I/O accesses to 0, I/O Space always @ PCI low 64Kbytes */ + REGSTORE(regs->iomap, REGLOAD(regs->iomap) & 0x0000ffff); + + /* disable and clear pending interrupts */ + REGSTORE(regs->irq, 0); + + /* Setup BAR0 outside access range so that it does not conflict with + * peripheral DMA. There is no need to set up the PAGE0 register. + */ + grpci1_cfg_w32(priv, TGT, 0, PCI_BASE_ADDRESS_0, 0xffffffff); + grpci1_cfg_r32(priv, TGT, 0, PCI_BASE_ADDRESS_0, &bar_sz); + bar_sz = ~bar_sz + 1; + pciadr = priv->pci_area - bar_sz; + grpci1_cfg_w32(priv, TGT, 0, PCI_BASE_ADDRESS_0, pciadr); + + /* + * Setup the Host's PCI Target BAR1 for other peripherals to access, + * and do DMA to the host's memory. + */ + grpci1_cfg_w32(priv, TGT, 0, PCI_BASE_ADDRESS_1, ahbadr); + + /* + * Setup Latency Timer and cache line size. Default cache line + * size will result in poor performance (256 word fetches), 0xff + * will set it according to the max size of the PCI FIFO. + */ + grpci1_cfg_w8(priv, TGT, 0, PCI_CACHE_LINE_SIZE, 0xff); + grpci1_cfg_w8(priv, TGT, 0, PCI_LATENCY_TIMER, 0x40); + + /* set as bus master, enable pci memory responses, clear status bits */ + grpci1_cfg_r32(priv, TGT, 0, PCI_COMMAND, &data); + data |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + grpci1_cfg_w32(priv, TGT, 0, PCI_COMMAND, data); +} + +static irqreturn_t grpci1_jump_interrupt(int irq, void *arg) +{ + struct grpci1_priv *priv = arg; + dev_err(priv->dev, "Jump IRQ happened\n"); + return IRQ_NONE; +} + +/* Handle GRPCI1 Error Interrupt */ +static irqreturn_t grpci1_err_interrupt(int irq, void *arg) +{ + struct grpci1_priv *priv = arg; + u32 status; + + grpci1_cfg_r16(priv, TGT, 0, PCI_STATUS, &status); + status &= priv->pci_err_mask; + + if (status == 0) + return IRQ_NONE; + + if (status & PCI_STATUS_PARITY) + dev_err(priv->dev, "Data Parity Error\n"); + + if (status & PCI_STATUS_SIG_TARGET_ABORT) + dev_err(priv->dev, "Signalled Target Abort\n"); + + if (status & PCI_STATUS_REC_TARGET_ABORT) + dev_err(priv->dev, "Received Target Abort\n"); + + if (status & PCI_STATUS_REC_MASTER_ABORT) + dev_err(priv->dev, "Received Master Abort\n"); + + if (status & PCI_STATUS_SIG_SYSTEM_ERROR) + dev_err(priv->dev, "Signalled System Error\n"); + + if (status & PCI_STATUS_DETECTED_PARITY) + dev_err(priv->dev, "Parity Error\n"); + + /* Clear handled INT TYPE IRQs */ + grpci1_cfg_w16(priv, TGT, 0, PCI_STATUS, status); + + return IRQ_HANDLED; +} + +static int grpci1_of_probe(struct platform_device *ofdev) +{ + struct grpci1_regs __iomem *regs; + struct grpci1_priv *priv; + int err, len; + const int *tmp; + u32 cfg, size, err_mask; + struct resource *res; + + if (grpci1priv) { + dev_err(&ofdev->dev, "only one GRPCI1 supported\n"); + return -ENODEV; + } + + if (ofdev->num_resources < 3) { + dev_err(&ofdev->dev, "not enough APB/AHB resources\n"); + return -EIO; + } + + priv = devm_kzalloc(&ofdev->dev, sizeof(*priv), GFP_KERNEL); + if (!priv) { + dev_err(&ofdev->dev, "memory allocation failed\n"); + return -ENOMEM; + } + platform_set_drvdata(ofdev, priv); + priv->dev = &ofdev->dev; + + /* find device register base address */ + res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); + regs = devm_ioremap_resource(&ofdev->dev, res); + if (IS_ERR(regs)) + return PTR_ERR(regs); + + /* + * check that we're in Host Slot and that we can act as a Host Bridge + * and not only as target/peripheral. + */ + cfg = REGLOAD(regs->cfg_stat); + if ((cfg & CFGSTAT_HOST) == 0) { + dev_err(&ofdev->dev, "not in host system slot\n"); + return -EIO; + } + + /* check that BAR1 support 256 MByte so that we can map kernel space */ + REGSTORE(regs->page1, 0xffffffff); + size = ~REGLOAD(regs->page1) + 1; + if (size < 0x10000000) { + dev_err(&ofdev->dev, "BAR1 must be at least 256MByte\n"); + return -EIO; + } + + /* hardware must support little-endian PCI (byte-twisting) */ + if ((REGLOAD(regs->page0) & PAGE0_BTEN) == 0) { + dev_err(&ofdev->dev, "byte-twisting is required\n"); + return -EIO; + } + + priv->regs = regs; + priv->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0); + dev_info(&ofdev->dev, "host found at 0x%p, irq%d\n", regs, priv->irq); + + /* Find PCI Memory, I/O and Configuration Space Windows */ + priv->pci_area = ofdev->resource[1].start; + priv->pci_area_end = ofdev->resource[1].end+1; + priv->pci_io = ofdev->resource[2].start; + priv->pci_conf = ofdev->resource[2].start + 0x10000; + priv->pci_conf_end = priv->pci_conf + 0x10000; + priv->pci_io_va = (unsigned long)ioremap(priv->pci_io, 0x10000); + if (!priv->pci_io_va) { + dev_err(&ofdev->dev, "unable to map PCI I/O area\n"); + return -EIO; + } + + printk(KERN_INFO + "GRPCI1: MEMORY SPACE [0x%08lx - 0x%08lx]\n" + " I/O SPACE [0x%08lx - 0x%08lx]\n" + " CONFIG SPACE [0x%08lx - 0x%08lx]\n", + priv->pci_area, priv->pci_area_end-1, + priv->pci_io, priv->pci_conf-1, + priv->pci_conf, priv->pci_conf_end-1); + + /* + * I/O Space resources in I/O Window mapped into Virtual Adr Space + * We never use low 4KB because some devices seem have problems using + * address 0. + */ + priv->info.io_space.name = "GRPCI1 PCI I/O Space"; + priv->info.io_space.start = priv->pci_io_va + 0x1000; + priv->info.io_space.end = priv->pci_io_va + 0x10000 - 1; + priv->info.io_space.flags = IORESOURCE_IO; + + /* + * grpci1 has no prefetchable memory, map everything as + * non-prefetchable memory + */ + priv->info.mem_space.name = "GRPCI1 PCI MEM Space"; + priv->info.mem_space.start = priv->pci_area; + priv->info.mem_space.end = priv->pci_area_end - 1; + priv->info.mem_space.flags = IORESOURCE_MEM; + + if (request_resource(&iomem_resource, &priv->info.mem_space) < 0) { + dev_err(&ofdev->dev, "unable to request PCI memory area\n"); + err = -ENOMEM; + goto err1; + } + + if (request_resource(&ioport_resource, &priv->info.io_space) < 0) { + dev_err(&ofdev->dev, "unable to request PCI I/O area\n"); + err = -ENOMEM; + goto err2; + } + + /* setup maximum supported PCI buses */ + priv->info.busn.name = "GRPCI1 busn"; + priv->info.busn.start = 0; + priv->info.busn.end = 15; + + grpci1priv = priv; + + /* Initialize hardware */ + grpci1_hw_init(priv); + + /* + * Get PCI Interrupt to System IRQ mapping and setup IRQ handling + * Error IRQ. All PCI and PCI-Error interrupts are shared using the + * same system IRQ. + */ + leon_update_virq_handling(priv->irq, grpci1_pci_flow_irq, "pcilvl", 0); + + priv->irq_map[0] = grpci1_build_device_irq(1); + priv->irq_map[1] = grpci1_build_device_irq(2); + priv->irq_map[2] = grpci1_build_device_irq(3); + priv->irq_map[3] = grpci1_build_device_irq(4); + priv->irq_err = grpci1_build_device_irq(5); + + printk(KERN_INFO " PCI INTA..D#: IRQ%d, IRQ%d, IRQ%d, IRQ%d\n", + priv->irq_map[0], priv->irq_map[1], priv->irq_map[2], + priv->irq_map[3]); + + /* Enable IRQs on LEON IRQ controller */ + err = devm_request_irq(&ofdev->dev, priv->irq, grpci1_jump_interrupt, 0, + "GRPCI1_JUMP", priv); + if (err) { + dev_err(&ofdev->dev, "ERR IRQ request failed: %d\n", err); + goto err3; + } + + /* Setup IRQ handler for access errors */ + err = devm_request_irq(&ofdev->dev, priv->irq_err, + grpci1_err_interrupt, IRQF_SHARED, "GRPCI1_ERR", + priv); + if (err) { + dev_err(&ofdev->dev, "ERR VIRQ request failed: %d\n", err); + goto err3; + } + + tmp = of_get_property(ofdev->dev.of_node, "all_pci_errors", &len); + if (tmp && (len == 4)) { + priv->pci_err_mask = ALL_PCI_ERRORS; + err_mask = IRQ_ALL_ERRORS << IRQ_MASK_BIT; + } else { + priv->pci_err_mask = DEF_PCI_ERRORS; + err_mask = IRQ_DEF_ERRORS << IRQ_MASK_BIT; + } + + /* + * Enable Error Interrupts. PCI interrupts are unmasked once request_irq + * is called by the PCI Device drivers + */ + REGSTORE(regs->irq, err_mask); + + /* Init common layer and scan buses */ + priv->info.ops = &grpci1_ops; + priv->info.map_irq = grpci1_map_irq; + leon_pci_init(ofdev, &priv->info); + + return 0; + +err3: + release_resource(&priv->info.io_space); +err2: + release_resource(&priv->info.mem_space); +err1: + iounmap((void __iomem *)priv->pci_io_va); + grpci1priv = NULL; + return err; +} + +static const struct of_device_id grpci1_of_match[] = { + { + .name = "GAISLER_PCIFBRG", + }, + { + .name = "01_014", + }, + {}, +}; + +static struct platform_driver grpci1_of_driver = { + .driver = { + .name = "grpci1", + .of_match_table = grpci1_of_match, + }, + .probe = grpci1_of_probe, +}; + +static int __init grpci1_init(void) +{ + return platform_driver_register(&grpci1_of_driver); +} + +subsys_initcall(grpci1_init); diff --git a/arch/sparc/kernel/leon_pci_grpci2.c b/arch/sparc/kernel/leon_pci_grpci2.c new file mode 100644 index 000000000..c1cbdf881 --- /dev/null +++ b/arch/sparc/kernel/leon_pci_grpci2.c @@ -0,0 +1,915 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * leon_pci_grpci2.c: GRPCI2 Host PCI driver + * + * Copyright (C) 2011 Aeroflex Gaisler AB, Daniel Hellstrom + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "irq.h" + +struct grpci2_barcfg { + unsigned long pciadr; /* PCI Space Address */ + unsigned long ahbadr; /* PCI Base address mapped to this AHB addr */ +}; + +/* Device Node Configuration options: + * - barcfgs : Custom Configuration of Host's 6 target BARs + * - irq_mask : Limit which PCI interrupts are enabled + * - do_reset : Force PCI Reset on startup + * + * barcfgs + * ======= + * + * Optional custom Target BAR configuration (see struct grpci2_barcfg). All + * addresses are physical. Array always contains 6 elements (len=2*4*6 bytes) + * + * -1 means not configured (let host driver do default setup). + * + * [i*2+0] = PCI Address of BAR[i] on target interface + * [i*2+1] = Accessing PCI address of BAR[i] result in this AMBA address + * + * + * irq_mask + * ======== + * + * Limit which PCI interrupts are enabled. 0=Disable, 1=Enable. By default + * all are enabled. Use this when PCI interrupt pins are floating on PCB. + * int, len=4. + * bit0 = PCI INTA# + * bit1 = PCI INTB# + * bit2 = PCI INTC# + * bit3 = PCI INTD# + * + * + * reset + * ===== + * + * Force PCI reset on startup. int, len=4 + */ + +/* Enable Debugging Configuration Space Access */ +#undef GRPCI2_DEBUG_CFGACCESS + +/* + * GRPCI2 APB Register MAP + */ +struct grpci2_regs { + unsigned int ctrl; /* 0x00 Control */ + unsigned int sts_cap; /* 0x04 Status / Capabilities */ + int res1; /* 0x08 */ + unsigned int io_map; /* 0x0C I/O Map address */ + unsigned int dma_ctrl; /* 0x10 DMA */ + unsigned int dma_bdbase; /* 0x14 DMA */ + int res2[2]; /* 0x18 */ + unsigned int bars[6]; /* 0x20 read-only PCI BARs */ + int res3[2]; /* 0x38 */ + unsigned int ahbmst_map[16]; /* 0x40 AHB->PCI Map per AHB Master */ + + /* PCI Trace Buffer Registers (OPTIONAL) */ + unsigned int t_ctrl; /* 0x80 */ + unsigned int t_cnt; /* 0x84 */ + unsigned int t_adpat; /* 0x88 */ + unsigned int t_admask; /* 0x8C */ + unsigned int t_sigpat; /* 0x90 */ + unsigned int t_sigmask; /* 0x94 */ + unsigned int t_adstate; /* 0x98 */ + unsigned int t_sigstate; /* 0x9C */ +}; + +#define REGLOAD(a) (be32_to_cpu(__raw_readl(&(a)))) +#define REGSTORE(a, v) (__raw_writel(cpu_to_be32(v), &(a))) + +#define CTRL_BUS_BIT 16 + +#define CTRL_RESET (1<<31) +#define CTRL_SI (1<<27) +#define CTRL_PE (1<<26) +#define CTRL_EI (1<<25) +#define CTRL_ER (1<<24) +#define CTRL_BUS (0xff<bus->sysdata; + int irq_group; + + /* Use default IRQ decoding on PCI BUS0 according slot numbering */ + irq_group = slot & 0x3; + pin = ((pin - 1) + irq_group) & 0x3; + + return priv->irq_map[pin]; +} + +static int grpci2_cfg_r32(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + unsigned int *pci_conf; + unsigned long flags; + u32 tmp; + + if (where & 0x3) + return -EINVAL; + + if (bus == 0) { + devfn += (0x8 * 6); /* start at AD16=Device0 */ + } else if (bus == TGT) { + bus = 0; + devfn = 0; /* special case: bridge controller itself */ + } + + /* Select bus */ + spin_lock_irqsave(&grpci2_dev_lock, flags); + REGSTORE(priv->regs->ctrl, (REGLOAD(priv->regs->ctrl) & ~(0xff << 16)) | + (bus << 16)); + spin_unlock_irqrestore(&grpci2_dev_lock, flags); + + /* clear old status */ + REGSTORE(priv->regs->sts_cap, (STS_CFGERR | STS_CFGERRVALID)); + + pci_conf = (unsigned int *) (priv->pci_conf | + (devfn << 8) | (where & 0xfc)); + tmp = LEON3_BYPASS_LOAD_PA(pci_conf); + + /* Wait until GRPCI2 signals that CFG access is done, it should be + * done instantaneously unless a DMA operation is ongoing... + */ + while ((REGLOAD(priv->regs->sts_cap) & STS_CFGERRVALID) == 0) + ; + + if (REGLOAD(priv->regs->sts_cap) & STS_CFGERR) { + *val = 0xffffffff; + } else { + /* Bus always little endian (unaffected by byte-swapping) */ + *val = swab32(tmp); + } + + return 0; +} + +static int grpci2_cfg_r16(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 v; + int ret; + + if (where & 0x1) + return -EINVAL; + ret = grpci2_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + *val = 0xffff & (v >> (8 * (where & 0x3))); + return ret; +} + +static int grpci2_cfg_r8(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 *val) +{ + u32 v; + int ret; + + ret = grpci2_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + *val = 0xff & (v >> (8 * (where & 3))); + + return ret; +} + +static int grpci2_cfg_w32(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + unsigned int *pci_conf; + unsigned long flags; + + if (where & 0x3) + return -EINVAL; + + if (bus == 0) { + devfn += (0x8 * 6); /* start at AD16=Device0 */ + } else if (bus == TGT) { + bus = 0; + devfn = 0; /* special case: bridge controller itself */ + } + + /* Select bus */ + spin_lock_irqsave(&grpci2_dev_lock, flags); + REGSTORE(priv->regs->ctrl, (REGLOAD(priv->regs->ctrl) & ~(0xff << 16)) | + (bus << 16)); + spin_unlock_irqrestore(&grpci2_dev_lock, flags); + + /* clear old status */ + REGSTORE(priv->regs->sts_cap, (STS_CFGERR | STS_CFGERRVALID)); + + pci_conf = (unsigned int *) (priv->pci_conf | + (devfn << 8) | (where & 0xfc)); + LEON3_BYPASS_STORE_PA(pci_conf, swab32(val)); + + /* Wait until GRPCI2 signals that CFG access is done, it should be + * done instantaneously unless a DMA operation is ongoing... + */ + while ((REGLOAD(priv->regs->sts_cap) & STS_CFGERRVALID) == 0) + ; + + return 0; +} + +static int grpci2_cfg_w16(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + int ret; + u32 v; + + if (where & 0x1) + return -EINVAL; + ret = grpci2_cfg_r32(priv, bus, devfn, where&~3, &v); + if (ret) + return ret; + v = (v & ~(0xffff << (8 * (where & 0x3)))) | + ((0xffff & val) << (8 * (where & 0x3))); + return grpci2_cfg_w32(priv, bus, devfn, where & ~0x3, v); +} + +static int grpci2_cfg_w8(struct grpci2_priv *priv, unsigned int bus, + unsigned int devfn, int where, u32 val) +{ + int ret; + u32 v; + + ret = grpci2_cfg_r32(priv, bus, devfn, where & ~0x3, &v); + if (ret != 0) + return ret; + v = (v & ~(0xff << (8 * (where & 0x3)))) | + ((0xff & val) << (8 * (where & 0x3))); + return grpci2_cfg_w32(priv, bus, devfn, where & ~0x3, v); +} + +/* Read from Configuration Space. When entering here the PCI layer has taken + * the pci_lock spinlock and IRQ is off. + */ +static int grpci2_read_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + struct grpci2_priv *priv = grpci2priv; + unsigned int busno = bus->number; + int ret; + + if (PCI_SLOT(devfn) > 15 || busno > 255) { + *val = ~0; + return 0; + } + + switch (size) { + case 1: + ret = grpci2_cfg_r8(priv, busno, devfn, where, val); + break; + case 2: + ret = grpci2_cfg_r16(priv, busno, devfn, where, val); + break; + case 4: + ret = grpci2_cfg_r32(priv, busno, devfn, where, val); + break; + default: + ret = -EINVAL; + break; + } + +#ifdef GRPCI2_DEBUG_CFGACCESS + printk(KERN_INFO "grpci2_read_config: [%02x:%02x:%x] ofs=%d val=%x " + "size=%d\n", busno, PCI_SLOT(devfn), PCI_FUNC(devfn), where, + *val, size); +#endif + + return ret; +} + +/* Write to Configuration Space. When entering here the PCI layer has taken + * the pci_lock spinlock and IRQ is off. + */ +static int grpci2_write_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + struct grpci2_priv *priv = grpci2priv; + unsigned int busno = bus->number; + + if (PCI_SLOT(devfn) > 15 || busno > 255) + return 0; + +#ifdef GRPCI2_DEBUG_CFGACCESS + printk(KERN_INFO "grpci2_write_config: [%02x:%02x:%x] ofs=%d size=%d " + "val=%x\n", busno, PCI_SLOT(devfn), PCI_FUNC(devfn), + where, size, val); +#endif + + switch (size) { + default: + return -EINVAL; + case 1: + return grpci2_cfg_w8(priv, busno, devfn, where, val); + case 2: + return grpci2_cfg_w16(priv, busno, devfn, where, val); + case 4: + return grpci2_cfg_w32(priv, busno, devfn, where, val); + } +} + +static struct pci_ops grpci2_ops = { + .read = grpci2_read_config, + .write = grpci2_write_config, +}; + +/* GENIRQ IRQ chip implementation for GRPCI2 irqmode=0..2. In configuration + * 3 where all PCI Interrupts has a separate IRQ on the system IRQ controller + * this is not needed and the standard IRQ controller can be used. + */ + +static void grpci2_mask_irq(struct irq_data *data) +{ + unsigned long flags; + unsigned int irqidx; + struct grpci2_priv *priv = grpci2priv; + + irqidx = (unsigned int)data->chip_data - 1; + if (irqidx > 3) /* only mask PCI interrupts here */ + return; + + spin_lock_irqsave(&grpci2_dev_lock, flags); + REGSTORE(priv->regs->ctrl, REGLOAD(priv->regs->ctrl) & ~(1 << irqidx)); + spin_unlock_irqrestore(&grpci2_dev_lock, flags); +} + +static void grpci2_unmask_irq(struct irq_data *data) +{ + unsigned long flags; + unsigned int irqidx; + struct grpci2_priv *priv = grpci2priv; + + irqidx = (unsigned int)data->chip_data - 1; + if (irqidx > 3) /* only unmask PCI interrupts here */ + return; + + spin_lock_irqsave(&grpci2_dev_lock, flags); + REGSTORE(priv->regs->ctrl, REGLOAD(priv->regs->ctrl) | (1 << irqidx)); + spin_unlock_irqrestore(&grpci2_dev_lock, flags); +} + +static unsigned int grpci2_startup_irq(struct irq_data *data) +{ + grpci2_unmask_irq(data); + return 0; +} + +static void grpci2_shutdown_irq(struct irq_data *data) +{ + grpci2_mask_irq(data); +} + +static struct irq_chip grpci2_irq = { + .name = "grpci2", + .irq_startup = grpci2_startup_irq, + .irq_shutdown = grpci2_shutdown_irq, + .irq_mask = grpci2_mask_irq, + .irq_unmask = grpci2_unmask_irq, +}; + +/* Handle one or multiple IRQs from the PCI core */ +static void grpci2_pci_flow_irq(struct irq_desc *desc) +{ + struct grpci2_priv *priv = grpci2priv; + int i, ack = 0; + unsigned int ctrl, sts_cap, pci_ints; + + ctrl = REGLOAD(priv->regs->ctrl); + sts_cap = REGLOAD(priv->regs->sts_cap); + + /* Error Interrupt? */ + if (sts_cap & STS_ERR_IRQ) { + generic_handle_irq(priv->virq_err); + ack = 1; + } + + /* PCI Interrupt? */ + pci_ints = ((~sts_cap) >> STS_INTSTS_BIT) & ctrl & CTRL_HOSTINT; + if (pci_ints) { + /* Call respective PCI Interrupt handler */ + for (i = 0; i < 4; i++) { + if (pci_ints & (1 << i)) + generic_handle_irq(priv->irq_map[i]); + } + ack = 1; + } + + /* + * Decode DMA Interrupt only when shared with Err and PCI INTX#, when + * the DMA is a unique IRQ the DMA interrupts doesn't end up here, they + * goes directly to DMA ISR. + */ + if ((priv->irq_mode == 0) && (sts_cap & (STS_IDMA | STS_IDMAERR))) { + generic_handle_irq(priv->virq_dma); + ack = 1; + } + + /* + * Call "first level" IRQ chip end-of-irq handler. It will ACK LEON IRQ + * Controller, this must be done after IRQ sources have been handled to + * avoid double IRQ generation + */ + if (ack) + desc->irq_data.chip->irq_eoi(&desc->irq_data); +} + +/* Create a virtual IRQ */ +static unsigned int grpci2_build_device_irq(unsigned int irq) +{ + unsigned int virq = 0, pil; + + pil = 1 << 8; + virq = irq_alloc(irq, pil); + if (virq == 0) + goto out; + + irq_set_chip_and_handler_name(virq, &grpci2_irq, handle_simple_irq, + "pcilvl"); + irq_set_chip_data(virq, (void *)irq); + +out: + return virq; +} + +static void grpci2_hw_init(struct grpci2_priv *priv) +{ + u32 ahbadr, pciadr, bar_sz, capptr, io_map, data; + struct grpci2_regs __iomem *regs = priv->regs; + int i; + struct grpci2_barcfg *barcfg = priv->tgtbars; + + /* Reset any earlier setup */ + if (priv->do_reset) { + printk(KERN_INFO "GRPCI2: Resetting PCI bus\n"); + REGSTORE(regs->ctrl, CTRL_RESET); + ssleep(1); /* Wait for boards to settle */ + } + REGSTORE(regs->ctrl, 0); + REGSTORE(regs->sts_cap, ~0); /* Clear Status */ + REGSTORE(regs->dma_ctrl, 0); + REGSTORE(regs->dma_bdbase, 0); + + /* Translate I/O accesses to 0, I/O Space always @ PCI low 64Kbytes */ + REGSTORE(regs->io_map, REGLOAD(regs->io_map) & 0x0000ffff); + + /* set 1:1 mapping between AHB -> PCI memory space, for all Masters + * Each AHB master has its own mapping registers. Max 16 AHB masters. + */ + for (i = 0; i < 16; i++) + REGSTORE(regs->ahbmst_map[i], priv->pci_area); + + /* Get the GRPCI2 Host PCI ID */ + grpci2_cfg_r32(priv, TGT, 0, PCI_VENDOR_ID, &priv->pciid); + + /* Get address to first (always defined) capability structure */ + grpci2_cfg_r8(priv, TGT, 0, PCI_CAPABILITY_LIST, &capptr); + + /* Enable/Disable Byte twisting */ + grpci2_cfg_r32(priv, TGT, 0, capptr+CAP9_IOMAP_OFS, &io_map); + io_map = (io_map & ~0x1) | (priv->bt_enabled ? 1 : 0); + grpci2_cfg_w32(priv, TGT, 0, capptr+CAP9_IOMAP_OFS, io_map); + + /* Setup the Host's PCI Target BARs for other peripherals to access, + * and do DMA to the host's memory. The target BARs can be sized and + * enabled individually. + * + * User may set custom target BARs, but default is: + * The first BARs is used to map kernel low (DMA is part of normal + * region on sparc which is SRMMU_MAXMEM big) main memory 1:1 to the + * PCI bus, the other BARs are disabled. We assume that the first BAR + * is always available. + */ + for (i = 0; i < 6; i++) { + if (barcfg[i].pciadr != ~0 && barcfg[i].ahbadr != ~0) { + /* Target BARs must have the proper alignment */ + ahbadr = barcfg[i].ahbadr; + pciadr = barcfg[i].pciadr; + bar_sz = ((pciadr - 1) & ~pciadr) + 1; + } else { + if (i == 0) { + /* Map main memory */ + bar_sz = 0xf0000008; /* 256MB prefetchable */ + ahbadr = 0xf0000000 & (u32)__pa(PAGE_ALIGN( + (unsigned long) &_end)); + pciadr = ahbadr; + } else { + bar_sz = 0; + ahbadr = 0; + pciadr = 0; + } + } + grpci2_cfg_w32(priv, TGT, 0, capptr+CAP9_BARSIZE_OFS+i*4, + bar_sz); + grpci2_cfg_w32(priv, TGT, 0, PCI_BASE_ADDRESS_0+i*4, pciadr); + grpci2_cfg_w32(priv, TGT, 0, capptr+CAP9_BAR_OFS+i*4, ahbadr); + printk(KERN_INFO " TGT BAR[%d]: 0x%08x (PCI)-> 0x%08x\n", + i, pciadr, ahbadr); + } + + /* set as bus master and enable pci memory responses */ + grpci2_cfg_r32(priv, TGT, 0, PCI_COMMAND, &data); + data |= (PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER); + grpci2_cfg_w32(priv, TGT, 0, PCI_COMMAND, data); + + /* Enable Error respone (CPU-TRAP) on illegal memory access. */ + REGSTORE(regs->ctrl, CTRL_ER | CTRL_PE); +} + +static irqreturn_t grpci2_jump_interrupt(int irq, void *arg) +{ + printk(KERN_ERR "GRPCI2: Jump IRQ happened\n"); + return IRQ_NONE; +} + +/* Handle GRPCI2 Error Interrupt */ +static irqreturn_t grpci2_err_interrupt(int irq, void *arg) +{ + struct grpci2_priv *priv = arg; + struct grpci2_regs __iomem *regs = priv->regs; + unsigned int status; + + status = REGLOAD(regs->sts_cap); + if ((status & STS_ERR_IRQ) == 0) + return IRQ_NONE; + + if (status & STS_IPARERR) + printk(KERN_ERR "GRPCI2: Parity Error\n"); + + if (status & STS_ITGTABRT) + printk(KERN_ERR "GRPCI2: Target Abort\n"); + + if (status & STS_IMSTABRT) + printk(KERN_ERR "GRPCI2: Master Abort\n"); + + if (status & STS_ISYSERR) + printk(KERN_ERR "GRPCI2: System Error\n"); + + /* Clear handled INT TYPE IRQs */ + REGSTORE(regs->sts_cap, status & STS_ERR_IRQ); + + return IRQ_HANDLED; +} + +static int grpci2_of_probe(struct platform_device *ofdev) +{ + struct grpci2_regs __iomem *regs; + struct grpci2_priv *priv; + int err, i, len; + const int *tmp; + unsigned int capability; + + if (grpci2priv) { + printk(KERN_ERR "GRPCI2: only one GRPCI2 core supported\n"); + return -ENODEV; + } + + if (ofdev->num_resources < 3) { + printk(KERN_ERR "GRPCI2: not enough APB/AHB resources\n"); + return -EIO; + } + + /* Find Device Address */ + regs = of_ioremap(&ofdev->resource[0], 0, + resource_size(&ofdev->resource[0]), + "grlib-grpci2 regs"); + if (regs == NULL) { + printk(KERN_ERR "GRPCI2: ioremap failed\n"); + return -EIO; + } + + /* + * Check that we're in Host Slot and that we can act as a Host Bridge + * and not only as target. + */ + capability = REGLOAD(regs->sts_cap); + if ((capability & STS_HOST) || !(capability & STS_MST)) { + printk(KERN_INFO "GRPCI2: not in host system slot\n"); + err = -EIO; + goto err1; + } + + priv = grpci2priv = kzalloc_obj(struct grpci2_priv); + if (grpci2priv == NULL) { + err = -ENOMEM; + goto err1; + } + priv->regs = regs; + priv->irq = ofdev->archdata.irqs[0]; /* BASE IRQ */ + priv->irq_mode = (capability & STS_IRQMODE) >> STS_IRQMODE_BIT; + + printk(KERN_INFO "GRPCI2: host found at %p, irq%d\n", regs, priv->irq); + + /* Byte twisting should be made configurable from kernel command line */ + priv->bt_enabled = 1; + + /* Let user do custom Target BAR assignment */ + tmp = of_get_property(ofdev->dev.of_node, "barcfg", &len); + if (tmp && (len == 2*4*6)) + memcpy(priv->tgtbars, tmp, 2*4*6); + else + memset(priv->tgtbars, -1, 2*4*6); + + /* Limit IRQ unmasking in irq_mode 2 and 3 */ + tmp = of_get_property(ofdev->dev.of_node, "irq_mask", &len); + if (tmp && (len == 4)) + priv->do_reset = *tmp; + else + priv->irq_mask = 0xf; + + /* Optional PCI reset. Force PCI reset on startup */ + tmp = of_get_property(ofdev->dev.of_node, "reset", &len); + if (tmp && (len == 4)) + priv->do_reset = *tmp; + else + priv->do_reset = 0; + + /* Find PCI Memory, I/O and Configuration Space Windows */ + priv->pci_area = ofdev->resource[1].start; + priv->pci_area_end = ofdev->resource[1].end+1; + priv->pci_io = ofdev->resource[2].start; + priv->pci_conf = ofdev->resource[2].start + 0x10000; + priv->pci_conf_end = priv->pci_conf + 0x10000; + priv->pci_io_va = (unsigned long)ioremap(priv->pci_io, 0x10000); + if (!priv->pci_io_va) { + err = -EIO; + goto err2; + } + + printk(KERN_INFO + "GRPCI2: MEMORY SPACE [0x%08lx - 0x%08lx]\n" + " I/O SPACE [0x%08lx - 0x%08lx]\n" + " CONFIG SPACE [0x%08lx - 0x%08lx]\n", + priv->pci_area, priv->pci_area_end-1, + priv->pci_io, priv->pci_conf-1, + priv->pci_conf, priv->pci_conf_end-1); + + /* + * I/O Space resources in I/O Window mapped into Virtual Adr Space + * We never use low 4KB because some devices seem have problems using + * address 0. + */ + memset(&priv->info.io_space, 0, sizeof(struct resource)); + priv->info.io_space.name = "GRPCI2 PCI I/O Space"; + priv->info.io_space.start = priv->pci_io_va + 0x1000; + priv->info.io_space.end = priv->pci_io_va + 0x10000 - 1; + priv->info.io_space.flags = IORESOURCE_IO; + + /* + * GRPCI2 has no prefetchable memory, map everything as + * non-prefetchable memory + */ + memset(&priv->info.mem_space, 0, sizeof(struct resource)); + priv->info.mem_space.name = "GRPCI2 PCI MEM Space"; + priv->info.mem_space.start = priv->pci_area; + priv->info.mem_space.end = priv->pci_area_end - 1; + priv->info.mem_space.flags = IORESOURCE_MEM; + + if (request_resource(&iomem_resource, &priv->info.mem_space) < 0) + goto err3; + if (request_resource(&ioport_resource, &priv->info.io_space) < 0) + goto err4; + + /* setup maximum supported PCI buses */ + priv->info.busn.name = "GRPCI2 busn"; + priv->info.busn.start = 0; + priv->info.busn.end = 255; + + grpci2_hw_init(priv); + + /* + * Get PCI Interrupt to System IRQ mapping and setup IRQ handling + * Error IRQ always on PCI INTA. + */ + if (priv->irq_mode < 2) { + /* All PCI interrupts are shared using the same system IRQ */ + leon_update_virq_handling(priv->irq, grpci2_pci_flow_irq, + "pcilvl", 0); + + priv->irq_map[0] = grpci2_build_device_irq(1); + priv->irq_map[1] = grpci2_build_device_irq(2); + priv->irq_map[2] = grpci2_build_device_irq(3); + priv->irq_map[3] = grpci2_build_device_irq(4); + + priv->virq_err = grpci2_build_device_irq(5); + if (priv->irq_mode & 1) + priv->virq_dma = ofdev->archdata.irqs[1]; + else + priv->virq_dma = grpci2_build_device_irq(6); + + /* Enable IRQs on LEON IRQ controller */ + err = request_irq(priv->irq, grpci2_jump_interrupt, 0, + "GRPCI2_JUMP", priv); + if (err) + printk(KERN_ERR "GRPCI2: ERR IRQ request failed\n"); + } else { + /* All PCI interrupts have an unique IRQ interrupt */ + for (i = 0; i < 4; i++) { + /* Make LEON IRQ layer handle level IRQ by acking */ + leon_update_virq_handling(ofdev->archdata.irqs[i], + handle_fasteoi_irq, "pcilvl", + 1); + priv->irq_map[i] = ofdev->archdata.irqs[i]; + } + priv->virq_err = priv->irq_map[0]; + if (priv->irq_mode & 1) + priv->virq_dma = ofdev->archdata.irqs[4]; + else + priv->virq_dma = priv->irq_map[0]; + + /* Unmask all PCI interrupts, request_irq will not do that */ + REGSTORE(regs->ctrl, REGLOAD(regs->ctrl)|(priv->irq_mask&0xf)); + } + + /* Setup IRQ handler for non-configuration space access errors */ + err = request_irq(priv->virq_err, grpci2_err_interrupt, IRQF_SHARED, + "GRPCI2_ERR", priv); + if (err) { + printk(KERN_DEBUG "GRPCI2: ERR VIRQ request failed: %d\n", err); + goto err5; + } + + /* + * Enable Error Interrupts. PCI interrupts are unmasked once request_irq + * is called by the PCI Device drivers + */ + REGSTORE(regs->ctrl, REGLOAD(regs->ctrl) | CTRL_EI | CTRL_SI); + + /* Init common layer and scan buses */ + priv->info.ops = &grpci2_ops; + priv->info.map_irq = grpci2_map_irq; + leon_pci_init(ofdev, &priv->info); + + return 0; + +err5: + release_resource(&priv->info.io_space); +err4: + release_resource(&priv->info.mem_space); +err3: + err = -ENOMEM; + iounmap((void __iomem *)priv->pci_io_va); +err2: + kfree(priv); +err1: + of_iounmap(&ofdev->resource[0], regs, + resource_size(&ofdev->resource[0])); + return err; +} + +static const struct of_device_id grpci2_of_match[] = { + { + .name = "GAISLER_GRPCI2", + }, + { + .name = "01_07c", + }, + {}, +}; + +static struct platform_driver grpci2_of_driver = { + .driver = { + .name = "grpci2", + .of_match_table = grpci2_of_match, + }, + .probe = grpci2_of_probe, +}; + +static int __init grpci2_init(void) +{ + return platform_driver_register(&grpci2_of_driver); +} + +subsys_initcall(grpci2_init); diff --git a/arch/sparc/kernel/leon_pmc.c b/arch/sparc/kernel/leon_pmc.c new file mode 100644 index 000000000..6c00cbad7 --- /dev/null +++ b/arch/sparc/kernel/leon_pmc.c @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: GPL-2.0 +/* leon_pmc.c: LEON Power-down cpu_idle() handler + * + * Copyright (C) 2011 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB + */ + +#include +#include + +#include +#include +#include +#include + +/* List of Systems that need fixup instructions around power-down instruction */ +static unsigned int pmc_leon_fixup_ids[] = { + AEROFLEX_UT699, + GAISLER_GR712RC, + LEON4_NEXTREME1, + 0 +}; + +static int pmc_leon_need_fixup(void) +{ + unsigned int systemid = amba_system_id >> 16; + unsigned int *id; + + id = &pmc_leon_fixup_ids[0]; + while (*id != 0) { + if (*id == systemid) + return 1; + id++; + } + + return 0; +} + +/* + * CPU idle callback function for systems that need some extra handling + * See .../arch/sparc/kernel/process.c + */ +static void pmc_leon_idle_fixup(void) +{ + /* Prepare an address to a non-cachable region. APB is always + * none-cachable. One instruction is executed after the Sleep + * instruction, we make sure to read the bus and throw away the + * value by accessing a non-cachable area, also we make sure the + * MMU does not get a TLB miss here by using the MMU BYPASS ASI. + */ + register unsigned int address = (unsigned int)leon3_irqctrl_regs; + + /* Interrupts need to be enabled to not hang the CPU */ + raw_local_irq_enable(); + + __asm__ __volatile__ ( + "wr %%g0, %%asr19\n" + "lda [%0] %1, %%g0\n" + : + : "r"(address), "i"(ASI_LEON_BYPASS)); + + raw_local_irq_disable(); +} + +/* + * CPU idle callback function + * See .../arch/sparc/kernel/process.c + */ +static void pmc_leon_idle(void) +{ + /* Interrupts need to be enabled to not hang the CPU */ + raw_local_irq_enable(); + + /* For systems without power-down, this will be no-op */ + __asm__ __volatile__ ("wr %g0, %asr19\n\t"); + + raw_local_irq_disable(); +} + +/* Install LEON Power Down function */ +static int __init leon_pmc_install(void) +{ + if (sparc_cpu_model == sparc_leon) { + /* Assign power management IDLE handler */ + if (pmc_leon_need_fixup()) + sparc_idle = pmc_leon_idle_fixup; + else + sparc_idle = pmc_leon_idle; + + printk(KERN_INFO "leon: power management initialized\n"); + } + + return 0; +} + +/* This driver is not critical to the boot process, don't care + * if initialized late. + */ +late_initcall(leon_pmc_install); diff --git a/arch/sparc/kernel/leon_smp.c b/arch/sparc/kernel/leon_smp.c new file mode 100644 index 000000000..1ee393abc --- /dev/null +++ b/arch/sparc/kernel/leon_smp.c @@ -0,0 +1,470 @@ +// SPDX-License-Identifier: GPL-2.0 +/* leon_smp.c: Sparc-Leon SMP support. + * + * based on sun4m_smp.c + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 2009 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB + * Copyright (C) 2009 Konrad Eisele (konrad@gaisler.com) Aeroflex Gaisler AB + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" + +#include "irq.h" + +extern ctxd_t *srmmu_ctx_table_phys; +static int smp_processors_ready; +extern volatile unsigned long cpu_callin_map[NR_CPUS]; +extern cpumask_t smp_commenced_mask; +void leon_configure_cache_smp(void); +static void leon_ipi_init(void); + +/* IRQ number of LEON IPIs */ +int leon_ipi_irq = LEON3_IRQ_IPI_DEFAULT; + +static inline unsigned long do_swap(volatile unsigned long *ptr, + unsigned long val) +{ + __asm__ __volatile__("swapa [%2] %3, %0\n\t" : "=&r"(val) + : "0"(val), "r"(ptr), "i"(ASI_LEON_DCACHE_MISS) + : "memory"); + return val; +} + +void leon_cpu_pre_starting(void *arg) +{ + leon_configure_cache_smp(); +} + +void leon_cpu_pre_online(void *arg) +{ + int cpuid = hard_smp_processor_id(); + + /* Allow master to continue. The master will then give us the + * go-ahead by setting the smp_commenced_mask and will wait without + * timeouts until our setup is completed fully (signified by + * our bit being set in the cpu_online_mask). + */ + do_swap(&cpu_callin_map[cpuid], 1); + + local_ops->cache_all(); + local_ops->tlb_all(); + + /* Fix idle thread fields. */ + __asm__ __volatile__("ld [%0], %%g6\n\t" : : "r"(¤t_set[cpuid]) + : "memory" /* paranoid */); + + /* Attach to the address space of init_task. */ + mmgrab(&init_mm); + current->active_mm = &init_mm; + + while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) + mb(); +} + +/* + * Cycle through the processors asking the PROM to start each one. + */ + +extern struct linux_prom_registers smp_penguin_ctable; + +void leon_configure_cache_smp(void) +{ + unsigned long cfg = sparc_leon3_get_dcachecfg(); + int me = smp_processor_id(); + + if (ASI_LEON3_SYSCTRL_CFG_SSIZE(cfg) > 4) { + printk(KERN_INFO "Note: SMP with snooping only works on 4k cache, found %dk(0x%x) on cpu %d, disabling caches\n", + (unsigned int)ASI_LEON3_SYSCTRL_CFG_SSIZE(cfg), + (unsigned int)cfg, (unsigned int)me); + sparc_leon3_disable_cache(); + } else { + if (cfg & ASI_LEON3_SYSCTRL_CFG_SNOOPING) { + sparc_leon3_enable_snooping(); + } else { + printk(KERN_INFO "Note: You have to enable snooping in the vhdl model cpu %d, disabling caches\n", + me); + sparc_leon3_disable_cache(); + } + } + + local_ops->cache_all(); + local_ops->tlb_all(); +} + +static void leon_smp_setbroadcast(unsigned int mask) +{ + int broadcast = + ((LEON3_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpstatus)) >> + LEON3_IRQMPSTATUS_BROADCAST) & 1); + if (!broadcast) { + prom_printf("######## !!!! The irqmp-ctrl must have broadcast enabled, smp wont work !!!!! ####### nr cpus: %d\n", + leon_smp_nrcpus()); + if (leon_smp_nrcpus() > 1) { + BUG(); + } else { + prom_printf("continue anyway\n"); + return; + } + } + LEON_BYPASS_STORE_PA(&(leon3_irqctrl_regs->mpbroadcast), mask); +} + +int leon_smp_nrcpus(void) +{ + int nrcpu = + ((LEON3_BYPASS_LOAD_PA(&(leon3_irqctrl_regs->mpstatus)) >> + LEON3_IRQMPSTATUS_CPUNR) & 0xf) + 1; + return nrcpu; +} + +void __init leon_boot_cpus(void) +{ + int nrcpu = leon_smp_nrcpus(); + int me = smp_processor_id(); + + /* Setup IPI */ + leon_ipi_init(); + + printk(KERN_INFO "%d:(%d:%d) cpus mpirq at 0x%x\n", (unsigned int)me, + (unsigned int)nrcpu, (unsigned int)NR_CPUS, + (unsigned int)&(leon3_irqctrl_regs->mpstatus)); + + leon_enable_irq_cpu(LEON3_IRQ_CROSS_CALL, me); + leon_enable_irq_cpu(LEON3_IRQ_TICKER, me); + leon_enable_irq_cpu(leon_ipi_irq, me); + + leon_smp_setbroadcast(1 << LEON3_IRQ_TICKER); + + leon_configure_cache_smp(); + local_ops->cache_all(); + +} + +int leon_boot_one_cpu(int i, struct task_struct *idle) +{ + int timeout; + + current_set[i] = task_thread_info(idle); + + /* See trampoline.S:leon_smp_cpu_startup for details... + * Initialize the contexts table + * Since the call to prom_startcpu() trashes the structure, + * we need to re-initialize it for each cpu + */ + smp_penguin_ctable.which_io = 0; + smp_penguin_ctable.phys_addr = (unsigned int)srmmu_ctx_table_phys; + smp_penguin_ctable.reg_size = 0; + + /* whirrr, whirrr, whirrrrrrrrr... */ + printk(KERN_INFO "Starting CPU %d : (irqmp: 0x%x)\n", (unsigned int)i, + (unsigned int)&leon3_irqctrl_regs->mpstatus); + local_ops->cache_all(); + + /* Make sure all IRQs are of from the start for this new CPU */ + LEON_BYPASS_STORE_PA(&leon3_irqctrl_regs->mask[i], 0); + + /* Wake one CPU */ + LEON_BYPASS_STORE_PA(&(leon3_irqctrl_regs->mpstatus), 1 << i); + + /* wheee... it's going... */ + for (timeout = 0; timeout < 10000; timeout++) { + if (cpu_callin_map[i]) + break; + udelay(200); + } + printk(KERN_INFO "Started CPU %d\n", (unsigned int)i); + + if (!(cpu_callin_map[i])) { + printk(KERN_ERR "Processor %d is stuck.\n", i); + return -ENODEV; + } else { + leon_enable_irq_cpu(LEON3_IRQ_CROSS_CALL, i); + leon_enable_irq_cpu(LEON3_IRQ_TICKER, i); + leon_enable_irq_cpu(leon_ipi_irq, i); + } + + local_ops->cache_all(); + return 0; +} + +void __init leon_smp_done(void) +{ + + int i, first; + int *prev; + + /* setup cpu list for irq rotation */ + first = 0; + prev = &first; + for (i = 0; i < NR_CPUS; i++) { + if (cpu_online(i)) { + *prev = i; + prev = &cpu_data(i).next; + } + } + *prev = first; + local_ops->cache_all(); + + /* Free unneeded trap tables */ + if (!cpu_present(1)) { + free_reserved_page(virt_to_page(&trapbase_cpu1[0])); + } + if (!cpu_present(2)) { + free_reserved_page(virt_to_page(&trapbase_cpu2[0])); + } + if (!cpu_present(3)) { + free_reserved_page(virt_to_page(&trapbase_cpu3[0])); + } + /* Ok, they are spinning and ready to go. */ + smp_processors_ready = 1; + +} + +struct leon_ipi_work { + int single; + int msk; + int resched; +}; + +static DEFINE_PER_CPU_SHARED_ALIGNED(struct leon_ipi_work, leon_ipi_work); + +/* Initialize IPIs on the LEON, in order to save IRQ resources only one IRQ + * is used for all three types of IPIs. + */ +static void __init leon_ipi_init(void) +{ + int cpu, len; + struct leon_ipi_work *work; + struct property *pp; + struct device_node *rootnp; + struct tt_entry *trap_table; + unsigned long flags; + + /* Find IPI IRQ or stick with default value */ + rootnp = of_find_node_by_path("/ambapp0"); + if (rootnp) { + pp = of_find_property(rootnp, "ipi_num", &len); + if (pp && (*(int *)pp->value)) + leon_ipi_irq = *(int *)pp->value; + } + printk(KERN_INFO "leon: SMP IPIs at IRQ %d\n", leon_ipi_irq); + + /* Adjust so that we jump directly to smpleon_ipi */ + local_irq_save(flags); + trap_table = &sparc_ttable[SP_TRAP_IRQ1 + (leon_ipi_irq - 1)]; + trap_table->inst_three += smpleon_ipi - real_irq_entry; + local_ops->cache_all(); + local_irq_restore(flags); + + for_each_possible_cpu(cpu) { + work = &per_cpu(leon_ipi_work, cpu); + work->single = work->msk = work->resched = 0; + } +} + +static void leon_send_ipi(int cpu, int level) +{ + unsigned long mask; + mask = leon_get_irqmask(level); + LEON3_BYPASS_STORE_PA(&leon3_irqctrl_regs->force[cpu], mask); +} + +static void leon_ipi_single(int cpu) +{ + struct leon_ipi_work *work = &per_cpu(leon_ipi_work, cpu); + + /* Mark work */ + work->single = 1; + + /* Generate IRQ on the CPU */ + leon_send_ipi(cpu, leon_ipi_irq); +} + +static void leon_ipi_mask_one(int cpu) +{ + struct leon_ipi_work *work = &per_cpu(leon_ipi_work, cpu); + + /* Mark work */ + work->msk = 1; + + /* Generate IRQ on the CPU */ + leon_send_ipi(cpu, leon_ipi_irq); +} + +static void leon_ipi_resched(int cpu) +{ + struct leon_ipi_work *work = &per_cpu(leon_ipi_work, cpu); + + /* Mark work */ + work->resched = 1; + + /* Generate IRQ on the CPU (any IRQ will cause resched) */ + leon_send_ipi(cpu, leon_ipi_irq); +} + +void leonsmp_ipi_interrupt(void) +{ + struct leon_ipi_work *work = this_cpu_ptr(&leon_ipi_work); + + if (work->single) { + work->single = 0; + smp_call_function_single_interrupt(); + } + if (work->msk) { + work->msk = 0; + smp_call_function_interrupt(); + } + if (work->resched) { + work->resched = 0; + smp_resched_interrupt(); + } +} + +static struct smp_funcall { + void *func; + unsigned long arg1; + unsigned long arg2; + unsigned long arg3; + unsigned long arg4; + unsigned long arg5; + unsigned long processors_in[NR_CPUS]; /* Set when ipi entered. */ + unsigned long processors_out[NR_CPUS]; /* Set when ipi exited. */ +} ccall_info __attribute__((aligned(8))); + +static DEFINE_SPINLOCK(cross_call_lock); + +/* Cross calls must be serialized, at least currently. */ +static void leon_cross_call(void *func, cpumask_t mask, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4) +{ + if (smp_processors_ready) { + register int high = NR_CPUS - 1; + unsigned long flags; + + spin_lock_irqsave(&cross_call_lock, flags); + + { + /* If you make changes here, make sure gcc generates proper code... */ + register void *f asm("i0") = func; + register unsigned long a1 asm("i1") = arg1; + register unsigned long a2 asm("i2") = arg2; + register unsigned long a3 asm("i3") = arg3; + register unsigned long a4 asm("i4") = arg4; + register unsigned long a5 asm("i5") = 0; + + __asm__ __volatile__("std %0, [%6]\n\t" + "std %2, [%6 + 8]\n\t" + "std %4, [%6 + 16]\n\t" : : + "r"(f), "r"(a1), "r"(a2), "r"(a3), + "r"(a4), "r"(a5), + "r"(&ccall_info.func)); + } + + /* Init receive/complete mapping, plus fire the IPI's off. */ + { + register int i; + + cpumask_clear_cpu(smp_processor_id(), &mask); + cpumask_and(&mask, cpu_online_mask, &mask); + for (i = 0; i <= high; i++) { + if (cpumask_test_cpu(i, &mask)) { + ccall_info.processors_in[i] = 0; + ccall_info.processors_out[i] = 0; + leon_send_ipi(i, LEON3_IRQ_CROSS_CALL); + + } + } + } + + { + register int i; + + i = 0; + do { + if (!cpumask_test_cpu(i, &mask)) + continue; + + while (!ccall_info.processors_in[i]) + barrier(); + } while (++i <= high); + + i = 0; + do { + if (!cpumask_test_cpu(i, &mask)) + continue; + + while (!ccall_info.processors_out[i]) + barrier(); + } while (++i <= high); + } + + spin_unlock_irqrestore(&cross_call_lock, flags); + } +} + +/* Running cross calls. */ +void leon_cross_call_irq(void) +{ + void (*func)(unsigned long, unsigned long, unsigned long, unsigned long, + unsigned long) = ccall_info.func; + int i = smp_processor_id(); + + ccall_info.processors_in[i] = 1; + func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4, + ccall_info.arg5); + ccall_info.processors_out[i] = 1; +} + +static const struct sparc32_ipi_ops leon_ipi_ops = { + .cross_call = leon_cross_call, + .resched = leon_ipi_resched, + .single = leon_ipi_single, + .mask_one = leon_ipi_mask_one, +}; + +void __init leon_init_smp(void) +{ + /* Patch ipi15 trap table */ + t_nmi[1] = t_nmi[1] + (linux_trap_ipi15_leon - linux_trap_ipi15_sun4m); + + sparc32_ipi_ops = &leon_ipi_ops; +} diff --git a/arch/sparc/kernel/mdesc.c b/arch/sparc/kernel/mdesc.c new file mode 100644 index 000000000..ecd6c8ae4 --- /dev/null +++ b/arch/sparc/kernel/mdesc.c @@ -0,0 +1,1351 @@ +// SPDX-License-Identifier: GPL-2.0 +/* mdesc.c: Sun4V machine description handling. + * + * Copyright (C) 2007, 2008 David S. Miller + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/* Unlike the OBP device tree, the machine description is a full-on + * DAG. An arbitrary number of ARCs are possible from one + * node to other nodes and thus we can't use the OBP device_node + * data structure to represent these nodes inside of the kernel. + * + * Actually, it isn't even a DAG, because there are back pointers + * which create cycles in the graph. + * + * mdesc_hdr and mdesc_elem describe the layout of the data structure + * we get from the Hypervisor. + */ +struct mdesc_hdr { + u32 version; /* Transport version */ + u32 node_sz; /* node block size */ + u32 name_sz; /* name block size */ + u32 data_sz; /* data block size */ + char data[]; +} __attribute__((aligned(16))); + +struct mdesc_elem { + u8 tag; +#define MD_LIST_END 0x00 +#define MD_NODE 0x4e +#define MD_NODE_END 0x45 +#define MD_NOOP 0x20 +#define MD_PROP_ARC 0x61 +#define MD_PROP_VAL 0x76 +#define MD_PROP_STR 0x73 +#define MD_PROP_DATA 0x64 + u8 name_len; + u16 resv; + u32 name_offset; + union { + struct { + u32 data_len; + u32 data_offset; + } data; + u64 val; + } d; +}; + +struct mdesc_mem_ops { + struct mdesc_handle *(*alloc)(unsigned int mdesc_size); + void (*free)(struct mdesc_handle *handle); +}; + +struct mdesc_handle { + struct list_head list; + struct mdesc_mem_ops *mops; + void *self_base; + refcount_t refcnt; + unsigned int handle_size; + struct mdesc_hdr mdesc; +}; + +typedef int (*mdesc_node_info_get_f)(struct mdesc_handle *, u64, + union md_node_info *); +typedef void (*mdesc_node_info_rel_f)(union md_node_info *); +typedef bool (*mdesc_node_match_f)(union md_node_info *, union md_node_info *); + +struct md_node_ops { + char *name; + mdesc_node_info_get_f get_info; + mdesc_node_info_rel_f rel_info; + mdesc_node_match_f node_match; +}; + +static int get_vdev_port_node_info(struct mdesc_handle *md, u64 node, + union md_node_info *node_info); +static void rel_vdev_port_node_info(union md_node_info *node_info); +static bool vdev_port_node_match(union md_node_info *a_node_info, + union md_node_info *b_node_info); + +static int get_ds_port_node_info(struct mdesc_handle *md, u64 node, + union md_node_info *node_info); +static void rel_ds_port_node_info(union md_node_info *node_info); +static bool ds_port_node_match(union md_node_info *a_node_info, + union md_node_info *b_node_info); + +/* supported node types which can be registered */ +static struct md_node_ops md_node_ops_table[] = { + {"virtual-device-port", get_vdev_port_node_info, + rel_vdev_port_node_info, vdev_port_node_match}, + {"domain-services-port", get_ds_port_node_info, + rel_ds_port_node_info, ds_port_node_match}, + {NULL, NULL, NULL, NULL} +}; + +static void mdesc_get_node_ops(const char *node_name, + mdesc_node_info_get_f *get_info_f, + mdesc_node_info_rel_f *rel_info_f, + mdesc_node_match_f *match_f) +{ + int i; + + if (get_info_f) + *get_info_f = NULL; + + if (rel_info_f) + *rel_info_f = NULL; + + if (match_f) + *match_f = NULL; + + if (!node_name) + return; + + for (i = 0; md_node_ops_table[i].name != NULL; i++) { + if (strcmp(md_node_ops_table[i].name, node_name) == 0) { + if (get_info_f) + *get_info_f = md_node_ops_table[i].get_info; + + if (rel_info_f) + *rel_info_f = md_node_ops_table[i].rel_info; + + if (match_f) + *match_f = md_node_ops_table[i].node_match; + + break; + } + } +} + +static void mdesc_handle_init(struct mdesc_handle *hp, + unsigned int handle_size, + void *base) +{ + BUG_ON(((unsigned long)&hp->mdesc) & (16UL - 1)); + + memset(hp, 0, handle_size); + INIT_LIST_HEAD(&hp->list); + hp->self_base = base; + refcount_set(&hp->refcnt, 1); + hp->handle_size = handle_size; +} + +static struct mdesc_handle * __init mdesc_memblock_alloc(unsigned int mdesc_size) +{ + unsigned int handle_size, alloc_size; + struct mdesc_handle *hp; + unsigned long paddr; + + handle_size = (sizeof(struct mdesc_handle) - + sizeof(struct mdesc_hdr) + + mdesc_size); + alloc_size = PAGE_ALIGN(handle_size); + + paddr = memblock_phys_alloc(alloc_size, PAGE_SIZE); + + hp = NULL; + if (paddr) { + hp = __va(paddr); + mdesc_handle_init(hp, handle_size, hp); + } + return hp; +} + +static void __init mdesc_memblock_free(struct mdesc_handle *hp) +{ + unsigned int alloc_size; + + BUG_ON(refcount_read(&hp->refcnt) != 0); + BUG_ON(!list_empty(&hp->list)); + + alloc_size = PAGE_ALIGN(hp->handle_size); + memblock_free(hp, alloc_size); +} + +static struct mdesc_mem_ops memblock_mdesc_ops = { + .alloc = mdesc_memblock_alloc, + .free = mdesc_memblock_free, +}; + +static struct mdesc_handle *mdesc_kmalloc(unsigned int mdesc_size) +{ + unsigned int handle_size; + struct mdesc_handle *hp; + unsigned long addr; + void *base; + + handle_size = (sizeof(struct mdesc_handle) - + sizeof(struct mdesc_hdr) + + mdesc_size); + base = kmalloc(handle_size + 15, GFP_KERNEL | __GFP_RETRY_MAYFAIL); + if (!base) + return NULL; + + addr = (unsigned long)base; + addr = (addr + 15UL) & ~15UL; + hp = (struct mdesc_handle *) addr; + + mdesc_handle_init(hp, handle_size, base); + + return hp; +} + +static void mdesc_kfree(struct mdesc_handle *hp) +{ + BUG_ON(refcount_read(&hp->refcnt) != 0); + BUG_ON(!list_empty(&hp->list)); + + kfree(hp->self_base); +} + +static struct mdesc_mem_ops kmalloc_mdesc_memops = { + .alloc = mdesc_kmalloc, + .free = mdesc_kfree, +}; + +static struct mdesc_handle *mdesc_alloc(unsigned int mdesc_size, + struct mdesc_mem_ops *mops) +{ + struct mdesc_handle *hp = mops->alloc(mdesc_size); + + if (hp) + hp->mops = mops; + + return hp; +} + +static void mdesc_free(struct mdesc_handle *hp) +{ + hp->mops->free(hp); +} + +static struct mdesc_handle *cur_mdesc; +static LIST_HEAD(mdesc_zombie_list); +static DEFINE_SPINLOCK(mdesc_lock); + +struct mdesc_handle *mdesc_grab(void) +{ + struct mdesc_handle *hp; + unsigned long flags; + + spin_lock_irqsave(&mdesc_lock, flags); + hp = cur_mdesc; + if (hp) + refcount_inc(&hp->refcnt); + spin_unlock_irqrestore(&mdesc_lock, flags); + + return hp; +} +EXPORT_SYMBOL(mdesc_grab); + +void mdesc_release(struct mdesc_handle *hp) +{ + unsigned long flags; + + spin_lock_irqsave(&mdesc_lock, flags); + if (refcount_dec_and_test(&hp->refcnt)) { + list_del_init(&hp->list); + hp->mops->free(hp); + } + spin_unlock_irqrestore(&mdesc_lock, flags); +} +EXPORT_SYMBOL(mdesc_release); + +static DEFINE_MUTEX(mdesc_mutex); +static struct mdesc_notifier_client *client_list; + +void mdesc_register_notifier(struct mdesc_notifier_client *client) +{ + bool supported = false; + u64 node; + int i; + + mutex_lock(&mdesc_mutex); + + /* check to see if the node is supported for registration */ + for (i = 0; md_node_ops_table[i].name != NULL; i++) { + if (strcmp(md_node_ops_table[i].name, client->node_name) == 0) { + supported = true; + break; + } + } + + if (!supported) { + pr_err("MD: %s node not supported\n", client->node_name); + mutex_unlock(&mdesc_mutex); + return; + } + + client->next = client_list; + client_list = client; + + mdesc_for_each_node_by_name(cur_mdesc, node, client->node_name) + client->add(cur_mdesc, node, client->node_name); + + mutex_unlock(&mdesc_mutex); +} + +static const u64 *parent_cfg_handle(struct mdesc_handle *hp, u64 node) +{ + const u64 *id; + u64 a; + + id = NULL; + mdesc_for_each_arc(a, hp, node, MDESC_ARC_TYPE_BACK) { + u64 target; + + target = mdesc_arc_target(hp, a); + id = mdesc_get_property(hp, target, + "cfg-handle", NULL); + if (id) + break; + } + + return id; +} + +static int get_vdev_port_node_info(struct mdesc_handle *md, u64 node, + union md_node_info *node_info) +{ + const u64 *parent_cfg_hdlp; + const char *name; + const u64 *idp; + + /* + * Virtual device nodes are distinguished by: + * 1. "id" property + * 2. "name" property + * 3. parent node "cfg-handle" property + */ + idp = mdesc_get_property(md, node, "id", NULL); + name = mdesc_get_property(md, node, "name", NULL); + parent_cfg_hdlp = parent_cfg_handle(md, node); + + if (!idp || !name || !parent_cfg_hdlp) + return -1; + + node_info->vdev_port.id = *idp; + node_info->vdev_port.name = kstrdup_const(name, GFP_KERNEL); + if (!node_info->vdev_port.name) + return -1; + node_info->vdev_port.parent_cfg_hdl = *parent_cfg_hdlp; + + return 0; +} + +static void rel_vdev_port_node_info(union md_node_info *node_info) +{ + if (node_info && node_info->vdev_port.name) { + kfree_const(node_info->vdev_port.name); + node_info->vdev_port.name = NULL; + } +} + +static bool vdev_port_node_match(union md_node_info *a_node_info, + union md_node_info *b_node_info) +{ + if (a_node_info->vdev_port.id != b_node_info->vdev_port.id) + return false; + + if (a_node_info->vdev_port.parent_cfg_hdl != + b_node_info->vdev_port.parent_cfg_hdl) + return false; + + if (strncmp(a_node_info->vdev_port.name, + b_node_info->vdev_port.name, MDESC_MAX_STR_LEN) != 0) + return false; + + return true; +} + +static int get_ds_port_node_info(struct mdesc_handle *md, u64 node, + union md_node_info *node_info) +{ + const u64 *idp; + + /* DS port nodes use the "id" property to distinguish them */ + idp = mdesc_get_property(md, node, "id", NULL); + if (!idp) + return -1; + + node_info->ds_port.id = *idp; + + return 0; +} + +static void rel_ds_port_node_info(union md_node_info *node_info) +{ +} + +static bool ds_port_node_match(union md_node_info *a_node_info, + union md_node_info *b_node_info) +{ + if (a_node_info->ds_port.id != b_node_info->ds_port.id) + return false; + + return true; +} + +/* Run 'func' on nodes which are in A but not in B. */ +static void invoke_on_missing(const char *name, + struct mdesc_handle *a, + struct mdesc_handle *b, + void (*func)(struct mdesc_handle *, u64, + const char *node_name)) +{ + mdesc_node_info_get_f get_info_func; + mdesc_node_info_rel_f rel_info_func; + mdesc_node_match_f node_match_func; + union md_node_info a_node_info; + union md_node_info b_node_info; + bool found; + u64 a_node; + u64 b_node; + int rv; + + /* + * Find the get_info, rel_info and node_match ops for the given + * node name + */ + mdesc_get_node_ops(name, &get_info_func, &rel_info_func, + &node_match_func); + + /* If we didn't find a match, the node type is not supported */ + if (!get_info_func || !rel_info_func || !node_match_func) { + pr_err("MD: %s node type is not supported\n", name); + return; + } + + mdesc_for_each_node_by_name(a, a_node, name) { + found = false; + + rv = get_info_func(a, a_node, &a_node_info); + if (rv != 0) { + pr_err("MD: Cannot find 1 or more required match properties for %s node.\n", + name); + continue; + } + + /* Check each node in B for node matching a_node */ + mdesc_for_each_node_by_name(b, b_node, name) { + rv = get_info_func(b, b_node, &b_node_info); + if (rv != 0) + continue; + + if (node_match_func(&a_node_info, &b_node_info)) { + found = true; + rel_info_func(&b_node_info); + break; + } + + rel_info_func(&b_node_info); + } + + rel_info_func(&a_node_info); + + if (!found) + func(a, a_node, name); + } +} + +static void notify_one(struct mdesc_notifier_client *p, + struct mdesc_handle *old_hp, + struct mdesc_handle *new_hp) +{ + invoke_on_missing(p->node_name, old_hp, new_hp, p->remove); + invoke_on_missing(p->node_name, new_hp, old_hp, p->add); +} + +static void mdesc_notify_clients(struct mdesc_handle *old_hp, + struct mdesc_handle *new_hp) +{ + struct mdesc_notifier_client *p = client_list; + + while (p) { + notify_one(p, old_hp, new_hp); + p = p->next; + } +} + +void mdesc_update(void) +{ + unsigned long len, real_len, status; + struct mdesc_handle *hp, *orig_hp; + unsigned long flags; + + mutex_lock(&mdesc_mutex); + + (void) sun4v_mach_desc(0UL, 0UL, &len); + + hp = mdesc_alloc(len, &kmalloc_mdesc_memops); + if (!hp) { + printk(KERN_ERR "MD: mdesc alloc fails\n"); + goto out; + } + + status = sun4v_mach_desc(__pa(&hp->mdesc), len, &real_len); + if (status != HV_EOK || real_len > len) { + printk(KERN_ERR "MD: mdesc reread fails with %lu\n", + status); + refcount_dec(&hp->refcnt); + mdesc_free(hp); + goto out; + } + + spin_lock_irqsave(&mdesc_lock, flags); + orig_hp = cur_mdesc; + cur_mdesc = hp; + spin_unlock_irqrestore(&mdesc_lock, flags); + + mdesc_notify_clients(orig_hp, hp); + + spin_lock_irqsave(&mdesc_lock, flags); + if (refcount_dec_and_test(&orig_hp->refcnt)) + mdesc_free(orig_hp); + else + list_add(&orig_hp->list, &mdesc_zombie_list); + spin_unlock_irqrestore(&mdesc_lock, flags); + +out: + mutex_unlock(&mdesc_mutex); +} + +u64 mdesc_get_node(struct mdesc_handle *hp, const char *node_name, + union md_node_info *node_info) +{ + mdesc_node_info_get_f get_info_func; + mdesc_node_info_rel_f rel_info_func; + mdesc_node_match_f node_match_func; + union md_node_info hp_node_info; + u64 hp_node; + int rv; + + if (hp == NULL || node_name == NULL || node_info == NULL) + return MDESC_NODE_NULL; + + /* Find the ops for the given node name */ + mdesc_get_node_ops(node_name, &get_info_func, &rel_info_func, + &node_match_func); + + /* If we didn't find ops for the given node name, it is not supported */ + if (!get_info_func || !rel_info_func || !node_match_func) { + pr_err("MD: %s node is not supported\n", node_name); + return -EINVAL; + } + + mdesc_for_each_node_by_name(hp, hp_node, node_name) { + rv = get_info_func(hp, hp_node, &hp_node_info); + if (rv != 0) + continue; + + if (node_match_func(node_info, &hp_node_info)) + break; + + rel_info_func(&hp_node_info); + } + + rel_info_func(&hp_node_info); + + return hp_node; +} +EXPORT_SYMBOL(mdesc_get_node); + +int mdesc_get_node_info(struct mdesc_handle *hp, u64 node, + const char *node_name, union md_node_info *node_info) +{ + mdesc_node_info_get_f get_info_func; + int rv; + + if (hp == NULL || node == MDESC_NODE_NULL || + node_name == NULL || node_info == NULL) + return -EINVAL; + + /* Find the get_info op for the given node name */ + mdesc_get_node_ops(node_name, &get_info_func, NULL, NULL); + + /* If we didn't find a get_info_func, the node name is not supported */ + if (get_info_func == NULL) { + pr_err("MD: %s node is not supported\n", node_name); + return -EINVAL; + } + + rv = get_info_func(hp, node, node_info); + if (rv != 0) { + pr_err("MD: Cannot find 1 or more required match properties for %s node.\n", + node_name); + return -1; + } + + return 0; +} +EXPORT_SYMBOL(mdesc_get_node_info); + +static struct mdesc_elem *node_block(struct mdesc_hdr *mdesc) +{ + return (struct mdesc_elem *) mdesc->data; +} + +static void *name_block(struct mdesc_hdr *mdesc) +{ + return ((void *) node_block(mdesc)) + mdesc->node_sz; +} + +static void *data_block(struct mdesc_hdr *mdesc) +{ + return ((void *) name_block(mdesc)) + mdesc->name_sz; +} + +u64 mdesc_node_by_name(struct mdesc_handle *hp, + u64 from_node, const char *name) +{ + struct mdesc_elem *ep = node_block(&hp->mdesc); + const char *names = name_block(&hp->mdesc); + u64 last_node = hp->mdesc.node_sz / 16; + u64 ret; + + if (from_node == MDESC_NODE_NULL) { + ret = from_node = 0; + } else if (from_node >= last_node) { + return MDESC_NODE_NULL; + } else { + ret = ep[from_node].d.val; + } + + while (ret < last_node) { + if (ep[ret].tag != MD_NODE) + return MDESC_NODE_NULL; + if (!strcmp(names + ep[ret].name_offset, name)) + break; + ret = ep[ret].d.val; + } + if (ret >= last_node) + ret = MDESC_NODE_NULL; + return ret; +} +EXPORT_SYMBOL(mdesc_node_by_name); + +const void *mdesc_get_property(struct mdesc_handle *hp, u64 node, + const char *name, int *lenp) +{ + const char *names = name_block(&hp->mdesc); + u64 last_node = hp->mdesc.node_sz / 16; + void *data = data_block(&hp->mdesc); + struct mdesc_elem *ep; + + if (node == MDESC_NODE_NULL || node >= last_node) + return NULL; + + ep = node_block(&hp->mdesc) + node; + ep++; + for (; ep->tag != MD_NODE_END; ep++) { + void *val = NULL; + int len = 0; + + switch (ep->tag) { + case MD_PROP_VAL: + val = &ep->d.val; + len = 8; + break; + + case MD_PROP_STR: + case MD_PROP_DATA: + val = data + ep->d.data.data_offset; + len = ep->d.data.data_len; + break; + + default: + break; + } + if (!val) + continue; + + if (!strcmp(names + ep->name_offset, name)) { + if (lenp) + *lenp = len; + return val; + } + } + + return NULL; +} +EXPORT_SYMBOL(mdesc_get_property); + +u64 mdesc_next_arc(struct mdesc_handle *hp, u64 from, const char *arc_type) +{ + struct mdesc_elem *ep, *base = node_block(&hp->mdesc); + const char *names = name_block(&hp->mdesc); + u64 last_node = hp->mdesc.node_sz / 16; + + if (from == MDESC_NODE_NULL || from >= last_node) + return MDESC_NODE_NULL; + + ep = base + from; + + ep++; + for (; ep->tag != MD_NODE_END; ep++) { + if (ep->tag != MD_PROP_ARC) + continue; + + if (strcmp(names + ep->name_offset, arc_type)) + continue; + + return ep - base; + } + + return MDESC_NODE_NULL; +} +EXPORT_SYMBOL(mdesc_next_arc); + +u64 mdesc_arc_target(struct mdesc_handle *hp, u64 arc) +{ + struct mdesc_elem *ep, *base = node_block(&hp->mdesc); + + ep = base + arc; + + return ep->d.val; +} +EXPORT_SYMBOL(mdesc_arc_target); + +const char *mdesc_node_name(struct mdesc_handle *hp, u64 node) +{ + struct mdesc_elem *ep, *base = node_block(&hp->mdesc); + const char *names = name_block(&hp->mdesc); + u64 last_node = hp->mdesc.node_sz / 16; + + if (node == MDESC_NODE_NULL || node >= last_node) + return NULL; + + ep = base + node; + if (ep->tag != MD_NODE) + return NULL; + + return names + ep->name_offset; +} +EXPORT_SYMBOL(mdesc_node_name); + +static u64 max_cpus = 64; + +static void __init report_platform_properties(void) +{ + struct mdesc_handle *hp = mdesc_grab(); + u64 pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); + const char *s; + const u64 *v; + + if (pn == MDESC_NODE_NULL) { + prom_printf("No platform node in machine-description.\n"); + prom_halt(); + } + + s = mdesc_get_property(hp, pn, "banner-name", NULL); + printk("PLATFORM: banner-name [%s]\n", s); + s = mdesc_get_property(hp, pn, "name", NULL); + printk("PLATFORM: name [%s]\n", s); + + v = mdesc_get_property(hp, pn, "hostid", NULL); + if (v) + printk("PLATFORM: hostid [%08llx]\n", *v); + v = mdesc_get_property(hp, pn, "serial#", NULL); + if (v) + printk("PLATFORM: serial# [%08llx]\n", *v); + v = mdesc_get_property(hp, pn, "stick-frequency", NULL); + printk("PLATFORM: stick-frequency [%08llx]\n", *v); + v = mdesc_get_property(hp, pn, "mac-address", NULL); + if (v) + printk("PLATFORM: mac-address [%llx]\n", *v); + v = mdesc_get_property(hp, pn, "watchdog-resolution", NULL); + if (v) + printk("PLATFORM: watchdog-resolution [%llu ms]\n", *v); + v = mdesc_get_property(hp, pn, "watchdog-max-timeout", NULL); + if (v) + printk("PLATFORM: watchdog-max-timeout [%llu ms]\n", *v); + v = mdesc_get_property(hp, pn, "max-cpus", NULL); + if (v) { + max_cpus = *v; + printk("PLATFORM: max-cpus [%llu]\n", max_cpus); + } + +#ifdef CONFIG_SMP + { + int max_cpu, i; + + if (v) { + max_cpu = *v; + if (max_cpu > NR_CPUS) + max_cpu = NR_CPUS; + } else { + max_cpu = NR_CPUS; + } + for (i = 0; i < max_cpu; i++) + set_cpu_possible(i, true); + } +#endif + + mdesc_release(hp); +} + +static void fill_in_one_cache(cpuinfo_sparc *c, struct mdesc_handle *hp, u64 mp) +{ + const u64 *level = mdesc_get_property(hp, mp, "level", NULL); + const u64 *size = mdesc_get_property(hp, mp, "size", NULL); + const u64 *line_size = mdesc_get_property(hp, mp, "line-size", NULL); + const char *type; + int type_len; + + type = mdesc_get_property(hp, mp, "type", &type_len); + + switch (*level) { + case 1: + if (of_find_in_proplist(type, "instn", type_len)) { + c->icache_size = *size; + c->icache_line_size = *line_size; + } else if (of_find_in_proplist(type, "data", type_len)) { + c->dcache_size = *size; + c->dcache_line_size = *line_size; + } + break; + + case 2: + c->ecache_size = *size; + c->ecache_line_size = *line_size; + break; + + default: + break; + } + + if (*level == 1) { + u64 a; + + mdesc_for_each_arc(a, hp, mp, MDESC_ARC_TYPE_FWD) { + u64 target = mdesc_arc_target(hp, a); + const char *name = mdesc_node_name(hp, target); + + if (!strcmp(name, "cache")) + fill_in_one_cache(c, hp, target); + } + } +} + +static void find_back_node_value(struct mdesc_handle *hp, u64 node, + char *srch_val, + void (*func)(struct mdesc_handle *, u64, int), + u64 val, int depth) +{ + u64 arc; + + /* Since we have an estimate of recursion depth, do a sanity check. */ + if (depth == 0) + return; + + mdesc_for_each_arc(arc, hp, node, MDESC_ARC_TYPE_BACK) { + u64 n = mdesc_arc_target(hp, arc); + const char *name = mdesc_node_name(hp, n); + + if (!strcmp(srch_val, name)) + (*func)(hp, n, val); + + find_back_node_value(hp, n, srch_val, func, val, depth-1); + } +} + +static void __mark_core_id(struct mdesc_handle *hp, u64 node, + int core_id) +{ + const u64 *id = mdesc_get_property(hp, node, "id", NULL); + + if (*id < num_possible_cpus()) + cpu_data(*id).core_id = core_id; +} + +static void __mark_max_cache_id(struct mdesc_handle *hp, u64 node, + int max_cache_id) +{ + const u64 *id = mdesc_get_property(hp, node, "id", NULL); + + if (*id < num_possible_cpus()) { + cpu_data(*id).max_cache_id = max_cache_id; + + /** + * On systems without explicit socket descriptions socket + * is max_cache_id + */ + cpu_data(*id).sock_id = max_cache_id; + } +} + +static void mark_core_ids(struct mdesc_handle *hp, u64 mp, + int core_id) +{ + find_back_node_value(hp, mp, "cpu", __mark_core_id, core_id, 10); +} + +static void mark_max_cache_ids(struct mdesc_handle *hp, u64 mp, + int max_cache_id) +{ + find_back_node_value(hp, mp, "cpu", __mark_max_cache_id, + max_cache_id, 10); +} + +static void set_core_ids(struct mdesc_handle *hp) +{ + int idx; + u64 mp; + + idx = 1; + + /* Identify unique cores by looking for cpus backpointed to by + * level 1 instruction caches. + */ + mdesc_for_each_node_by_name(hp, mp, "cache") { + const u64 *level; + const char *type; + int len; + + level = mdesc_get_property(hp, mp, "level", NULL); + if (*level != 1) + continue; + + type = mdesc_get_property(hp, mp, "type", &len); + if (!of_find_in_proplist(type, "instn", len)) + continue; + + mark_core_ids(hp, mp, idx); + idx++; + } +} + +static int set_max_cache_ids_by_cache(struct mdesc_handle *hp, int level) +{ + u64 mp; + int idx = 1; + int fnd = 0; + + /** + * Identify unique highest level of shared cache by looking for cpus + * backpointed to by shared level N caches. + */ + mdesc_for_each_node_by_name(hp, mp, "cache") { + const u64 *cur_lvl; + + cur_lvl = mdesc_get_property(hp, mp, "level", NULL); + if (*cur_lvl != level) + continue; + mark_max_cache_ids(hp, mp, idx); + idx++; + fnd = 1; + } + return fnd; +} + +static void set_sock_ids_by_socket(struct mdesc_handle *hp, u64 mp) +{ + int idx = 1; + + mdesc_for_each_node_by_name(hp, mp, "socket") { + u64 a; + + mdesc_for_each_arc(a, hp, mp, MDESC_ARC_TYPE_FWD) { + u64 t = mdesc_arc_target(hp, a); + const char *name; + const u64 *id; + + name = mdesc_node_name(hp, t); + if (strcmp(name, "cpu")) + continue; + + id = mdesc_get_property(hp, t, "id", NULL); + if (*id < num_possible_cpus()) + cpu_data(*id).sock_id = idx; + } + idx++; + } +} + +static void set_sock_ids(struct mdesc_handle *hp) +{ + u64 mp; + + /** + * Find the highest level of shared cache which pre-T7 is also + * the socket. + */ + if (!set_max_cache_ids_by_cache(hp, 3)) + set_max_cache_ids_by_cache(hp, 2); + + /* If machine description exposes sockets data use it.*/ + mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "sockets"); + if (mp != MDESC_NODE_NULL) + set_sock_ids_by_socket(hp, mp); +} + +static void mark_proc_ids(struct mdesc_handle *hp, u64 mp, int proc_id) +{ + u64 a; + + mdesc_for_each_arc(a, hp, mp, MDESC_ARC_TYPE_BACK) { + u64 t = mdesc_arc_target(hp, a); + const char *name; + const u64 *id; + + name = mdesc_node_name(hp, t); + if (strcmp(name, "cpu")) + continue; + + id = mdesc_get_property(hp, t, "id", NULL); + if (*id < NR_CPUS) + cpu_data(*id).proc_id = proc_id; + } +} + +static void __set_proc_ids(struct mdesc_handle *hp, const char *exec_unit_name) +{ + int idx; + u64 mp; + + idx = 0; + mdesc_for_each_node_by_name(hp, mp, exec_unit_name) { + const char *type; + int len; + + type = mdesc_get_property(hp, mp, "type", &len); + if (!of_find_in_proplist(type, "int", len) && + !of_find_in_proplist(type, "integer", len)) + continue; + + mark_proc_ids(hp, mp, idx); + idx++; + } +} + +static void set_proc_ids(struct mdesc_handle *hp) +{ + __set_proc_ids(hp, "exec_unit"); + __set_proc_ids(hp, "exec-unit"); +} + +static void get_one_mondo_bits(const u64 *p, unsigned int *mask, + unsigned long def, unsigned long max) +{ + u64 val; + + if (!p) + goto use_default; + val = *p; + + if (!val || val >= 64) + goto use_default; + + if (val > max) + val = max; + + *mask = ((1U << val) * 64U) - 1U; + return; + +use_default: + *mask = ((1U << def) * 64U) - 1U; +} + +static void get_mondo_data(struct mdesc_handle *hp, u64 mp, + struct trap_per_cpu *tb) +{ + static int printed; + const u64 *val; + + val = mdesc_get_property(hp, mp, "q-cpu-mondo-#bits", NULL); + get_one_mondo_bits(val, &tb->cpu_mondo_qmask, 7, ilog2(max_cpus * 2)); + + val = mdesc_get_property(hp, mp, "q-dev-mondo-#bits", NULL); + get_one_mondo_bits(val, &tb->dev_mondo_qmask, 7, 8); + + val = mdesc_get_property(hp, mp, "q-resumable-#bits", NULL); + get_one_mondo_bits(val, &tb->resum_qmask, 6, 7); + + val = mdesc_get_property(hp, mp, "q-nonresumable-#bits", NULL); + get_one_mondo_bits(val, &tb->nonresum_qmask, 2, 2); + if (!printed++) { + pr_info("SUN4V: Mondo queue sizes " + "[cpu(%u) dev(%u) r(%u) nr(%u)]\n", + tb->cpu_mondo_qmask + 1, + tb->dev_mondo_qmask + 1, + tb->resum_qmask + 1, + tb->nonresum_qmask + 1); + } +} + +static void *mdesc_iterate_over_cpus(void *(*func)(struct mdesc_handle *, u64, int, void *), void *arg, cpumask_t *mask) +{ + struct mdesc_handle *hp = mdesc_grab(); + void *ret = NULL; + u64 mp; + + mdesc_for_each_node_by_name(hp, mp, "cpu") { + const u64 *id = mdesc_get_property(hp, mp, "id", NULL); + int cpuid = *id; + +#ifdef CONFIG_SMP + if (cpuid >= NR_CPUS) { + printk(KERN_WARNING "Ignoring CPU %d which is " + ">= NR_CPUS (%d)\n", + cpuid, NR_CPUS); + continue; + } + if (!cpumask_test_cpu(cpuid, mask)) + continue; +#endif + + ret = func(hp, mp, cpuid, arg); + if (ret) + goto out; + } +out: + mdesc_release(hp); + return ret; +} + +static void *record_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, + void *arg) +{ + ncpus_probed++; +#ifdef CONFIG_SMP + set_cpu_present(cpuid, true); +#endif + return NULL; +} + +void mdesc_populate_present_mask(cpumask_t *mask) +{ + if (tlb_type != hypervisor) + return; + + ncpus_probed = 0; + mdesc_iterate_over_cpus(record_one_cpu, NULL, mask); +} + +static void * __init check_one_pgsz(struct mdesc_handle *hp, u64 mp, int cpuid, void *arg) +{ + const u64 *pgsz_prop = mdesc_get_property(hp, mp, "mmu-page-size-list", NULL); + unsigned long *pgsz_mask = arg; + u64 val; + + val = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K | + HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB); + if (pgsz_prop) + val = *pgsz_prop; + + if (!*pgsz_mask) + *pgsz_mask = val; + else + *pgsz_mask &= val; + return NULL; +} + +void __init mdesc_get_page_sizes(cpumask_t *mask, unsigned long *pgsz_mask) +{ + *pgsz_mask = 0; + mdesc_iterate_over_cpus(check_one_pgsz, pgsz_mask, mask); +} + +static void *fill_in_one_cpu(struct mdesc_handle *hp, u64 mp, int cpuid, + void *arg) +{ + const u64 *cfreq = mdesc_get_property(hp, mp, "clock-frequency", NULL); + struct trap_per_cpu *tb; + cpuinfo_sparc *c; + u64 a; + +#ifndef CONFIG_SMP + /* On uniprocessor we only want the values for the + * real physical cpu the kernel booted onto, however + * cpu_data() only has one entry at index 0. + */ + if (cpuid != real_hard_smp_processor_id()) + return NULL; + cpuid = 0; +#endif + + c = &cpu_data(cpuid); + c->clock_tick = *cfreq; + + tb = &trap_block[cpuid]; + get_mondo_data(hp, mp, tb); + + mdesc_for_each_arc(a, hp, mp, MDESC_ARC_TYPE_FWD) { + u64 j, t = mdesc_arc_target(hp, a); + const char *t_name; + + t_name = mdesc_node_name(hp, t); + if (!strcmp(t_name, "cache")) { + fill_in_one_cache(c, hp, t); + continue; + } + + mdesc_for_each_arc(j, hp, t, MDESC_ARC_TYPE_FWD) { + u64 n = mdesc_arc_target(hp, j); + const char *n_name; + + n_name = mdesc_node_name(hp, n); + if (!strcmp(n_name, "cache")) + fill_in_one_cache(c, hp, n); + } + } + + c->core_id = 0; + c->proc_id = -1; + + return NULL; +} + +void mdesc_fill_in_cpu_data(cpumask_t *mask) +{ + struct mdesc_handle *hp; + + mdesc_iterate_over_cpus(fill_in_one_cpu, NULL, mask); + + hp = mdesc_grab(); + + set_core_ids(hp); + set_proc_ids(hp); + set_sock_ids(hp); + + mdesc_release(hp); + + smp_fill_in_sib_core_maps(); +} + +/* mdesc_open() - Grab a reference to mdesc_handle when /dev/mdesc is + * opened. Hold this reference until /dev/mdesc is closed to ensure + * mdesc data structure is not released underneath us. Store the + * pointer to mdesc structure in private_data for read and seek to use + */ +static int mdesc_open(struct inode *inode, struct file *file) +{ + struct mdesc_handle *hp = mdesc_grab(); + + if (!hp) + return -ENODEV; + + file->private_data = hp; + + return 0; +} + +static ssize_t mdesc_read(struct file *file, char __user *buf, + size_t len, loff_t *offp) +{ + struct mdesc_handle *hp = file->private_data; + unsigned char *mdesc; + int bytes_left, count = len; + + if (*offp >= hp->handle_size) + return 0; + + bytes_left = hp->handle_size - *offp; + if (count > bytes_left) + count = bytes_left; + + mdesc = (unsigned char *)&hp->mdesc; + mdesc += *offp; + if (!copy_to_user(buf, mdesc, count)) { + *offp += count; + return count; + } else { + return -EFAULT; + } +} + +static loff_t mdesc_llseek(struct file *file, loff_t offset, int whence) +{ + struct mdesc_handle *hp = file->private_data; + + return no_seek_end_llseek_size(file, offset, whence, hp->handle_size); +} + +/* mdesc_close() - /dev/mdesc is being closed, release the reference to + * mdesc structure. + */ +static int mdesc_close(struct inode *inode, struct file *file) +{ + mdesc_release(file->private_data); + return 0; +} + +static const struct file_operations mdesc_fops = { + .open = mdesc_open, + .read = mdesc_read, + .llseek = mdesc_llseek, + .release = mdesc_close, + .owner = THIS_MODULE, +}; + +static struct miscdevice mdesc_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "mdesc", + .fops = &mdesc_fops, +}; + +static int __init mdesc_misc_init(void) +{ + return misc_register(&mdesc_misc); +} + +__initcall(mdesc_misc_init); + +void __init sun4v_mdesc_init(void) +{ + struct mdesc_handle *hp; + unsigned long len, real_len, status; + + (void) sun4v_mach_desc(0UL, 0UL, &len); + + printk("MDESC: Size is %lu bytes.\n", len); + + hp = mdesc_alloc(len, &memblock_mdesc_ops); + if (hp == NULL) { + prom_printf("MDESC: alloc of %lu bytes failed.\n", len); + prom_halt(); + } + + status = sun4v_mach_desc(__pa(&hp->mdesc), len, &real_len); + if (status != HV_EOK || real_len > len) { + prom_printf("sun4v_mach_desc fails, err(%lu), " + "len(%lu), real_len(%lu)\n", + status, len, real_len); + mdesc_free(hp); + prom_halt(); + } + + cur_mdesc = hp; + + mdesc_adi_init(); + report_platform_properties(); +} diff --git a/arch/sparc/kernel/misctrap.S b/arch/sparc/kernel/misctrap.S new file mode 100644 index 000000000..b5c841775 --- /dev/null +++ b/arch/sparc/kernel/misctrap.S @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifdef CONFIG_KGDB + .globl arch_kgdb_breakpoint + .type arch_kgdb_breakpoint,#function +arch_kgdb_breakpoint: + ta 0x72 + retl + nop + .size arch_kgdb_breakpoint,.-arch_kgdb_breakpoint +#endif + + .type __do_privact,#function +__do_privact: + mov TLB_SFSR, %g3 + stxa %g0, [%g3] ASI_DMMU ! Clear FaultValid bit + membar #Sync + sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + call do_privact + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + .size __do_privact,.-__do_privact + + .type do_mna,#function +do_mna: + rdpr %tl, %g3 + cmp %g3, 1 + + /* Setup %g4/%g5 now as they are used in the + * winfixup code. + */ + mov TLB_SFSR, %g3 + mov DMMU_SFAR, %g4 + ldxa [%g4] ASI_DMMU, %g4 + ldxa [%g3] ASI_DMMU, %g5 + stxa %g0, [%g3] ASI_DMMU ! Clear FaultValid bit + membar #Sync + bgu,pn %icc, winfix_mna + rdpr %tpc, %g3 + +1: sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call mem_address_unaligned + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + .size do_mna,.-do_mna + + .type do_lddfmna,#function +do_lddfmna: + sethi %hi(109f), %g7 + mov TLB_SFSR, %g4 + ldxa [%g4] ASI_DMMU, %g5 + stxa %g0, [%g4] ASI_DMMU ! Clear FaultValid bit + membar #Sync + mov DMMU_SFAR, %g4 + ldxa [%g4] ASI_DMMU, %g4 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call handle_lddfmna + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + .size do_lddfmna,.-do_lddfmna + + .type do_stdfmna,#function +do_stdfmna: + sethi %hi(109f), %g7 + mov TLB_SFSR, %g4 + ldxa [%g4] ASI_DMMU, %g5 + stxa %g0, [%g4] ASI_DMMU ! Clear FaultValid bit + membar #Sync + mov DMMU_SFAR, %g4 + ldxa [%g4] ASI_DMMU, %g4 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call handle_stdfmna + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + nop + .size do_stdfmna,.-do_stdfmna + + .type breakpoint_trap,#function +breakpoint_trap: + call sparc_breakpoint + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + .size breakpoint_trap,.-breakpoint_trap diff --git a/arch/sparc/kernel/module.c b/arch/sparc/kernel/module.c new file mode 100644 index 000000000..2a8770369 --- /dev/null +++ b/arch/sparc/kernel/module.c @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Kernel module help for sparc64. + * + * Copyright (C) 2001 Rusty Russell. + * Copyright (C) 2002 David S. Miller. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "entry.h" + +/* Make generic code ignore STT_REGISTER dummy undefined symbols. */ +int module_frob_arch_sections(Elf_Ehdr *hdr, + Elf_Shdr *sechdrs, + char *secstrings, + struct module *mod) +{ + unsigned int symidx; + Elf_Sym *sym; + int i; + + for (symidx = 0; sechdrs[symidx].sh_type != SHT_SYMTAB; symidx++) { + if (symidx == hdr->e_shnum-1) { + printk("%s: no symtab found.\n", mod->name); + return -ENOEXEC; + } + } + sym = (Elf_Sym *)sechdrs[symidx].sh_addr; + + for (i = 1; i < sechdrs[symidx].sh_size / sizeof(Elf_Sym); i++) { + if (sym[i].st_shndx == SHN_UNDEF) { + if (ELF_ST_TYPE(sym[i].st_info) == STT_REGISTER) + sym[i].st_shndx = SHN_ABS; + } + } + return 0; +} + +int apply_relocate_add(Elf_Shdr *sechdrs, + const char *strtab, + unsigned int symindex, + unsigned int relsec, + struct module *me) +{ + unsigned int i; + Elf_Rela *rel = (void *)sechdrs[relsec].sh_addr; + Elf_Sym *sym; + u8 *location; + u32 *loc32; + + for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { + Elf_Addr v; + + /* This is where to make the change */ + location = (u8 *)sechdrs[sechdrs[relsec].sh_info].sh_addr + + rel[i].r_offset; + loc32 = (u32 *) location; + +#ifdef CONFIG_SPARC64 + BUG_ON(((u64)location >> (u64)32) != (u64)0); +#endif /* CONFIG_SPARC64 */ + + /* This is the symbol it is referring to. Note that all + undefined symbols have been resolved. */ + sym = (Elf_Sym *)sechdrs[symindex].sh_addr + + ELF_R_SYM(rel[i].r_info); + v = sym->st_value + rel[i].r_addend; + + switch (ELF_R_TYPE(rel[i].r_info) & 0xff) { + case R_SPARC_DISP32: + v -= (Elf_Addr) location; + *loc32 = v; + break; +#ifdef CONFIG_SPARC64 + case R_SPARC_64: + case R_SPARC_UA64: + location[0] = v >> 56; + location[1] = v >> 48; + location[2] = v >> 40; + location[3] = v >> 32; + location[4] = v >> 24; + location[5] = v >> 16; + location[6] = v >> 8; + location[7] = v >> 0; + break; + + case R_SPARC_WDISP19: + v -= (Elf_Addr) location; + *loc32 = (*loc32 & ~0x7ffff) | + ((v >> 2) & 0x7ffff); + break; + + case R_SPARC_OLO10: + *loc32 = (*loc32 & ~0x1fff) | + (((v & 0x3ff) + + (ELF_R_TYPE(rel[i].r_info) >> 8)) + & 0x1fff); + break; +#endif /* CONFIG_SPARC64 */ + + case R_SPARC_32: + case R_SPARC_UA32: + location[0] = v >> 24; + location[1] = v >> 16; + location[2] = v >> 8; + location[3] = v >> 0; + break; + + case R_SPARC_WDISP30: + v -= (Elf_Addr) location; + *loc32 = (*loc32 & ~0x3fffffff) | + ((v >> 2) & 0x3fffffff); + break; + + case R_SPARC_WDISP22: + v -= (Elf_Addr) location; + *loc32 = (*loc32 & ~0x3fffff) | + ((v >> 2) & 0x3fffff); + break; + + case R_SPARC_LO10: + *loc32 = (*loc32 & ~0x3ff) | (v & 0x3ff); + break; + + case R_SPARC_HI22: + *loc32 = (*loc32 & ~0x3fffff) | + ((v >> 10) & 0x3fffff); + break; + + default: + printk(KERN_ERR "module %s: Unknown relocation: 0x%x\n", + me->name, + (int) (ELF_R_TYPE(rel[i].r_info) & 0xff)); + return -ENOEXEC; + } + } + return 0; +} + +#ifdef CONFIG_SPARC64 +static void do_patch_sections(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs) +{ + const Elf_Shdr *s, *sun4v_1insn = NULL, *sun4v_2insn = NULL; + char *secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; + + for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { + if (!strcmp(".sun4v_1insn_patch", secstrings + s->sh_name)) + sun4v_1insn = s; + if (!strcmp(".sun4v_2insn_patch", secstrings + s->sh_name)) + sun4v_2insn = s; + } + + if (sun4v_1insn && tlb_type == hypervisor) { + void *p = (void *) sun4v_1insn->sh_addr; + sun4v_patch_1insn_range(p, p + sun4v_1insn->sh_size); + } + if (sun4v_2insn && tlb_type == hypervisor) { + void *p = (void *) sun4v_2insn->sh_addr; + sun4v_patch_2insn_range(p, p + sun4v_2insn->sh_size); + } +} + +int module_finalize(const Elf_Ehdr *hdr, + const Elf_Shdr *sechdrs, + struct module *me) +{ + do_patch_sections(hdr, sechdrs); + + /* Cheetah's I-cache is fully coherent. */ + if (tlb_type == spitfire) { + unsigned long va; + + flushw_all(); + for (va = 0; va < (PAGE_SIZE << 1); va += 32) + spitfire_put_icache_tag(va, 0x0); + __asm__ __volatile__("flush %g6"); + } + + return 0; +} +#endif /* CONFIG_SPARC64 */ diff --git a/arch/sparc/kernel/nmi.c b/arch/sparc/kernel/nmi.c new file mode 100644 index 000000000..149adc094 --- /dev/null +++ b/arch/sparc/kernel/nmi.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Pseudo NMI support on sparc64 systems. + * + * Copyright (C) 2009 David S. Miller + * + * The NMI watchdog support and infrastructure is based almost + * entirely upon the x86 NMI support code. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "kstack.h" + +/* We don't have a real NMI on sparc64, but we can fake one + * up using profiling counter overflow interrupts and interrupt + * levels. + * + * The profile overflow interrupts at level 15, so we use + * level 14 as our IRQ off level. + */ + +static int panic_on_timeout; + +/* nmi_active: + * >0: the NMI watchdog is active, but can be disabled + * <0: the NMI watchdog has not been set up, and cannot be enabled + * 0: the NMI watchdog is disabled, but can be enabled + */ +atomic_t nmi_active = ATOMIC_INIT(0); /* oprofile uses this */ +EXPORT_SYMBOL(nmi_active); +static int nmi_init_done; +static unsigned int nmi_hz = HZ; +static DEFINE_PER_CPU(short, wd_enabled); +static int endflag __initdata; + +static DEFINE_PER_CPU(unsigned int, last_irq_sum); +static DEFINE_PER_CPU(long, alert_counter); +static DEFINE_PER_CPU(int, nmi_touch); + +void arch_touch_nmi_watchdog(void) +{ + if (atomic_read(&nmi_active)) { + int cpu; + + for_each_present_cpu(cpu) { + if (per_cpu(nmi_touch, cpu) != 1) + per_cpu(nmi_touch, cpu) = 1; + } + } +} +EXPORT_SYMBOL(arch_touch_nmi_watchdog); + +int __init watchdog_hardlockup_probe(void) +{ + return 0; +} + +static void die_nmi(const char *str, struct pt_regs *regs, int do_panic) +{ + int this_cpu = smp_processor_id(); + + if (notify_die(DIE_NMIWATCHDOG, str, regs, 0, + pt_regs_trap_type(regs), SIGINT) == NOTIFY_STOP) + return; + + if (do_panic || panic_on_oops) + panic("Watchdog detected hard LOCKUP on cpu %d", this_cpu); + else + WARN(1, "Watchdog detected hard LOCKUP on cpu %d", this_cpu); +} + +notrace __kprobes void perfctr_irq(int irq, struct pt_regs *regs) +{ + unsigned int sum, touched = 0; + void *orig_sp; + + clear_softint(1 << irq); + + local_cpu_data().__nmi_count++; + + nmi_enter(); + + orig_sp = set_hardirq_stack(); + + if (notify_die(DIE_NMI, "nmi", regs, 0, + pt_regs_trap_type(regs), SIGINT) == NOTIFY_STOP) + touched = 1; + else + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); + + sum = local_cpu_data().irq0_irqs; + if (__this_cpu_read(nmi_touch)) { + __this_cpu_write(nmi_touch, 0); + touched = 1; + } + if (!touched && __this_cpu_read(last_irq_sum) == sum) { + __this_cpu_inc(alert_counter); + if (__this_cpu_read(alert_counter) == 30 * nmi_hz) + die_nmi("BUG: NMI Watchdog detected LOCKUP", + regs, panic_on_timeout); + } else { + __this_cpu_write(last_irq_sum, sum); + __this_cpu_write(alert_counter, 0); + } + if (__this_cpu_read(wd_enabled)) { + pcr_ops->write_pic(0, pcr_ops->nmi_picl_value(nmi_hz)); + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_enable); + } + + restore_hardirq_stack(orig_sp); + + nmi_exit(); +} + +static inline unsigned int get_nmi_count(int cpu) +{ + return cpu_data(cpu).__nmi_count; +} + +static __init void nmi_cpu_busy(void *data) +{ + while (endflag == 0) + mb(); +} + +static void report_broken_nmi(int cpu, int *prev_nmi_count) +{ + printk(KERN_CONT "\n"); + + printk(KERN_WARNING + "WARNING: CPU#%d: NMI appears to be stuck (%d->%d)!\n", + cpu, prev_nmi_count[cpu], get_nmi_count(cpu)); + + printk(KERN_WARNING + "Please report this to bugzilla.kernel.org,\n"); + printk(KERN_WARNING + "and attach the output of the 'dmesg' command.\n"); + + per_cpu(wd_enabled, cpu) = 0; + atomic_dec(&nmi_active); +} + +void stop_nmi_watchdog(void *unused) +{ + if (!__this_cpu_read(wd_enabled)) + return; + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); + __this_cpu_write(wd_enabled, 0); + atomic_dec(&nmi_active); +} + +static int __init check_nmi_watchdog(void) +{ + unsigned int *prev_nmi_count; + int cpu, err; + + if (!atomic_read(&nmi_active)) + return 0; + + prev_nmi_count = kmalloc_array(nr_cpu_ids, sizeof(unsigned int), + GFP_KERNEL); + if (!prev_nmi_count) { + err = -ENOMEM; + goto error; + } + + printk(KERN_INFO "Testing NMI watchdog ... "); + + smp_call_function(nmi_cpu_busy, (void *)&endflag, 0); + + for_each_possible_cpu(cpu) + prev_nmi_count[cpu] = get_nmi_count(cpu); + local_irq_enable(); + mdelay((20 * 1000) / nmi_hz); /* wait 20 ticks */ + + for_each_online_cpu(cpu) { + if (!per_cpu(wd_enabled, cpu)) + continue; + if (get_nmi_count(cpu) - prev_nmi_count[cpu] <= 5) + report_broken_nmi(cpu, prev_nmi_count); + } + endflag = 1; + if (!atomic_read(&nmi_active)) { + kfree(prev_nmi_count); + atomic_set(&nmi_active, -1); + err = -ENODEV; + goto error; + } + printk("OK.\n"); + + nmi_hz = 1; + + kfree(prev_nmi_count); + return 0; +error: + on_each_cpu(stop_nmi_watchdog, NULL, 1); + return err; +} + +void start_nmi_watchdog(void *unused) +{ + if (__this_cpu_read(wd_enabled)) + return; + + __this_cpu_write(wd_enabled, 1); + atomic_inc(&nmi_active); + + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); + pcr_ops->write_pic(0, pcr_ops->nmi_picl_value(nmi_hz)); + + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_enable); +} + +static void nmi_adjust_hz_one(void *unused) +{ + if (!__this_cpu_read(wd_enabled)) + return; + + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_disable); + pcr_ops->write_pic(0, pcr_ops->nmi_picl_value(nmi_hz)); + + pcr_ops->write_pcr(0, pcr_ops->pcr_nmi_enable); +} + +void nmi_adjust_hz(unsigned int new_hz) +{ + nmi_hz = new_hz; + on_each_cpu(nmi_adjust_hz_one, NULL, 1); +} +EXPORT_SYMBOL_GPL(nmi_adjust_hz); + +static int nmi_shutdown(struct notifier_block *nb, unsigned long cmd, void *p) +{ + on_each_cpu(stop_nmi_watchdog, NULL, 1); + return 0; +} + +static struct notifier_block nmi_reboot_notifier = { + .notifier_call = nmi_shutdown, +}; + +int __init nmi_init(void) +{ + int err; + + on_each_cpu(start_nmi_watchdog, NULL, 1); + + err = check_nmi_watchdog(); + if (!err) { + err = register_reboot_notifier(&nmi_reboot_notifier); + if (err) { + on_each_cpu(stop_nmi_watchdog, NULL, 1); + atomic_set(&nmi_active, -1); + } + } + + nmi_init_done = 1; + + return err; +} + +static int __init setup_nmi_watchdog(char *str) +{ + if (!strncmp(str, "panic", 5)) + panic_on_timeout = 1; + + return 1; +} +__setup("nmi_watchdog=", setup_nmi_watchdog); + +/* + * sparc specific NMI watchdog enable function. + * Enables watchdog if it is not enabled already. + */ +void watchdog_hardlockup_enable(unsigned int cpu) +{ + if (atomic_read(&nmi_active) == -1) { + pr_warn("NMI watchdog cannot be enabled or disabled\n"); + return; + } + + /* + * watchdog thread could start even before nmi_init is called. + * Just Return in that case. Let nmi_init finish the init + * process first. + */ + if (!nmi_init_done) + return; + + smp_call_function_single(cpu, start_nmi_watchdog, NULL, 1); +} +/* + * sparc specific NMI watchdog disable function. + * Disables watchdog if it is not disabled already. + */ +void watchdog_hardlockup_disable(unsigned int cpu) +{ + if (atomic_read(&nmi_active) == -1) + pr_warn_once("NMI watchdog cannot be enabled or disabled\n"); + else + smp_call_function_single(cpu, stop_nmi_watchdog, NULL, 1); +} diff --git a/arch/sparc/kernel/of_device_32.c b/arch/sparc/kernel/of_device_32.c new file mode 100644 index 000000000..d18f548bb --- /dev/null +++ b/arch/sparc/kernel/of_device_32.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "of_device_common.h" +#include "irq.h" + +/* + * PCI bus specific translator + */ + +static int of_bus_pci_match(struct device_node *np) +{ + if (of_node_is_type(np, "pci") || of_node_is_type(np, "pciex")) { + /* Do not do PCI specific frobbing if the + * PCI bridge lacks a ranges property. We + * want to pass it through up to the next + * parent as-is, not with the PCI translate + * method which chops off the top address cell. + */ + if (!of_property_present(np, "ranges")) + return 0; + + return 1; + } + + return 0; +} + +static void of_bus_pci_count_cells(struct device_node *np, + int *addrc, int *sizec) +{ + if (addrc) + *addrc = 3; + if (sizec) + *sizec = 2; +} + +static int of_bus_pci_map(u32 *addr, const u32 *range, + int na, int ns, int pna) +{ + u32 result[OF_MAX_ADDR_CELLS]; + int i; + + /* Check address type match */ + if ((addr[0] ^ range[0]) & 0x03000000) + return -EINVAL; + + if (of_out_of_range(addr + 1, range + 1, range + na + pna, + na - 1, ns)) + return -EINVAL; + + /* Start with the parent range base. */ + memcpy(result, range + na, pna * 4); + + /* Add in the child address offset, skipping high cell. */ + for (i = 0; i < na - 1; i++) + result[pna - 1 - i] += + (addr[na - 1 - i] - + range[na - 1 - i]); + + memcpy(addr, result, pna * 4); + + return 0; +} + +static unsigned long of_bus_pci_get_flags(const u32 *addr, unsigned long flags) +{ + u32 w = addr[0]; + + /* For PCI, we override whatever child busses may have used. */ + flags = 0; + switch((w >> 24) & 0x03) { + case 0x01: + flags |= IORESOURCE_IO; + break; + + case 0x02: /* 32 bits */ + case 0x03: /* 64 bits */ + flags |= IORESOURCE_MEM; + break; + } + if (w & 0x40000000) + flags |= IORESOURCE_PREFETCH; + return flags; +} + +static unsigned long of_bus_sbus_get_flags(const u32 *addr, unsigned long flags) +{ + return IORESOURCE_MEM; +} + + /* + * AMBAPP bus specific translator + */ + +static int of_bus_ambapp_match(struct device_node *np) +{ + return of_node_is_type(np, "ambapp"); +} + +static void of_bus_ambapp_count_cells(struct device_node *child, + int *addrc, int *sizec) +{ + if (addrc) + *addrc = 1; + if (sizec) + *sizec = 1; +} + +static int of_bus_ambapp_map(u32 *addr, const u32 *range, + int na, int ns, int pna) +{ + return of_bus_default_map(addr, range, na, ns, pna); +} + +static unsigned long of_bus_ambapp_get_flags(const u32 *addr, + unsigned long flags) +{ + return IORESOURCE_MEM; +} + +/* + * Array of bus specific translators + */ + +static struct of_bus of_busses[] = { + /* PCI */ + { + .name = "pci", + .addr_prop_name = "assigned-addresses", + .match = of_bus_pci_match, + .count_cells = of_bus_pci_count_cells, + .map = of_bus_pci_map, + .get_flags = of_bus_pci_get_flags, + }, + /* SBUS */ + { + .name = "sbus", + .addr_prop_name = "reg", + .match = of_bus_sbus_match, + .count_cells = of_bus_sbus_count_cells, + .map = of_bus_default_map, + .get_flags = of_bus_sbus_get_flags, + }, + /* AMBA */ + { + .name = "ambapp", + .addr_prop_name = "reg", + .match = of_bus_ambapp_match, + .count_cells = of_bus_ambapp_count_cells, + .map = of_bus_ambapp_map, + .get_flags = of_bus_ambapp_get_flags, + }, + /* Default */ + { + .name = "default", + .addr_prop_name = "reg", + .match = NULL, + .count_cells = of_bus_default_count_cells, + .map = of_bus_default_map, + .get_flags = of_bus_default_get_flags, + }, +}; + +static struct of_bus *of_match_bus(struct device_node *np) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(of_busses); i ++) + if (!of_busses[i].match || of_busses[i].match(np)) + return &of_busses[i]; + BUG(); + return NULL; +} + +static int __init build_one_resource(struct device_node *parent, + struct of_bus *bus, + struct of_bus *pbus, + u32 *addr, + int na, int ns, int pna) +{ + const u32 *ranges; + unsigned int rlen; + int rone; + + ranges = of_get_property(parent, "ranges", &rlen); + if (ranges == NULL || rlen == 0) { + u32 result[OF_MAX_ADDR_CELLS]; + int i; + + memset(result, 0, pna * 4); + for (i = 0; i < na; i++) + result[pna - 1 - i] = + addr[na - 1 - i]; + + memcpy(addr, result, pna * 4); + return 0; + } + + /* Now walk through the ranges */ + rlen /= 4; + rone = na + pna + ns; + for (; rlen >= rone; rlen -= rone, ranges += rone) { + if (!bus->map(addr, ranges, na, ns, pna)) + return 0; + } + + return 1; +} + +static int __init use_1to1_mapping(struct device_node *pp) +{ + /* If we have a ranges property in the parent, use it. */ + if (of_property_present(pp, "ranges")) + return 0; + + /* Some SBUS devices use intermediate nodes to express + * hierarchy within the device itself. These aren't + * real bus nodes, and don't have a 'ranges' property. + * But, we should still pass the translation work up + * to the SBUS itself. + */ + if (of_node_name_eq(pp, "dma") || + of_node_name_eq(pp, "espdma") || + of_node_name_eq(pp, "ledma") || + of_node_name_eq(pp, "lebuffer")) + return 0; + + return 1; +} + +static int of_resource_verbose; + +static void __init build_device_resources(struct platform_device *op, + struct device *parent) +{ + struct platform_device *p_op; + struct of_bus *bus; + int na, ns; + int index, num_reg; + const void *preg; + + if (!parent) + return; + + p_op = to_platform_device(parent); + bus = of_match_bus(p_op->dev.of_node); + bus->count_cells(op->dev.of_node, &na, &ns); + + preg = of_get_property(op->dev.of_node, bus->addr_prop_name, &num_reg); + if (!preg || num_reg == 0) + return; + + /* Convert to num-cells. */ + num_reg /= 4; + + /* Conver to num-entries. */ + num_reg /= na + ns; + + op->resource = op->archdata.resource; + op->num_resources = num_reg; + for (index = 0; index < num_reg; index++) { + struct resource *r = &op->resource[index]; + u32 addr[OF_MAX_ADDR_CELLS]; + const u32 *reg = (preg + (index * ((na + ns) * 4))); + struct device_node *dp = op->dev.of_node; + struct device_node *pp = p_op->dev.of_node; + struct of_bus *pbus, *dbus; + u64 size, result = OF_BAD_ADDR; + unsigned long flags; + int dna, dns; + int pna, pns; + + size = of_read_addr(reg + na, ns); + + memcpy(addr, reg, na * 4); + + flags = bus->get_flags(reg, 0); + + if (use_1to1_mapping(pp)) { + result = of_read_addr(addr, na); + goto build_res; + } + + dna = na; + dns = ns; + dbus = bus; + + while (1) { + dp = pp; + pp = dp->parent; + if (!pp) { + result = of_read_addr(addr, dna); + break; + } + + pbus = of_match_bus(pp); + pbus->count_cells(dp, &pna, &pns); + + if (build_one_resource(dp, dbus, pbus, addr, + dna, dns, pna)) + break; + + flags = pbus->get_flags(addr, flags); + + dna = pna; + dns = pns; + dbus = pbus; + } + + build_res: + memset(r, 0, sizeof(*r)); + + if (of_resource_verbose) + printk("%pOF reg[%d] -> %llx\n", + op->dev.of_node, index, + result); + + if (result != OF_BAD_ADDR) { + r->start = result & 0xffffffff; + r->end = result + size - 1; + r->flags = flags | ((result >> 32ULL) & 0xffUL); + } + r->name = op->dev.of_node->full_name; + } +} + +static struct platform_device * __init scan_one_device(struct device_node *dp, + struct device *parent) +{ + struct platform_device *op = kzalloc_obj(*op); + const struct linux_prom_irqs *intr; + struct dev_archdata *sd; + int len, i; + + if (!op) + return NULL; + + sd = &op->dev.archdata; + sd->op = op; + + op->dev.of_node = dp; + + intr = of_get_property(dp, "intr", &len); + if (intr) { + op->archdata.num_irqs = len / sizeof(struct linux_prom_irqs); + for (i = 0; i < op->archdata.num_irqs; i++) + op->archdata.irqs[i] = + sparc_config.build_device_irq(op, intr[i].pri); + } else { + const unsigned int *irq = + of_get_property(dp, "interrupts", &len); + + if (irq) { + op->archdata.num_irqs = len / sizeof(unsigned int); + for (i = 0; i < op->archdata.num_irqs; i++) + op->archdata.irqs[i] = + sparc_config.build_device_irq(op, irq[i]); + } else { + op->archdata.num_irqs = 0; + } + } + + build_device_resources(op, parent); + + op->dev.parent = parent; + op->dev.bus = &platform_bus_type; + if (!parent) + dev_set_name(&op->dev, "root"); + else + dev_set_name(&op->dev, "%08x", dp->phandle); + + op->dev.coherent_dma_mask = DMA_BIT_MASK(32); + op->dev.dma_mask = &op->dev.coherent_dma_mask; + + if (of_device_register(op)) { + printk("%pOF: Could not register of device.\n", dp); + put_device(&op->dev); + kfree(op); + op = NULL; + } + + return op; +} + +static void __init scan_tree(struct device_node *dp, struct device *parent) +{ + while (dp) { + struct platform_device *op = scan_one_device(dp, parent); + + if (op) + scan_tree(dp->child, &op->dev); + + dp = dp->sibling; + } +} + +static int __init scan_of_devices(void) +{ + struct device_node *root = of_find_node_by_path("/"); + struct platform_device *parent; + + parent = scan_one_device(root, NULL); + if (!parent) + return 0; + + scan_tree(root->child, &parent->dev); + return 0; +} +postcore_initcall(scan_of_devices); + +static int __init of_debug(char *str) +{ + int val = 0; + + get_option(&str, &val); + if (val & 1) + of_resource_verbose = 1; + return 1; +} + +__setup("of_debug=", of_debug); diff --git a/arch/sparc/kernel/of_device_64.c b/arch/sparc/kernel/of_device_64.c new file mode 100644 index 000000000..8c4df3367 --- /dev/null +++ b/arch/sparc/kernel/of_device_64.c @@ -0,0 +1,725 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "of_device_common.h" + +void __iomem *of_ioremap(struct resource *res, unsigned long offset, unsigned long size, char *name) +{ + unsigned long ret = res->start + offset; + struct resource *r; + + if (res->flags & IORESOURCE_MEM) + r = request_mem_region(ret, size, name); + else + r = request_region(ret, size, name); + if (!r) + ret = 0; + + return (void __iomem *) ret; +} +EXPORT_SYMBOL(of_ioremap); + +void of_iounmap(struct resource *res, void __iomem *base, unsigned long size) +{ + if (res->flags & IORESOURCE_MEM) + release_mem_region((unsigned long) base, size); + else + release_region((unsigned long) base, size); +} +EXPORT_SYMBOL(of_iounmap); + +/* + * PCI bus specific translator + */ + +static int of_bus_pci_match(struct device_node *np) +{ + if (of_node_name_eq(np, "pci")) { + const char *model = of_get_property(np, "model", NULL); + + if (model && !strcmp(model, "SUNW,simba")) + return 0; + + /* Do not do PCI specific frobbing if the + * PCI bridge lacks a ranges property. We + * want to pass it through up to the next + * parent as-is, not with the PCI translate + * method which chops off the top address cell. + */ + if (!of_property_present(np, "ranges")) + return 0; + + return 1; + } + + return 0; +} + +static int of_bus_simba_match(struct device_node *np) +{ + const char *model = of_get_property(np, "model", NULL); + + if (model && !strcmp(model, "SUNW,simba")) + return 1; + + /* Treat PCI busses lacking ranges property just like + * simba. + */ + if (of_node_name_eq(np, "pci")) { + if (!of_property_present(np, "ranges")) + return 1; + } + + return 0; +} + +static int of_bus_simba_map(u32 *addr, const u32 *range, + int na, int ns, int pna) +{ + return 0; +} + +static void of_bus_pci_count_cells(struct device_node *np, + int *addrc, int *sizec) +{ + if (addrc) + *addrc = 3; + if (sizec) + *sizec = 2; +} + +static int of_bus_pci_map(u32 *addr, const u32 *range, + int na, int ns, int pna) +{ + u32 result[OF_MAX_ADDR_CELLS]; + int i; + + /* Check address type match */ + if (!((addr[0] ^ range[0]) & 0x03000000)) + goto type_match; + + /* Special exception, we can map a 64-bit address into + * a 32-bit range. + */ + if ((addr[0] & 0x03000000) == 0x03000000 && + (range[0] & 0x03000000) == 0x02000000) + goto type_match; + + return -EINVAL; + +type_match: + if (of_out_of_range(addr + 1, range + 1, range + na + pna, + na - 1, ns)) + return -EINVAL; + + /* Start with the parent range base. */ + memcpy(result, range + na, pna * 4); + + /* Add in the child address offset, skipping high cell. */ + for (i = 0; i < na - 1; i++) + result[pna - 1 - i] += + (addr[na - 1 - i] - + range[na - 1 - i]); + + memcpy(addr, result, pna * 4); + + return 0; +} + +static unsigned long of_bus_pci_get_flags(const u32 *addr, unsigned long flags) +{ + u32 w = addr[0]; + + /* For PCI, we override whatever child busses may have used. */ + flags = 0; + switch((w >> 24) & 0x03) { + case 0x01: + flags |= IORESOURCE_IO; + break; + + case 0x02: /* 32 bits */ + case 0x03: /* 64 bits */ + flags |= IORESOURCE_MEM; + break; + } + if (w & 0x40000000) + flags |= IORESOURCE_PREFETCH; + return flags; +} + +/* + * FHC/Central bus specific translator. + * + * This is just needed to hard-code the address and size cell + * counts. 'fhc' and 'central' nodes lack the #address-cells and + * #size-cells properties, and if you walk to the root on such + * Enterprise boxes all you'll get is a #size-cells of 2 which is + * not what we want to use. + */ +static int of_bus_fhc_match(struct device_node *np) +{ + return of_node_name_eq(np, "fhc") || + of_node_name_eq(np, "central"); +} + +#define of_bus_fhc_count_cells of_bus_sbus_count_cells + +/* + * Array of bus specific translators + */ + +static struct of_bus of_busses[] = { + /* PCI */ + { + .name = "pci", + .addr_prop_name = "assigned-addresses", + .match = of_bus_pci_match, + .count_cells = of_bus_pci_count_cells, + .map = of_bus_pci_map, + .get_flags = of_bus_pci_get_flags, + }, + /* SIMBA */ + { + .name = "simba", + .addr_prop_name = "assigned-addresses", + .match = of_bus_simba_match, + .count_cells = of_bus_pci_count_cells, + .map = of_bus_simba_map, + .get_flags = of_bus_pci_get_flags, + }, + /* SBUS */ + { + .name = "sbus", + .addr_prop_name = "reg", + .match = of_bus_sbus_match, + .count_cells = of_bus_sbus_count_cells, + .map = of_bus_default_map, + .get_flags = of_bus_default_get_flags, + }, + /* FHC */ + { + .name = "fhc", + .addr_prop_name = "reg", + .match = of_bus_fhc_match, + .count_cells = of_bus_fhc_count_cells, + .map = of_bus_default_map, + .get_flags = of_bus_default_get_flags, + }, + /* Default */ + { + .name = "default", + .addr_prop_name = "reg", + .match = NULL, + .count_cells = of_bus_default_count_cells, + .map = of_bus_default_map, + .get_flags = of_bus_default_get_flags, + }, +}; + +static struct of_bus *of_match_bus(struct device_node *np) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(of_busses); i ++) + if (!of_busses[i].match || of_busses[i].match(np)) + return &of_busses[i]; + BUG(); + return NULL; +} + +static int __init build_one_resource(struct device_node *parent, + struct of_bus *bus, + struct of_bus *pbus, + u32 *addr, + int na, int ns, int pna) +{ + const u32 *ranges; + int rone, rlen; + + ranges = of_get_property(parent, "ranges", &rlen); + if (ranges == NULL || rlen == 0) { + u32 result[OF_MAX_ADDR_CELLS]; + int i; + + memset(result, 0, pna * 4); + for (i = 0; i < na; i++) + result[pna - 1 - i] = + addr[na - 1 - i]; + + memcpy(addr, result, pna * 4); + return 0; + } + + /* Now walk through the ranges */ + rlen /= 4; + rone = na + pna + ns; + for (; rlen >= rone; rlen -= rone, ranges += rone) { + if (!bus->map(addr, ranges, na, ns, pna)) + return 0; + } + + /* When we miss an I/O space match on PCI, just pass it up + * to the next PCI bridge and/or controller. + */ + if (!strcmp(bus->name, "pci") && + (addr[0] & 0x03000000) == 0x01000000) + return 0; + + return 1; +} + +static int __init use_1to1_mapping(struct device_node *pp) +{ + /* If we have a ranges property in the parent, use it. */ + if (of_property_present(pp, "ranges")) + return 0; + + /* If the parent is the dma node of an ISA bus, pass + * the translation up to the root. + * + * Some SBUS devices use intermediate nodes to express + * hierarchy within the device itself. These aren't + * real bus nodes, and don't have a 'ranges' property. + * But, we should still pass the translation work up + * to the SBUS itself. + */ + if (of_node_name_eq(pp, "dma") || + of_node_name_eq(pp, "espdma") || + of_node_name_eq(pp, "ledma") || + of_node_name_eq(pp, "lebuffer")) + return 0; + + /* Similarly for all PCI bridges, if we get this far + * it lacks a ranges property, and this will include + * cases like Simba. + */ + if (of_node_name_eq(pp, "pci")) + return 0; + + return 1; +} + +static int of_resource_verbose; + +static void __init build_device_resources(struct platform_device *op, + struct device *parent) +{ + struct platform_device *p_op; + struct of_bus *bus; + int na, ns; + int index, num_reg; + const void *preg; + + if (!parent) + return; + + p_op = to_platform_device(parent); + bus = of_match_bus(p_op->dev.of_node); + bus->count_cells(op->dev.of_node, &na, &ns); + + preg = of_get_property(op->dev.of_node, bus->addr_prop_name, &num_reg); + if (!preg || num_reg == 0) + return; + + /* Convert to num-cells. */ + num_reg /= 4; + + /* Convert to num-entries. */ + num_reg /= na + ns; + + /* Prevent overrunning the op->resources[] array. */ + if (num_reg > PROMREG_MAX) { + printk(KERN_WARNING "%pOF: Too many regs (%d), " + "limiting to %d.\n", + op->dev.of_node, num_reg, PROMREG_MAX); + num_reg = PROMREG_MAX; + } + + op->resource = op->archdata.resource; + op->num_resources = num_reg; + for (index = 0; index < num_reg; index++) { + struct resource *r = &op->resource[index]; + u32 addr[OF_MAX_ADDR_CELLS]; + const u32 *reg = (preg + (index * ((na + ns) * 4))); + struct device_node *dp = op->dev.of_node; + struct device_node *pp = p_op->dev.of_node; + struct of_bus *pbus, *dbus; + u64 size, result = OF_BAD_ADDR; + unsigned long flags; + int dna, dns; + int pna, pns; + + size = of_read_addr(reg + na, ns); + memcpy(addr, reg, na * 4); + + flags = bus->get_flags(addr, 0); + + if (use_1to1_mapping(pp)) { + result = of_read_addr(addr, na); + goto build_res; + } + + dna = na; + dns = ns; + dbus = bus; + + while (1) { + dp = pp; + pp = dp->parent; + if (!pp) { + result = of_read_addr(addr, dna); + break; + } + + pbus = of_match_bus(pp); + pbus->count_cells(dp, &pna, &pns); + + if (build_one_resource(dp, dbus, pbus, addr, + dna, dns, pna)) + break; + + flags = pbus->get_flags(addr, flags); + + dna = pna; + dns = pns; + dbus = pbus; + } + + build_res: + memset(r, 0, sizeof(*r)); + + if (of_resource_verbose) + printk("%pOF reg[%d] -> %llx\n", + op->dev.of_node, index, + result); + + if (result != OF_BAD_ADDR) { + if (tlb_type == hypervisor) + result &= 0x0fffffffffffffffUL; + + r->start = result; + r->end = result + size - 1; + r->flags = flags; + } + r->name = op->dev.of_node->full_name; + } +} + +static struct device_node * __init +apply_interrupt_map(struct device_node *dp, struct device_node *pp, + const u32 *imap, int imlen, const u32 *imask, + unsigned int *irq_p) +{ + struct device_node *cp; + unsigned int irq = *irq_p; + struct of_bus *bus; + phandle handle; + const u32 *reg; + int na, num_reg, i; + + bus = of_match_bus(pp); + bus->count_cells(dp, &na, NULL); + + reg = of_get_property(dp, "reg", &num_reg); + if (!reg || !num_reg) + return NULL; + + imlen /= ((na + 3) * 4); + handle = 0; + for (i = 0; i < imlen; i++) { + int j; + + for (j = 0; j < na; j++) { + if ((reg[j] & imask[j]) != imap[j]) + goto next; + } + if (imap[na] == irq) { + handle = imap[na + 1]; + irq = imap[na + 2]; + break; + } + + next: + imap += (na + 3); + } + if (i == imlen) { + /* Psycho and Sabre PCI controllers can have 'interrupt-map' + * properties that do not include the on-board device + * interrupts. Instead, the device's 'interrupts' property + * is already a fully specified INO value. + * + * Handle this by deciding that, if we didn't get a + * match in the parent's 'interrupt-map', and the + * parent is an IRQ translator, then use the parent as + * our IRQ controller. + */ + if (pp->irq_trans) + return pp; + + return NULL; + } + + *irq_p = irq; + cp = of_find_node_by_phandle(handle); + + return cp; +} + +static unsigned int __init pci_irq_swizzle(struct device_node *dp, + struct device_node *pp, + unsigned int irq) +{ + const struct linux_prom_pci_registers *regs; + unsigned int bus, devfn, slot, ret; + + if (irq < 1 || irq > 4) + return irq; + + regs = of_get_property(dp, "reg", NULL); + if (!regs) + return irq; + + bus = (regs->phys_hi >> 16) & 0xff; + devfn = (regs->phys_hi >> 8) & 0xff; + slot = (devfn >> 3) & 0x1f; + + if (pp->irq_trans) { + /* Derived from Table 8-3, U2P User's Manual. This branch + * is handling a PCI controller that lacks a proper set of + * interrupt-map and interrupt-map-mask properties. The + * Ultra-E450 is one example. + * + * The bit layout is BSSLL, where: + * B: 0 on bus A, 1 on bus B + * D: 2-bit slot number, derived from PCI device number as + * (dev - 1) for bus A, or (dev - 2) for bus B + * L: 2-bit line number + */ + if (bus & 0x80) { + /* PBM-A */ + bus = 0x00; + slot = (slot - 1) << 2; + } else { + /* PBM-B */ + bus = 0x10; + slot = (slot - 2) << 2; + } + irq -= 1; + + ret = (bus | slot | irq); + } else { + /* Going through a PCI-PCI bridge that lacks a set of + * interrupt-map and interrupt-map-mask properties. + */ + ret = ((irq - 1 + (slot & 3)) & 3) + 1; + } + + return ret; +} + +static int of_irq_verbose; + +static unsigned int __init build_one_device_irq(struct platform_device *op, + struct device *parent, + unsigned int irq) +{ + struct device_node *dp = op->dev.of_node; + struct device_node *pp, *ip; + unsigned int orig_irq = irq; + int nid; + + if (irq == 0xffffffff) + return irq; + + if (dp->irq_trans) { + irq = dp->irq_trans->irq_build(dp, irq, + dp->irq_trans->data); + + if (of_irq_verbose) + printk("%pOF: direct translate %x --> %x\n", + dp, orig_irq, irq); + + goto out; + } + + /* Something more complicated. Walk up to the root, applying + * interrupt-map or bus specific translations, until we hit + * an IRQ translator. + * + * If we hit a bus type or situation we cannot handle, we + * stop and assume that the original IRQ number was in a + * format which has special meaning to its immediate parent. + */ + pp = dp->parent; + ip = NULL; + while (pp) { + const void *imap, *imsk; + int imlen; + + imap = of_get_property(pp, "interrupt-map", &imlen); + imsk = of_get_property(pp, "interrupt-map-mask", NULL); + if (imap && imsk) { + struct device_node *iret; + int this_orig_irq = irq; + + iret = apply_interrupt_map(dp, pp, + imap, imlen, imsk, + &irq); + + if (of_irq_verbose) + printk("%pOF: Apply [%pOF:%x] imap --> [%pOF:%x]\n", + op->dev.of_node, + pp, this_orig_irq, iret, irq); + + if (!iret) + break; + + if (iret->irq_trans) { + ip = iret; + break; + } + } else { + if (of_node_name_eq(pp, "pci")) { + unsigned int this_orig_irq = irq; + + irq = pci_irq_swizzle(dp, pp, irq); + if (of_irq_verbose) + printk("%pOF: PCI swizzle [%pOF] " + "%x --> %x\n", + op->dev.of_node, + pp, this_orig_irq, + irq); + + } + + if (pp->irq_trans) { + ip = pp; + break; + } + } + dp = pp; + pp = pp->parent; + } + if (!ip) + return orig_irq; + + irq = ip->irq_trans->irq_build(op->dev.of_node, irq, + ip->irq_trans->data); + if (of_irq_verbose) + printk("%pOF: Apply IRQ trans [%pOF] %x --> %x\n", + op->dev.of_node, ip, orig_irq, irq); + +out: + nid = of_node_to_nid(dp); + if (nid != -1) { + irq_set_affinity(irq, cpumask_of_node(nid)); + } + + return irq; +} + +static struct platform_device * __init scan_one_device(struct device_node *dp, + struct device *parent) +{ + struct platform_device *op = kzalloc_obj(*op); + const unsigned int *irq; + struct dev_archdata *sd; + int len, i; + + if (!op) + return NULL; + + sd = &op->dev.archdata; + sd->op = op; + + op->dev.of_node = dp; + + irq = of_get_property(dp, "interrupts", &len); + if (irq) { + op->archdata.num_irqs = len / 4; + + /* Prevent overrunning the op->irqs[] array. */ + if (op->archdata.num_irqs > PROMINTR_MAX) { + printk(KERN_WARNING "%pOF: Too many irqs (%d), " + "limiting to %d.\n", + dp, op->archdata.num_irqs, PROMINTR_MAX); + op->archdata.num_irqs = PROMINTR_MAX; + } + memcpy(op->archdata.irqs, irq, op->archdata.num_irqs * 4); + } else { + op->archdata.num_irqs = 0; + } + + build_device_resources(op, parent); + for (i = 0; i < op->archdata.num_irqs; i++) + op->archdata.irqs[i] = build_one_device_irq(op, parent, op->archdata.irqs[i]); + + op->dev.parent = parent; + op->dev.bus = &platform_bus_type; + if (!parent) + dev_set_name(&op->dev, "root"); + else + dev_set_name(&op->dev, "%08x", dp->phandle); + op->dev.coherent_dma_mask = DMA_BIT_MASK(32); + op->dev.dma_mask = &op->dev.coherent_dma_mask; + + if (of_device_register(op)) { + printk("%pOF: Could not register of device.\n", dp); + put_device(&op->dev); + kfree(op); + op = NULL; + } + + return op; +} + +static void __init scan_tree(struct device_node *dp, struct device *parent) +{ + while (dp) { + struct platform_device *op = scan_one_device(dp, parent); + + if (op) + scan_tree(dp->child, &op->dev); + + dp = dp->sibling; + } +} + +static int __init scan_of_devices(void) +{ + struct device_node *root = of_find_node_by_path("/"); + struct platform_device *parent; + + parent = scan_one_device(root, NULL); + if (!parent) + return 0; + + scan_tree(root->child, &parent->dev); + return 0; +} +postcore_initcall(scan_of_devices); + +static int __init of_debug(char *str) +{ + int val = 0; + + get_option(&str, &val); + if (val & 1) + of_resource_verbose = 1; + if (val & 2) + of_irq_verbose = 1; + return 1; +} + +__setup("of_debug=", of_debug); diff --git a/arch/sparc/kernel/of_device_common.c b/arch/sparc/kernel/of_device_common.c new file mode 100644 index 000000000..7f2f9972b --- /dev/null +++ b/arch/sparc/kernel/of_device_common.c @@ -0,0 +1,179 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "of_device_common.h" + +unsigned int irq_of_parse_and_map(struct device_node *node, int index) +{ + struct platform_device *op = of_find_device_by_node(node); + + if (!op || index >= op->archdata.num_irqs) + return 0; + + return op->archdata.irqs[index]; +} +EXPORT_SYMBOL(irq_of_parse_and_map); + +int of_address_to_resource(struct device_node *node, int index, + struct resource *r) +{ + struct platform_device *op = of_find_device_by_node(node); + + if (!op || index >= op->num_resources) + return -EINVAL; + + memcpy(r, &op->archdata.resource[index], sizeof(*r)); + return 0; +} +EXPORT_SYMBOL_GPL(of_address_to_resource); + +void __iomem *of_iomap(struct device_node *node, int index) +{ + struct platform_device *op = of_find_device_by_node(node); + struct resource *r; + + if (!op || index >= op->num_resources) + return NULL; + + r = &op->archdata.resource[index]; + + return of_ioremap(r, 0, resource_size(r), (char *) r->name); +} +EXPORT_SYMBOL(of_iomap); + +/* Take the archdata values for IOMMU, STC, and HOSTDATA found in + * BUS and propagate to all child platform_device objects. + */ +void of_propagate_archdata(struct platform_device *bus) +{ + struct dev_archdata *bus_sd = &bus->dev.archdata; + struct device_node *bus_dp = bus->dev.of_node; + struct device_node *dp; + + for (dp = bus_dp->child; dp; dp = dp->sibling) { + struct platform_device *op = of_find_device_by_node(dp); + + op->dev.archdata.iommu = bus_sd->iommu; + op->dev.archdata.stc = bus_sd->stc; + op->dev.archdata.host_controller = bus_sd->host_controller; + op->dev.archdata.numa_node = bus_sd->numa_node; + op->dev.dma_ops = bus->dev.dma_ops; + + if (dp->child) + of_propagate_archdata(op); + } +} + +static void get_cells(struct device_node *dp, int *addrc, int *sizec) +{ + if (addrc) + *addrc = of_n_addr_cells(dp); + if (sizec) + *sizec = of_n_size_cells(dp); +} + +/* + * Default translator (generic bus) + */ + +void of_bus_default_count_cells(struct device_node *dev, int *addrc, int *sizec) +{ + get_cells(dev, addrc, sizec); +} + +/* Make sure the least significant 64-bits are in-range. Even + * for 3 or 4 cell values it is a good enough approximation. + */ +int of_out_of_range(const u32 *addr, const u32 *base, + const u32 *size, int na, int ns) +{ + u64 a = of_read_addr(addr, na); + u64 b = of_read_addr(base, na); + + if (a < b) + return 1; + + b += of_read_addr(size, ns); + if (a >= b) + return 1; + + return 0; +} + +int of_bus_default_map(u32 *addr, const u32 *range, int na, int ns, int pna) +{ + u32 result[OF_MAX_ADDR_CELLS]; + int i; + + if (ns > 2) { + printk("of_device: Cannot handle size cells (%d) > 2.", ns); + return -EINVAL; + } + + if (of_out_of_range(addr, range, range + na + pna, na, ns)) + return -EINVAL; + + /* Start with the parent range base. */ + memcpy(result, range + na, pna * 4); + + /* Add in the child address offset. */ + for (i = 0; i < na; i++) + result[pna - 1 - i] += + (addr[na - 1 - i] - + range[na - 1 - i]); + + memcpy(addr, result, pna * 4); + + return 0; +} + +unsigned long of_bus_default_get_flags(const u32 *addr, unsigned long flags) +{ + if (flags) + return flags; + return IORESOURCE_MEM; +} + +/* + * SBUS bus specific translator + */ + +int of_bus_sbus_match(struct device_node *np) +{ + struct device_node *dp = np; + + while (dp) { + if (of_node_name_eq(dp, "sbus") || + of_node_name_eq(dp, "sbi")) + return 1; + + /* Have a look at use_1to1_mapping(). We're trying + * to match SBUS if that's the top-level bus and we + * don't have some intervening real bus that provides + * ranges based translations. + */ + if (of_property_present(dp, "ranges")) + break; + + dp = dp->parent; + } + + return 0; +} + +void of_bus_sbus_count_cells(struct device_node *child, int *addrc, int *sizec) +{ + if (addrc) + *addrc = 2; + if (sizec) + *sizec = 1; +} diff --git a/arch/sparc/kernel/of_device_common.h b/arch/sparc/kernel/of_device_common.h new file mode 100644 index 000000000..3d66230c6 --- /dev/null +++ b/arch/sparc/kernel/of_device_common.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _OF_DEVICE_COMMON_H +#define _OF_DEVICE_COMMON_H + +static inline u64 of_read_addr(const u32 *cell, int size) +{ + u64 r = 0; + while (size--) + r = (r << 32) | *(cell++); + return r; +} + +void of_bus_default_count_cells(struct device_node *dev, int *addrc, + int *sizec); +int of_out_of_range(const u32 *addr, const u32 *base, + const u32 *size, int na, int ns); +int of_bus_default_map(u32 *addr, const u32 *range, int na, int ns, int pna); +unsigned long of_bus_default_get_flags(const u32 *addr, unsigned long flags); + +int of_bus_sbus_match(struct device_node *np); +void of_bus_sbus_count_cells(struct device_node *child, int *addrc, int *sizec); + +/* Max address size we deal with */ +#define OF_MAX_ADDR_CELLS 4 + +struct of_bus { + const char *name; + const char *addr_prop_name; + int (*match)(struct device_node *parent); + void (*count_cells)(struct device_node *child, + int *addrc, int *sizec); + int (*map)(u32 *addr, const u32 *range, + int na, int ns, int pna); + unsigned long (*get_flags)(const u32 *addr, unsigned long); +}; + +#endif /* _OF_DEVICE_COMMON_H */ diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c new file mode 100644 index 000000000..1603d50fd --- /dev/null +++ b/arch/sparc/kernel/pci.c @@ -0,0 +1,1013 @@ +// SPDX-License-Identifier: GPL-2.0 +/* pci.c: UltraSparc PCI controller support. + * + * Copyright (C) 1997, 1998, 1999 David S. Miller (davem@redhat.com) + * Copyright (C) 1998, 1999 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz) + * + * OF tree based PCI bus probing taken from the PowerPC port + * with minor modifications, see there for credits. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "pci_impl.h" +#include "kernel.h" + +/* List of all PCI controllers found in the system. */ +struct pci_pbm_info *pci_pbm_root = NULL; + +/* Each PBM found gets a unique index. */ +int pci_num_pbms = 0; + +volatile int pci_poke_in_progress; +volatile int pci_poke_cpu = -1; +volatile int pci_poke_faulted; + +static DEFINE_SPINLOCK(pci_poke_lock); + +void pci_config_read8(u8 *addr, u8 *ret) +{ + unsigned long flags; + u8 byte; + + spin_lock_irqsave(&pci_poke_lock, flags); + pci_poke_cpu = smp_processor_id(); + pci_poke_in_progress = 1; + pci_poke_faulted = 0; + __asm__ __volatile__("membar #Sync\n\t" + "lduba [%1] %2, %0\n\t" + "membar #Sync" + : "=r" (byte) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + pci_poke_in_progress = 0; + pci_poke_cpu = -1; + if (!pci_poke_faulted) + *ret = byte; + spin_unlock_irqrestore(&pci_poke_lock, flags); +} + +void pci_config_read16(u16 *addr, u16 *ret) +{ + unsigned long flags; + u16 word; + + spin_lock_irqsave(&pci_poke_lock, flags); + pci_poke_cpu = smp_processor_id(); + pci_poke_in_progress = 1; + pci_poke_faulted = 0; + __asm__ __volatile__("membar #Sync\n\t" + "lduha [%1] %2, %0\n\t" + "membar #Sync" + : "=r" (word) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + pci_poke_in_progress = 0; + pci_poke_cpu = -1; + if (!pci_poke_faulted) + *ret = word; + spin_unlock_irqrestore(&pci_poke_lock, flags); +} + +void pci_config_read32(u32 *addr, u32 *ret) +{ + unsigned long flags; + u32 dword; + + spin_lock_irqsave(&pci_poke_lock, flags); + pci_poke_cpu = smp_processor_id(); + pci_poke_in_progress = 1; + pci_poke_faulted = 0; + __asm__ __volatile__("membar #Sync\n\t" + "lduwa [%1] %2, %0\n\t" + "membar #Sync" + : "=r" (dword) + : "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + pci_poke_in_progress = 0; + pci_poke_cpu = -1; + if (!pci_poke_faulted) + *ret = dword; + spin_unlock_irqrestore(&pci_poke_lock, flags); +} + +void pci_config_write8(u8 *addr, u8 val) +{ + unsigned long flags; + + spin_lock_irqsave(&pci_poke_lock, flags); + pci_poke_cpu = smp_processor_id(); + pci_poke_in_progress = 1; + pci_poke_faulted = 0; + __asm__ __volatile__("membar #Sync\n\t" + "stba %0, [%1] %2\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (val), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + pci_poke_in_progress = 0; + pci_poke_cpu = -1; + spin_unlock_irqrestore(&pci_poke_lock, flags); +} + +void pci_config_write16(u16 *addr, u16 val) +{ + unsigned long flags; + + spin_lock_irqsave(&pci_poke_lock, flags); + pci_poke_cpu = smp_processor_id(); + pci_poke_in_progress = 1; + pci_poke_faulted = 0; + __asm__ __volatile__("membar #Sync\n\t" + "stha %0, [%1] %2\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (val), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + pci_poke_in_progress = 0; + pci_poke_cpu = -1; + spin_unlock_irqrestore(&pci_poke_lock, flags); +} + +void pci_config_write32(u32 *addr, u32 val) +{ + unsigned long flags; + + spin_lock_irqsave(&pci_poke_lock, flags); + pci_poke_cpu = smp_processor_id(); + pci_poke_in_progress = 1; + pci_poke_faulted = 0; + __asm__ __volatile__("membar #Sync\n\t" + "stwa %0, [%1] %2\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (val), "r" (addr), "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + pci_poke_in_progress = 0; + pci_poke_cpu = -1; + spin_unlock_irqrestore(&pci_poke_lock, flags); +} + +static int ofpci_verbose; + +static int __init ofpci_debug(char *str) +{ + int val = 0; + + get_option(&str, &val); + if (val) + ofpci_verbose = 1; + return 1; +} + +__setup("ofpci_debug=", ofpci_debug); + +static void of_fixup_pci_pref(struct pci_dev *dev, int index, + struct resource *res) +{ + struct pci_bus_region region; + + if (!(res->flags & IORESOURCE_MEM_64)) + return; + + if (!resource_size(res)) + return; + + pcibios_resource_to_bus(dev->bus, ®ion, res); + if (region.end <= ~((u32)0)) + return; + + if (!(res->flags & IORESOURCE_PREFETCH)) { + res->flags |= IORESOURCE_PREFETCH; + pci_info(dev, "reg 0x%x: fixup: pref added to 64-bit resource\n", + index); + } +} + +static unsigned long pci_parse_of_flags(u32 addr0) +{ + unsigned long flags = 0; + + if (addr0 & 0x02000000) { + flags = IORESOURCE_MEM | PCI_BASE_ADDRESS_SPACE_MEMORY; + flags |= (addr0 >> 28) & PCI_BASE_ADDRESS_MEM_TYPE_1M; + if (addr0 & 0x01000000) + flags |= IORESOURCE_MEM_64 + | PCI_BASE_ADDRESS_MEM_TYPE_64; + if (addr0 & 0x40000000) + flags |= IORESOURCE_PREFETCH + | PCI_BASE_ADDRESS_MEM_PREFETCH; + } else if (addr0 & 0x01000000) + flags = IORESOURCE_IO | PCI_BASE_ADDRESS_SPACE_IO; + return flags; +} + +/* The of_device layer has translated all of the assigned-address properties + * into physical address resources, we only have to figure out the register + * mapping. + */ +static void pci_parse_of_addrs(struct platform_device *op, + struct device_node *node, + struct pci_dev *dev) +{ + struct resource *op_res; + const u32 *addrs; + int proplen; + + addrs = of_get_property(node, "assigned-addresses", &proplen); + if (!addrs) + return; + if (ofpci_verbose) + pci_info(dev, " parse addresses (%d bytes) @ %p\n", + proplen, addrs); + op_res = &op->resource[0]; + for (; proplen >= 20; proplen -= 20, addrs += 5, op_res++) { + struct resource *res; + unsigned long flags; + int i; + + flags = pci_parse_of_flags(addrs[0]); + if (!flags) + continue; + i = addrs[0] & 0xff; + if (ofpci_verbose) + pci_info(dev, " start: %llx, end: %llx, i: %x\n", + op_res->start, op_res->end, i); + + if (PCI_BASE_ADDRESS_0 <= i && i <= PCI_BASE_ADDRESS_5) { + res = &dev->resource[(i - PCI_BASE_ADDRESS_0) >> 2]; + } else if (i == dev->rom_base_reg) { + res = &dev->resource[PCI_ROM_RESOURCE]; + flags |= IORESOURCE_READONLY | IORESOURCE_SIZEALIGN; + } else { + pci_err(dev, "bad cfg reg num 0x%x\n", i); + continue; + } + res->start = op_res->start; + res->end = op_res->end; + res->flags = flags; + res->name = pci_name(dev); + of_fixup_pci_pref(dev, i, res); + + pci_info(dev, "reg 0x%x: %pR\n", i, res); + } +} + +static void pci_init_dev_archdata(struct dev_archdata *sd, void *iommu, + void *stc, void *host_controller, + struct platform_device *op, + int numa_node) +{ + sd->iommu = iommu; + sd->stc = stc; + sd->host_controller = host_controller; + sd->op = op; + sd->numa_node = numa_node; +} + +static struct pci_dev *of_create_pci_dev(struct pci_pbm_info *pbm, + struct device_node *node, + struct pci_bus *bus, int devfn) +{ + struct dev_archdata *sd; + struct platform_device *op; + struct pci_dev *dev; + u32 class; + + dev = pci_alloc_dev(bus); + if (!dev) + return NULL; + + op = of_find_device_by_node(node); + sd = &dev->dev.archdata; + pci_init_dev_archdata(sd, pbm->iommu, &pbm->stc, pbm, op, + pbm->numa_node); + sd = &op->dev.archdata; + sd->iommu = pbm->iommu; + sd->stc = &pbm->stc; + sd->numa_node = pbm->numa_node; + + if (of_node_name_eq(node, "ebus")) + of_propagate_archdata(op); + + if (ofpci_verbose) + pci_info(bus," create device, devfn: %x, type: %s\n", + devfn, of_node_get_device_type(node)); + + dev->sysdata = node; + dev->dev.parent = bus->bridge; + dev->dev.bus = &pci_bus_type; + dev->dev.of_node = of_node_get(node); + dev->devfn = devfn; + dev->multifunction = 0; /* maybe a lie? */ + set_pcie_port_type(dev); + + pci_dev_assign_slot(dev); + dev->vendor = of_getintprop_default(node, "vendor-id", 0xffff); + dev->device = of_getintprop_default(node, "device-id", 0xffff); + dev->subsystem_vendor = + of_getintprop_default(node, "subsystem-vendor-id", 0); + dev->subsystem_device = + of_getintprop_default(node, "subsystem-id", 0); + + dev->cfg_size = pci_cfg_space_size(dev); + + /* We can't actually use the firmware value, we have + * to read what is in the register right now. One + * reason is that in the case of IDE interfaces the + * firmware can sample the value before the IDE + * interface is programmed into native mode. + */ + pci_read_config_dword(dev, PCI_CLASS_REVISION, &class); + dev->class = class >> 8; + dev->revision = class & 0xff; + + dev_set_name(&dev->dev, "%04x:%02x:%02x.%d", pci_domain_nr(bus), + dev->bus->number, PCI_SLOT(devfn), PCI_FUNC(devfn)); + + /* I have seen IDE devices which will not respond to + * the bmdma simplex check reads if bus mastering is + * disabled. + */ + if ((dev->class >> 8) == PCI_CLASS_STORAGE_IDE) + pci_set_master(dev); + + dev->current_state = PCI_UNKNOWN; /* unknown power state */ + dev->error_state = pci_channel_io_normal; + dev->dma_mask = 0xffffffff; + + /* + * Assume 64-bit addresses for MSI initially. Will be changed to 32-bit + * if MSI (rather than MSI-X) capability does not have + * PCI_MSI_FLAGS_64BIT. Can also be overridden by driver. + */ + dev->msi_addr_mask = DMA_BIT_MASK(64); + + if (of_node_name_eq(node, "pci")) { + /* a PCI-PCI bridge */ + dev->hdr_type = PCI_HEADER_TYPE_BRIDGE; + dev->rom_base_reg = PCI_ROM_ADDRESS1; + } else if (of_node_is_type(node, "cardbus")) { + dev->hdr_type = PCI_HEADER_TYPE_CARDBUS; + } else { + dev->hdr_type = PCI_HEADER_TYPE_NORMAL; + dev->rom_base_reg = PCI_ROM_ADDRESS; + + dev->irq = sd->op->archdata.irqs[0]; + if (dev->irq == 0xffffffff) + dev->irq = PCI_IRQ_NONE; + } + + pci_info(dev, "[%04x:%04x] type %02x class %#08x\n", + dev->vendor, dev->device, dev->hdr_type, dev->class); + + pci_parse_of_addrs(sd->op, node, dev); + + if (ofpci_verbose) + pci_info(dev, " adding to system ...\n"); + + pci_device_add(dev, bus); + + return dev; +} + +static void apb_calc_first_last(u8 map, u32 *first_p, u32 *last_p) +{ + u32 idx, first, last; + + first = 8; + last = 0; + for (idx = 0; idx < 8; idx++) { + if ((map & (1 << idx)) != 0) { + if (first > idx) + first = idx; + if (last < idx) + last = idx; + } + } + + *first_p = first; + *last_p = last; +} + +/* Cook up fake bus resources for SUNW,simba PCI bridges which lack + * a proper 'ranges' property. + */ +static void apb_fake_ranges(struct pci_dev *dev, + struct pci_bus *bus, + struct pci_pbm_info *pbm) +{ + struct pci_bus_region region; + struct resource *res; + u32 first, last; + u8 map; + + pci_read_config_byte(dev, APB_IO_ADDRESS_MAP, &map); + apb_calc_first_last(map, &first, &last); + res = bus->resource[0]; + res->flags = IORESOURCE_IO; + region.start = (first << 21); + region.end = (last << 21) + ((1 << 21) - 1); + pcibios_bus_to_resource(dev->bus, res, ®ion); + + pci_read_config_byte(dev, APB_MEM_ADDRESS_MAP, &map); + apb_calc_first_last(map, &first, &last); + res = bus->resource[1]; + res->flags = IORESOURCE_MEM; + region.start = (first << 29); + region.end = (last << 29) + ((1 << 29) - 1); + pcibios_bus_to_resource(dev->bus, res, ®ion); +} + +static void pci_of_scan_bus(struct pci_pbm_info *pbm, + struct device_node *node, + struct pci_bus *bus); + +#define GET_64BIT(prop, i) ((((u64) (prop)[(i)]) << 32) | (prop)[(i)+1]) + +static void of_scan_pci_bridge(struct pci_pbm_info *pbm, + struct device_node *node, + struct pci_dev *dev) +{ + struct pci_bus *bus; + const u32 *busrange, *ranges; + int len, i, simba; + struct pci_bus_region region; + struct resource *res; + unsigned int flags; + u64 size; + + if (ofpci_verbose) + pci_info(dev, "of_scan_pci_bridge(%pOF)\n", node); + + /* parse bus-range property */ + busrange = of_get_property(node, "bus-range", &len); + if (busrange == NULL || len != 8) { + pci_info(dev, "Can't get bus-range for PCI-PCI bridge %pOF\n", + node); + return; + } + + if (ofpci_verbose) + pci_info(dev, " Bridge bus range [%u --> %u]\n", + busrange[0], busrange[1]); + + ranges = of_get_property(node, "ranges", &len); + simba = 0; + if (ranges == NULL) { + const char *model = of_get_property(node, "model", NULL); + if (model && !strcmp(model, "SUNW,simba")) + simba = 1; + } + + bus = pci_add_new_bus(dev->bus, dev, busrange[0]); + if (!bus) { + pci_err(dev, "Failed to create pci bus for %pOF\n", + node); + return; + } + + bus->primary = dev->bus->number; + pci_bus_insert_busn_res(bus, busrange[0], busrange[1]); + bus->bridge_ctl = 0; + + if (ofpci_verbose) + pci_info(dev, " Bridge ranges[%p] simba[%d]\n", + ranges, simba); + + /* parse ranges property, or cook one up by hand for Simba */ + /* PCI #address-cells == 3 and #size-cells == 2 always */ + res = &dev->resource[PCI_BRIDGE_RESOURCES]; + for (i = 0; i < PCI_NUM_RESOURCES - PCI_BRIDGE_RESOURCES; ++i) { + res->flags = 0; + bus->resource[i] = res; + ++res; + } + if (simba) { + apb_fake_ranges(dev, bus, pbm); + goto after_ranges; + } else if (ranges == NULL) { + pci_read_bridge_bases(bus); + goto after_ranges; + } + i = 1; + for (; len >= 32; len -= 32, ranges += 8) { + u64 start; + + if (ofpci_verbose) + pci_info(dev, " RAW Range[%08x:%08x:%08x:%08x:%08x:%08x:" + "%08x:%08x]\n", + ranges[0], ranges[1], ranges[2], ranges[3], + ranges[4], ranges[5], ranges[6], ranges[7]); + + flags = pci_parse_of_flags(ranges[0]); + size = GET_64BIT(ranges, 6); + if (flags == 0 || size == 0) + continue; + + /* On PCI-Express systems, PCI bridges that have no devices downstream + * have a bogus size value where the first 32-bit cell is 0xffffffff. + * This results in a bogus range where start + size overflows. + * + * Just skip these otherwise the kernel will complain when the resource + * tries to be claimed. + */ + if (size >> 32 == 0xffffffff) + continue; + + if (flags & IORESOURCE_IO) { + res = bus->resource[0]; + if (res->flags) { + pci_err(dev, "ignoring extra I/O range" + " for bridge %pOF\n", node); + continue; + } + } else { + if (i >= PCI_NUM_RESOURCES - PCI_BRIDGE_RESOURCES) { + pci_err(dev, "too many memory ranges" + " for bridge %pOF\n", node); + continue; + } + res = bus->resource[i]; + ++i; + } + + res->flags = flags; + region.start = start = GET_64BIT(ranges, 1); + region.end = region.start + size - 1; + + if (ofpci_verbose) + pci_info(dev, " Using flags[%08x] start[%016llx] size[%016llx]\n", + flags, start, size); + + pcibios_bus_to_resource(dev->bus, res, ®ion); + } +after_ranges: + sprintf(bus->name, "PCI Bus %04x:%02x", pci_domain_nr(bus), + bus->number); + if (ofpci_verbose) + pci_info(dev, " bus name: %s\n", bus->name); + + pci_of_scan_bus(pbm, node, bus); +} + +static void pci_of_scan_bus(struct pci_pbm_info *pbm, + struct device_node *node, + struct pci_bus *bus) +{ + struct device_node *child; + const u32 *reg; + int reglen, devfn, prev_devfn; + struct pci_dev *dev; + + if (ofpci_verbose) + pci_info(bus, "scan_bus[%pOF] bus no %d\n", + node, bus->number); + + prev_devfn = -1; + for_each_child_of_node(node, child) { + if (ofpci_verbose) + pci_info(bus, " * %pOF\n", child); + reg = of_get_property(child, "reg", ®len); + if (reg == NULL || reglen < 20) + continue; + + devfn = (reg[0] >> 8) & 0xff; + + /* This is a workaround for some device trees + * which list PCI devices twice. On the V100 + * for example, device number 3 is listed twice. + * Once as "pm" and once again as "lomp". + */ + if (devfn == prev_devfn) + continue; + prev_devfn = devfn; + + /* create a new pci_dev for this device */ + dev = of_create_pci_dev(pbm, child, bus, devfn); + if (!dev) + continue; + if (ofpci_verbose) + pci_info(dev, "dev header type: %x\n", dev->hdr_type); + + if (pci_is_bridge(dev)) + of_scan_pci_bridge(pbm, child, dev); + } +} + +static ssize_t +show_pciobppath_attr(struct device * dev, struct device_attribute * attr, char * buf) +{ + struct pci_dev *pdev; + struct device_node *dp; + + pdev = to_pci_dev(dev); + dp = pdev->dev.of_node; + + return scnprintf(buf, PAGE_SIZE, "%pOF\n", dp); +} + +static DEVICE_ATTR(obppath, S_IRUSR | S_IRGRP | S_IROTH, show_pciobppath_attr, NULL); + +static void pci_bus_register_of_sysfs(struct pci_bus *bus) +{ + struct pci_dev *dev; + struct pci_bus *child_bus; + int err; + + list_for_each_entry(dev, &bus->devices, bus_list) { + /* we don't really care if we can create this file or + * not, but we need to assign the result of the call + * or the world will fall under alien invasion and + * everybody will be frozen on a spaceship ready to be + * eaten on alpha centauri by some green and jelly + * humanoid. + */ + err = sysfs_create_file(&dev->dev.kobj, &dev_attr_obppath.attr); + (void) err; + } + list_for_each_entry(child_bus, &bus->children, node) + pci_bus_register_of_sysfs(child_bus); +} + +static void pci_claim_legacy_resources(struct pci_dev *dev) +{ + struct pci_bus_region region; + struct resource *p, *root, *conflict; + + if ((dev->class >> 8) != PCI_CLASS_DISPLAY_VGA) + return; + + p = kzalloc_obj(*p); + if (!p) + return; + + p->name = "Video RAM area"; + p->flags = IORESOURCE_MEM | IORESOURCE_BUSY; + + region.start = 0xa0000UL; + region.end = region.start + 0x1ffffUL; + pcibios_bus_to_resource(dev->bus, p, ®ion); + + root = pci_find_parent_resource(dev, p); + if (!root) { + pci_info(dev, "can't claim VGA legacy %pR: no compatible bridge window\n", p); + goto err; + } + + conflict = request_resource_conflict(root, p); + if (conflict) { + pci_info(dev, "can't claim VGA legacy %pR: address conflict with %s %pR\n", + p, conflict->name, conflict); + goto err; + } + + pci_info(dev, "VGA legacy framebuffer %pR\n", p); + return; + +err: + kfree(p); +} + +static void pci_claim_bus_resources(struct pci_bus *bus) +{ + struct pci_bus *child_bus; + struct pci_dev *dev; + + list_for_each_entry(dev, &bus->devices, bus_list) { + struct resource *r; + int i; + + pci_dev_for_each_resource(dev, r, i) { + if (r->parent || !r->start || !r->flags) + continue; + + if (ofpci_verbose) + pci_info(dev, "Claiming Resource %d: %pR\n", + i, r); + + pci_claim_resource(dev, i); + } + + pci_claim_legacy_resources(dev); + } + + list_for_each_entry(child_bus, &bus->children, node) + pci_claim_bus_resources(child_bus); +} + +struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, + struct device *parent) +{ + LIST_HEAD(resources); + struct device_node *node = pbm->op->dev.of_node; + struct pci_bus *bus; + + printk("PCI: Scanning PBM %pOF\n", node); + + pci_add_resource_offset(&resources, &pbm->io_space, + pbm->io_offset); + pci_add_resource_offset(&resources, &pbm->mem_space, + pbm->mem_offset); + if (pbm->mem64_space.flags) + pci_add_resource_offset(&resources, &pbm->mem64_space, + pbm->mem64_offset); + pbm->busn.start = pbm->pci_first_busno; + pbm->busn.end = pbm->pci_last_busno; + pbm->busn.flags = IORESOURCE_BUS; + pci_add_resource(&resources, &pbm->busn); + bus = pci_create_root_bus(parent, pbm->pci_first_busno, pbm->pci_ops, + pbm, &resources); + if (!bus) { + printk(KERN_ERR "Failed to create bus for %pOF\n", node); + pci_free_resource_list(&resources); + return NULL; + } + + pci_of_scan_bus(pbm, node, bus); + pci_bus_register_of_sysfs(bus); + + pci_claim_bus_resources(bus); + + pci_bus_add_devices(bus); + return bus; +} + +/* Platform support for /proc/bus/pci/X/Y mmap()s. */ +int pci_iobar_pfn(struct pci_dev *pdev, int bar, struct vm_area_struct *vma) +{ + struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; + resource_size_t ioaddr = pci_resource_start(pdev, bar); + + if (!pbm) + return -EINVAL; + + vma->vm_pgoff += (ioaddr + pbm->io_space.start) >> PAGE_SHIFT; + + return 0; +} + +#ifdef CONFIG_NUMA +int pcibus_to_node(struct pci_bus *pbus) +{ + struct pci_pbm_info *pbm = pbus->sysdata; + + return pbm->numa_node; +} +EXPORT_SYMBOL(pcibus_to_node); +#endif + +/* Return the domain number for this pci bus */ + +int pci_domain_nr(struct pci_bus *pbus) +{ + struct pci_pbm_info *pbm = pbus->sysdata; + int ret; + + if (!pbm) { + ret = -ENXIO; + } else { + ret = pbm->index; + } + + return ret; +} +EXPORT_SYMBOL(pci_domain_nr); + +#ifdef CONFIG_PCI_MSI +int arch_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) +{ + struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; + unsigned int irq; + + if (!pbm->setup_msi_irq) + return -EINVAL; + + return pbm->setup_msi_irq(&irq, pdev, desc); +} + +void arch_teardown_msi_irq(unsigned int irq) +{ + struct msi_desc *entry = irq_get_msi_desc(irq); + struct pci_dev *pdev = msi_desc_to_pci_dev(entry); + struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; + + if (pbm->teardown_msi_irq) + pbm->teardown_msi_irq(irq, pdev); +} +#endif /* !(CONFIG_PCI_MSI) */ + +/* ALI sound chips generate 31-bits of DMA, a special register + * determines what bit 31 is emitted as. + */ +int ali_sound_dma_hack(struct device *dev, u64 device_mask) +{ + struct iommu *iommu = dev->archdata.iommu; + struct pci_dev *ali_isa_bridge; + u8 val; + + if (!dev_is_pci(dev)) + return 0; + + if (to_pci_dev(dev)->vendor != PCI_VENDOR_ID_AL || + to_pci_dev(dev)->device != PCI_DEVICE_ID_AL_M5451 || + device_mask != 0x7fffffff) + return 0; + + ali_isa_bridge = pci_get_device(PCI_VENDOR_ID_AL, + PCI_DEVICE_ID_AL_M1533, + NULL); + + pci_read_config_byte(ali_isa_bridge, 0x7e, &val); + if (iommu->dma_addr_mask & 0x80000000) + val |= 0x01; + else + val &= ~0x01; + pci_write_config_byte(ali_isa_bridge, 0x7e, val); + pci_dev_put(ali_isa_bridge); + return 1; +} + +void pci_resource_to_user(const struct pci_dev *pdev, int bar, + const struct resource *rp, resource_size_t *start, + resource_size_t *end) +{ + struct pci_bus_region region; + + /* + * "User" addresses are shown in /sys/devices/pci.../.../resource + * and /proc/bus/pci/devices and used as mmap offsets for + * /proc/bus/pci/BB/DD.F files (see proc_bus_pci_mmap()). + * + * On sparc, these are PCI bus addresses, i.e., raw BAR values. + */ + pcibios_resource_to_bus(pdev->bus, ®ion, (struct resource *) rp); + *start = region.start; + *end = region.end; +} + +void pcibios_set_master(struct pci_dev *dev) +{ + /* No special bus mastering setup handling */ +} + +#ifdef CONFIG_PCI_IOV +int pcibios_device_add(struct pci_dev *dev) +{ + struct pci_dev *pdev; + + /* Add sriov arch specific initialization here. + * Copy dev_archdata from PF to VF + */ + if (dev->is_virtfn) { + struct dev_archdata *psd; + + pdev = dev->physfn; + psd = &pdev->dev.archdata; + pci_init_dev_archdata(&dev->dev.archdata, psd->iommu, + psd->stc, psd->host_controller, NULL, + psd->numa_node); + } + return 0; +} +#endif /* CONFIG_PCI_IOV */ + +static int __init pcibios_init(void) +{ + pci_dfl_cache_line_size = 64 >> 2; + return 0; +} +subsys_initcall(pcibios_init); + +#ifdef CONFIG_SYSFS + +#define SLOT_NAME_SIZE 11 /* Max decimal digits + null in u32 */ + +static void pcie_bus_slot_names(struct pci_bus *pbus) +{ + struct pci_dev *pdev; + struct pci_bus *bus; + + list_for_each_entry(pdev, &pbus->devices, bus_list) { + char name[SLOT_NAME_SIZE]; + struct pci_slot *pci_slot; + const u32 *slot_num; + int len; + + slot_num = of_get_property(pdev->dev.of_node, + "physical-slot#", &len); + + if (slot_num == NULL || len != 4) + continue; + + snprintf(name, sizeof(name), "%u", slot_num[0]); + pci_slot = pci_create_slot(pbus, slot_num[0], name, NULL); + + if (IS_ERR(pci_slot)) + pr_err("PCI: pci_create_slot returned %ld.\n", + PTR_ERR(pci_slot)); + } + + list_for_each_entry(bus, &pbus->children, node) + pcie_bus_slot_names(bus); +} + +static void pci_bus_slot_names(struct device_node *node, struct pci_bus *bus) +{ + const struct pci_slot_names { + u32 slot_mask; + char names[]; + } *prop; + const char *sp; + int len, i; + u32 mask; + + prop = of_get_property(node, "slot-names", &len); + if (!prop) + return; + + mask = prop->slot_mask; + sp = prop->names; + + if (ofpci_verbose) + pci_info(bus, "Making slots for [%pOF] mask[0x%02x]\n", + node, mask); + + i = 0; + while (mask) { + struct pci_slot *pci_slot; + u32 this_bit = 1 << i; + + if (!(mask & this_bit)) { + i++; + continue; + } + + if (ofpci_verbose) + pci_info(bus, "Making slot [%s]\n", sp); + + pci_slot = pci_create_slot(bus, i, sp, NULL); + if (IS_ERR(pci_slot)) + pci_err(bus, "pci_create_slot returned %ld\n", + PTR_ERR(pci_slot)); + + sp += strlen(sp) + 1; + mask &= ~this_bit; + i++; + } +} + +static int __init of_pci_slot_init(void) +{ + struct pci_bus *pbus = NULL; + + while ((pbus = pci_find_next_bus(pbus)) != NULL) { + struct device_node *node; + struct pci_dev *pdev; + + pdev = list_first_entry(&pbus->devices, struct pci_dev, + bus_list); + + if (pdev && pci_is_pcie(pdev)) { + pcie_bus_slot_names(pbus); + } else { + + if (pbus->self) { + + /* PCI->PCI bridge */ + node = pbus->self->dev.of_node; + + } else { + struct pci_pbm_info *pbm = pbus->sysdata; + + /* Host PCI controller */ + node = pbm->op->dev.of_node; + } + + pci_bus_slot_names(node, pbus); + } + } + + return 0; +} +device_initcall(of_pci_slot_init); +#endif diff --git a/arch/sparc/kernel/pci_common.c b/arch/sparc/kernel/pci_common.c new file mode 100644 index 000000000..0eafcb2d8 --- /dev/null +++ b/arch/sparc/kernel/pci_common.c @@ -0,0 +1,547 @@ +// SPDX-License-Identifier: GPL-2.0 +/* pci_common.c: PCI controller common support. + * + * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "pci_impl.h" +#include "pci_sun4v.h" + +static int config_out_of_range(struct pci_pbm_info *pbm, + unsigned long bus, + unsigned long devfn, + unsigned long reg) +{ + if (bus < pbm->pci_first_busno || + bus > pbm->pci_last_busno) + return 1; + return 0; +} + +static void *sun4u_config_mkaddr(struct pci_pbm_info *pbm, + unsigned long bus, + unsigned long devfn, + unsigned long reg) +{ + unsigned long rbits = pbm->config_space_reg_bits; + + if (config_out_of_range(pbm, bus, devfn, reg)) + return NULL; + + reg = (reg & ((1 << rbits) - 1)); + devfn <<= rbits; + bus <<= rbits + 8; + + return (void *) (pbm->config_space | bus | devfn | reg); +} + +/* At least on Sabre, it is necessary to access all PCI host controller + * registers at their natural size, otherwise zeros are returned. + * Strange but true, and I see no language in the UltraSPARC-IIi + * programmer's manual that mentions this even indirectly. + */ +static int sun4u_read_pci_cfg_host(struct pci_pbm_info *pbm, + unsigned char bus, unsigned int devfn, + int where, int size, u32 *value) +{ + u32 tmp32, *addr; + u16 tmp16; + u8 tmp8; + + addr = sun4u_config_mkaddr(pbm, bus, devfn, where); + if (!addr) + return PCIBIOS_SUCCESSFUL; + + switch (size) { + case 1: + if (where < 8) { + unsigned long align = (unsigned long) addr; + + align &= ~1; + pci_config_read16((u16 *)align, &tmp16); + if (where & 1) + *value = tmp16 >> 8; + else + *value = tmp16 & 0xff; + } else { + pci_config_read8((u8 *)addr, &tmp8); + *value = (u32) tmp8; + } + break; + + case 2: + if (where < 8) { + pci_config_read16((u16 *)addr, &tmp16); + *value = (u32) tmp16; + } else { + pci_config_read8((u8 *)addr, &tmp8); + *value = (u32) tmp8; + pci_config_read8(((u8 *)addr) + 1, &tmp8); + *value |= ((u32) tmp8) << 8; + } + break; + + case 4: + tmp32 = 0xffffffff; + sun4u_read_pci_cfg_host(pbm, bus, devfn, + where, 2, &tmp32); + *value = tmp32; + + tmp32 = 0xffffffff; + sun4u_read_pci_cfg_host(pbm, bus, devfn, + where + 2, 2, &tmp32); + *value |= tmp32 << 16; + break; + } + return PCIBIOS_SUCCESSFUL; +} + +static int sun4u_read_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, + int where, int size, u32 *value) +{ + struct pci_pbm_info *pbm = bus_dev->sysdata; + unsigned char bus = bus_dev->number; + u32 *addr; + u16 tmp16; + u8 tmp8; + + switch (size) { + case 1: + *value = 0xff; + break; + case 2: + *value = 0xffff; + break; + case 4: + *value = 0xffffffff; + break; + } + + if (!bus_dev->number && !PCI_SLOT(devfn)) + return sun4u_read_pci_cfg_host(pbm, bus, devfn, where, + size, value); + + addr = sun4u_config_mkaddr(pbm, bus, devfn, where); + if (!addr) + return PCIBIOS_SUCCESSFUL; + + switch (size) { + case 1: + pci_config_read8((u8 *)addr, &tmp8); + *value = (u32) tmp8; + break; + + case 2: + if (where & 0x01) { + printk("pci_read_config_word: misaligned reg [%x]\n", + where); + return PCIBIOS_SUCCESSFUL; + } + pci_config_read16((u16 *)addr, &tmp16); + *value = (u32) tmp16; + break; + + case 4: + if (where & 0x03) { + printk("pci_read_config_dword: misaligned reg [%x]\n", + where); + return PCIBIOS_SUCCESSFUL; + } + pci_config_read32(addr, value); + break; + } + return PCIBIOS_SUCCESSFUL; +} + +static int sun4u_write_pci_cfg_host(struct pci_pbm_info *pbm, + unsigned char bus, unsigned int devfn, + int where, int size, u32 value) +{ + u32 *addr; + + addr = sun4u_config_mkaddr(pbm, bus, devfn, where); + if (!addr) + return PCIBIOS_SUCCESSFUL; + + switch (size) { + case 1: + if (where < 8) { + unsigned long align = (unsigned long) addr; + u16 tmp16; + + align &= ~1; + pci_config_read16((u16 *)align, &tmp16); + if (where & 1) { + tmp16 &= 0x00ff; + tmp16 |= value << 8; + } else { + tmp16 &= 0xff00; + tmp16 |= value; + } + pci_config_write16((u16 *)align, tmp16); + } else + pci_config_write8((u8 *)addr, value); + break; + case 2: + if (where < 8) { + pci_config_write16((u16 *)addr, value); + } else { + pci_config_write8((u8 *)addr, value & 0xff); + pci_config_write8(((u8 *)addr) + 1, value >> 8); + } + break; + case 4: + sun4u_write_pci_cfg_host(pbm, bus, devfn, + where, 2, value & 0xffff); + sun4u_write_pci_cfg_host(pbm, bus, devfn, + where + 2, 2, value >> 16); + break; + } + return PCIBIOS_SUCCESSFUL; +} + +static int sun4u_write_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, + int where, int size, u32 value) +{ + struct pci_pbm_info *pbm = bus_dev->sysdata; + unsigned char bus = bus_dev->number; + u32 *addr; + + if (!bus_dev->number && !PCI_SLOT(devfn)) + return sun4u_write_pci_cfg_host(pbm, bus, devfn, where, + size, value); + + addr = sun4u_config_mkaddr(pbm, bus, devfn, where); + if (!addr) + return PCIBIOS_SUCCESSFUL; + + switch (size) { + case 1: + pci_config_write8((u8 *)addr, value); + break; + + case 2: + if (where & 0x01) { + printk("pci_write_config_word: misaligned reg [%x]\n", + where); + return PCIBIOS_SUCCESSFUL; + } + pci_config_write16((u16 *)addr, value); + break; + + case 4: + if (where & 0x03) { + printk("pci_write_config_dword: misaligned reg [%x]\n", + where); + return PCIBIOS_SUCCESSFUL; + } + pci_config_write32(addr, value); + } + return PCIBIOS_SUCCESSFUL; +} + +struct pci_ops sun4u_pci_ops = { + .read = sun4u_read_pci_cfg, + .write = sun4u_write_pci_cfg, +}; + +static int sun4v_read_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, + int where, int size, u32 *value) +{ + struct pci_pbm_info *pbm = bus_dev->sysdata; + u32 devhandle = pbm->devhandle; + unsigned int bus = bus_dev->number; + unsigned int device = PCI_SLOT(devfn); + unsigned int func = PCI_FUNC(devfn); + unsigned long ret; + + if (config_out_of_range(pbm, bus, devfn, where)) { + ret = ~0UL; + } else { + ret = pci_sun4v_config_get(devhandle, + HV_PCI_DEVICE_BUILD(bus, device, func), + where, size); + } + switch (size) { + case 1: + *value = ret & 0xff; + break; + case 2: + *value = ret & 0xffff; + break; + case 4: + *value = ret & 0xffffffff; + break; + } + + + return PCIBIOS_SUCCESSFUL; +} + +static int sun4v_write_pci_cfg(struct pci_bus *bus_dev, unsigned int devfn, + int where, int size, u32 value) +{ + struct pci_pbm_info *pbm = bus_dev->sysdata; + u32 devhandle = pbm->devhandle; + unsigned int bus = bus_dev->number; + unsigned int device = PCI_SLOT(devfn); + unsigned int func = PCI_FUNC(devfn); + + if (config_out_of_range(pbm, bus, devfn, where)) { + /* Do nothing. */ + } else { + /* We don't check for hypervisor errors here, but perhaps + * we should and influence our return value depending upon + * what kind of error is thrown. + */ + pci_sun4v_config_put(devhandle, + HV_PCI_DEVICE_BUILD(bus, device, func), + where, size, value); + } + return PCIBIOS_SUCCESSFUL; +} + +struct pci_ops sun4v_pci_ops = { + .read = sun4v_read_pci_cfg, + .write = sun4v_write_pci_cfg, +}; + +void pci_get_pbm_props(struct pci_pbm_info *pbm) +{ + const u32 *val = of_get_property(pbm->op->dev.of_node, "bus-range", NULL); + + pbm->pci_first_busno = val[0]; + pbm->pci_last_busno = val[1]; + + val = of_get_property(pbm->op->dev.of_node, "ino-bitmap", NULL); + if (val) { + pbm->ino_bitmap = (((u64)val[1] << 32UL) | + ((u64)val[0] << 0UL)); + } +} + +static void pci_register_iommu_region(struct pci_pbm_info *pbm) +{ + const u32 *vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma", + NULL); + + if (vdma) { + struct resource *rp = kzalloc_obj(*rp); + + if (!rp) { + pr_info("%s: Cannot allocate IOMMU resource.\n", + pbm->name); + return; + } + rp->name = "IOMMU"; + rp->start = pbm->mem_space.start + (unsigned long) vdma[0]; + rp->end = rp->start + (unsigned long) vdma[1] - 1UL; + rp->flags = IORESOURCE_BUSY; + if (request_resource(&pbm->mem_space, rp)) { + pr_info("%s: Unable to request IOMMU resource.\n", + pbm->name); + kfree(rp); + } + } +} + +void pci_determine_mem_io_space(struct pci_pbm_info *pbm) +{ + const struct linux_prom_pci_ranges *pbm_ranges; + int i, saw_mem, saw_io; + int num_pbm_ranges; + + /* Corresponds to generic devm_of_pci_get_host_bridge_resources() */ + + saw_mem = saw_io = 0; + pbm_ranges = of_get_property(pbm->op->dev.of_node, "ranges", &i); + if (!pbm_ranges) { + prom_printf("PCI: Fatal error, missing PBM ranges property " + " for %s\n", + pbm->name); + prom_halt(); + } + + num_pbm_ranges = i / sizeof(*pbm_ranges); + memset(&pbm->mem64_space, 0, sizeof(struct resource)); + + for (i = 0; i < num_pbm_ranges; i++) { + const struct linux_prom_pci_ranges *pr = &pbm_ranges[i]; + unsigned long a, size, region_a; + u32 parent_phys_hi, parent_phys_lo; + u32 child_phys_mid, child_phys_lo; + u32 size_hi, size_lo; + int type; + + parent_phys_hi = pr->parent_phys_hi; + parent_phys_lo = pr->parent_phys_lo; + child_phys_mid = pr->child_phys_mid; + child_phys_lo = pr->child_phys_lo; + if (tlb_type == hypervisor) + parent_phys_hi &= 0x0fffffff; + + size_hi = pr->size_hi; + size_lo = pr->size_lo; + + type = (pr->child_phys_hi >> 24) & 0x3; + a = (((unsigned long)parent_phys_hi << 32UL) | + ((unsigned long)parent_phys_lo << 0UL)); + region_a = (((unsigned long)child_phys_mid << 32UL) | + ((unsigned long)child_phys_lo << 0UL)); + size = (((unsigned long)size_hi << 32UL) | + ((unsigned long)size_lo << 0UL)); + + switch (type) { + case 0: + /* PCI config space, 16MB */ + pbm->config_space = a; + break; + + case 1: + /* 16-bit IO space, 16MB */ + pbm->io_space.start = a; + pbm->io_space.end = a + size - 1UL; + pbm->io_space.flags = IORESOURCE_IO; + pbm->io_offset = a - region_a; + saw_io = 1; + break; + + case 2: + /* 32-bit MEM space, 2GB */ + pbm->mem_space.start = a; + pbm->mem_space.end = a + size - 1UL; + pbm->mem_space.flags = IORESOURCE_MEM; + pbm->mem_offset = a - region_a; + saw_mem = 1; + break; + + case 3: + /* 64-bit MEM handling */ + pbm->mem64_space.start = a; + pbm->mem64_space.end = a + size - 1UL; + pbm->mem64_space.flags = IORESOURCE_MEM; + pbm->mem64_offset = a - region_a; + saw_mem = 1; + break; + + default: + break; + } + } + + if (!saw_io || !saw_mem) { + prom_printf("%s: Fatal error, missing %s PBM range.\n", + pbm->name, + (!saw_io ? "IO" : "MEM")); + prom_halt(); + } + + if (pbm->io_space.flags) + printk("%s: PCI IO %pR offset %llx\n", + pbm->name, &pbm->io_space, pbm->io_offset); + if (pbm->mem_space.flags) + printk("%s: PCI MEM %pR offset %llx\n", + pbm->name, &pbm->mem_space, pbm->mem_offset); + if (pbm->mem64_space.flags && pbm->mem_space.flags) { + if (pbm->mem64_space.start <= pbm->mem_space.end) + pbm->mem64_space.start = pbm->mem_space.end + 1; + if (pbm->mem64_space.start > pbm->mem64_space.end) + pbm->mem64_space.flags = 0; + } + + if (pbm->mem64_space.flags) + printk("%s: PCI MEM64 %pR offset %llx\n", + pbm->name, &pbm->mem64_space, pbm->mem64_offset); + + pbm->io_space.name = pbm->mem_space.name = pbm->name; + pbm->mem64_space.name = pbm->name; + + request_resource(&ioport_resource, &pbm->io_space); + request_resource(&iomem_resource, &pbm->mem_space); + if (pbm->mem64_space.flags) + request_resource(&iomem_resource, &pbm->mem64_space); + + pci_register_iommu_region(pbm); +} + +/* Generic helper routines for PCI error reporting. */ +void pci_scan_for_target_abort(struct pci_pbm_info *pbm, + struct pci_bus *pbus) +{ + struct pci_dev *pdev; + struct pci_bus *bus; + + list_for_each_entry(pdev, &pbus->devices, bus_list) { + u16 status, error_bits; + + pci_read_config_word(pdev, PCI_STATUS, &status); + error_bits = + (status & (PCI_STATUS_SIG_TARGET_ABORT | + PCI_STATUS_REC_TARGET_ABORT)); + if (error_bits) { + pci_write_config_word(pdev, PCI_STATUS, error_bits); + pci_info(pdev, "%s: Device saw Target Abort [%016x]\n", + pbm->name, status); + } + } + + list_for_each_entry(bus, &pbus->children, node) + pci_scan_for_target_abort(pbm, bus); +} + +void pci_scan_for_master_abort(struct pci_pbm_info *pbm, + struct pci_bus *pbus) +{ + struct pci_dev *pdev; + struct pci_bus *bus; + + list_for_each_entry(pdev, &pbus->devices, bus_list) { + u16 status, error_bits; + + pci_read_config_word(pdev, PCI_STATUS, &status); + error_bits = + (status & (PCI_STATUS_REC_MASTER_ABORT)); + if (error_bits) { + pci_write_config_word(pdev, PCI_STATUS, error_bits); + pci_info(pdev, "%s: Device received Master Abort " + "[%016x]\n", pbm->name, status); + } + } + + list_for_each_entry(bus, &pbus->children, node) + pci_scan_for_master_abort(pbm, bus); +} + +void pci_scan_for_parity_error(struct pci_pbm_info *pbm, + struct pci_bus *pbus) +{ + struct pci_dev *pdev; + struct pci_bus *bus; + + list_for_each_entry(pdev, &pbus->devices, bus_list) { + u16 status, error_bits; + + pci_read_config_word(pdev, PCI_STATUS, &status); + error_bits = + (status & (PCI_STATUS_PARITY | + PCI_STATUS_DETECTED_PARITY)); + if (error_bits) { + pci_write_config_word(pdev, PCI_STATUS, error_bits); + pci_info(pdev, "%s: Device saw Parity Error [%016x]\n", + pbm->name, status); + } + } + + list_for_each_entry(bus, &pbus->children, node) + pci_scan_for_parity_error(pbm, bus); +} diff --git a/arch/sparc/kernel/pci_fire.c b/arch/sparc/kernel/pci_fire.c new file mode 100644 index 000000000..c35a8e3c1 --- /dev/null +++ b/arch/sparc/kernel/pci_fire.c @@ -0,0 +1,524 @@ +// SPDX-License-Identifier: GPL-2.0 +/* pci_fire.c: Sun4u platform PCI-E controller support. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "pci_impl.h" + +#define DRIVER_NAME "fire" +#define PFX DRIVER_NAME ": " + +#define FIRE_IOMMU_CONTROL 0x40000UL +#define FIRE_IOMMU_TSBBASE 0x40008UL +#define FIRE_IOMMU_FLUSH 0x40100UL +#define FIRE_IOMMU_FLUSHINV 0x40108UL + +static int pci_fire_pbm_iommu_init(struct pci_pbm_info *pbm) +{ + struct iommu *iommu = pbm->iommu; + u32 vdma[2], dma_mask; + u64 control; + int tsbsize, err; + + /* No virtual-dma property on these guys, use largest size. */ + vdma[0] = 0xc0000000; /* base */ + vdma[1] = 0x40000000; /* size */ + dma_mask = 0xffffffff; + tsbsize = 128; + + /* Register addresses. */ + iommu->iommu_control = pbm->pbm_regs + FIRE_IOMMU_CONTROL; + iommu->iommu_tsbbase = pbm->pbm_regs + FIRE_IOMMU_TSBBASE; + iommu->iommu_flush = pbm->pbm_regs + FIRE_IOMMU_FLUSH; + iommu->iommu_flushinv = pbm->pbm_regs + FIRE_IOMMU_FLUSHINV; + + /* We use the main control/status register of FIRE as the write + * completion register. + */ + iommu->write_complete_reg = pbm->controller_regs + 0x410000UL; + + /* + * Invalidate TLB Entries. + */ + upa_writeq(~(u64)0, iommu->iommu_flushinv); + + err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask, + pbm->numa_node); + if (err) + return err; + + upa_writeq(__pa(iommu->page_table) | 0x7UL, iommu->iommu_tsbbase); + + control = upa_readq(iommu->iommu_control); + control |= (0x00000400 /* TSB cache snoop enable */ | + 0x00000300 /* Cache mode */ | + 0x00000002 /* Bypass enable */ | + 0x00000001 /* Translation enable */); + upa_writeq(control, iommu->iommu_control); + + return 0; +} + +#ifdef CONFIG_PCI_MSI +struct pci_msiq_entry { + u64 word0; +#define MSIQ_WORD0_RESV 0x8000000000000000UL +#define MSIQ_WORD0_FMT_TYPE 0x7f00000000000000UL +#define MSIQ_WORD0_FMT_TYPE_SHIFT 56 +#define MSIQ_WORD0_LEN 0x00ffc00000000000UL +#define MSIQ_WORD0_LEN_SHIFT 46 +#define MSIQ_WORD0_ADDR0 0x00003fff00000000UL +#define MSIQ_WORD0_ADDR0_SHIFT 32 +#define MSIQ_WORD0_RID 0x00000000ffff0000UL +#define MSIQ_WORD0_RID_SHIFT 16 +#define MSIQ_WORD0_DATA0 0x000000000000ffffUL +#define MSIQ_WORD0_DATA0_SHIFT 0 + +#define MSIQ_TYPE_MSG 0x6 +#define MSIQ_TYPE_MSI32 0xb +#define MSIQ_TYPE_MSI64 0xf + + u64 word1; +#define MSIQ_WORD1_ADDR1 0xffffffffffff0000UL +#define MSIQ_WORD1_ADDR1_SHIFT 16 +#define MSIQ_WORD1_DATA1 0x000000000000ffffUL +#define MSIQ_WORD1_DATA1_SHIFT 0 + + u64 resv[6]; +}; + +/* All MSI registers are offset from pbm->pbm_regs */ +#define EVENT_QUEUE_BASE_ADDR_REG 0x010000UL +#define EVENT_QUEUE_BASE_ADDR_ALL_ONES 0xfffc000000000000UL + +#define EVENT_QUEUE_CONTROL_SET(EQ) (0x011000UL + (EQ) * 0x8UL) +#define EVENT_QUEUE_CONTROL_SET_OFLOW 0x0200000000000000UL +#define EVENT_QUEUE_CONTROL_SET_EN 0x0000100000000000UL + +#define EVENT_QUEUE_CONTROL_CLEAR(EQ) (0x011200UL + (EQ) * 0x8UL) +#define EVENT_QUEUE_CONTROL_CLEAR_OF 0x0200000000000000UL +#define EVENT_QUEUE_CONTROL_CLEAR_E2I 0x0000800000000000UL +#define EVENT_QUEUE_CONTROL_CLEAR_DIS 0x0000100000000000UL + +#define EVENT_QUEUE_STATE(EQ) (0x011400UL + (EQ) * 0x8UL) +#define EVENT_QUEUE_STATE_MASK 0x0000000000000007UL +#define EVENT_QUEUE_STATE_IDLE 0x0000000000000001UL +#define EVENT_QUEUE_STATE_ACTIVE 0x0000000000000002UL +#define EVENT_QUEUE_STATE_ERROR 0x0000000000000004UL + +#define EVENT_QUEUE_TAIL(EQ) (0x011600UL + (EQ) * 0x8UL) +#define EVENT_QUEUE_TAIL_OFLOW 0x0200000000000000UL +#define EVENT_QUEUE_TAIL_VAL 0x000000000000007fUL + +#define EVENT_QUEUE_HEAD(EQ) (0x011800UL + (EQ) * 0x8UL) +#define EVENT_QUEUE_HEAD_VAL 0x000000000000007fUL + +#define MSI_MAP(MSI) (0x020000UL + (MSI) * 0x8UL) +#define MSI_MAP_VALID 0x8000000000000000UL +#define MSI_MAP_EQWR_N 0x4000000000000000UL +#define MSI_MAP_EQNUM 0x000000000000003fUL + +#define MSI_CLEAR(MSI) (0x028000UL + (MSI) * 0x8UL) +#define MSI_CLEAR_EQWR_N 0x4000000000000000UL + +#define IMONDO_DATA0 0x02C000UL +#define IMONDO_DATA0_DATA 0xffffffffffffffc0UL + +#define IMONDO_DATA1 0x02C008UL +#define IMONDO_DATA1_DATA 0xffffffffffffffffUL + +#define MSI_32BIT_ADDR 0x034000UL +#define MSI_32BIT_ADDR_VAL 0x00000000ffff0000UL + +#define MSI_64BIT_ADDR 0x034008UL +#define MSI_64BIT_ADDR_VAL 0xffffffffffff0000UL + +static int pci_fire_get_head(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long *head) +{ + *head = upa_readq(pbm->pbm_regs + EVENT_QUEUE_HEAD(msiqid)); + return 0; +} + +static int pci_fire_dequeue_msi(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long *head, unsigned long *msi) +{ + unsigned long type_fmt, type, msi_num; + struct pci_msiq_entry *base, *ep; + + base = (pbm->msi_queues + ((msiqid - pbm->msiq_first) * 8192)); + ep = &base[*head]; + + if ((ep->word0 & MSIQ_WORD0_FMT_TYPE) == 0) + return 0; + + type_fmt = ((ep->word0 & MSIQ_WORD0_FMT_TYPE) >> + MSIQ_WORD0_FMT_TYPE_SHIFT); + type = (type_fmt >> 3); + if (unlikely(type != MSIQ_TYPE_MSI32 && + type != MSIQ_TYPE_MSI64)) + return -EINVAL; + + *msi = msi_num = ((ep->word0 & MSIQ_WORD0_DATA0) >> + MSIQ_WORD0_DATA0_SHIFT); + + upa_writeq(MSI_CLEAR_EQWR_N, pbm->pbm_regs + MSI_CLEAR(msi_num)); + + /* Clear the entry. */ + ep->word0 &= ~MSIQ_WORD0_FMT_TYPE; + + /* Go to next entry in ring. */ + (*head)++; + if (*head >= pbm->msiq_ent_count) + *head = 0; + + return 1; +} + +static int pci_fire_set_head(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long head) +{ + upa_writeq(head, pbm->pbm_regs + EVENT_QUEUE_HEAD(msiqid)); + return 0; +} + +static int pci_fire_msi_setup(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long msi, int is_msi64) +{ + u64 val; + + val = upa_readq(pbm->pbm_regs + MSI_MAP(msi)); + val &= ~(MSI_MAP_EQNUM); + val |= msiqid; + upa_writeq(val, pbm->pbm_regs + MSI_MAP(msi)); + + upa_writeq(MSI_CLEAR_EQWR_N, pbm->pbm_regs + MSI_CLEAR(msi)); + + val = upa_readq(pbm->pbm_regs + MSI_MAP(msi)); + val |= MSI_MAP_VALID; + upa_writeq(val, pbm->pbm_regs + MSI_MAP(msi)); + + return 0; +} + +static int pci_fire_msi_teardown(struct pci_pbm_info *pbm, unsigned long msi) +{ + u64 val; + + val = upa_readq(pbm->pbm_regs + MSI_MAP(msi)); + + val &= ~MSI_MAP_VALID; + + upa_writeq(val, pbm->pbm_regs + MSI_MAP(msi)); + + return 0; +} + +static int pci_fire_msiq_alloc(struct pci_pbm_info *pbm) +{ + unsigned long pages, order, i; + + order = get_order(512 * 1024); + pages = __get_free_pages(GFP_KERNEL | __GFP_COMP, order); + if (pages == 0UL) { + printk(KERN_ERR "MSI: Cannot allocate MSI queues (o=%lu).\n", + order); + return -ENOMEM; + } + memset((char *)pages, 0, PAGE_SIZE << order); + pbm->msi_queues = (void *) pages; + + upa_writeq((EVENT_QUEUE_BASE_ADDR_ALL_ONES | + __pa(pbm->msi_queues)), + pbm->pbm_regs + EVENT_QUEUE_BASE_ADDR_REG); + + upa_writeq(pbm->portid << 6, pbm->pbm_regs + IMONDO_DATA0); + upa_writeq(0, pbm->pbm_regs + IMONDO_DATA1); + + upa_writeq(pbm->msi32_start, pbm->pbm_regs + MSI_32BIT_ADDR); + upa_writeq(pbm->msi64_start, pbm->pbm_regs + MSI_64BIT_ADDR); + + for (i = 0; i < pbm->msiq_num; i++) { + upa_writeq(0, pbm->pbm_regs + EVENT_QUEUE_HEAD(i)); + upa_writeq(0, pbm->pbm_regs + EVENT_QUEUE_TAIL(i)); + } + + return 0; +} + +static void pci_fire_msiq_free(struct pci_pbm_info *pbm) +{ + unsigned long pages, order; + + order = get_order(512 * 1024); + pages = (unsigned long) pbm->msi_queues; + + free_pages(pages, order); + + pbm->msi_queues = NULL; +} + +static int pci_fire_msiq_build_irq(struct pci_pbm_info *pbm, + unsigned long msiqid, + unsigned long devino) +{ + unsigned long cregs = (unsigned long) pbm->pbm_regs; + unsigned long imap_reg, iclr_reg, int_ctrlr; + unsigned int irq; + int fixup; + u64 val; + + imap_reg = cregs + (0x001000UL + (devino * 0x08UL)); + iclr_reg = cregs + (0x001400UL + (devino * 0x08UL)); + + /* XXX iterate amongst the 4 IRQ controllers XXX */ + int_ctrlr = (1UL << 6); + + val = upa_readq(imap_reg); + val |= (1UL << 63) | int_ctrlr; + upa_writeq(val, imap_reg); + + fixup = ((pbm->portid << 6) | devino) - int_ctrlr; + + irq = build_irq(fixup, iclr_reg, imap_reg); + if (!irq) + return -ENOMEM; + + upa_writeq(EVENT_QUEUE_CONTROL_SET_EN, + pbm->pbm_regs + EVENT_QUEUE_CONTROL_SET(msiqid)); + + return irq; +} + +static const struct sparc64_msiq_ops pci_fire_msiq_ops = { + .get_head = pci_fire_get_head, + .dequeue_msi = pci_fire_dequeue_msi, + .set_head = pci_fire_set_head, + .msi_setup = pci_fire_msi_setup, + .msi_teardown = pci_fire_msi_teardown, + .msiq_alloc = pci_fire_msiq_alloc, + .msiq_free = pci_fire_msiq_free, + .msiq_build_irq = pci_fire_msiq_build_irq, +}; + +static void pci_fire_msi_init(struct pci_pbm_info *pbm) +{ + sparc64_pbm_msi_init(pbm, &pci_fire_msiq_ops); +} +#else /* CONFIG_PCI_MSI */ +static void pci_fire_msi_init(struct pci_pbm_info *pbm) +{ +} +#endif /* !(CONFIG_PCI_MSI) */ + +/* Based at pbm->controller_regs */ +#define FIRE_PARITY_CONTROL 0x470010UL +#define FIRE_PARITY_ENAB 0x8000000000000000UL +#define FIRE_FATAL_RESET_CTL 0x471028UL +#define FIRE_FATAL_RESET_SPARE 0x0000000004000000UL +#define FIRE_FATAL_RESET_MB 0x0000000002000000UL +#define FIRE_FATAL_RESET_CPE 0x0000000000008000UL +#define FIRE_FATAL_RESET_APE 0x0000000000004000UL +#define FIRE_FATAL_RESET_PIO 0x0000000000000040UL +#define FIRE_FATAL_RESET_JW 0x0000000000000004UL +#define FIRE_FATAL_RESET_JI 0x0000000000000002UL +#define FIRE_FATAL_RESET_JR 0x0000000000000001UL +#define FIRE_CORE_INTR_ENABLE 0x471800UL + +/* Based at pbm->pbm_regs */ +#define FIRE_TLU_CTRL 0x80000UL +#define FIRE_TLU_CTRL_TIM 0x00000000da000000UL +#define FIRE_TLU_CTRL_QDET 0x0000000000000100UL +#define FIRE_TLU_CTRL_CFG 0x0000000000000001UL +#define FIRE_TLU_DEV_CTRL 0x90008UL +#define FIRE_TLU_LINK_CTRL 0x90020UL +#define FIRE_TLU_LINK_CTRL_CLK 0x0000000000000040UL +#define FIRE_LPU_RESET 0xe2008UL +#define FIRE_LPU_LLCFG 0xe2200UL +#define FIRE_LPU_LLCFG_VC0 0x0000000000000100UL +#define FIRE_LPU_FCTRL_UCTRL 0xe2240UL +#define FIRE_LPU_FCTRL_UCTRL_N 0x0000000000000002UL +#define FIRE_LPU_FCTRL_UCTRL_P 0x0000000000000001UL +#define FIRE_LPU_TXL_FIFOP 0xe2430UL +#define FIRE_LPU_LTSSM_CFG2 0xe2788UL +#define FIRE_LPU_LTSSM_CFG3 0xe2790UL +#define FIRE_LPU_LTSSM_CFG4 0xe2798UL +#define FIRE_LPU_LTSSM_CFG5 0xe27a0UL +#define FIRE_DMC_IENAB 0x31800UL +#define FIRE_DMC_DBG_SEL_A 0x53000UL +#define FIRE_DMC_DBG_SEL_B 0x53008UL +#define FIRE_PEC_IENAB 0x51800UL + +static void pci_fire_hw_init(struct pci_pbm_info *pbm) +{ + u64 val; + + upa_writeq(FIRE_PARITY_ENAB, + pbm->controller_regs + FIRE_PARITY_CONTROL); + + upa_writeq((FIRE_FATAL_RESET_SPARE | + FIRE_FATAL_RESET_MB | + FIRE_FATAL_RESET_CPE | + FIRE_FATAL_RESET_APE | + FIRE_FATAL_RESET_PIO | + FIRE_FATAL_RESET_JW | + FIRE_FATAL_RESET_JI | + FIRE_FATAL_RESET_JR), + pbm->controller_regs + FIRE_FATAL_RESET_CTL); + + upa_writeq(~(u64)0, pbm->controller_regs + FIRE_CORE_INTR_ENABLE); + + val = upa_readq(pbm->pbm_regs + FIRE_TLU_CTRL); + val |= (FIRE_TLU_CTRL_TIM | + FIRE_TLU_CTRL_QDET | + FIRE_TLU_CTRL_CFG); + upa_writeq(val, pbm->pbm_regs + FIRE_TLU_CTRL); + upa_writeq(0, pbm->pbm_regs + FIRE_TLU_DEV_CTRL); + upa_writeq(FIRE_TLU_LINK_CTRL_CLK, + pbm->pbm_regs + FIRE_TLU_LINK_CTRL); + + upa_writeq(0, pbm->pbm_regs + FIRE_LPU_RESET); + upa_writeq(FIRE_LPU_LLCFG_VC0, pbm->pbm_regs + FIRE_LPU_LLCFG); + upa_writeq((FIRE_LPU_FCTRL_UCTRL_N | FIRE_LPU_FCTRL_UCTRL_P), + pbm->pbm_regs + FIRE_LPU_FCTRL_UCTRL); + upa_writeq(((0xffff << 16) | (0x0000 << 0)), + pbm->pbm_regs + FIRE_LPU_TXL_FIFOP); + upa_writeq(3000000, pbm->pbm_regs + FIRE_LPU_LTSSM_CFG2); + upa_writeq(500000, pbm->pbm_regs + FIRE_LPU_LTSSM_CFG3); + upa_writeq((2 << 16) | (140 << 8), + pbm->pbm_regs + FIRE_LPU_LTSSM_CFG4); + upa_writeq(0, pbm->pbm_regs + FIRE_LPU_LTSSM_CFG5); + + upa_writeq(~(u64)0, pbm->pbm_regs + FIRE_DMC_IENAB); + upa_writeq(0, pbm->pbm_regs + FIRE_DMC_DBG_SEL_A); + upa_writeq(0, pbm->pbm_regs + FIRE_DMC_DBG_SEL_B); + + upa_writeq(~(u64)0, pbm->pbm_regs + FIRE_PEC_IENAB); +} + +static int pci_fire_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op, u32 portid) +{ + const struct linux_prom64_registers *regs; + struct device_node *dp = op->dev.of_node; + int err; + + pbm->numa_node = NUMA_NO_NODE; + + pbm->pci_ops = &sun4u_pci_ops; + pbm->config_space_reg_bits = 12; + + pbm->index = pci_num_pbms++; + + pbm->portid = portid; + pbm->op = op; + pbm->name = dp->full_name; + + regs = of_get_property(dp, "reg", NULL); + pbm->pbm_regs = regs[0].phys_addr; + pbm->controller_regs = regs[1].phys_addr - 0x410000UL; + + printk("%s: SUN4U PCIE Bus Module\n", pbm->name); + + pci_determine_mem_io_space(pbm); + + pci_get_pbm_props(pbm); + + pci_fire_hw_init(pbm); + + err = pci_fire_pbm_iommu_init(pbm); + if (err) + return err; + + pci_fire_msi_init(pbm); + + pbm->pci_bus = pci_scan_one_pbm(pbm, &op->dev); + + /* XXX register error interrupt handlers XXX */ + + pbm->next = pci_pbm_root; + pci_pbm_root = pbm; + + return 0; +} + +static int fire_probe(struct platform_device *op) +{ + struct device_node *dp = op->dev.of_node; + struct pci_pbm_info *pbm; + struct iommu *iommu; + u32 portid; + int err; + + portid = of_getintprop_default(dp, "portid", 0xff); + + err = -ENOMEM; + pbm = kzalloc_obj(*pbm); + if (!pbm) { + printk(KERN_ERR PFX "Cannot allocate pci_pbminfo.\n"); + goto out_err; + } + + iommu = kzalloc_obj(struct iommu); + if (!iommu) { + printk(KERN_ERR PFX "Cannot allocate PBM iommu.\n"); + goto out_free_controller; + } + + pbm->iommu = iommu; + + err = pci_fire_pbm_init(pbm, op, portid); + if (err) + goto out_free_iommu; + + dev_set_drvdata(&op->dev, pbm); + + return 0; + +out_free_iommu: + kfree(pbm->iommu); + +out_free_controller: + kfree(pbm); + +out_err: + return err; +} + +static const struct of_device_id fire_match[] = { + { + .name = "pci", + .compatible = "pciex108e,80f0", + }, + {}, +}; + +static struct platform_driver fire_driver = { + .driver = { + .name = DRIVER_NAME, + .of_match_table = fire_match, + }, + .probe = fire_probe, +}; + +static int __init fire_init(void) +{ + return platform_driver_register(&fire_driver); +} + +subsys_initcall(fire_init); diff --git a/arch/sparc/kernel/pci_impl.h b/arch/sparc/kernel/pci_impl.h new file mode 100644 index 000000000..83718876f --- /dev/null +++ b/arch/sparc/kernel/pci_impl.h @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* pci_impl.h: Helper definitions for PCI controller support. + * + * Copyright (C) 1999, 2007 David S. Miller (davem@davemloft.net) + */ + +#ifndef PCI_IMPL_H +#define PCI_IMPL_H + +#include +#include +#include +#include +#include +#include +#include + +/* The abstraction used here is that there are PCI controllers, + * each with one (Sabre) or two (PSYCHO/SCHIZO) PCI bus modules + * underneath. Each PCI bus module uses an IOMMU (shared by both + * PBMs of a controller, or per-PBM), and if a streaming buffer + * is present, each PCI bus module has its own. (ie. the IOMMU + * might be shared between PBMs, the STC is never shared) + * Furthermore, each PCI bus module controls its own autonomous + * PCI bus. + */ + +#define PCI_STC_FLUSHFLAG_INIT(STC) \ + (*((STC)->strbuf_flushflag) = 0UL) +#define PCI_STC_FLUSHFLAG_SET(STC) \ + (*((STC)->strbuf_flushflag) != 0UL) + +#ifdef CONFIG_PCI_MSI +struct pci_pbm_info; +struct sparc64_msiq_ops { + int (*get_head)(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long *head); + int (*dequeue_msi)(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long *head, unsigned long *msi); + int (*set_head)(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long head); + int (*msi_setup)(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long msi, int is_msi64); + int (*msi_teardown)(struct pci_pbm_info *pbm, unsigned long msi); + int (*msiq_alloc)(struct pci_pbm_info *pbm); + void (*msiq_free)(struct pci_pbm_info *pbm); + int (*msiq_build_irq)(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long devino); +}; + +void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, + const struct sparc64_msiq_ops *ops); + +struct sparc64_msiq_cookie { + struct pci_pbm_info *pbm; + unsigned long msiqid; +}; +#endif + +struct pci_pbm_info { + struct pci_pbm_info *next; + struct pci_pbm_info *sibling; + int index; + + /* Physical address base of controller registers. */ + unsigned long controller_regs; + + /* Physical address base of PBM registers. */ + unsigned long pbm_regs; + + /* Physical address of DMA sync register, if any. */ + unsigned long sync_reg; + + /* Opaque 32-bit system bus Port ID. */ + u32 portid; + + /* Opaque 32-bit handle used for hypervisor calls. */ + u32 devhandle; + + /* Chipset version information. */ + int chip_type; +#define PBM_CHIP_TYPE_SABRE 1 +#define PBM_CHIP_TYPE_PSYCHO 2 +#define PBM_CHIP_TYPE_SCHIZO 3 +#define PBM_CHIP_TYPE_SCHIZO_PLUS 4 +#define PBM_CHIP_TYPE_TOMATILLO 5 + int chip_version; + int chip_revision; + + /* Name used for top-level resources. */ + const char *name; + + /* OBP specific information. */ + struct platform_device *op; + u64 ino_bitmap; + + /* PBM I/O and Memory space resources. */ + struct resource io_space; + struct resource mem_space; + struct resource mem64_space; + struct resource busn; + /* offset */ + resource_size_t io_offset; + resource_size_t mem_offset; + resource_size_t mem64_offset; + + /* Base of PCI Config space, can be per-PBM or shared. */ + unsigned long config_space; + + /* This will be 12 on PCI-E controllers, 8 elsewhere. */ + unsigned long config_space_reg_bits; + + unsigned long pci_afsr; + unsigned long pci_afar; + unsigned long pci_csr; + + /* State of 66MHz capabilities on this PBM. */ + int is_66mhz_capable; + int all_devs_66mhz; + +#ifdef CONFIG_PCI_MSI + /* MSI info. */ + u32 msiq_num; + u32 msiq_ent_count; + u32 msiq_first; + u32 msiq_first_devino; + u32 msiq_rotor; + struct sparc64_msiq_cookie *msiq_irq_cookies; + u32 msi_num; + u32 msi_first; + u32 msi_data_mask; + u32 msix_data_width; + u64 msi32_start; + u64 msi64_start; + u32 msi32_len; + u32 msi64_len; + void *msi_queues; + unsigned long *msi_bitmap; + unsigned int *msi_irq_table; + int (*setup_msi_irq)(unsigned int *irq_p, struct pci_dev *pdev, + struct msi_desc *entry); + void (*teardown_msi_irq)(unsigned int irq, struct pci_dev *pdev); + const struct sparc64_msiq_ops *msi_ops; +#endif /* !(CONFIG_PCI_MSI) */ + + /* This PBM's streaming buffer. */ + struct strbuf stc; + + /* IOMMU state, potentially shared by both PBM segments. */ + struct iommu *iommu; + + /* Now things for the actual PCI bus probes. */ + unsigned int pci_first_busno; + unsigned int pci_last_busno; + struct pci_bus *pci_bus; + struct pci_ops *pci_ops; + + int numa_node; +}; + +extern struct pci_pbm_info *pci_pbm_root; + +extern int pci_num_pbms; + +/* PCI bus scanning and fixup support. */ +void pci_get_pbm_props(struct pci_pbm_info *pbm); +struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, + struct device *parent); +void pci_determine_mem_io_space(struct pci_pbm_info *pbm); + +/* Error reporting support. */ +void pci_scan_for_target_abort(struct pci_pbm_info *, struct pci_bus *); +void pci_scan_for_master_abort(struct pci_pbm_info *, struct pci_bus *); +void pci_scan_for_parity_error(struct pci_pbm_info *, struct pci_bus *); + +/* Configuration space access. */ +void pci_config_read8(u8 *addr, u8 *ret); +void pci_config_read16(u16 *addr, u16 *ret); +void pci_config_read32(u32 *addr, u32 *ret); +void pci_config_write8(u8 *addr, u8 val); +void pci_config_write16(u16 *addr, u16 val); +void pci_config_write32(u32 *addr, u32 val); + +extern struct pci_ops sun4u_pci_ops; +extern struct pci_ops sun4v_pci_ops; + +extern volatile int pci_poke_in_progress; +extern volatile int pci_poke_cpu; +extern volatile int pci_poke_faulted; + +#endif /* !(PCI_IMPL_H) */ diff --git a/arch/sparc/kernel/pci_msi.c b/arch/sparc/kernel/pci_msi.c new file mode 100644 index 000000000..acb2f83a1 --- /dev/null +++ b/arch/sparc/kernel/pci_msi.c @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: GPL-2.0 +/* pci_msi.c: Sparc64 MSI support common layer. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ +#include +#include +#include +#include +#include +#include + +#include "pci_impl.h" + +static irqreturn_t sparc64_msiq_interrupt(int irq, void *cookie) +{ + struct sparc64_msiq_cookie *msiq_cookie = cookie; + struct pci_pbm_info *pbm = msiq_cookie->pbm; + unsigned long msiqid = msiq_cookie->msiqid; + const struct sparc64_msiq_ops *ops; + unsigned long orig_head, head; + int err; + + ops = pbm->msi_ops; + + err = ops->get_head(pbm, msiqid, &head); + if (unlikely(err < 0)) + goto err_get_head; + + orig_head = head; + for (;;) { + unsigned long msi; + + err = ops->dequeue_msi(pbm, msiqid, &head, &msi); + if (likely(err > 0)) { + unsigned int irq; + + irq = pbm->msi_irq_table[msi - pbm->msi_first]; + generic_handle_irq(irq); + } + + if (unlikely(err < 0)) + goto err_dequeue; + + if (err == 0) + break; + } + if (likely(head != orig_head)) { + err = ops->set_head(pbm, msiqid, head); + if (unlikely(err < 0)) + goto err_set_head; + } + return IRQ_HANDLED; + +err_get_head: + printk(KERN_EMERG "MSI: Get head on msiqid[%lu] gives error %d\n", + msiqid, err); + goto err_out; + +err_dequeue: + printk(KERN_EMERG "MSI: Dequeue head[%lu] from msiqid[%lu] " + "gives error %d\n", + head, msiqid, err); + goto err_out; + +err_set_head: + printk(KERN_EMERG "MSI: Set head[%lu] on msiqid[%lu] " + "gives error %d\n", + head, msiqid, err); + goto err_out; + +err_out: + return IRQ_NONE; +} + +static u32 pick_msiq(struct pci_pbm_info *pbm) +{ + static DEFINE_SPINLOCK(rotor_lock); + unsigned long flags; + u32 ret, rotor; + + spin_lock_irqsave(&rotor_lock, flags); + + rotor = pbm->msiq_rotor; + ret = pbm->msiq_first + rotor; + + if (++rotor >= pbm->msiq_num) + rotor = 0; + pbm->msiq_rotor = rotor; + + spin_unlock_irqrestore(&rotor_lock, flags); + + return ret; +} + + +static int alloc_msi(struct pci_pbm_info *pbm) +{ + int i; + + for (i = 0; i < pbm->msi_num; i++) { + if (!test_and_set_bit(i, pbm->msi_bitmap)) + return i + pbm->msi_first; + } + + return -ENOENT; +} + +static void free_msi(struct pci_pbm_info *pbm, int msi_num) +{ + msi_num -= pbm->msi_first; + clear_bit(msi_num, pbm->msi_bitmap); +} + +static struct irq_chip msi_irq = { + .name = "PCI-MSI", + .irq_mask = pci_msi_mask_irq, + .irq_unmask = pci_msi_unmask_irq, + .irq_enable = pci_msi_unmask_irq, + .irq_disable = pci_msi_mask_irq, + /* XXX affinity XXX */ +}; + +static int sparc64_setup_msi_irq(unsigned int *irq_p, + struct pci_dev *pdev, + struct msi_desc *entry) +{ + struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; + const struct sparc64_msiq_ops *ops = pbm->msi_ops; + struct msi_msg msg; + int msi, err; + u32 msiqid; + + *irq_p = irq_alloc(0, 0); + err = -ENOMEM; + if (!*irq_p) + goto out_err; + + irq_set_chip_and_handler_name(*irq_p, &msi_irq, handle_simple_irq, + "MSI"); + + err = alloc_msi(pbm); + if (unlikely(err < 0)) + goto out_irq_free; + + msi = err; + + msiqid = pick_msiq(pbm); + + err = ops->msi_setup(pbm, msiqid, msi, + (entry->pci.msi_attrib.is_64 ? 1 : 0)); + if (err) + goto out_msi_free; + + pbm->msi_irq_table[msi - pbm->msi_first] = *irq_p; + + if (entry->pci.msi_attrib.is_64) { + msg.address_hi = pbm->msi64_start >> 32; + msg.address_lo = pbm->msi64_start & 0xffffffff; + } else { + msg.address_hi = 0; + msg.address_lo = pbm->msi32_start; + } + msg.data = msi; + + irq_set_msi_desc(*irq_p, entry); + pci_write_msi_msg(*irq_p, &msg); + + return 0; + +out_msi_free: + free_msi(pbm, msi); + +out_irq_free: + irq_set_chip(*irq_p, NULL); + irq_free(*irq_p); + *irq_p = 0; + +out_err: + return err; +} + +static void sparc64_teardown_msi_irq(unsigned int irq, + struct pci_dev *pdev) +{ + struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; + const struct sparc64_msiq_ops *ops = pbm->msi_ops; + unsigned int msi_num; + int i, err; + + for (i = 0; i < pbm->msi_num; i++) { + if (pbm->msi_irq_table[i] == irq) + break; + } + if (i >= pbm->msi_num) { + pci_err(pdev, "%s: teardown: No MSI for irq %u\n", pbm->name, + irq); + return; + } + + msi_num = pbm->msi_first + i; + pbm->msi_irq_table[i] = ~0U; + + err = ops->msi_teardown(pbm, msi_num); + if (err) { + pci_err(pdev, "%s: teardown: ops->teardown() on MSI %u, " + "irq %u, gives error %d\n", pbm->name, msi_num, irq, + err); + return; + } + + free_msi(pbm, msi_num); + + irq_set_chip(irq, NULL); + irq_free(irq); +} + +static int msi_bitmap_alloc(struct pci_pbm_info *pbm) +{ + unsigned long size, bits_per_ulong; + + bits_per_ulong = sizeof(unsigned long) * 8; + size = (pbm->msi_num + (bits_per_ulong - 1)) & ~(bits_per_ulong - 1); + size /= 8; + BUG_ON(size % sizeof(unsigned long)); + + pbm->msi_bitmap = kzalloc(size, GFP_KERNEL); + if (!pbm->msi_bitmap) + return -ENOMEM; + + return 0; +} + +static void msi_bitmap_free(struct pci_pbm_info *pbm) +{ + kfree(pbm->msi_bitmap); + pbm->msi_bitmap = NULL; +} + +static int msi_table_alloc(struct pci_pbm_info *pbm) +{ + int size, i; + + size = pbm->msiq_num * sizeof(struct sparc64_msiq_cookie); + pbm->msiq_irq_cookies = kzalloc(size, GFP_KERNEL); + if (!pbm->msiq_irq_cookies) + return -ENOMEM; + + for (i = 0; i < pbm->msiq_num; i++) { + struct sparc64_msiq_cookie *p; + + p = &pbm->msiq_irq_cookies[i]; + p->pbm = pbm; + p->msiqid = pbm->msiq_first + i; + } + + size = pbm->msi_num * sizeof(unsigned int); + pbm->msi_irq_table = kzalloc(size, GFP_KERNEL); + if (!pbm->msi_irq_table) { + kfree(pbm->msiq_irq_cookies); + pbm->msiq_irq_cookies = NULL; + return -ENOMEM; + } + + return 0; +} + +static void msi_table_free(struct pci_pbm_info *pbm) +{ + kfree(pbm->msiq_irq_cookies); + pbm->msiq_irq_cookies = NULL; + + kfree(pbm->msi_irq_table); + pbm->msi_irq_table = NULL; +} + +static int bringup_one_msi_queue(struct pci_pbm_info *pbm, + const struct sparc64_msiq_ops *ops, + unsigned long msiqid, + unsigned long devino) +{ + int irq = ops->msiq_build_irq(pbm, msiqid, devino); + int err, nid; + + if (irq < 0) + return irq; + + nid = pbm->numa_node; + if (nid != -1) { + irq_set_affinity(irq, cpumask_of_node(nid)); + } + err = request_irq(irq, sparc64_msiq_interrupt, 0, + "MSIQ", + &pbm->msiq_irq_cookies[msiqid - pbm->msiq_first]); + if (err) + return err; + + return 0; +} + +static int sparc64_bringup_msi_queues(struct pci_pbm_info *pbm, + const struct sparc64_msiq_ops *ops) +{ + int i; + + for (i = 0; i < pbm->msiq_num; i++) { + unsigned long msiqid = i + pbm->msiq_first; + unsigned long devino = i + pbm->msiq_first_devino; + int err; + + err = bringup_one_msi_queue(pbm, ops, msiqid, devino); + if (err) + return err; + } + + return 0; +} + +void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, + const struct sparc64_msiq_ops *ops) +{ + const u32 *val; + int len; + + val = of_get_property(pbm->op->dev.of_node, "#msi-eqs", &len); + if (!val || len != 4) + goto no_msi; + pbm->msiq_num = *val; + if (pbm->msiq_num) { + const struct msiq_prop { + u32 first_msiq; + u32 num_msiq; + u32 first_devino; + } *mqp; + const struct msi_range_prop { + u32 first_msi; + u32 num_msi; + } *mrng; + const struct addr_range_prop { + u32 msi32_high; + u32 msi32_low; + u32 msi32_len; + u32 msi64_high; + u32 msi64_low; + u32 msi64_len; + } *arng; + + val = of_get_property(pbm->op->dev.of_node, "msi-eq-size", &len); + if (!val || len != 4) + goto no_msi; + + pbm->msiq_ent_count = *val; + + mqp = of_get_property(pbm->op->dev.of_node, + "msi-eq-to-devino", &len); + if (!mqp) + mqp = of_get_property(pbm->op->dev.of_node, + "msi-eq-devino", &len); + if (!mqp || len != sizeof(struct msiq_prop)) + goto no_msi; + + pbm->msiq_first = mqp->first_msiq; + pbm->msiq_first_devino = mqp->first_devino; + + val = of_get_property(pbm->op->dev.of_node, "#msi", &len); + if (!val || len != 4) + goto no_msi; + pbm->msi_num = *val; + + mrng = of_get_property(pbm->op->dev.of_node, "msi-ranges", &len); + if (!mrng || len != sizeof(struct msi_range_prop)) + goto no_msi; + pbm->msi_first = mrng->first_msi; + + val = of_get_property(pbm->op->dev.of_node, "msi-data-mask", &len); + if (!val || len != 4) + goto no_msi; + pbm->msi_data_mask = *val; + + val = of_get_property(pbm->op->dev.of_node, "msix-data-width", &len); + if (!val || len != 4) + goto no_msi; + pbm->msix_data_width = *val; + + arng = of_get_property(pbm->op->dev.of_node, "msi-address-ranges", + &len); + if (!arng || len != sizeof(struct addr_range_prop)) + goto no_msi; + pbm->msi32_start = ((u64)arng->msi32_high << 32) | + (u64) arng->msi32_low; + pbm->msi64_start = ((u64)arng->msi64_high << 32) | + (u64) arng->msi64_low; + pbm->msi32_len = arng->msi32_len; + pbm->msi64_len = arng->msi64_len; + + if (msi_bitmap_alloc(pbm)) + goto no_msi; + + if (msi_table_alloc(pbm)) { + msi_bitmap_free(pbm); + goto no_msi; + } + + if (ops->msiq_alloc(pbm)) { + msi_table_free(pbm); + msi_bitmap_free(pbm); + goto no_msi; + } + + if (sparc64_bringup_msi_queues(pbm, ops)) { + ops->msiq_free(pbm); + msi_table_free(pbm); + msi_bitmap_free(pbm); + goto no_msi; + } + + printk(KERN_INFO "%s: MSI Queue first[%u] num[%u] count[%u] " + "devino[0x%x]\n", + pbm->name, + pbm->msiq_first, pbm->msiq_num, + pbm->msiq_ent_count, + pbm->msiq_first_devino); + printk(KERN_INFO "%s: MSI first[%u] num[%u] mask[0x%x] " + "width[%u]\n", + pbm->name, + pbm->msi_first, pbm->msi_num, pbm->msi_data_mask, + pbm->msix_data_width); + printk(KERN_INFO "%s: MSI addr32[0x%llx:0x%x] " + "addr64[0x%llx:0x%x]\n", + pbm->name, + pbm->msi32_start, pbm->msi32_len, + pbm->msi64_start, pbm->msi64_len); + printk(KERN_INFO "%s: MSI queues at RA [%016lx]\n", + pbm->name, + __pa(pbm->msi_queues)); + + pbm->msi_ops = ops; + pbm->setup_msi_irq = sparc64_setup_msi_irq; + pbm->teardown_msi_irq = sparc64_teardown_msi_irq; + } + return; + +no_msi: + pbm->msiq_num = 0; + printk(KERN_INFO "%s: No MSI support.\n", pbm->name); +} diff --git a/arch/sparc/kernel/pci_psycho.c b/arch/sparc/kernel/pci_psycho.c new file mode 100644 index 000000000..199ab73f8 --- /dev/null +++ b/arch/sparc/kernel/pci_psycho.c @@ -0,0 +1,620 @@ +// SPDX-License-Identifier: GPL-2.0 +/* pci_psycho.c: PSYCHO/U2P specific PCI controller support. + * + * Copyright (C) 1997, 1998, 1999, 2007 David S. Miller (davem@davemloft.net) + * Copyright (C) 1998, 1999 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1999 Jakub Jelinek (jakub@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "pci_impl.h" +#include "iommu_common.h" +#include "psycho_common.h" + +#define DRIVER_NAME "psycho" +#define PFX DRIVER_NAME ": " + +/* Misc. PSYCHO PCI controller register offsets and definitions. */ +#define PSYCHO_CONTROL 0x0010UL +#define PSYCHO_CONTROL_IMPL 0xf000000000000000UL /* Implementation of this PSYCHO*/ +#define PSYCHO_CONTROL_VER 0x0f00000000000000UL /* Version of this PSYCHO */ +#define PSYCHO_CONTROL_MID 0x00f8000000000000UL /* UPA Module ID of PSYCHO */ +#define PSYCHO_CONTROL_IGN 0x0007c00000000000UL /* Interrupt Group Number */ +#define PSYCHO_CONTROL_RESV 0x00003ffffffffff0UL /* Reserved */ +#define PSYCHO_CONTROL_APCKEN 0x0000000000000008UL /* Address Parity Check Enable */ +#define PSYCHO_CONTROL_APERR 0x0000000000000004UL /* Incoming System Addr Parerr */ +#define PSYCHO_CONTROL_IAP 0x0000000000000002UL /* Invert UPA Parity */ +#define PSYCHO_CONTROL_MODE 0x0000000000000001UL /* PSYCHO clock mode */ +#define PSYCHO_PCIA_CTRL 0x2000UL +#define PSYCHO_PCIB_CTRL 0x4000UL +#define PSYCHO_PCICTRL_RESV1 0xfffffff000000000UL /* Reserved */ +#define PSYCHO_PCICTRL_SBH_ERR 0x0000000800000000UL /* Streaming byte hole error */ +#define PSYCHO_PCICTRL_SERR 0x0000000400000000UL /* SERR signal asserted */ +#define PSYCHO_PCICTRL_SPEED 0x0000000200000000UL /* PCI speed (1 is U2P clock) */ +#define PSYCHO_PCICTRL_RESV2 0x00000001ffc00000UL /* Reserved */ +#define PSYCHO_PCICTRL_ARB_PARK 0x0000000000200000UL /* PCI arbitration parking */ +#define PSYCHO_PCICTRL_RESV3 0x00000000001ff800UL /* Reserved */ +#define PSYCHO_PCICTRL_SBH_INT 0x0000000000000400UL /* Streaming byte hole int enab */ +#define PSYCHO_PCICTRL_WEN 0x0000000000000200UL /* Power Mgmt Wake Enable */ +#define PSYCHO_PCICTRL_EEN 0x0000000000000100UL /* PCI Error Interrupt Enable */ +#define PSYCHO_PCICTRL_RESV4 0x00000000000000c0UL /* Reserved */ +#define PSYCHO_PCICTRL_AEN 0x000000000000003fUL /* PCI DVMA Arbitration Enable */ + +/* PSYCHO error handling support. */ + +/* Helper function of IOMMU error checking, which checks out + * the state of the streaming buffers. The IOMMU lock is + * held when this is called. + * + * For the PCI error case we know which PBM (and thus which + * streaming buffer) caused the error, but for the uncorrectable + * error case we do not. So we always check both streaming caches. + */ +#define PSYCHO_STRBUF_CONTROL_A 0x2800UL +#define PSYCHO_STRBUF_CONTROL_B 0x4800UL +#define PSYCHO_STRBUF_CTRL_LPTR 0x00000000000000f0UL /* LRU Lock Pointer */ +#define PSYCHO_STRBUF_CTRL_LENAB 0x0000000000000008UL /* LRU Lock Enable */ +#define PSYCHO_STRBUF_CTRL_RRDIS 0x0000000000000004UL /* Rerun Disable */ +#define PSYCHO_STRBUF_CTRL_DENAB 0x0000000000000002UL /* Diagnostic Mode Enable */ +#define PSYCHO_STRBUF_CTRL_ENAB 0x0000000000000001UL /* Streaming Buffer Enable */ +#define PSYCHO_STRBUF_FLUSH_A 0x2808UL +#define PSYCHO_STRBUF_FLUSH_B 0x4808UL +#define PSYCHO_STRBUF_FSYNC_A 0x2810UL +#define PSYCHO_STRBUF_FSYNC_B 0x4810UL +#define PSYCHO_STC_DATA_A 0xb000UL +#define PSYCHO_STC_DATA_B 0xc000UL +#define PSYCHO_STC_ERR_A 0xb400UL +#define PSYCHO_STC_ERR_B 0xc400UL +#define PSYCHO_STC_TAG_A 0xb800UL +#define PSYCHO_STC_TAG_B 0xc800UL +#define PSYCHO_STC_LINE_A 0xb900UL +#define PSYCHO_STC_LINE_B 0xc900UL + +/* When an Uncorrectable Error or a PCI Error happens, we + * interrogate the IOMMU state to see if it is the cause. + */ +#define PSYCHO_IOMMU_CONTROL 0x0200UL +#define PSYCHO_IOMMU_CTRL_RESV 0xfffffffff9000000UL /* Reserved */ +#define PSYCHO_IOMMU_CTRL_XLTESTAT 0x0000000006000000UL /* Translation Error Status */ +#define PSYCHO_IOMMU_CTRL_XLTEERR 0x0000000001000000UL /* Translation Error encountered */ +#define PSYCHO_IOMMU_CTRL_LCKEN 0x0000000000800000UL /* Enable translation locking */ +#define PSYCHO_IOMMU_CTRL_LCKPTR 0x0000000000780000UL /* Translation lock pointer */ +#define PSYCHO_IOMMU_CTRL_TSBSZ 0x0000000000070000UL /* TSB Size */ +#define PSYCHO_IOMMU_TSBSZ_1K 0x0000000000000000UL /* TSB Table 1024 8-byte entries */ +#define PSYCHO_IOMMU_TSBSZ_2K 0x0000000000010000UL /* TSB Table 2048 8-byte entries */ +#define PSYCHO_IOMMU_TSBSZ_4K 0x0000000000020000UL /* TSB Table 4096 8-byte entries */ +#define PSYCHO_IOMMU_TSBSZ_8K 0x0000000000030000UL /* TSB Table 8192 8-byte entries */ +#define PSYCHO_IOMMU_TSBSZ_16K 0x0000000000040000UL /* TSB Table 16k 8-byte entries */ +#define PSYCHO_IOMMU_TSBSZ_32K 0x0000000000050000UL /* TSB Table 32k 8-byte entries */ +#define PSYCHO_IOMMU_TSBSZ_64K 0x0000000000060000UL /* TSB Table 64k 8-byte entries */ +#define PSYCHO_IOMMU_TSBSZ_128K 0x0000000000070000UL /* TSB Table 128k 8-byte entries */ +#define PSYCHO_IOMMU_CTRL_RESV2 0x000000000000fff8UL /* Reserved */ +#define PSYCHO_IOMMU_CTRL_TBWSZ 0x0000000000000004UL /* Assumed page size, 0=8k 1=64k */ +#define PSYCHO_IOMMU_CTRL_DENAB 0x0000000000000002UL /* Diagnostic mode enable */ +#define PSYCHO_IOMMU_CTRL_ENAB 0x0000000000000001UL /* IOMMU Enable */ +#define PSYCHO_IOMMU_TSBBASE 0x0208UL +#define PSYCHO_IOMMU_FLUSH 0x0210UL +#define PSYCHO_IOMMU_TAG 0xa580UL +#define PSYCHO_IOMMU_DATA 0xa600UL + +/* Uncorrectable Errors. Cause of the error and the address are + * recorded in the UE_AFSR and UE_AFAR of PSYCHO. They are errors + * relating to UPA interface transactions. + */ +#define PSYCHO_UE_AFSR 0x0030UL +#define PSYCHO_UEAFSR_PPIO 0x8000000000000000UL /* Primary PIO is cause */ +#define PSYCHO_UEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read is cause */ +#define PSYCHO_UEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write is cause */ +#define PSYCHO_UEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO is cause */ +#define PSYCHO_UEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read is cause */ +#define PSYCHO_UEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write is cause*/ +#define PSYCHO_UEAFSR_RESV1 0x03ff000000000000UL /* Reserved */ +#define PSYCHO_UEAFSR_BMSK 0x0000ffff00000000UL /* Bytemask of failed transfer */ +#define PSYCHO_UEAFSR_DOFF 0x00000000e0000000UL /* Doubleword Offset */ +#define PSYCHO_UEAFSR_MID 0x000000001f000000UL /* UPA MID causing the fault */ +#define PSYCHO_UEAFSR_BLK 0x0000000000800000UL /* Trans was block operation */ +#define PSYCHO_UEAFSR_RESV2 0x00000000007fffffUL /* Reserved */ +#define PSYCHO_UE_AFAR 0x0038UL + +static irqreturn_t psycho_ue_intr(int irq, void *dev_id) +{ + struct pci_pbm_info *pbm = dev_id; + unsigned long afsr_reg = pbm->controller_regs + PSYCHO_UE_AFSR; + unsigned long afar_reg = pbm->controller_regs + PSYCHO_UE_AFAR; + unsigned long afsr, afar, error_bits; + int reported; + + /* Latch uncorrectable error status. */ + afar = upa_readq(afar_reg); + afsr = upa_readq(afsr_reg); + + /* Clear the primary/secondary error status bits. */ + error_bits = afsr & + (PSYCHO_UEAFSR_PPIO | PSYCHO_UEAFSR_PDRD | PSYCHO_UEAFSR_PDWR | + PSYCHO_UEAFSR_SPIO | PSYCHO_UEAFSR_SDRD | PSYCHO_UEAFSR_SDWR); + if (!error_bits) + return IRQ_NONE; + upa_writeq(error_bits, afsr_reg); + + /* Log the error. */ + printk("%s: Uncorrectable Error, primary error type[%s]\n", + pbm->name, + (((error_bits & PSYCHO_UEAFSR_PPIO) ? + "PIO" : + ((error_bits & PSYCHO_UEAFSR_PDRD) ? + "DMA Read" : + ((error_bits & PSYCHO_UEAFSR_PDWR) ? + "DMA Write" : "???"))))); + printk("%s: bytemask[%04lx] dword_offset[%lx] UPA_MID[%02lx] was_block(%d)\n", + pbm->name, + (afsr & PSYCHO_UEAFSR_BMSK) >> 32UL, + (afsr & PSYCHO_UEAFSR_DOFF) >> 29UL, + (afsr & PSYCHO_UEAFSR_MID) >> 24UL, + ((afsr & PSYCHO_UEAFSR_BLK) ? 1 : 0)); + printk("%s: UE AFAR [%016lx]\n", pbm->name, afar); + printk("%s: UE Secondary errors [", pbm->name); + reported = 0; + if (afsr & PSYCHO_UEAFSR_SPIO) { + reported++; + printk("(PIO)"); + } + if (afsr & PSYCHO_UEAFSR_SDRD) { + reported++; + printk("(DMA Read)"); + } + if (afsr & PSYCHO_UEAFSR_SDWR) { + reported++; + printk("(DMA Write)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + /* Interrogate both IOMMUs for error status. */ + psycho_check_iommu_error(pbm, afsr, afar, UE_ERR); + if (pbm->sibling) + psycho_check_iommu_error(pbm->sibling, afsr, afar, UE_ERR); + + return IRQ_HANDLED; +} + +/* Correctable Errors. */ +#define PSYCHO_CE_AFSR 0x0040UL +#define PSYCHO_CEAFSR_PPIO 0x8000000000000000UL /* Primary PIO is cause */ +#define PSYCHO_CEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read is cause */ +#define PSYCHO_CEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write is cause */ +#define PSYCHO_CEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO is cause */ +#define PSYCHO_CEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read is cause */ +#define PSYCHO_CEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write is cause*/ +#define PSYCHO_CEAFSR_RESV1 0x0300000000000000UL /* Reserved */ +#define PSYCHO_CEAFSR_ESYND 0x00ff000000000000UL /* Syndrome Bits */ +#define PSYCHO_CEAFSR_BMSK 0x0000ffff00000000UL /* Bytemask of failed transfer */ +#define PSYCHO_CEAFSR_DOFF 0x00000000e0000000UL /* Double Offset */ +#define PSYCHO_CEAFSR_MID 0x000000001f000000UL /* UPA MID causing the fault */ +#define PSYCHO_CEAFSR_BLK 0x0000000000800000UL /* Trans was block operation */ +#define PSYCHO_CEAFSR_RESV2 0x00000000007fffffUL /* Reserved */ +#define PSYCHO_CE_AFAR 0x0040UL + +static irqreturn_t psycho_ce_intr(int irq, void *dev_id) +{ + struct pci_pbm_info *pbm = dev_id; + unsigned long afsr_reg = pbm->controller_regs + PSYCHO_CE_AFSR; + unsigned long afar_reg = pbm->controller_regs + PSYCHO_CE_AFAR; + unsigned long afsr, afar, error_bits; + int reported; + + /* Latch error status. */ + afar = upa_readq(afar_reg); + afsr = upa_readq(afsr_reg); + + /* Clear primary/secondary error status bits. */ + error_bits = afsr & + (PSYCHO_CEAFSR_PPIO | PSYCHO_CEAFSR_PDRD | PSYCHO_CEAFSR_PDWR | + PSYCHO_CEAFSR_SPIO | PSYCHO_CEAFSR_SDRD | PSYCHO_CEAFSR_SDWR); + if (!error_bits) + return IRQ_NONE; + upa_writeq(error_bits, afsr_reg); + + /* Log the error. */ + printk("%s: Correctable Error, primary error type[%s]\n", + pbm->name, + (((error_bits & PSYCHO_CEAFSR_PPIO) ? + "PIO" : + ((error_bits & PSYCHO_CEAFSR_PDRD) ? + "DMA Read" : + ((error_bits & PSYCHO_CEAFSR_PDWR) ? + "DMA Write" : "???"))))); + + /* XXX Use syndrome and afar to print out module string just like + * XXX UDB CE trap handler does... -DaveM + */ + printk("%s: syndrome[%02lx] bytemask[%04lx] dword_offset[%lx] " + "UPA_MID[%02lx] was_block(%d)\n", + pbm->name, + (afsr & PSYCHO_CEAFSR_ESYND) >> 48UL, + (afsr & PSYCHO_CEAFSR_BMSK) >> 32UL, + (afsr & PSYCHO_CEAFSR_DOFF) >> 29UL, + (afsr & PSYCHO_CEAFSR_MID) >> 24UL, + ((afsr & PSYCHO_CEAFSR_BLK) ? 1 : 0)); + printk("%s: CE AFAR [%016lx]\n", pbm->name, afar); + printk("%s: CE Secondary errors [", pbm->name); + reported = 0; + if (afsr & PSYCHO_CEAFSR_SPIO) { + reported++; + printk("(PIO)"); + } + if (afsr & PSYCHO_CEAFSR_SDRD) { + reported++; + printk("(DMA Read)"); + } + if (afsr & PSYCHO_CEAFSR_SDWR) { + reported++; + printk("(DMA Write)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + return IRQ_HANDLED; +} + +/* PCI Errors. They are signalled by the PCI bus module since they + * are associated with a specific bus segment. + */ +#define PSYCHO_PCI_AFSR_A 0x2010UL +#define PSYCHO_PCI_AFSR_B 0x4010UL +#define PSYCHO_PCI_AFAR_A 0x2018UL +#define PSYCHO_PCI_AFAR_B 0x4018UL + +/* XXX What about PowerFail/PowerManagement??? -DaveM */ +#define PSYCHO_ECC_CTRL 0x0020 +#define PSYCHO_ECCCTRL_EE 0x8000000000000000UL /* Enable ECC Checking */ +#define PSYCHO_ECCCTRL_UE 0x4000000000000000UL /* Enable UE Interrupts */ +#define PSYCHO_ECCCTRL_CE 0x2000000000000000UL /* Enable CE INterrupts */ +static void psycho_register_error_handlers(struct pci_pbm_info *pbm) +{ + struct platform_device *op = of_find_device_by_node(pbm->op->dev.of_node); + unsigned long base = pbm->controller_regs; + u64 tmp; + int err; + + if (!op) + return; + + /* Psycho interrupt property order is: + * 0: PCIERR INO for this PBM + * 1: UE ERR + * 2: CE ERR + * 3: POWER FAIL + * 4: SPARE HARDWARE + * 5: POWER MANAGEMENT + */ + + if (op->archdata.num_irqs < 6) + return; + + /* We really mean to ignore the return result here. Two + * PCI controller share the same interrupt numbers and + * drive the same front-end hardware. + */ + err = request_irq(op->archdata.irqs[1], psycho_ue_intr, IRQF_SHARED, + "PSYCHO_UE", pbm); + err = request_irq(op->archdata.irqs[2], psycho_ce_intr, IRQF_SHARED, + "PSYCHO_CE", pbm); + + /* This one, however, ought not to fail. We can just warn + * about it since the system can still operate properly even + * if this fails. + */ + err = request_irq(op->archdata.irqs[0], psycho_pcierr_intr, IRQF_SHARED, + "PSYCHO_PCIERR", pbm); + if (err) + printk(KERN_WARNING "%s: Could not register PCIERR, " + "err=%d\n", pbm->name, err); + + /* Enable UE and CE interrupts for controller. */ + upa_writeq((PSYCHO_ECCCTRL_EE | + PSYCHO_ECCCTRL_UE | + PSYCHO_ECCCTRL_CE), base + PSYCHO_ECC_CTRL); + + /* Enable PCI Error interrupts and clear error + * bits for each PBM. + */ + tmp = upa_readq(base + PSYCHO_PCIA_CTRL); + tmp |= (PSYCHO_PCICTRL_SERR | + PSYCHO_PCICTRL_SBH_ERR | + PSYCHO_PCICTRL_EEN); + tmp &= ~(PSYCHO_PCICTRL_SBH_INT); + upa_writeq(tmp, base + PSYCHO_PCIA_CTRL); + + tmp = upa_readq(base + PSYCHO_PCIB_CTRL); + tmp |= (PSYCHO_PCICTRL_SERR | + PSYCHO_PCICTRL_SBH_ERR | + PSYCHO_PCICTRL_EEN); + tmp &= ~(PSYCHO_PCICTRL_SBH_INT); + upa_writeq(tmp, base + PSYCHO_PCIB_CTRL); +} + +/* PSYCHO boot time probing and initialization. */ +static void pbm_config_busmastering(struct pci_pbm_info *pbm) +{ + u8 *addr; + + /* Set cache-line size to 64 bytes, this is actually + * a nop but I do it for completeness. + */ + addr = psycho_pci_config_mkaddr(pbm, pbm->pci_first_busno, + 0, PCI_CACHE_LINE_SIZE); + pci_config_write8(addr, 64 / sizeof(u32)); + + /* Set PBM latency timer to 64 PCI clocks. */ + addr = psycho_pci_config_mkaddr(pbm, pbm->pci_first_busno, + 0, PCI_LATENCY_TIMER); + pci_config_write8(addr, 64); +} + +static void psycho_scan_bus(struct pci_pbm_info *pbm, + struct device *parent) +{ + pbm_config_busmastering(pbm); + pbm->is_66mhz_capable = 0; + pbm->pci_bus = pci_scan_one_pbm(pbm, parent); + + /* After the PCI bus scan is complete, we can register + * the error interrupt handlers. + */ + psycho_register_error_handlers(pbm); +} + +#define PSYCHO_IRQ_RETRY 0x1a00UL +#define PSYCHO_PCIA_DIAG 0x2020UL +#define PSYCHO_PCIB_DIAG 0x4020UL +#define PSYCHO_PCIDIAG_RESV 0xffffffffffffff80UL /* Reserved */ +#define PSYCHO_PCIDIAG_DRETRY 0x0000000000000040UL /* Disable retry limit */ +#define PSYCHO_PCIDIAG_DISYNC 0x0000000000000020UL /* Disable DMA wr / irq sync */ +#define PSYCHO_PCIDIAG_DDWSYNC 0x0000000000000010UL /* Disable DMA wr / PIO rd sync */ +#define PSYCHO_PCIDIAG_IDDPAR 0x0000000000000008UL /* Invert DMA data parity */ +#define PSYCHO_PCIDIAG_IPDPAR 0x0000000000000004UL /* Invert PIO data parity */ +#define PSYCHO_PCIDIAG_IPAPAR 0x0000000000000002UL /* Invert PIO address parity */ +#define PSYCHO_PCIDIAG_LPBACK 0x0000000000000001UL /* Enable loopback mode */ + +static void psycho_controller_hwinit(struct pci_pbm_info *pbm) +{ + u64 tmp; + + upa_writeq(5, pbm->controller_regs + PSYCHO_IRQ_RETRY); + + /* Enable arbiter for all PCI slots. */ + tmp = upa_readq(pbm->controller_regs + PSYCHO_PCIA_CTRL); + tmp |= PSYCHO_PCICTRL_AEN; + upa_writeq(tmp, pbm->controller_regs + PSYCHO_PCIA_CTRL); + + tmp = upa_readq(pbm->controller_regs + PSYCHO_PCIB_CTRL); + tmp |= PSYCHO_PCICTRL_AEN; + upa_writeq(tmp, pbm->controller_regs + PSYCHO_PCIB_CTRL); + + /* Disable DMA write / PIO read synchronization on + * both PCI bus segments. + * [ U2P Erratum 1243770, STP2223BGA data sheet ] + */ + tmp = upa_readq(pbm->controller_regs + PSYCHO_PCIA_DIAG); + tmp |= PSYCHO_PCIDIAG_DDWSYNC; + upa_writeq(tmp, pbm->controller_regs + PSYCHO_PCIA_DIAG); + + tmp = upa_readq(pbm->controller_regs + PSYCHO_PCIB_DIAG); + tmp |= PSYCHO_PCIDIAG_DDWSYNC; + upa_writeq(tmp, pbm->controller_regs + PSYCHO_PCIB_DIAG); +} + +static void psycho_pbm_strbuf_init(struct pci_pbm_info *pbm, + int is_pbm_a) +{ + unsigned long base = pbm->controller_regs; + u64 control; + + if (is_pbm_a) { + pbm->stc.strbuf_control = base + PSYCHO_STRBUF_CONTROL_A; + pbm->stc.strbuf_pflush = base + PSYCHO_STRBUF_FLUSH_A; + pbm->stc.strbuf_fsync = base + PSYCHO_STRBUF_FSYNC_A; + pbm->stc.strbuf_err_stat = base + PSYCHO_STC_ERR_A; + pbm->stc.strbuf_tag_diag = base + PSYCHO_STC_TAG_A; + pbm->stc.strbuf_line_diag= base + PSYCHO_STC_LINE_A; + } else { + pbm->stc.strbuf_control = base + PSYCHO_STRBUF_CONTROL_B; + pbm->stc.strbuf_pflush = base + PSYCHO_STRBUF_FLUSH_B; + pbm->stc.strbuf_fsync = base + PSYCHO_STRBUF_FSYNC_B; + pbm->stc.strbuf_err_stat = base + PSYCHO_STC_ERR_B; + pbm->stc.strbuf_tag_diag = base + PSYCHO_STC_TAG_B; + pbm->stc.strbuf_line_diag= base + PSYCHO_STC_LINE_B; + } + /* PSYCHO's streaming buffer lacks ctx flushing. */ + pbm->stc.strbuf_ctxflush = 0; + pbm->stc.strbuf_ctxmatch_base = 0; + + pbm->stc.strbuf_flushflag = (volatile unsigned long *) + ((((unsigned long)&pbm->stc.__flushflag_buf[0]) + + 63UL) + & ~63UL); + pbm->stc.strbuf_flushflag_pa = (unsigned long) + __pa(pbm->stc.strbuf_flushflag); + + /* Enable the streaming buffer. We have to be careful + * just in case OBP left it with LRU locking enabled. + * + * It is possible to control if PBM will be rerun on + * line misses. Currently I just retain whatever setting + * OBP left us with. All checks so far show it having + * a value of zero. + */ +#undef PSYCHO_STRBUF_RERUN_ENABLE +#undef PSYCHO_STRBUF_RERUN_DISABLE + control = upa_readq(pbm->stc.strbuf_control); + control |= PSYCHO_STRBUF_CTRL_ENAB; + control &= ~(PSYCHO_STRBUF_CTRL_LENAB | PSYCHO_STRBUF_CTRL_LPTR); +#ifdef PSYCHO_STRBUF_RERUN_ENABLE + control &= ~(PSYCHO_STRBUF_CTRL_RRDIS); +#else +#ifdef PSYCHO_STRBUF_RERUN_DISABLE + control |= PSYCHO_STRBUF_CTRL_RRDIS; +#endif +#endif + upa_writeq(control, pbm->stc.strbuf_control); + + pbm->stc.strbuf_enabled = 1; +} + +#define PSYCHO_IOSPACE_A 0x002000000UL +#define PSYCHO_IOSPACE_B 0x002010000UL +#define PSYCHO_IOSPACE_SIZE 0x00000ffffUL +#define PSYCHO_MEMSPACE_A 0x100000000UL +#define PSYCHO_MEMSPACE_B 0x180000000UL +#define PSYCHO_MEMSPACE_SIZE 0x07fffffffUL + +static void psycho_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op, int is_pbm_a) +{ + psycho_pbm_init_common(pbm, op, "PSYCHO", PBM_CHIP_TYPE_PSYCHO); + psycho_pbm_strbuf_init(pbm, is_pbm_a); + psycho_scan_bus(pbm, &op->dev); +} + +static struct pci_pbm_info *psycho_find_sibling(u32 upa_portid) +{ + struct pci_pbm_info *pbm; + + for (pbm = pci_pbm_root; pbm; pbm = pbm->next) { + if (pbm->portid == upa_portid) + return pbm; + } + return NULL; +} + +#define PSYCHO_CONFIGSPACE 0x001000000UL + +static int psycho_probe(struct platform_device *op) +{ + const struct linux_prom64_registers *pr_regs; + struct device_node *dp = op->dev.of_node; + struct pci_pbm_info *pbm; + struct iommu *iommu; + int is_pbm_a, err; + u32 upa_portid; + + upa_portid = of_getintprop_default(dp, "upa-portid", 0xff); + + err = -ENOMEM; + pbm = kzalloc_obj(*pbm); + if (!pbm) { + printk(KERN_ERR PFX "Cannot allocate pci_pbm_info.\n"); + goto out_err; + } + + pbm->sibling = psycho_find_sibling(upa_portid); + if (pbm->sibling) { + iommu = pbm->sibling->iommu; + } else { + iommu = kzalloc_obj(struct iommu); + if (!iommu) { + printk(KERN_ERR PFX "Cannot allocate PBM iommu.\n"); + goto out_free_controller; + } + } + + pbm->iommu = iommu; + pbm->portid = upa_portid; + + pr_regs = of_get_property(dp, "reg", NULL); + err = -ENODEV; + if (!pr_regs) { + printk(KERN_ERR PFX "No reg property.\n"); + goto out_free_iommu; + } + + is_pbm_a = ((pr_regs[0].phys_addr & 0x6000) == 0x2000); + + pbm->controller_regs = pr_regs[2].phys_addr; + pbm->config_space = (pr_regs[2].phys_addr + PSYCHO_CONFIGSPACE); + + if (is_pbm_a) { + pbm->pci_afsr = pbm->controller_regs + PSYCHO_PCI_AFSR_A; + pbm->pci_afar = pbm->controller_regs + PSYCHO_PCI_AFAR_A; + pbm->pci_csr = pbm->controller_regs + PSYCHO_PCIA_CTRL; + } else { + pbm->pci_afsr = pbm->controller_regs + PSYCHO_PCI_AFSR_B; + pbm->pci_afar = pbm->controller_regs + PSYCHO_PCI_AFAR_B; + pbm->pci_csr = pbm->controller_regs + PSYCHO_PCIB_CTRL; + } + + psycho_controller_hwinit(pbm); + if (!pbm->sibling) { + err = psycho_iommu_init(pbm, 128, 0xc0000000, + 0xffffffff, PSYCHO_CONTROL); + if (err) + goto out_free_iommu; + + /* If necessary, hook us up for starfire IRQ translations. */ + if (this_is_starfire) + starfire_hookup(pbm->portid); + } + + psycho_pbm_init(pbm, op, is_pbm_a); + + pbm->next = pci_pbm_root; + pci_pbm_root = pbm; + + if (pbm->sibling) + pbm->sibling->sibling = pbm; + + dev_set_drvdata(&op->dev, pbm); + + return 0; + +out_free_iommu: + if (!pbm->sibling) + kfree(pbm->iommu); + +out_free_controller: + kfree(pbm); + +out_err: + return err; +} + +static const struct of_device_id psycho_match[] = { + { + .name = "pci", + .compatible = "pci108e,8000", + }, + {}, +}; + +static struct platform_driver psycho_driver = { + .driver = { + .name = DRIVER_NAME, + .of_match_table = psycho_match, + }, + .probe = psycho_probe, +}; + +static int __init psycho_init(void) +{ + return platform_driver_register(&psycho_driver); +} + +subsys_initcall(psycho_init); diff --git a/arch/sparc/kernel/pci_sabre.c b/arch/sparc/kernel/pci_sabre.c new file mode 100644 index 000000000..1109d5b34 --- /dev/null +++ b/arch/sparc/kernel/pci_sabre.c @@ -0,0 +1,615 @@ +// SPDX-License-Identifier: GPL-2.0 +/* pci_sabre.c: Sabre specific PCI controller support. + * + * Copyright (C) 1997, 1998, 1999, 2007 David S. Miller (davem@davemloft.net) + * Copyright (C) 1998, 1999 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1999 Jakub Jelinek (jakub@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "pci_impl.h" +#include "iommu_common.h" +#include "psycho_common.h" + +#define DRIVER_NAME "sabre" +#define PFX DRIVER_NAME ": " + +/* SABRE PCI controller register offsets and definitions. */ +#define SABRE_UE_AFSR 0x0030UL +#define SABRE_UEAFSR_PDRD 0x4000000000000000UL /* Primary PCI DMA Read */ +#define SABRE_UEAFSR_PDWR 0x2000000000000000UL /* Primary PCI DMA Write */ +#define SABRE_UEAFSR_SDRD 0x0800000000000000UL /* Secondary PCI DMA Read */ +#define SABRE_UEAFSR_SDWR 0x0400000000000000UL /* Secondary PCI DMA Write */ +#define SABRE_UEAFSR_SDTE 0x0200000000000000UL /* Secondary DMA Translation Error */ +#define SABRE_UEAFSR_PDTE 0x0100000000000000UL /* Primary DMA Translation Error */ +#define SABRE_UEAFSR_BMSK 0x0000ffff00000000UL /* Bytemask */ +#define SABRE_UEAFSR_OFF 0x00000000e0000000UL /* Offset (AFAR bits [5:3] */ +#define SABRE_UEAFSR_BLK 0x0000000000800000UL /* Was block operation */ +#define SABRE_UECE_AFAR 0x0038UL +#define SABRE_CE_AFSR 0x0040UL +#define SABRE_CEAFSR_PDRD 0x4000000000000000UL /* Primary PCI DMA Read */ +#define SABRE_CEAFSR_PDWR 0x2000000000000000UL /* Primary PCI DMA Write */ +#define SABRE_CEAFSR_SDRD 0x0800000000000000UL /* Secondary PCI DMA Read */ +#define SABRE_CEAFSR_SDWR 0x0400000000000000UL /* Secondary PCI DMA Write */ +#define SABRE_CEAFSR_ESYND 0x00ff000000000000UL /* ECC Syndrome */ +#define SABRE_CEAFSR_BMSK 0x0000ffff00000000UL /* Bytemask */ +#define SABRE_CEAFSR_OFF 0x00000000e0000000UL /* Offset */ +#define SABRE_CEAFSR_BLK 0x0000000000800000UL /* Was block operation */ +#define SABRE_UECE_AFAR_ALIAS 0x0048UL /* Aliases to 0x0038 */ +#define SABRE_IOMMU_CONTROL 0x0200UL +#define SABRE_IOMMUCTRL_ERRSTS 0x0000000006000000UL /* Error status bits */ +#define SABRE_IOMMUCTRL_ERR 0x0000000001000000UL /* Error present in IOTLB */ +#define SABRE_IOMMUCTRL_LCKEN 0x0000000000800000UL /* IOTLB lock enable */ +#define SABRE_IOMMUCTRL_LCKPTR 0x0000000000780000UL /* IOTLB lock pointer */ +#define SABRE_IOMMUCTRL_TSBSZ 0x0000000000070000UL /* TSB Size */ +#define SABRE_IOMMU_TSBSZ_1K 0x0000000000000000 +#define SABRE_IOMMU_TSBSZ_2K 0x0000000000010000 +#define SABRE_IOMMU_TSBSZ_4K 0x0000000000020000 +#define SABRE_IOMMU_TSBSZ_8K 0x0000000000030000 +#define SABRE_IOMMU_TSBSZ_16K 0x0000000000040000 +#define SABRE_IOMMU_TSBSZ_32K 0x0000000000050000 +#define SABRE_IOMMU_TSBSZ_64K 0x0000000000060000 +#define SABRE_IOMMU_TSBSZ_128K 0x0000000000070000 +#define SABRE_IOMMUCTRL_TBWSZ 0x0000000000000004UL /* TSB assumed page size */ +#define SABRE_IOMMUCTRL_DENAB 0x0000000000000002UL /* Diagnostic Mode Enable */ +#define SABRE_IOMMUCTRL_ENAB 0x0000000000000001UL /* IOMMU Enable */ +#define SABRE_IOMMU_TSBBASE 0x0208UL +#define SABRE_IOMMU_FLUSH 0x0210UL +#define SABRE_IMAP_A_SLOT0 0x0c00UL +#define SABRE_IMAP_B_SLOT0 0x0c20UL +#define SABRE_IMAP_SCSI 0x1000UL +#define SABRE_IMAP_ETH 0x1008UL +#define SABRE_IMAP_BPP 0x1010UL +#define SABRE_IMAP_AU_REC 0x1018UL +#define SABRE_IMAP_AU_PLAY 0x1020UL +#define SABRE_IMAP_PFAIL 0x1028UL +#define SABRE_IMAP_KMS 0x1030UL +#define SABRE_IMAP_FLPY 0x1038UL +#define SABRE_IMAP_SHW 0x1040UL +#define SABRE_IMAP_KBD 0x1048UL +#define SABRE_IMAP_MS 0x1050UL +#define SABRE_IMAP_SER 0x1058UL +#define SABRE_IMAP_UE 0x1070UL +#define SABRE_IMAP_CE 0x1078UL +#define SABRE_IMAP_PCIERR 0x1080UL +#define SABRE_IMAP_GFX 0x1098UL +#define SABRE_IMAP_EUPA 0x10a0UL +#define SABRE_ICLR_A_SLOT0 0x1400UL +#define SABRE_ICLR_B_SLOT0 0x1480UL +#define SABRE_ICLR_SCSI 0x1800UL +#define SABRE_ICLR_ETH 0x1808UL +#define SABRE_ICLR_BPP 0x1810UL +#define SABRE_ICLR_AU_REC 0x1818UL +#define SABRE_ICLR_AU_PLAY 0x1820UL +#define SABRE_ICLR_PFAIL 0x1828UL +#define SABRE_ICLR_KMS 0x1830UL +#define SABRE_ICLR_FLPY 0x1838UL +#define SABRE_ICLR_SHW 0x1840UL +#define SABRE_ICLR_KBD 0x1848UL +#define SABRE_ICLR_MS 0x1850UL +#define SABRE_ICLR_SER 0x1858UL +#define SABRE_ICLR_UE 0x1870UL +#define SABRE_ICLR_CE 0x1878UL +#define SABRE_ICLR_PCIERR 0x1880UL +#define SABRE_WRSYNC 0x1c20UL +#define SABRE_PCICTRL 0x2000UL +#define SABRE_PCICTRL_MRLEN 0x0000001000000000UL /* Use MemoryReadLine for block loads/stores */ +#define SABRE_PCICTRL_SERR 0x0000000400000000UL /* Set when SERR asserted on PCI bus */ +#define SABRE_PCICTRL_ARBPARK 0x0000000000200000UL /* Bus Parking 0=Ultra-IIi 1=prev-bus-owner */ +#define SABRE_PCICTRL_CPUPRIO 0x0000000000100000UL /* Ultra-IIi granted every other bus cycle */ +#define SABRE_PCICTRL_ARBPRIO 0x00000000000f0000UL /* Slot which is granted every other bus cycle */ +#define SABRE_PCICTRL_ERREN 0x0000000000000100UL /* PCI Error Interrupt Enable */ +#define SABRE_PCICTRL_RTRYWE 0x0000000000000080UL /* DMA Flow Control 0=wait-if-possible 1=retry */ +#define SABRE_PCICTRL_AEN 0x000000000000000fUL /* Slot PCI arbitration enables */ +#define SABRE_PIOAFSR 0x2010UL +#define SABRE_PIOAFSR_PMA 0x8000000000000000UL /* Primary Master Abort */ +#define SABRE_PIOAFSR_PTA 0x4000000000000000UL /* Primary Target Abort */ +#define SABRE_PIOAFSR_PRTRY 0x2000000000000000UL /* Primary Excessive Retries */ +#define SABRE_PIOAFSR_PPERR 0x1000000000000000UL /* Primary Parity Error */ +#define SABRE_PIOAFSR_SMA 0x0800000000000000UL /* Secondary Master Abort */ +#define SABRE_PIOAFSR_STA 0x0400000000000000UL /* Secondary Target Abort */ +#define SABRE_PIOAFSR_SRTRY 0x0200000000000000UL /* Secondary Excessive Retries */ +#define SABRE_PIOAFSR_SPERR 0x0100000000000000UL /* Secondary Parity Error */ +#define SABRE_PIOAFSR_BMSK 0x0000ffff00000000UL /* Byte Mask */ +#define SABRE_PIOAFSR_BLK 0x0000000080000000UL /* Was Block Operation */ +#define SABRE_PIOAFAR 0x2018UL +#define SABRE_PCIDIAG 0x2020UL +#define SABRE_PCIDIAG_DRTRY 0x0000000000000040UL /* Disable PIO Retry Limit */ +#define SABRE_PCIDIAG_IPAPAR 0x0000000000000008UL /* Invert PIO Address Parity */ +#define SABRE_PCIDIAG_IPDPAR 0x0000000000000004UL /* Invert PIO Data Parity */ +#define SABRE_PCIDIAG_IDDPAR 0x0000000000000002UL /* Invert DMA Data Parity */ +#define SABRE_PCIDIAG_ELPBK 0x0000000000000001UL /* Loopback Enable - not supported */ +#define SABRE_PCITASR 0x2028UL +#define SABRE_PCITASR_EF 0x0000000000000080UL /* Respond to 0xe0000000-0xffffffff */ +#define SABRE_PCITASR_CD 0x0000000000000040UL /* Respond to 0xc0000000-0xdfffffff */ +#define SABRE_PCITASR_AB 0x0000000000000020UL /* Respond to 0xa0000000-0xbfffffff */ +#define SABRE_PCITASR_89 0x0000000000000010UL /* Respond to 0x80000000-0x9fffffff */ +#define SABRE_PCITASR_67 0x0000000000000008UL /* Respond to 0x60000000-0x7fffffff */ +#define SABRE_PCITASR_45 0x0000000000000004UL /* Respond to 0x40000000-0x5fffffff */ +#define SABRE_PCITASR_23 0x0000000000000002UL /* Respond to 0x20000000-0x3fffffff */ +#define SABRE_PCITASR_01 0x0000000000000001UL /* Respond to 0x00000000-0x1fffffff */ +#define SABRE_PIOBUF_DIAG 0x5000UL +#define SABRE_DMABUF_DIAGLO 0x5100UL +#define SABRE_DMABUF_DIAGHI 0x51c0UL +#define SABRE_IMAP_GFX_ALIAS 0x6000UL /* Aliases to 0x1098 */ +#define SABRE_IMAP_EUPA_ALIAS 0x8000UL /* Aliases to 0x10a0 */ +#define SABRE_IOMMU_VADIAG 0xa400UL +#define SABRE_IOMMU_TCDIAG 0xa408UL +#define SABRE_IOMMU_TAG 0xa580UL +#define SABRE_IOMMUTAG_ERRSTS 0x0000000001800000UL /* Error status bits */ +#define SABRE_IOMMUTAG_ERR 0x0000000000400000UL /* Error present */ +#define SABRE_IOMMUTAG_WRITE 0x0000000000200000UL /* Page is writable */ +#define SABRE_IOMMUTAG_STREAM 0x0000000000100000UL /* Streamable bit - unused */ +#define SABRE_IOMMUTAG_SIZE 0x0000000000080000UL /* 0=8k 1=16k */ +#define SABRE_IOMMUTAG_VPN 0x000000000007ffffUL /* Virtual Page Number [31:13] */ +#define SABRE_IOMMU_DATA 0xa600UL +#define SABRE_IOMMUDATA_VALID 0x0000000040000000UL /* Valid */ +#define SABRE_IOMMUDATA_USED 0x0000000020000000UL /* Used (for LRU algorithm) */ +#define SABRE_IOMMUDATA_CACHE 0x0000000010000000UL /* Cacheable */ +#define SABRE_IOMMUDATA_PPN 0x00000000001fffffUL /* Physical Page Number [33:13] */ +#define SABRE_PCI_IRQSTATE 0xa800UL +#define SABRE_OBIO_IRQSTATE 0xa808UL +#define SABRE_FFBCFG 0xf000UL +#define SABRE_FFBCFG_SPRQS 0x000000000f000000 /* Slave P_RQST queue size */ +#define SABRE_FFBCFG_ONEREAD 0x0000000000004000 /* Slave supports one outstanding read */ +#define SABRE_MCCTRL0 0xf010UL +#define SABRE_MCCTRL0_RENAB 0x0000000080000000 /* Refresh Enable */ +#define SABRE_MCCTRL0_EENAB 0x0000000010000000 /* Enable all ECC functions */ +#define SABRE_MCCTRL0_11BIT 0x0000000000001000 /* Enable 11-bit column addressing */ +#define SABRE_MCCTRL0_DPP 0x0000000000000f00 /* DIMM Pair Present Bits */ +#define SABRE_MCCTRL0_RINTVL 0x00000000000000ff /* Refresh Interval */ +#define SABRE_MCCTRL1 0xf018UL +#define SABRE_MCCTRL1_AMDC 0x0000000038000000 /* Advance Memdata Clock */ +#define SABRE_MCCTRL1_ARDC 0x0000000007000000 /* Advance DRAM Read Data Clock */ +#define SABRE_MCCTRL1_CSR 0x0000000000e00000 /* CAS to RAS delay for CBR refresh */ +#define SABRE_MCCTRL1_CASRW 0x00000000001c0000 /* CAS length for read/write */ +#define SABRE_MCCTRL1_RCD 0x0000000000038000 /* RAS to CAS delay */ +#define SABRE_MCCTRL1_CP 0x0000000000007000 /* CAS Precharge */ +#define SABRE_MCCTRL1_RP 0x0000000000000e00 /* RAS Precharge */ +#define SABRE_MCCTRL1_RAS 0x00000000000001c0 /* Length of RAS for refresh */ +#define SABRE_MCCTRL1_CASRW2 0x0000000000000038 /* Must be same as CASRW */ +#define SABRE_MCCTRL1_RSC 0x0000000000000007 /* RAS after CAS hold time */ +#define SABRE_RESETCTRL 0xf020UL + +#define SABRE_CONFIGSPACE 0x001000000UL +#define SABRE_IOSPACE 0x002000000UL +#define SABRE_IOSPACE_SIZE 0x000ffffffUL +#define SABRE_MEMSPACE 0x100000000UL +#define SABRE_MEMSPACE_SIZE 0x07fffffffUL + +static int hummingbird_p; +static struct pci_bus *sabre_root_bus; + +static irqreturn_t sabre_ue_intr(int irq, void *dev_id) +{ + struct pci_pbm_info *pbm = dev_id; + unsigned long afsr_reg = pbm->controller_regs + SABRE_UE_AFSR; + unsigned long afar_reg = pbm->controller_regs + SABRE_UECE_AFAR; + unsigned long afsr, afar, error_bits; + int reported; + + /* Latch uncorrectable error status. */ + afar = upa_readq(afar_reg); + afsr = upa_readq(afsr_reg); + + /* Clear the primary/secondary error status bits. */ + error_bits = afsr & + (SABRE_UEAFSR_PDRD | SABRE_UEAFSR_PDWR | + SABRE_UEAFSR_SDRD | SABRE_UEAFSR_SDWR | + SABRE_UEAFSR_SDTE | SABRE_UEAFSR_PDTE); + if (!error_bits) + return IRQ_NONE; + upa_writeq(error_bits, afsr_reg); + + /* Log the error. */ + printk("%s: Uncorrectable Error, primary error type[%s%s]\n", + pbm->name, + ((error_bits & SABRE_UEAFSR_PDRD) ? + "DMA Read" : + ((error_bits & SABRE_UEAFSR_PDWR) ? + "DMA Write" : "???")), + ((error_bits & SABRE_UEAFSR_PDTE) ? + ":Translation Error" : "")); + printk("%s: bytemask[%04lx] dword_offset[%lx] was_block(%d)\n", + pbm->name, + (afsr & SABRE_UEAFSR_BMSK) >> 32UL, + (afsr & SABRE_UEAFSR_OFF) >> 29UL, + ((afsr & SABRE_UEAFSR_BLK) ? 1 : 0)); + printk("%s: UE AFAR [%016lx]\n", pbm->name, afar); + printk("%s: UE Secondary errors [", pbm->name); + reported = 0; + if (afsr & SABRE_UEAFSR_SDRD) { + reported++; + printk("(DMA Read)"); + } + if (afsr & SABRE_UEAFSR_SDWR) { + reported++; + printk("(DMA Write)"); + } + if (afsr & SABRE_UEAFSR_SDTE) { + reported++; + printk("(Translation Error)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + /* Interrogate IOMMU for error status. */ + psycho_check_iommu_error(pbm, afsr, afar, UE_ERR); + + return IRQ_HANDLED; +} + +static irqreturn_t sabre_ce_intr(int irq, void *dev_id) +{ + struct pci_pbm_info *pbm = dev_id; + unsigned long afsr_reg = pbm->controller_regs + SABRE_CE_AFSR; + unsigned long afar_reg = pbm->controller_regs + SABRE_UECE_AFAR; + unsigned long afsr, afar, error_bits; + int reported; + + /* Latch error status. */ + afar = upa_readq(afar_reg); + afsr = upa_readq(afsr_reg); + + /* Clear primary/secondary error status bits. */ + error_bits = afsr & + (SABRE_CEAFSR_PDRD | SABRE_CEAFSR_PDWR | + SABRE_CEAFSR_SDRD | SABRE_CEAFSR_SDWR); + if (!error_bits) + return IRQ_NONE; + upa_writeq(error_bits, afsr_reg); + + /* Log the error. */ + printk("%s: Correctable Error, primary error type[%s]\n", + pbm->name, + ((error_bits & SABRE_CEAFSR_PDRD) ? + "DMA Read" : + ((error_bits & SABRE_CEAFSR_PDWR) ? + "DMA Write" : "???"))); + + /* XXX Use syndrome and afar to print out module string just like + * XXX UDB CE trap handler does... -DaveM + */ + printk("%s: syndrome[%02lx] bytemask[%04lx] dword_offset[%lx] " + "was_block(%d)\n", + pbm->name, + (afsr & SABRE_CEAFSR_ESYND) >> 48UL, + (afsr & SABRE_CEAFSR_BMSK) >> 32UL, + (afsr & SABRE_CEAFSR_OFF) >> 29UL, + ((afsr & SABRE_CEAFSR_BLK) ? 1 : 0)); + printk("%s: CE AFAR [%016lx]\n", pbm->name, afar); + printk("%s: CE Secondary errors [", pbm->name); + reported = 0; + if (afsr & SABRE_CEAFSR_SDRD) { + reported++; + printk("(DMA Read)"); + } + if (afsr & SABRE_CEAFSR_SDWR) { + reported++; + printk("(DMA Write)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + return IRQ_HANDLED; +} + +static void sabre_register_error_handlers(struct pci_pbm_info *pbm) +{ + struct device_node *dp = pbm->op->dev.of_node; + struct platform_device *op; + unsigned long base = pbm->controller_regs; + u64 tmp; + int err; + + if (pbm->chip_type == PBM_CHIP_TYPE_SABRE) + dp = dp->parent; + + op = of_find_device_by_node(dp); + if (!op) + return; + + /* Sabre/Hummingbird IRQ property layout is: + * 0: PCI ERR + * 1: UE ERR + * 2: CE ERR + * 3: POWER FAIL + */ + if (op->archdata.num_irqs < 4) + return; + + /* We clear the error bits in the appropriate AFSR before + * registering the handler so that we don't get spurious + * interrupts. + */ + upa_writeq((SABRE_UEAFSR_PDRD | SABRE_UEAFSR_PDWR | + SABRE_UEAFSR_SDRD | SABRE_UEAFSR_SDWR | + SABRE_UEAFSR_SDTE | SABRE_UEAFSR_PDTE), + base + SABRE_UE_AFSR); + + err = request_irq(op->archdata.irqs[1], sabre_ue_intr, 0, "SABRE_UE", pbm); + if (err) + printk(KERN_WARNING "%s: Couldn't register UE, err=%d.\n", + pbm->name, err); + + upa_writeq((SABRE_CEAFSR_PDRD | SABRE_CEAFSR_PDWR | + SABRE_CEAFSR_SDRD | SABRE_CEAFSR_SDWR), + base + SABRE_CE_AFSR); + + + err = request_irq(op->archdata.irqs[2], sabre_ce_intr, 0, "SABRE_CE", pbm); + if (err) + printk(KERN_WARNING "%s: Couldn't register CE, err=%d.\n", + pbm->name, err); + err = request_irq(op->archdata.irqs[0], psycho_pcierr_intr, 0, + "SABRE_PCIERR", pbm); + if (err) + printk(KERN_WARNING "%s: Couldn't register PCIERR, err=%d.\n", + pbm->name, err); + + tmp = upa_readq(base + SABRE_PCICTRL); + tmp |= SABRE_PCICTRL_ERREN; + upa_writeq(tmp, base + SABRE_PCICTRL); +} + +static void apb_init(struct pci_bus *sabre_bus) +{ + struct pci_dev *pdev; + + list_for_each_entry(pdev, &sabre_bus->devices, bus_list) { + if (pdev->vendor == PCI_VENDOR_ID_SUN && + pdev->device == PCI_DEVICE_ID_SUN_SIMBA) { + u16 word16; + + pci_read_config_word(pdev, PCI_COMMAND, &word16); + word16 |= PCI_COMMAND_SERR | PCI_COMMAND_PARITY | + PCI_COMMAND_MASTER | PCI_COMMAND_MEMORY | + PCI_COMMAND_IO; + pci_write_config_word(pdev, PCI_COMMAND, word16); + + /* Status register bits are "write 1 to clear". */ + pci_write_config_word(pdev, PCI_STATUS, 0xffff); + pci_write_config_word(pdev, PCI_SEC_STATUS, 0xffff); + + /* Use a primary/seconday latency timer value + * of 64. + */ + pci_write_config_byte(pdev, PCI_LATENCY_TIMER, 64); + pci_write_config_byte(pdev, PCI_SEC_LATENCY_TIMER, 64); + + /* Enable reporting/forwarding of master aborts, + * parity, and SERR. + */ + pci_write_config_byte(pdev, PCI_BRIDGE_CONTROL, + (PCI_BRIDGE_CTL_PARITY | + PCI_BRIDGE_CTL_SERR | + PCI_BRIDGE_CTL_MASTER_ABORT)); + } + } +} + +static void sabre_scan_bus(struct pci_pbm_info *pbm, struct device *parent) +{ + static int once; + + /* The APB bridge speaks to the Sabre host PCI bridge + * at 66Mhz, but the front side of APB runs at 33Mhz + * for both segments. + * + * Hummingbird systems do not use APB, so they run + * at 66MHZ. + */ + if (hummingbird_p) + pbm->is_66mhz_capable = 1; + else + pbm->is_66mhz_capable = 0; + + /* This driver has not been verified to handle + * multiple SABREs yet, so trap this. + * + * Also note that the SABRE host bridge is hardwired + * to live at bus 0. + */ + if (once != 0) { + printk(KERN_ERR PFX "Multiple controllers unsupported.\n"); + return; + } + once++; + + pbm->pci_bus = pci_scan_one_pbm(pbm, parent); + if (!pbm->pci_bus) + return; + + sabre_root_bus = pbm->pci_bus; + + apb_init(pbm->pci_bus); + + sabre_register_error_handlers(pbm); +} + +static void sabre_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op) +{ + psycho_pbm_init_common(pbm, op, "SABRE", PBM_CHIP_TYPE_SABRE); + pbm->pci_afsr = pbm->controller_regs + SABRE_PIOAFSR; + pbm->pci_afar = pbm->controller_regs + SABRE_PIOAFAR; + pbm->pci_csr = pbm->controller_regs + SABRE_PCICTRL; + sabre_scan_bus(pbm, &op->dev); +} + +static const struct of_device_id sabre_match[]; +static int sabre_probe(struct platform_device *op) +{ + const struct linux_prom64_registers *pr_regs; + struct device_node *dp = op->dev.of_node; + struct pci_pbm_info *pbm; + u32 upa_portid, dma_mask; + struct iommu *iommu; + int tsbsize, err; + const u32 *vdma; + u64 clear_irq; + + hummingbird_p = (uintptr_t)device_get_match_data(&op->dev); + if (!hummingbird_p) { + struct device_node *cpu_dp; + + /* Of course, Sun has to encode things a thousand + * different ways, inconsistently. + */ + for_each_node_by_type(cpu_dp, "cpu") { + if (of_node_name_eq(cpu_dp, "SUNW,UltraSPARC-IIe")) + hummingbird_p = 1; + } + } + + err = -ENOMEM; + pbm = kzalloc_obj(*pbm); + if (!pbm) { + printk(KERN_ERR PFX "Cannot allocate pci_pbm_info.\n"); + goto out_err; + } + + iommu = kzalloc_obj(*iommu); + if (!iommu) { + printk(KERN_ERR PFX "Cannot allocate PBM iommu.\n"); + goto out_free_controller; + } + + pbm->iommu = iommu; + + upa_portid = of_getintprop_default(dp, "upa-portid", 0xff); + + pbm->portid = upa_portid; + + /* + * Map in SABRE register set and report the presence of this SABRE. + */ + + pr_regs = of_get_property(dp, "reg", NULL); + err = -ENODEV; + if (!pr_regs) { + printk(KERN_ERR PFX "No reg property\n"); + goto out_free_iommu; + } + + /* + * First REG in property is base of entire SABRE register space. + */ + pbm->controller_regs = pr_regs[0].phys_addr; + + /* Clear interrupts */ + + /* PCI first */ + for (clear_irq = SABRE_ICLR_A_SLOT0; clear_irq < SABRE_ICLR_B_SLOT0 + 0x80; clear_irq += 8) + upa_writeq(0x0UL, pbm->controller_regs + clear_irq); + + /* Then OBIO */ + for (clear_irq = SABRE_ICLR_SCSI; clear_irq < SABRE_ICLR_SCSI + 0x80; clear_irq += 8) + upa_writeq(0x0UL, pbm->controller_regs + clear_irq); + + /* Error interrupts are enabled later after the bus scan. */ + upa_writeq((SABRE_PCICTRL_MRLEN | SABRE_PCICTRL_SERR | + SABRE_PCICTRL_ARBPARK | SABRE_PCICTRL_AEN), + pbm->controller_regs + SABRE_PCICTRL); + + /* Now map in PCI config space for entire SABRE. */ + pbm->config_space = pbm->controller_regs + SABRE_CONFIGSPACE; + + vdma = of_get_property(dp, "virtual-dma", NULL); + if (!vdma) { + printk(KERN_ERR PFX "No virtual-dma property\n"); + goto out_free_iommu; + } + + dma_mask = vdma[0]; + switch(vdma[1]) { + case 0x20000000: + dma_mask |= 0x1fffffff; + tsbsize = 64; + break; + case 0x40000000: + dma_mask |= 0x3fffffff; + tsbsize = 128; + break; + + case 0x80000000: + dma_mask |= 0x7fffffff; + tsbsize = 128; + break; + default: + printk(KERN_ERR PFX "Strange virtual-dma size.\n"); + goto out_free_iommu; + } + + err = psycho_iommu_init(pbm, tsbsize, vdma[0], dma_mask, SABRE_WRSYNC); + if (err) + goto out_free_iommu; + + /* + * Look for APB underneath. + */ + sabre_pbm_init(pbm, op); + + pbm->next = pci_pbm_root; + pci_pbm_root = pbm; + + dev_set_drvdata(&op->dev, pbm); + + return 0; + +out_free_iommu: + kfree(pbm->iommu); + +out_free_controller: + kfree(pbm); + +out_err: + return err; +} + +static const struct of_device_id sabre_match[] = { + { + .name = "pci", + .compatible = "pci108e,a001", + .data = (void *) 1, + }, + { + .name = "pci", + .compatible = "pci108e,a000", + }, + {}, +}; + +static struct platform_driver sabre_driver = { + .driver = { + .name = DRIVER_NAME, + .of_match_table = sabre_match, + }, + .probe = sabre_probe, +}; + +static int __init sabre_init(void) +{ + return platform_driver_register(&sabre_driver); +} + +subsys_initcall(sabre_init); diff --git a/arch/sparc/kernel/pci_schizo.c b/arch/sparc/kernel/pci_schizo.c new file mode 100644 index 000000000..4348ca2c4 --- /dev/null +++ b/arch/sparc/kernel/pci_schizo.c @@ -0,0 +1,1511 @@ +// SPDX-License-Identifier: GPL-2.0 +/* pci_schizo.c: SCHIZO/TOMATILLO specific PCI controller support. + * + * Copyright (C) 2001, 2002, 2003, 2007, 2008 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "pci_impl.h" +#include "iommu_common.h" + +#define DRIVER_NAME "schizo" +#define PFX DRIVER_NAME ": " + +/* This is a convention that at least Excalibur and Merlin + * follow. I suppose the SCHIZO used in Starcat and friends + * will do similar. + * + * The only way I could see this changing is if the newlink + * block requires more space in Schizo's address space than + * they predicted, thus requiring an address space reorg when + * the newer Schizo is taped out. + */ + +/* Streaming buffer control register. */ +#define SCHIZO_STRBUF_CTRL_LPTR 0x00000000000000f0UL /* LRU Lock Pointer */ +#define SCHIZO_STRBUF_CTRL_LENAB 0x0000000000000008UL /* LRU Lock Enable */ +#define SCHIZO_STRBUF_CTRL_RRDIS 0x0000000000000004UL /* Rerun Disable */ +#define SCHIZO_STRBUF_CTRL_DENAB 0x0000000000000002UL /* Diagnostic Mode Enable */ +#define SCHIZO_STRBUF_CTRL_ENAB 0x0000000000000001UL /* Streaming Buffer Enable */ + +/* IOMMU control register. */ +#define SCHIZO_IOMMU_CTRL_RESV 0xfffffffff9000000UL /* Reserved */ +#define SCHIZO_IOMMU_CTRL_XLTESTAT 0x0000000006000000UL /* Translation Error Status */ +#define SCHIZO_IOMMU_CTRL_XLTEERR 0x0000000001000000UL /* Translation Error encountered */ +#define SCHIZO_IOMMU_CTRL_LCKEN 0x0000000000800000UL /* Enable translation locking */ +#define SCHIZO_IOMMU_CTRL_LCKPTR 0x0000000000780000UL /* Translation lock pointer */ +#define SCHIZO_IOMMU_CTRL_TSBSZ 0x0000000000070000UL /* TSB Size */ +#define SCHIZO_IOMMU_TSBSZ_1K 0x0000000000000000UL /* TSB Table 1024 8-byte entries */ +#define SCHIZO_IOMMU_TSBSZ_2K 0x0000000000010000UL /* TSB Table 2048 8-byte entries */ +#define SCHIZO_IOMMU_TSBSZ_4K 0x0000000000020000UL /* TSB Table 4096 8-byte entries */ +#define SCHIZO_IOMMU_TSBSZ_8K 0x0000000000030000UL /* TSB Table 8192 8-byte entries */ +#define SCHIZO_IOMMU_TSBSZ_16K 0x0000000000040000UL /* TSB Table 16k 8-byte entries */ +#define SCHIZO_IOMMU_TSBSZ_32K 0x0000000000050000UL /* TSB Table 32k 8-byte entries */ +#define SCHIZO_IOMMU_TSBSZ_64K 0x0000000000060000UL /* TSB Table 64k 8-byte entries */ +#define SCHIZO_IOMMU_TSBSZ_128K 0x0000000000070000UL /* TSB Table 128k 8-byte entries */ +#define SCHIZO_IOMMU_CTRL_RESV2 0x000000000000fff8UL /* Reserved */ +#define SCHIZO_IOMMU_CTRL_TBWSZ 0x0000000000000004UL /* Assumed page size, 0=8k 1=64k */ +#define SCHIZO_IOMMU_CTRL_DENAB 0x0000000000000002UL /* Diagnostic mode enable */ +#define SCHIZO_IOMMU_CTRL_ENAB 0x0000000000000001UL /* IOMMU Enable */ + +/* Schizo config space address format is nearly identical to + * that of PSYCHO: + * + * 32 24 23 16 15 11 10 8 7 2 1 0 + * --------------------------------------------------------- + * |0 0 0 0 0 0 0 0 0| bus | device | function | reg | 0 0 | + * --------------------------------------------------------- + */ +#define SCHIZO_CONFIG_BASE(PBM) ((PBM)->config_space) +#define SCHIZO_CONFIG_ENCODE(BUS, DEVFN, REG) \ + (((unsigned long)(BUS) << 16) | \ + ((unsigned long)(DEVFN) << 8) | \ + ((unsigned long)(REG))) + +static void *schizo_pci_config_mkaddr(struct pci_pbm_info *pbm, + unsigned char bus, + unsigned int devfn, + int where) +{ + if (!pbm) + return NULL; + bus -= pbm->pci_first_busno; + return (void *) + (SCHIZO_CONFIG_BASE(pbm) | + SCHIZO_CONFIG_ENCODE(bus, devfn, where)); +} + +/* SCHIZO error handling support. */ +enum schizo_error_type { + UE_ERR, CE_ERR, PCI_ERR, SAFARI_ERR +}; + +static DEFINE_SPINLOCK(stc_buf_lock); +static unsigned long stc_error_buf[128]; +static unsigned long stc_tag_buf[16]; +static unsigned long stc_line_buf[16]; + +#define SCHIZO_UE_INO 0x30 /* Uncorrectable ECC error */ +#define SCHIZO_CE_INO 0x31 /* Correctable ECC error */ +#define SCHIZO_PCIERR_A_INO 0x32 /* PBM A PCI bus error */ +#define SCHIZO_PCIERR_B_INO 0x33 /* PBM B PCI bus error */ +#define SCHIZO_SERR_INO 0x34 /* Safari interface error */ + +#define SCHIZO_STC_ERR 0xb800UL /* --> 0xba00 */ +#define SCHIZO_STC_TAG 0xba00UL /* --> 0xba80 */ +#define SCHIZO_STC_LINE 0xbb00UL /* --> 0xbb80 */ + +#define SCHIZO_STCERR_WRITE 0x2UL +#define SCHIZO_STCERR_READ 0x1UL + +#define SCHIZO_STCTAG_PPN 0x3fffffff00000000UL +#define SCHIZO_STCTAG_VPN 0x00000000ffffe000UL +#define SCHIZO_STCTAG_VALID 0x8000000000000000UL +#define SCHIZO_STCTAG_READ 0x4000000000000000UL + +#define SCHIZO_STCLINE_LINDX 0x0000000007800000UL +#define SCHIZO_STCLINE_SPTR 0x000000000007e000UL +#define SCHIZO_STCLINE_LADDR 0x0000000000001fc0UL +#define SCHIZO_STCLINE_EPTR 0x000000000000003fUL +#define SCHIZO_STCLINE_VALID 0x0000000000600000UL +#define SCHIZO_STCLINE_FOFN 0x0000000000180000UL + +static void __schizo_check_stc_error_pbm(struct pci_pbm_info *pbm, + enum schizo_error_type type) +{ + struct strbuf *strbuf = &pbm->stc; + unsigned long regbase = pbm->pbm_regs; + unsigned long err_base, tag_base, line_base; + u64 control; + int i; + + err_base = regbase + SCHIZO_STC_ERR; + tag_base = regbase + SCHIZO_STC_TAG; + line_base = regbase + SCHIZO_STC_LINE; + + spin_lock(&stc_buf_lock); + + /* This is __REALLY__ dangerous. When we put the + * streaming buffer into diagnostic mode to probe + * its tags and error status, we _must_ clear all + * of the line tag valid bits before re-enabling + * the streaming buffer. If any dirty data lives + * in the STC when we do this, we will end up + * invalidating it before it has a chance to reach + * main memory. + */ + control = upa_readq(strbuf->strbuf_control); + upa_writeq((control | SCHIZO_STRBUF_CTRL_DENAB), + strbuf->strbuf_control); + for (i = 0; i < 128; i++) { + unsigned long val; + + val = upa_readq(err_base + (i * 8UL)); + upa_writeq(0UL, err_base + (i * 8UL)); + stc_error_buf[i] = val; + } + for (i = 0; i < 16; i++) { + stc_tag_buf[i] = upa_readq(tag_base + (i * 8UL)); + stc_line_buf[i] = upa_readq(line_base + (i * 8UL)); + upa_writeq(0UL, tag_base + (i * 8UL)); + upa_writeq(0UL, line_base + (i * 8UL)); + } + + /* OK, state is logged, exit diagnostic mode. */ + upa_writeq(control, strbuf->strbuf_control); + + for (i = 0; i < 16; i++) { + int j, saw_error, first, last; + + saw_error = 0; + first = i * 8; + last = first + 8; + for (j = first; j < last; j++) { + unsigned long errval = stc_error_buf[j]; + if (errval != 0) { + saw_error++; + printk("%s: STC_ERR(%d)[wr(%d)rd(%d)]\n", + pbm->name, + j, + (errval & SCHIZO_STCERR_WRITE) ? 1 : 0, + (errval & SCHIZO_STCERR_READ) ? 1 : 0); + } + } + if (saw_error != 0) { + unsigned long tagval = stc_tag_buf[i]; + unsigned long lineval = stc_line_buf[i]; + printk("%s: STC_TAG(%d)[PA(%016lx)VA(%08lx)V(%d)R(%d)]\n", + pbm->name, + i, + ((tagval & SCHIZO_STCTAG_PPN) >> 19UL), + (tagval & SCHIZO_STCTAG_VPN), + ((tagval & SCHIZO_STCTAG_VALID) ? 1 : 0), + ((tagval & SCHIZO_STCTAG_READ) ? 1 : 0)); + + /* XXX Should spit out per-bank error information... -DaveM */ + printk("%s: STC_LINE(%d)[LIDX(%lx)SP(%lx)LADDR(%lx)EP(%lx)" + "V(%d)FOFN(%d)]\n", + pbm->name, + i, + ((lineval & SCHIZO_STCLINE_LINDX) >> 23UL), + ((lineval & SCHIZO_STCLINE_SPTR) >> 13UL), + ((lineval & SCHIZO_STCLINE_LADDR) >> 6UL), + ((lineval & SCHIZO_STCLINE_EPTR) >> 0UL), + ((lineval & SCHIZO_STCLINE_VALID) ? 1 : 0), + ((lineval & SCHIZO_STCLINE_FOFN) ? 1 : 0)); + } + } + + spin_unlock(&stc_buf_lock); +} + +/* IOMMU is per-PBM in Schizo, so interrogate both for anonymous + * controller level errors. + */ + +#define SCHIZO_IOMMU_TAG 0xa580UL +#define SCHIZO_IOMMU_DATA 0xa600UL + +#define SCHIZO_IOMMU_TAG_CTXT 0x0000001ffe000000UL +#define SCHIZO_IOMMU_TAG_ERRSTS 0x0000000001800000UL +#define SCHIZO_IOMMU_TAG_ERR 0x0000000000400000UL +#define SCHIZO_IOMMU_TAG_WRITE 0x0000000000200000UL +#define SCHIZO_IOMMU_TAG_STREAM 0x0000000000100000UL +#define SCHIZO_IOMMU_TAG_SIZE 0x0000000000080000UL +#define SCHIZO_IOMMU_TAG_VPAGE 0x000000000007ffffUL + +#define SCHIZO_IOMMU_DATA_VALID 0x0000000100000000UL +#define SCHIZO_IOMMU_DATA_CACHE 0x0000000040000000UL +#define SCHIZO_IOMMU_DATA_PPAGE 0x000000003fffffffUL + +static void schizo_check_iommu_error_pbm(struct pci_pbm_info *pbm, + enum schizo_error_type type) +{ + struct iommu *iommu = pbm->iommu; + unsigned long iommu_tag[16]; + unsigned long iommu_data[16]; + unsigned long flags; + u64 control; + int i; + + spin_lock_irqsave(&iommu->lock, flags); + control = upa_readq(iommu->iommu_control); + if (control & SCHIZO_IOMMU_CTRL_XLTEERR) { + unsigned long base; + char *type_string; + + /* Clear the error encountered bit. */ + control &= ~SCHIZO_IOMMU_CTRL_XLTEERR; + upa_writeq(control, iommu->iommu_control); + + switch((control & SCHIZO_IOMMU_CTRL_XLTESTAT) >> 25UL) { + case 0: + type_string = "Protection Error"; + break; + case 1: + type_string = "Invalid Error"; + break; + case 2: + type_string = "TimeOut Error"; + break; + case 3: + default: + type_string = "ECC Error"; + break; + } + printk("%s: IOMMU Error, type[%s]\n", + pbm->name, type_string); + + /* Put the IOMMU into diagnostic mode and probe + * its TLB for entries with error status. + * + * It is very possible for another DVMA to occur + * while we do this probe, and corrupt the system + * further. But we are so screwed at this point + * that we are likely to crash hard anyways, so + * get as much diagnostic information to the + * console as we can. + */ + upa_writeq(control | SCHIZO_IOMMU_CTRL_DENAB, + iommu->iommu_control); + + base = pbm->pbm_regs; + + for (i = 0; i < 16; i++) { + iommu_tag[i] = + upa_readq(base + SCHIZO_IOMMU_TAG + (i * 8UL)); + iommu_data[i] = + upa_readq(base + SCHIZO_IOMMU_DATA + (i * 8UL)); + + /* Now clear out the entry. */ + upa_writeq(0, base + SCHIZO_IOMMU_TAG + (i * 8UL)); + upa_writeq(0, base + SCHIZO_IOMMU_DATA + (i * 8UL)); + } + + /* Leave diagnostic mode. */ + upa_writeq(control, iommu->iommu_control); + + for (i = 0; i < 16; i++) { + unsigned long tag, data; + + tag = iommu_tag[i]; + if (!(tag & SCHIZO_IOMMU_TAG_ERR)) + continue; + + data = iommu_data[i]; + switch((tag & SCHIZO_IOMMU_TAG_ERRSTS) >> 23UL) { + case 0: + type_string = "Protection Error"; + break; + case 1: + type_string = "Invalid Error"; + break; + case 2: + type_string = "TimeOut Error"; + break; + case 3: + default: + type_string = "ECC Error"; + break; + } + printk("%s: IOMMU TAG(%d)[error(%s) ctx(%x) wr(%d) str(%d) " + "sz(%dK) vpg(%08lx)]\n", + pbm->name, i, type_string, + (int)((tag & SCHIZO_IOMMU_TAG_CTXT) >> 25UL), + ((tag & SCHIZO_IOMMU_TAG_WRITE) ? 1 : 0), + ((tag & SCHIZO_IOMMU_TAG_STREAM) ? 1 : 0), + ((tag & SCHIZO_IOMMU_TAG_SIZE) ? 64 : 8), + (tag & SCHIZO_IOMMU_TAG_VPAGE) << IOMMU_PAGE_SHIFT); + printk("%s: IOMMU DATA(%d)[valid(%d) cache(%d) ppg(%016lx)]\n", + pbm->name, i, + ((data & SCHIZO_IOMMU_DATA_VALID) ? 1 : 0), + ((data & SCHIZO_IOMMU_DATA_CACHE) ? 1 : 0), + (data & SCHIZO_IOMMU_DATA_PPAGE) << IOMMU_PAGE_SHIFT); + } + } + if (pbm->stc.strbuf_enabled) + __schizo_check_stc_error_pbm(pbm, type); + spin_unlock_irqrestore(&iommu->lock, flags); +} + +static void schizo_check_iommu_error(struct pci_pbm_info *pbm, + enum schizo_error_type type) +{ + schizo_check_iommu_error_pbm(pbm, type); + if (pbm->sibling) + schizo_check_iommu_error_pbm(pbm->sibling, type); +} + +/* Uncorrectable ECC error status gathering. */ +#define SCHIZO_UE_AFSR 0x10030UL +#define SCHIZO_UE_AFAR 0x10038UL + +#define SCHIZO_UEAFSR_PPIO 0x8000000000000000UL /* Safari */ +#define SCHIZO_UEAFSR_PDRD 0x4000000000000000UL /* Safari/Tomatillo */ +#define SCHIZO_UEAFSR_PDWR 0x2000000000000000UL /* Safari */ +#define SCHIZO_UEAFSR_SPIO 0x1000000000000000UL /* Safari */ +#define SCHIZO_UEAFSR_SDMA 0x0800000000000000UL /* Safari/Tomatillo */ +#define SCHIZO_UEAFSR_ERRPNDG 0x0300000000000000UL /* Safari */ +#define SCHIZO_UEAFSR_BMSK 0x000003ff00000000UL /* Safari */ +#define SCHIZO_UEAFSR_QOFF 0x00000000c0000000UL /* Safari/Tomatillo */ +#define SCHIZO_UEAFSR_AID 0x000000001f000000UL /* Safari/Tomatillo */ +#define SCHIZO_UEAFSR_PARTIAL 0x0000000000800000UL /* Safari */ +#define SCHIZO_UEAFSR_OWNEDIN 0x0000000000400000UL /* Safari */ +#define SCHIZO_UEAFSR_MTAGSYND 0x00000000000f0000UL /* Safari */ +#define SCHIZO_UEAFSR_MTAG 0x000000000000e000UL /* Safari */ +#define SCHIZO_UEAFSR_ECCSYND 0x00000000000001ffUL /* Safari */ + +static irqreturn_t schizo_ue_intr(int irq, void *dev_id) +{ + struct pci_pbm_info *pbm = dev_id; + unsigned long afsr_reg = pbm->controller_regs + SCHIZO_UE_AFSR; + unsigned long afar_reg = pbm->controller_regs + SCHIZO_UE_AFAR; + unsigned long afsr, afar, error_bits; + int reported, limit; + + /* Latch uncorrectable error status. */ + afar = upa_readq(afar_reg); + + /* If either of the error pending bits are set in the + * AFSR, the error status is being actively updated by + * the hardware and we must re-read to get a clean value. + */ + limit = 1000; + do { + afsr = upa_readq(afsr_reg); + } while ((afsr & SCHIZO_UEAFSR_ERRPNDG) != 0 && --limit); + + /* Clear the primary/secondary error status bits. */ + error_bits = afsr & + (SCHIZO_UEAFSR_PPIO | SCHIZO_UEAFSR_PDRD | SCHIZO_UEAFSR_PDWR | + SCHIZO_UEAFSR_SPIO | SCHIZO_UEAFSR_SDMA); + if (!error_bits) + return IRQ_NONE; + upa_writeq(error_bits, afsr_reg); + + /* Log the error. */ + printk("%s: Uncorrectable Error, primary error type[%s]\n", + pbm->name, + (((error_bits & SCHIZO_UEAFSR_PPIO) ? + "PIO" : + ((error_bits & SCHIZO_UEAFSR_PDRD) ? + "DMA Read" : + ((error_bits & SCHIZO_UEAFSR_PDWR) ? + "DMA Write" : "???"))))); + printk("%s: bytemask[%04lx] qword_offset[%lx] SAFARI_AID[%02lx]\n", + pbm->name, + (afsr & SCHIZO_UEAFSR_BMSK) >> 32UL, + (afsr & SCHIZO_UEAFSR_QOFF) >> 30UL, + (afsr & SCHIZO_UEAFSR_AID) >> 24UL); + printk("%s: partial[%d] owned_in[%d] mtag[%lx] mtag_synd[%lx] ecc_sync[%lx]\n", + pbm->name, + (afsr & SCHIZO_UEAFSR_PARTIAL) ? 1 : 0, + (afsr & SCHIZO_UEAFSR_OWNEDIN) ? 1 : 0, + (afsr & SCHIZO_UEAFSR_MTAG) >> 13UL, + (afsr & SCHIZO_UEAFSR_MTAGSYND) >> 16UL, + (afsr & SCHIZO_UEAFSR_ECCSYND) >> 0UL); + printk("%s: UE AFAR [%016lx]\n", pbm->name, afar); + printk("%s: UE Secondary errors [", pbm->name); + reported = 0; + if (afsr & SCHIZO_UEAFSR_SPIO) { + reported++; + printk("(PIO)"); + } + if (afsr & SCHIZO_UEAFSR_SDMA) { + reported++; + printk("(DMA)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + /* Interrogate IOMMU for error status. */ + schizo_check_iommu_error(pbm, UE_ERR); + + return IRQ_HANDLED; +} + +#define SCHIZO_CE_AFSR 0x10040UL +#define SCHIZO_CE_AFAR 0x10048UL + +#define SCHIZO_CEAFSR_PPIO 0x8000000000000000UL +#define SCHIZO_CEAFSR_PDRD 0x4000000000000000UL +#define SCHIZO_CEAFSR_PDWR 0x2000000000000000UL +#define SCHIZO_CEAFSR_SPIO 0x1000000000000000UL +#define SCHIZO_CEAFSR_SDMA 0x0800000000000000UL +#define SCHIZO_CEAFSR_ERRPNDG 0x0300000000000000UL +#define SCHIZO_CEAFSR_BMSK 0x000003ff00000000UL +#define SCHIZO_CEAFSR_QOFF 0x00000000c0000000UL +#define SCHIZO_CEAFSR_AID 0x000000001f000000UL +#define SCHIZO_CEAFSR_PARTIAL 0x0000000000800000UL +#define SCHIZO_CEAFSR_OWNEDIN 0x0000000000400000UL +#define SCHIZO_CEAFSR_MTAGSYND 0x00000000000f0000UL +#define SCHIZO_CEAFSR_MTAG 0x000000000000e000UL +#define SCHIZO_CEAFSR_ECCSYND 0x00000000000001ffUL + +static irqreturn_t schizo_ce_intr(int irq, void *dev_id) +{ + struct pci_pbm_info *pbm = dev_id; + unsigned long afsr_reg = pbm->controller_regs + SCHIZO_CE_AFSR; + unsigned long afar_reg = pbm->controller_regs + SCHIZO_CE_AFAR; + unsigned long afsr, afar, error_bits; + int reported, limit; + + /* Latch error status. */ + afar = upa_readq(afar_reg); + + /* If either of the error pending bits are set in the + * AFSR, the error status is being actively updated by + * the hardware and we must re-read to get a clean value. + */ + limit = 1000; + do { + afsr = upa_readq(afsr_reg); + } while ((afsr & SCHIZO_UEAFSR_ERRPNDG) != 0 && --limit); + + /* Clear primary/secondary error status bits. */ + error_bits = afsr & + (SCHIZO_CEAFSR_PPIO | SCHIZO_CEAFSR_PDRD | SCHIZO_CEAFSR_PDWR | + SCHIZO_CEAFSR_SPIO | SCHIZO_CEAFSR_SDMA); + if (!error_bits) + return IRQ_NONE; + upa_writeq(error_bits, afsr_reg); + + /* Log the error. */ + printk("%s: Correctable Error, primary error type[%s]\n", + pbm->name, + (((error_bits & SCHIZO_CEAFSR_PPIO) ? + "PIO" : + ((error_bits & SCHIZO_CEAFSR_PDRD) ? + "DMA Read" : + ((error_bits & SCHIZO_CEAFSR_PDWR) ? + "DMA Write" : "???"))))); + + /* XXX Use syndrome and afar to print out module string just like + * XXX UDB CE trap handler does... -DaveM + */ + printk("%s: bytemask[%04lx] qword_offset[%lx] SAFARI_AID[%02lx]\n", + pbm->name, + (afsr & SCHIZO_UEAFSR_BMSK) >> 32UL, + (afsr & SCHIZO_UEAFSR_QOFF) >> 30UL, + (afsr & SCHIZO_UEAFSR_AID) >> 24UL); + printk("%s: partial[%d] owned_in[%d] mtag[%lx] mtag_synd[%lx] ecc_sync[%lx]\n", + pbm->name, + (afsr & SCHIZO_UEAFSR_PARTIAL) ? 1 : 0, + (afsr & SCHIZO_UEAFSR_OWNEDIN) ? 1 : 0, + (afsr & SCHIZO_UEAFSR_MTAG) >> 13UL, + (afsr & SCHIZO_UEAFSR_MTAGSYND) >> 16UL, + (afsr & SCHIZO_UEAFSR_ECCSYND) >> 0UL); + printk("%s: CE AFAR [%016lx]\n", pbm->name, afar); + printk("%s: CE Secondary errors [", pbm->name); + reported = 0; + if (afsr & SCHIZO_CEAFSR_SPIO) { + reported++; + printk("(PIO)"); + } + if (afsr & SCHIZO_CEAFSR_SDMA) { + reported++; + printk("(DMA)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + return IRQ_HANDLED; +} + +#define SCHIZO_PCI_AFSR 0x2010UL +#define SCHIZO_PCI_AFAR 0x2018UL + +#define SCHIZO_PCIAFSR_PMA 0x8000000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_PTA 0x4000000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_PRTRY 0x2000000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_PPERR 0x1000000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_PTTO 0x0800000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_PUNUS 0x0400000000000000UL /* Schizo */ +#define SCHIZO_PCIAFSR_SMA 0x0200000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_STA 0x0100000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_SRTRY 0x0080000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_SPERR 0x0040000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_STTO 0x0020000000000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_SUNUS 0x0010000000000000UL /* Schizo */ +#define SCHIZO_PCIAFSR_BMSK 0x000003ff00000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_BLK 0x0000000080000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_CFG 0x0000000040000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_MEM 0x0000000020000000UL /* Schizo/Tomatillo */ +#define SCHIZO_PCIAFSR_IO 0x0000000010000000UL /* Schizo/Tomatillo */ + +#define SCHIZO_PCI_CTRL (0x2000UL) +#define SCHIZO_PCICTRL_BUS_UNUS (1UL << 63UL) /* Safari */ +#define SCHIZO_PCICTRL_DTO_INT (1UL << 61UL) /* Tomatillo */ +#define SCHIZO_PCICTRL_ARB_PRIO (0x1ff << 52UL) /* Tomatillo */ +#define SCHIZO_PCICTRL_ESLCK (1UL << 51UL) /* Safari */ +#define SCHIZO_PCICTRL_ERRSLOT (7UL << 48UL) /* Safari */ +#define SCHIZO_PCICTRL_TTO_ERR (1UL << 38UL) /* Safari/Tomatillo */ +#define SCHIZO_PCICTRL_RTRY_ERR (1UL << 37UL) /* Safari/Tomatillo */ +#define SCHIZO_PCICTRL_DTO_ERR (1UL << 36UL) /* Safari/Tomatillo */ +#define SCHIZO_PCICTRL_SBH_ERR (1UL << 35UL) /* Safari */ +#define SCHIZO_PCICTRL_SERR (1UL << 34UL) /* Safari/Tomatillo */ +#define SCHIZO_PCICTRL_PCISPD (1UL << 33UL) /* Safari */ +#define SCHIZO_PCICTRL_MRM_PREF (1UL << 30UL) /* Tomatillo */ +#define SCHIZO_PCICTRL_RDO_PREF (1UL << 29UL) /* Tomatillo */ +#define SCHIZO_PCICTRL_RDL_PREF (1UL << 28UL) /* Tomatillo */ +#define SCHIZO_PCICTRL_PTO (3UL << 24UL) /* Safari/Tomatillo */ +#define SCHIZO_PCICTRL_PTO_SHIFT 24UL +#define SCHIZO_PCICTRL_TRWSW (7UL << 21UL) /* Tomatillo */ +#define SCHIZO_PCICTRL_F_TGT_A (1UL << 20UL) /* Tomatillo */ +#define SCHIZO_PCICTRL_S_DTO_INT (1UL << 19UL) /* Safari */ +#define SCHIZO_PCICTRL_F_TGT_RT (1UL << 19UL) /* Tomatillo */ +#define SCHIZO_PCICTRL_SBH_INT (1UL << 18UL) /* Safari */ +#define SCHIZO_PCICTRL_T_DTO_INT (1UL << 18UL) /* Tomatillo */ +#define SCHIZO_PCICTRL_EEN (1UL << 17UL) /* Safari/Tomatillo */ +#define SCHIZO_PCICTRL_PARK (1UL << 16UL) /* Safari/Tomatillo */ +#define SCHIZO_PCICTRL_PCIRST (1UL << 8UL) /* Safari */ +#define SCHIZO_PCICTRL_ARB_S (0x3fUL << 0UL) /* Safari */ +#define SCHIZO_PCICTRL_ARB_T (0xffUL << 0UL) /* Tomatillo */ + +static irqreturn_t schizo_pcierr_intr_other(struct pci_pbm_info *pbm) +{ + unsigned long csr_reg, csr, csr_error_bits; + irqreturn_t ret = IRQ_NONE; + u32 stat; + + csr_reg = pbm->pbm_regs + SCHIZO_PCI_CTRL; + csr = upa_readq(csr_reg); + csr_error_bits = + csr & (SCHIZO_PCICTRL_BUS_UNUS | + SCHIZO_PCICTRL_TTO_ERR | + SCHIZO_PCICTRL_RTRY_ERR | + SCHIZO_PCICTRL_DTO_ERR | + SCHIZO_PCICTRL_SBH_ERR | + SCHIZO_PCICTRL_SERR); + if (csr_error_bits) { + /* Clear the errors. */ + upa_writeq(csr, csr_reg); + + /* Log 'em. */ + if (csr_error_bits & SCHIZO_PCICTRL_BUS_UNUS) + printk("%s: Bus unusable error asserted.\n", + pbm->name); + if (csr_error_bits & SCHIZO_PCICTRL_TTO_ERR) + printk("%s: PCI TRDY# timeout error asserted.\n", + pbm->name); + if (csr_error_bits & SCHIZO_PCICTRL_RTRY_ERR) + printk("%s: PCI excessive retry error asserted.\n", + pbm->name); + if (csr_error_bits & SCHIZO_PCICTRL_DTO_ERR) + printk("%s: PCI discard timeout error asserted.\n", + pbm->name); + if (csr_error_bits & SCHIZO_PCICTRL_SBH_ERR) + printk("%s: PCI streaming byte hole error asserted.\n", + pbm->name); + if (csr_error_bits & SCHIZO_PCICTRL_SERR) + printk("%s: PCI SERR signal asserted.\n", + pbm->name); + ret = IRQ_HANDLED; + } + pbm->pci_ops->read(pbm->pci_bus, 0, PCI_STATUS, 2, &stat); + if (stat & (PCI_STATUS_PARITY | + PCI_STATUS_SIG_TARGET_ABORT | + PCI_STATUS_REC_TARGET_ABORT | + PCI_STATUS_REC_MASTER_ABORT | + PCI_STATUS_SIG_SYSTEM_ERROR)) { + printk("%s: PCI bus error, PCI_STATUS[%04x]\n", + pbm->name, stat); + pbm->pci_ops->write(pbm->pci_bus, 0, PCI_STATUS, 2, 0xffff); + ret = IRQ_HANDLED; + } + return ret; +} + +static irqreturn_t schizo_pcierr_intr(int irq, void *dev_id) +{ + struct pci_pbm_info *pbm = dev_id; + unsigned long afsr_reg, afar_reg, base; + unsigned long afsr, afar, error_bits; + int reported; + + base = pbm->pbm_regs; + + afsr_reg = base + SCHIZO_PCI_AFSR; + afar_reg = base + SCHIZO_PCI_AFAR; + + /* Latch error status. */ + afar = upa_readq(afar_reg); + afsr = upa_readq(afsr_reg); + + /* Clear primary/secondary error status bits. */ + error_bits = afsr & + (SCHIZO_PCIAFSR_PMA | SCHIZO_PCIAFSR_PTA | + SCHIZO_PCIAFSR_PRTRY | SCHIZO_PCIAFSR_PPERR | + SCHIZO_PCIAFSR_PTTO | SCHIZO_PCIAFSR_PUNUS | + SCHIZO_PCIAFSR_SMA | SCHIZO_PCIAFSR_STA | + SCHIZO_PCIAFSR_SRTRY | SCHIZO_PCIAFSR_SPERR | + SCHIZO_PCIAFSR_STTO | SCHIZO_PCIAFSR_SUNUS); + if (!error_bits) + return schizo_pcierr_intr_other(pbm); + upa_writeq(error_bits, afsr_reg); + + /* Log the error. */ + printk("%s: PCI Error, primary error type[%s]\n", + pbm->name, + (((error_bits & SCHIZO_PCIAFSR_PMA) ? + "Master Abort" : + ((error_bits & SCHIZO_PCIAFSR_PTA) ? + "Target Abort" : + ((error_bits & SCHIZO_PCIAFSR_PRTRY) ? + "Excessive Retries" : + ((error_bits & SCHIZO_PCIAFSR_PPERR) ? + "Parity Error" : + ((error_bits & SCHIZO_PCIAFSR_PTTO) ? + "Timeout" : + ((error_bits & SCHIZO_PCIAFSR_PUNUS) ? + "Bus Unusable" : "???")))))))); + printk("%s: bytemask[%04lx] was_block(%d) space(%s)\n", + pbm->name, + (afsr & SCHIZO_PCIAFSR_BMSK) >> 32UL, + (afsr & SCHIZO_PCIAFSR_BLK) ? 1 : 0, + ((afsr & SCHIZO_PCIAFSR_CFG) ? + "Config" : + ((afsr & SCHIZO_PCIAFSR_MEM) ? + "Memory" : + ((afsr & SCHIZO_PCIAFSR_IO) ? + "I/O" : "???")))); + printk("%s: PCI AFAR [%016lx]\n", + pbm->name, afar); + printk("%s: PCI Secondary errors [", + pbm->name); + reported = 0; + if (afsr & SCHIZO_PCIAFSR_SMA) { + reported++; + printk("(Master Abort)"); + } + if (afsr & SCHIZO_PCIAFSR_STA) { + reported++; + printk("(Target Abort)"); + } + if (afsr & SCHIZO_PCIAFSR_SRTRY) { + reported++; + printk("(Excessive Retries)"); + } + if (afsr & SCHIZO_PCIAFSR_SPERR) { + reported++; + printk("(Parity Error)"); + } + if (afsr & SCHIZO_PCIAFSR_STTO) { + reported++; + printk("(Timeout)"); + } + if (afsr & SCHIZO_PCIAFSR_SUNUS) { + reported++; + printk("(Bus Unusable)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + /* For the error types shown, scan PBM's PCI bus for devices + * which have logged that error type. + */ + + /* If we see a Target Abort, this could be the result of an + * IOMMU translation error of some sort. It is extremely + * useful to log this information as usually it indicates + * a bug in the IOMMU support code or a PCI device driver. + */ + if (error_bits & (SCHIZO_PCIAFSR_PTA | SCHIZO_PCIAFSR_STA)) { + schizo_check_iommu_error(pbm, PCI_ERR); + pci_scan_for_target_abort(pbm, pbm->pci_bus); + } + if (error_bits & (SCHIZO_PCIAFSR_PMA | SCHIZO_PCIAFSR_SMA)) + pci_scan_for_master_abort(pbm, pbm->pci_bus); + + /* For excessive retries, PSYCHO/PBM will abort the device + * and there is no way to specifically check for excessive + * retries in the config space status registers. So what + * we hope is that we'll catch it via the master/target + * abort events. + */ + + if (error_bits & (SCHIZO_PCIAFSR_PPERR | SCHIZO_PCIAFSR_SPERR)) + pci_scan_for_parity_error(pbm, pbm->pci_bus); + + return IRQ_HANDLED; +} + +#define SCHIZO_SAFARI_ERRLOG 0x10018UL + +#define SAFARI_ERRLOG_ERROUT 0x8000000000000000UL + +#define BUS_ERROR_BADCMD 0x4000000000000000UL /* Schizo/Tomatillo */ +#define BUS_ERROR_SSMDIS 0x2000000000000000UL /* Safari */ +#define BUS_ERROR_BADMA 0x1000000000000000UL /* Safari */ +#define BUS_ERROR_BADMB 0x0800000000000000UL /* Safari */ +#define BUS_ERROR_BADMC 0x0400000000000000UL /* Safari */ +#define BUS_ERROR_SNOOP_GR 0x0000000000200000UL /* Tomatillo */ +#define BUS_ERROR_SNOOP_PCI 0x0000000000100000UL /* Tomatillo */ +#define BUS_ERROR_SNOOP_RD 0x0000000000080000UL /* Tomatillo */ +#define BUS_ERROR_SNOOP_RDS 0x0000000000020000UL /* Tomatillo */ +#define BUS_ERROR_SNOOP_RDSA 0x0000000000010000UL /* Tomatillo */ +#define BUS_ERROR_SNOOP_OWN 0x0000000000008000UL /* Tomatillo */ +#define BUS_ERROR_SNOOP_RDO 0x0000000000004000UL /* Tomatillo */ +#define BUS_ERROR_CPU1PS 0x0000000000002000UL /* Safari */ +#define BUS_ERROR_WDATA_PERR 0x0000000000002000UL /* Tomatillo */ +#define BUS_ERROR_CPU1PB 0x0000000000001000UL /* Safari */ +#define BUS_ERROR_CTRL_PERR 0x0000000000001000UL /* Tomatillo */ +#define BUS_ERROR_CPU0PS 0x0000000000000800UL /* Safari */ +#define BUS_ERROR_SNOOP_ERR 0x0000000000000800UL /* Tomatillo */ +#define BUS_ERROR_CPU0PB 0x0000000000000400UL /* Safari */ +#define BUS_ERROR_JBUS_ILL_B 0x0000000000000400UL /* Tomatillo */ +#define BUS_ERROR_CIQTO 0x0000000000000200UL /* Safari */ +#define BUS_ERROR_LPQTO 0x0000000000000100UL /* Safari */ +#define BUS_ERROR_JBUS_ILL_C 0x0000000000000100UL /* Tomatillo */ +#define BUS_ERROR_SFPQTO 0x0000000000000080UL /* Safari */ +#define BUS_ERROR_UFPQTO 0x0000000000000040UL /* Safari */ +#define BUS_ERROR_RD_PERR 0x0000000000000040UL /* Tomatillo */ +#define BUS_ERROR_APERR 0x0000000000000020UL /* Safari/Tomatillo */ +#define BUS_ERROR_UNMAP 0x0000000000000010UL /* Safari/Tomatillo */ +#define BUS_ERROR_BUSERR 0x0000000000000004UL /* Safari/Tomatillo */ +#define BUS_ERROR_TIMEOUT 0x0000000000000002UL /* Safari/Tomatillo */ +#define BUS_ERROR_ILL 0x0000000000000001UL /* Safari */ + +/* We only expect UNMAP errors here. The rest of the Safari errors + * are marked fatal and thus cause a system reset. + */ +static irqreturn_t schizo_safarierr_intr(int irq, void *dev_id) +{ + struct pci_pbm_info *pbm = dev_id; + u64 errlog; + + errlog = upa_readq(pbm->controller_regs + SCHIZO_SAFARI_ERRLOG); + upa_writeq(errlog & ~(SAFARI_ERRLOG_ERROUT), + pbm->controller_regs + SCHIZO_SAFARI_ERRLOG); + + if (!(errlog & BUS_ERROR_UNMAP)) { + printk("%s: Unexpected Safari/JBUS error interrupt, errlog[%016llx]\n", + pbm->name, errlog); + + return IRQ_HANDLED; + } + + printk("%s: Safari/JBUS interrupt, UNMAPPED error, interrogating IOMMUs.\n", + pbm->name); + schizo_check_iommu_error(pbm, SAFARI_ERR); + + return IRQ_HANDLED; +} + +/* Nearly identical to PSYCHO equivalents... */ +#define SCHIZO_ECC_CTRL 0x10020UL +#define SCHIZO_ECCCTRL_EE 0x8000000000000000UL /* Enable ECC Checking */ +#define SCHIZO_ECCCTRL_UE 0x4000000000000000UL /* Enable UE Interrupts */ +#define SCHIZO_ECCCTRL_CE 0x2000000000000000UL /* Enable CE INterrupts */ + +#define SCHIZO_SAFARI_ERRCTRL 0x10008UL +#define SCHIZO_SAFERRCTRL_EN 0x8000000000000000UL +#define SCHIZO_SAFARI_IRQCTRL 0x10010UL +#define SCHIZO_SAFIRQCTRL_EN 0x8000000000000000UL + +static int pbm_routes_this_ino(struct pci_pbm_info *pbm, u32 ino) +{ + ino &= IMAP_INO; + + if (pbm->ino_bitmap & (1UL << ino)) + return 1; + + return 0; +} + +/* How the Tomatillo IRQs are routed around is pure guesswork here. + * + * All the Tomatillo devices I see in prtconf dumps seem to have only + * a single PCI bus unit attached to it. It would seem they are separate + * devices because their PortID (ie. JBUS ID) values are all different + * and thus the registers are mapped to totally different locations. + * + * However, two Tomatillo's look "similar" in that the only difference + * in their PortID is the lowest bit. + * + * So if we were to ignore this lower bit, it certainly looks like two + * PCI bus units of the same Tomatillo. I still have not really + * figured this out... + */ +static void tomatillo_register_error_handlers(struct pci_pbm_info *pbm) +{ + struct platform_device *op = of_find_device_by_node(pbm->op->dev.of_node); + u64 tmp, err_mask, err_no_mask; + int err; + + /* Tomatillo IRQ property layout is: + * 0: PCIERR + * 1: UE ERR + * 2: CE ERR + * 3: SERR + * 4: POWER FAIL? + */ + + if (pbm_routes_this_ino(pbm, SCHIZO_UE_INO)) { + err = request_irq(op->archdata.irqs[1], schizo_ue_intr, 0, + "TOMATILLO_UE", pbm); + if (err) + printk(KERN_WARNING "%s: Could not register UE, " + "err=%d\n", pbm->name, err); + } + if (pbm_routes_this_ino(pbm, SCHIZO_CE_INO)) { + err = request_irq(op->archdata.irqs[2], schizo_ce_intr, 0, + "TOMATILLO_CE", pbm); + if (err) + printk(KERN_WARNING "%s: Could not register CE, " + "err=%d\n", pbm->name, err); + } + err = 0; + if (pbm_routes_this_ino(pbm, SCHIZO_PCIERR_A_INO)) { + err = request_irq(op->archdata.irqs[0], schizo_pcierr_intr, 0, + "TOMATILLO_PCIERR", pbm); + } else if (pbm_routes_this_ino(pbm, SCHIZO_PCIERR_B_INO)) { + err = request_irq(op->archdata.irqs[0], schizo_pcierr_intr, 0, + "TOMATILLO_PCIERR", pbm); + } + if (err) + printk(KERN_WARNING "%s: Could not register PCIERR, " + "err=%d\n", pbm->name, err); + + if (pbm_routes_this_ino(pbm, SCHIZO_SERR_INO)) { + err = request_irq(op->archdata.irqs[3], schizo_safarierr_intr, 0, + "TOMATILLO_SERR", pbm); + if (err) + printk(KERN_WARNING "%s: Could not register SERR, " + "err=%d\n", pbm->name, err); + } + + /* Enable UE and CE interrupts for controller. */ + upa_writeq((SCHIZO_ECCCTRL_EE | + SCHIZO_ECCCTRL_UE | + SCHIZO_ECCCTRL_CE), pbm->controller_regs + SCHIZO_ECC_CTRL); + + /* Enable PCI Error interrupts and clear error + * bits. + */ + err_mask = (SCHIZO_PCICTRL_BUS_UNUS | + SCHIZO_PCICTRL_TTO_ERR | + SCHIZO_PCICTRL_RTRY_ERR | + SCHIZO_PCICTRL_SERR | + SCHIZO_PCICTRL_EEN); + + err_no_mask = SCHIZO_PCICTRL_DTO_ERR; + + tmp = upa_readq(pbm->pbm_regs + SCHIZO_PCI_CTRL); + tmp |= err_mask; + tmp &= ~err_no_mask; + upa_writeq(tmp, pbm->pbm_regs + SCHIZO_PCI_CTRL); + + err_mask = (SCHIZO_PCIAFSR_PMA | SCHIZO_PCIAFSR_PTA | + SCHIZO_PCIAFSR_PRTRY | SCHIZO_PCIAFSR_PPERR | + SCHIZO_PCIAFSR_PTTO | + SCHIZO_PCIAFSR_SMA | SCHIZO_PCIAFSR_STA | + SCHIZO_PCIAFSR_SRTRY | SCHIZO_PCIAFSR_SPERR | + SCHIZO_PCIAFSR_STTO); + + upa_writeq(err_mask, pbm->pbm_regs + SCHIZO_PCI_AFSR); + + err_mask = (BUS_ERROR_BADCMD | BUS_ERROR_SNOOP_GR | + BUS_ERROR_SNOOP_PCI | BUS_ERROR_SNOOP_RD | + BUS_ERROR_SNOOP_RDS | BUS_ERROR_SNOOP_RDSA | + BUS_ERROR_SNOOP_OWN | BUS_ERROR_SNOOP_RDO | + BUS_ERROR_WDATA_PERR | BUS_ERROR_CTRL_PERR | + BUS_ERROR_SNOOP_ERR | BUS_ERROR_JBUS_ILL_B | + BUS_ERROR_JBUS_ILL_C | BUS_ERROR_RD_PERR | + BUS_ERROR_APERR | BUS_ERROR_UNMAP | + BUS_ERROR_BUSERR | BUS_ERROR_TIMEOUT); + + upa_writeq((SCHIZO_SAFERRCTRL_EN | err_mask), + pbm->controller_regs + SCHIZO_SAFARI_ERRCTRL); + + upa_writeq((SCHIZO_SAFIRQCTRL_EN | (BUS_ERROR_UNMAP)), + pbm->controller_regs + SCHIZO_SAFARI_IRQCTRL); +} + +static void schizo_register_error_handlers(struct pci_pbm_info *pbm) +{ + struct platform_device *op = of_find_device_by_node(pbm->op->dev.of_node); + u64 tmp, err_mask, err_no_mask; + int err; + + /* Schizo IRQ property layout is: + * 0: PCIERR + * 1: UE ERR + * 2: CE ERR + * 3: SERR + * 4: POWER FAIL? + */ + + if (pbm_routes_this_ino(pbm, SCHIZO_UE_INO)) { + err = request_irq(op->archdata.irqs[1], schizo_ue_intr, 0, + "SCHIZO_UE", pbm); + if (err) + printk(KERN_WARNING "%s: Could not register UE, " + "err=%d\n", pbm->name, err); + } + if (pbm_routes_this_ino(pbm, SCHIZO_CE_INO)) { + err = request_irq(op->archdata.irqs[2], schizo_ce_intr, 0, + "SCHIZO_CE", pbm); + if (err) + printk(KERN_WARNING "%s: Could not register CE, " + "err=%d\n", pbm->name, err); + } + err = 0; + if (pbm_routes_this_ino(pbm, SCHIZO_PCIERR_A_INO)) { + err = request_irq(op->archdata.irqs[0], schizo_pcierr_intr, 0, + "SCHIZO_PCIERR", pbm); + } else if (pbm_routes_this_ino(pbm, SCHIZO_PCIERR_B_INO)) { + err = request_irq(op->archdata.irqs[0], schizo_pcierr_intr, 0, + "SCHIZO_PCIERR", pbm); + } + if (err) + printk(KERN_WARNING "%s: Could not register PCIERR, " + "err=%d\n", pbm->name, err); + + if (pbm_routes_this_ino(pbm, SCHIZO_SERR_INO)) { + err = request_irq(op->archdata.irqs[3], schizo_safarierr_intr, 0, + "SCHIZO_SERR", pbm); + if (err) + printk(KERN_WARNING "%s: Could not register SERR, " + "err=%d\n", pbm->name, err); + } + + /* Enable UE and CE interrupts for controller. */ + upa_writeq((SCHIZO_ECCCTRL_EE | + SCHIZO_ECCCTRL_UE | + SCHIZO_ECCCTRL_CE), pbm->controller_regs + SCHIZO_ECC_CTRL); + + err_mask = (SCHIZO_PCICTRL_BUS_UNUS | + SCHIZO_PCICTRL_ESLCK | + SCHIZO_PCICTRL_TTO_ERR | + SCHIZO_PCICTRL_RTRY_ERR | + SCHIZO_PCICTRL_SBH_ERR | + SCHIZO_PCICTRL_SERR | + SCHIZO_PCICTRL_EEN); + + err_no_mask = (SCHIZO_PCICTRL_DTO_ERR | + SCHIZO_PCICTRL_SBH_INT); + + /* Enable PCI Error interrupts and clear error + * bits for each PBM. + */ + tmp = upa_readq(pbm->pbm_regs + SCHIZO_PCI_CTRL); + tmp |= err_mask; + tmp &= ~err_no_mask; + upa_writeq(tmp, pbm->pbm_regs + SCHIZO_PCI_CTRL); + + upa_writeq((SCHIZO_PCIAFSR_PMA | SCHIZO_PCIAFSR_PTA | + SCHIZO_PCIAFSR_PRTRY | SCHIZO_PCIAFSR_PPERR | + SCHIZO_PCIAFSR_PTTO | SCHIZO_PCIAFSR_PUNUS | + SCHIZO_PCIAFSR_SMA | SCHIZO_PCIAFSR_STA | + SCHIZO_PCIAFSR_SRTRY | SCHIZO_PCIAFSR_SPERR | + SCHIZO_PCIAFSR_STTO | SCHIZO_PCIAFSR_SUNUS), + pbm->pbm_regs + SCHIZO_PCI_AFSR); + + /* Make all Safari error conditions fatal except unmapped + * errors which we make generate interrupts. + */ + err_mask = (BUS_ERROR_BADCMD | BUS_ERROR_SSMDIS | + BUS_ERROR_BADMA | BUS_ERROR_BADMB | + BUS_ERROR_BADMC | + BUS_ERROR_CPU1PS | BUS_ERROR_CPU1PB | + BUS_ERROR_CPU0PS | BUS_ERROR_CPU0PB | + BUS_ERROR_CIQTO | + BUS_ERROR_LPQTO | BUS_ERROR_SFPQTO | + BUS_ERROR_UFPQTO | BUS_ERROR_APERR | + BUS_ERROR_BUSERR | BUS_ERROR_TIMEOUT | + BUS_ERROR_ILL); +#if 1 + /* XXX Something wrong with some Excalibur systems + * XXX Sun is shipping. The behavior on a 2-cpu + * XXX machine is that both CPU1 parity error bits + * XXX are set and are immediately set again when + * XXX their error status bits are cleared. Just + * XXX ignore them for now. -DaveM + */ + err_mask &= ~(BUS_ERROR_CPU1PS | BUS_ERROR_CPU1PB | + BUS_ERROR_CPU0PS | BUS_ERROR_CPU0PB); +#endif + + upa_writeq((SCHIZO_SAFERRCTRL_EN | err_mask), + pbm->controller_regs + SCHIZO_SAFARI_ERRCTRL); +} + +static void pbm_config_busmastering(struct pci_pbm_info *pbm) +{ + u8 *addr; + + /* Set cache-line size to 64 bytes, this is actually + * a nop but I do it for completeness. + */ + addr = schizo_pci_config_mkaddr(pbm, pbm->pci_first_busno, + 0, PCI_CACHE_LINE_SIZE); + pci_config_write8(addr, 64 / sizeof(u32)); + + /* Set PBM latency timer to 64 PCI clocks. */ + addr = schizo_pci_config_mkaddr(pbm, pbm->pci_first_busno, + 0, PCI_LATENCY_TIMER); + pci_config_write8(addr, 64); +} + +static void schizo_scan_bus(struct pci_pbm_info *pbm, struct device *parent) +{ + pbm_config_busmastering(pbm); + pbm->is_66mhz_capable = + (of_find_property(pbm->op->dev.of_node, "66mhz-capable", NULL) + != NULL); + + pbm->pci_bus = pci_scan_one_pbm(pbm, parent); + + if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO) + tomatillo_register_error_handlers(pbm); + else + schizo_register_error_handlers(pbm); +} + +#define SCHIZO_STRBUF_CONTROL (0x02800UL) +#define SCHIZO_STRBUF_FLUSH (0x02808UL) +#define SCHIZO_STRBUF_FSYNC (0x02810UL) +#define SCHIZO_STRBUF_CTXFLUSH (0x02818UL) +#define SCHIZO_STRBUF_CTXMATCH (0x10000UL) + +static void schizo_pbm_strbuf_init(struct pci_pbm_info *pbm) +{ + unsigned long base = pbm->pbm_regs; + u64 control; + + if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO) { + /* TOMATILLO lacks streaming cache. */ + return; + } + + /* SCHIZO has context flushing. */ + pbm->stc.strbuf_control = base + SCHIZO_STRBUF_CONTROL; + pbm->stc.strbuf_pflush = base + SCHIZO_STRBUF_FLUSH; + pbm->stc.strbuf_fsync = base + SCHIZO_STRBUF_FSYNC; + pbm->stc.strbuf_ctxflush = base + SCHIZO_STRBUF_CTXFLUSH; + pbm->stc.strbuf_ctxmatch_base = base + SCHIZO_STRBUF_CTXMATCH; + + pbm->stc.strbuf_flushflag = (volatile unsigned long *) + ((((unsigned long)&pbm->stc.__flushflag_buf[0]) + + 63UL) + & ~63UL); + pbm->stc.strbuf_flushflag_pa = (unsigned long) + __pa(pbm->stc.strbuf_flushflag); + + /* Turn off LRU locking and diag mode, enable the + * streaming buffer and leave the rerun-disable + * setting however OBP set it. + */ + control = upa_readq(pbm->stc.strbuf_control); + control &= ~(SCHIZO_STRBUF_CTRL_LPTR | + SCHIZO_STRBUF_CTRL_LENAB | + SCHIZO_STRBUF_CTRL_DENAB); + control |= SCHIZO_STRBUF_CTRL_ENAB; + upa_writeq(control, pbm->stc.strbuf_control); + + pbm->stc.strbuf_enabled = 1; +} + +#define SCHIZO_IOMMU_CONTROL (0x00200UL) +#define SCHIZO_IOMMU_TSBBASE (0x00208UL) +#define SCHIZO_IOMMU_FLUSH (0x00210UL) +#define SCHIZO_IOMMU_CTXFLUSH (0x00218UL) + +static int schizo_pbm_iommu_init(struct pci_pbm_info *pbm) +{ + static const u32 vdma_default[] = { 0xc0000000, 0x40000000 }; + unsigned long i, tagbase, database; + struct iommu *iommu = pbm->iommu; + int tsbsize, err; + const u32 *vdma; + u32 dma_mask; + u64 control; + + vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma", NULL); + if (!vdma) + vdma = vdma_default; + + dma_mask = vdma[0]; + switch (vdma[1]) { + case 0x20000000: + dma_mask |= 0x1fffffff; + tsbsize = 64; + break; + + case 0x40000000: + dma_mask |= 0x3fffffff; + tsbsize = 128; + break; + + case 0x80000000: + dma_mask |= 0x7fffffff; + tsbsize = 128; + break; + + default: + printk(KERN_ERR PFX "Strange virtual-dma size.\n"); + return -EINVAL; + } + + /* Register addresses, SCHIZO has iommu ctx flushing. */ + iommu->iommu_control = pbm->pbm_regs + SCHIZO_IOMMU_CONTROL; + iommu->iommu_tsbbase = pbm->pbm_regs + SCHIZO_IOMMU_TSBBASE; + iommu->iommu_flush = pbm->pbm_regs + SCHIZO_IOMMU_FLUSH; + iommu->iommu_tags = iommu->iommu_flush + (0xa580UL - 0x0210UL); + iommu->iommu_ctxflush = pbm->pbm_regs + SCHIZO_IOMMU_CTXFLUSH; + + /* We use the main control/status register of SCHIZO as the write + * completion register. + */ + iommu->write_complete_reg = pbm->controller_regs + 0x10000UL; + + /* + * Invalidate TLB Entries. + */ + control = upa_readq(iommu->iommu_control); + control |= SCHIZO_IOMMU_CTRL_DENAB; + upa_writeq(control, iommu->iommu_control); + + tagbase = SCHIZO_IOMMU_TAG, database = SCHIZO_IOMMU_DATA; + + for (i = 0; i < 16; i++) { + upa_writeq(0, pbm->pbm_regs + tagbase + (i * 8UL)); + upa_writeq(0, pbm->pbm_regs + database + (i * 8UL)); + } + + /* Leave diag mode enabled for full-flushing done + * in pci_iommu.c + */ + err = iommu_table_init(iommu, tsbsize * 8 * 1024, vdma[0], dma_mask, + pbm->numa_node); + if (err) { + printk(KERN_ERR PFX "iommu_table_init() fails with %d\n", err); + return err; + } + + upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase); + + control = upa_readq(iommu->iommu_control); + control &= ~(SCHIZO_IOMMU_CTRL_TSBSZ | SCHIZO_IOMMU_CTRL_TBWSZ); + switch (tsbsize) { + case 64: + control |= SCHIZO_IOMMU_TSBSZ_64K; + break; + case 128: + control |= SCHIZO_IOMMU_TSBSZ_128K; + break; + } + + control |= SCHIZO_IOMMU_CTRL_ENAB; + upa_writeq(control, iommu->iommu_control); + + return 0; +} + +#define SCHIZO_PCI_IRQ_RETRY (0x1a00UL) +#define SCHIZO_IRQ_RETRY_INF 0xffUL + +#define SCHIZO_PCI_DIAG (0x2020UL) +#define SCHIZO_PCIDIAG_D_BADECC (1UL << 10UL) /* Disable BAD ECC errors (Schizo) */ +#define SCHIZO_PCIDIAG_D_BYPASS (1UL << 9UL) /* Disable MMU bypass mode (Schizo/Tomatillo) */ +#define SCHIZO_PCIDIAG_D_TTO (1UL << 8UL) /* Disable TTO errors (Schizo/Tomatillo) */ +#define SCHIZO_PCIDIAG_D_RTRYARB (1UL << 7UL) /* Disable retry arbitration (Schizo) */ +#define SCHIZO_PCIDIAG_D_RETRY (1UL << 6UL) /* Disable retry limit (Schizo/Tomatillo) */ +#define SCHIZO_PCIDIAG_D_INTSYNC (1UL << 5UL) /* Disable interrupt/DMA synch (Schizo/Tomatillo) */ +#define SCHIZO_PCIDIAG_I_DMA_PARITY (1UL << 3UL) /* Invert DMA parity (Schizo/Tomatillo) */ +#define SCHIZO_PCIDIAG_I_PIOD_PARITY (1UL << 2UL) /* Invert PIO data parity (Schizo/Tomatillo) */ +#define SCHIZO_PCIDIAG_I_PIOA_PARITY (1UL << 1UL) /* Invert PIO address parity (Schizo/Tomatillo) */ + +#define TOMATILLO_PCI_IOC_CSR (0x2248UL) +#define TOMATILLO_IOC_PART_WPENAB 0x0000000000080000UL +#define TOMATILLO_IOC_RDMULT_PENAB 0x0000000000040000UL +#define TOMATILLO_IOC_RDONE_PENAB 0x0000000000020000UL +#define TOMATILLO_IOC_RDLINE_PENAB 0x0000000000010000UL +#define TOMATILLO_IOC_RDMULT_PLEN 0x000000000000c000UL +#define TOMATILLO_IOC_RDMULT_PLEN_SHIFT 14UL +#define TOMATILLO_IOC_RDONE_PLEN 0x0000000000003000UL +#define TOMATILLO_IOC_RDONE_PLEN_SHIFT 12UL +#define TOMATILLO_IOC_RDLINE_PLEN 0x0000000000000c00UL +#define TOMATILLO_IOC_RDLINE_PLEN_SHIFT 10UL +#define TOMATILLO_IOC_PREF_OFF 0x00000000000003f8UL +#define TOMATILLO_IOC_PREF_OFF_SHIFT 3UL +#define TOMATILLO_IOC_RDMULT_CPENAB 0x0000000000000004UL +#define TOMATILLO_IOC_RDONE_CPENAB 0x0000000000000002UL +#define TOMATILLO_IOC_RDLINE_CPENAB 0x0000000000000001UL + +#define TOMATILLO_PCI_IOC_TDIAG (0x2250UL) +#define TOMATILLO_PCI_IOC_DDIAG (0x2290UL) + +static void schizo_pbm_hw_init(struct pci_pbm_info *pbm) +{ + u64 tmp; + + upa_writeq(5, pbm->pbm_regs + SCHIZO_PCI_IRQ_RETRY); + + tmp = upa_readq(pbm->pbm_regs + SCHIZO_PCI_CTRL); + + /* Enable arbiter for all PCI slots. */ + tmp |= 0xff; + + if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO && + pbm->chip_version >= 0x2) + tmp |= 0x3UL << SCHIZO_PCICTRL_PTO_SHIFT; + + if (!of_property_read_bool(pbm->op->dev.of_node, "no-bus-parking")) + tmp |= SCHIZO_PCICTRL_PARK; + else + tmp &= ~SCHIZO_PCICTRL_PARK; + + if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO && + pbm->chip_version <= 0x1) + tmp |= SCHIZO_PCICTRL_DTO_INT; + else + tmp &= ~SCHIZO_PCICTRL_DTO_INT; + + if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO) + tmp |= (SCHIZO_PCICTRL_MRM_PREF | + SCHIZO_PCICTRL_RDO_PREF | + SCHIZO_PCICTRL_RDL_PREF); + + upa_writeq(tmp, pbm->pbm_regs + SCHIZO_PCI_CTRL); + + tmp = upa_readq(pbm->pbm_regs + SCHIZO_PCI_DIAG); + tmp &= ~(SCHIZO_PCIDIAG_D_RTRYARB | + SCHIZO_PCIDIAG_D_RETRY | + SCHIZO_PCIDIAG_D_INTSYNC); + upa_writeq(tmp, pbm->pbm_regs + SCHIZO_PCI_DIAG); + + if (pbm->chip_type == PBM_CHIP_TYPE_TOMATILLO) { + /* Clear prefetch lengths to workaround a bug in + * Jalapeno... + */ + tmp = (TOMATILLO_IOC_PART_WPENAB | + (1 << TOMATILLO_IOC_PREF_OFF_SHIFT) | + TOMATILLO_IOC_RDMULT_CPENAB | + TOMATILLO_IOC_RDONE_CPENAB | + TOMATILLO_IOC_RDLINE_CPENAB); + + upa_writeq(tmp, pbm->pbm_regs + TOMATILLO_PCI_IOC_CSR); + } +} + +static int schizo_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op, u32 portid, + int chip_type) +{ + const struct linux_prom64_registers *regs; + struct device_node *dp = op->dev.of_node; + const char *chipset_name; + int err; + + switch (chip_type) { + case PBM_CHIP_TYPE_TOMATILLO: + chipset_name = "TOMATILLO"; + break; + + case PBM_CHIP_TYPE_SCHIZO_PLUS: + chipset_name = "SCHIZO+"; + break; + + case PBM_CHIP_TYPE_SCHIZO: + default: + chipset_name = "SCHIZO"; + break; + } + + /* For SCHIZO, three OBP regs: + * 1) PBM controller regs + * 2) Schizo front-end controller regs (same for both PBMs) + * 3) PBM PCI config space + * + * For TOMATILLO, four OBP regs: + * 1) PBM controller regs + * 2) Tomatillo front-end controller regs + * 3) PBM PCI config space + * 4) Ichip regs + */ + regs = of_get_property(dp, "reg", NULL); + + pbm->next = pci_pbm_root; + pci_pbm_root = pbm; + + pbm->numa_node = NUMA_NO_NODE; + + pbm->pci_ops = &sun4u_pci_ops; + pbm->config_space_reg_bits = 8; + + pbm->index = pci_num_pbms++; + + pbm->portid = portid; + pbm->op = op; + + pbm->chip_type = chip_type; + pbm->chip_version = of_getintprop_default(dp, "version#", 0); + pbm->chip_revision = of_getintprop_default(dp, "module-version#", 0); + + pbm->pbm_regs = regs[0].phys_addr; + pbm->controller_regs = regs[1].phys_addr - 0x10000UL; + + if (chip_type == PBM_CHIP_TYPE_TOMATILLO) + pbm->sync_reg = regs[3].phys_addr + 0x1a18UL; + + pbm->name = dp->full_name; + + printk("%s: %s PCI Bus Module ver[%x:%x]\n", + pbm->name, chipset_name, + pbm->chip_version, pbm->chip_revision); + + schizo_pbm_hw_init(pbm); + + pci_determine_mem_io_space(pbm); + + pci_get_pbm_props(pbm); + + err = schizo_pbm_iommu_init(pbm); + if (err) + return err; + + schizo_pbm_strbuf_init(pbm); + + schizo_scan_bus(pbm, &op->dev); + + return 0; +} + +static inline int portid_compare(u32 x, u32 y, int chip_type) +{ + if (chip_type == PBM_CHIP_TYPE_TOMATILLO) { + if (x == (y ^ 1)) + return 1; + return 0; + } + return (x == y); +} + +static struct pci_pbm_info *schizo_find_sibling(u32 portid, int chip_type) +{ + struct pci_pbm_info *pbm; + + for (pbm = pci_pbm_root; pbm; pbm = pbm->next) { + if (portid_compare(pbm->portid, portid, chip_type)) + return pbm; + } + return NULL; +} + +static int __schizo_init(struct platform_device *op, unsigned long chip_type) +{ + struct device_node *dp = op->dev.of_node; + struct pci_pbm_info *pbm; + struct iommu *iommu; + u32 portid; + int err; + + portid = of_getintprop_default(dp, "portid", 0xff); + + err = -ENOMEM; + pbm = kzalloc_obj(*pbm); + if (!pbm) { + printk(KERN_ERR PFX "Cannot allocate pci_pbm_info.\n"); + goto out_err; + } + + pbm->sibling = schizo_find_sibling(portid, chip_type); + + iommu = kzalloc_obj(struct iommu); + if (!iommu) { + printk(KERN_ERR PFX "Cannot allocate PBM A iommu.\n"); + goto out_free_pbm; + } + + pbm->iommu = iommu; + + if (schizo_pbm_init(pbm, op, portid, chip_type)) + goto out_free_iommu; + + if (pbm->sibling) + pbm->sibling->sibling = pbm; + + dev_set_drvdata(&op->dev, pbm); + + return 0; + +out_free_iommu: + kfree(pbm->iommu); + +out_free_pbm: + kfree(pbm); + +out_err: + return err; +} + +static int schizo_probe(struct platform_device *op) +{ + unsigned long chip_type = (unsigned long)device_get_match_data(&op->dev); + + if (!chip_type) + return -EINVAL; + return __schizo_init(op, chip_type); +} + +/* The ordering of this table is very important. Some Tomatillo + * nodes announce that they are compatible with both pci108e,a801 + * and pci108e,8001. So list the chips in reverse chronological + * order. + */ +static const struct of_device_id schizo_match[] = { + { + .name = "pci", + .compatible = "pci108e,a801", + .data = (void *) PBM_CHIP_TYPE_TOMATILLO, + }, + { + .name = "pci", + .compatible = "pci108e,8002", + .data = (void *) PBM_CHIP_TYPE_SCHIZO_PLUS, + }, + { + .name = "pci", + .compatible = "pci108e,8001", + .data = (void *) PBM_CHIP_TYPE_SCHIZO, + }, + {}, +}; + +static struct platform_driver schizo_driver = { + .driver = { + .name = DRIVER_NAME, + .of_match_table = schizo_match, + }, + .probe = schizo_probe, +}; + +static int __init schizo_init(void) +{ + return platform_driver_register(&schizo_driver); +} + +subsys_initcall(schizo_init); diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c new file mode 100644 index 000000000..61f14b4c8 --- /dev/null +++ b/arch/sparc/kernel/pci_sun4v.c @@ -0,0 +1,1359 @@ +// SPDX-License-Identifier: GPL-2.0 +/* pci_sun4v.c: SUN4V specific PCI controller support. + * + * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "pci_impl.h" +#include "iommu_common.h" +#include "kernel.h" + +#include "pci_sun4v.h" + +#define DRIVER_NAME "pci_sun4v" +#define PFX DRIVER_NAME ": " + +static unsigned long vpci_major; +static unsigned long vpci_minor; + +struct vpci_version { + unsigned long major; + unsigned long minor; +}; + +/* Ordered from largest major to lowest */ +static struct vpci_version vpci_versions[] = { + { .major = 2, .minor = 0 }, + { .major = 1, .minor = 1 }, +}; + +static unsigned long vatu_major = 1; +static unsigned long vatu_minor = 1; + +#define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) + +struct iommu_batch { + struct device *dev; /* Device mapping is for. */ + unsigned long prot; /* IOMMU page protections */ + unsigned long entry; /* Index into IOTSB. */ + u64 *pglist; /* List of physical pages */ + unsigned long npages; /* Number of pages in list. */ +}; + +static DEFINE_PER_CPU(struct iommu_batch, iommu_batch); +static int iommu_batch_initialized; + +/* Interrupts must be disabled. */ +static inline void iommu_batch_start(struct device *dev, unsigned long prot, unsigned long entry) +{ + struct iommu_batch *p = this_cpu_ptr(&iommu_batch); + + p->dev = dev; + p->prot = prot; + p->entry = entry; + p->npages = 0; +} + +static inline bool iommu_use_atu(struct iommu *iommu, u64 mask) +{ + return iommu->atu && mask > DMA_BIT_MASK(32); +} + +/* Interrupts must be disabled. */ +static long iommu_batch_flush(struct iommu_batch *p, u64 mask) +{ + struct pci_pbm_info *pbm = p->dev->archdata.host_controller; + u64 *pglist = p->pglist; + u64 index_count; + unsigned long devhandle = pbm->devhandle; + unsigned long prot = p->prot; + unsigned long entry = p->entry; + unsigned long npages = p->npages; + unsigned long iotsb_num; + unsigned long ret; + long num; + + /* VPCI maj=1, min=[0,1] only supports read and write */ + if (vpci_major < 2) + prot &= (HV_PCI_MAP_ATTR_READ | HV_PCI_MAP_ATTR_WRITE); + + while (npages != 0) { + if (!iommu_use_atu(pbm->iommu, mask)) { + num = pci_sun4v_iommu_map(devhandle, + HV_PCI_TSBID(0, entry), + npages, + prot, + __pa(pglist)); + if (unlikely(num < 0)) { + pr_err_ratelimited("%s: IOMMU map of [%08lx:%08llx:%lx:%lx:%lx] failed with status %ld\n", + __func__, + devhandle, + HV_PCI_TSBID(0, entry), + npages, prot, __pa(pglist), + num); + return -1; + } + } else { + index_count = HV_PCI_IOTSB_INDEX_COUNT(npages, entry), + iotsb_num = pbm->iommu->atu->iotsb->iotsb_num; + ret = pci_sun4v_iotsb_map(devhandle, + iotsb_num, + index_count, + prot, + __pa(pglist), + &num); + if (unlikely(ret != HV_EOK)) { + pr_err_ratelimited("%s: ATU map of [%08lx:%lx:%llx:%lx:%lx] failed with status %ld\n", + __func__, + devhandle, iotsb_num, + index_count, prot, + __pa(pglist), ret); + return -1; + } + } + entry += num; + npages -= num; + pglist += num; + } + + p->entry = entry; + p->npages = 0; + + return 0; +} + +static inline void iommu_batch_new_entry(unsigned long entry, u64 mask) +{ + struct iommu_batch *p = this_cpu_ptr(&iommu_batch); + + if (p->entry + p->npages == entry) + return; + if (p->entry != ~0UL) + iommu_batch_flush(p, mask); + p->entry = entry; +} + +/* Interrupts must be disabled. */ +static inline long iommu_batch_add(u64 phys_page, u64 mask) +{ + struct iommu_batch *p = this_cpu_ptr(&iommu_batch); + + BUG_ON(p->npages >= PGLIST_NENTS); + + p->pglist[p->npages++] = phys_page; + if (p->npages == PGLIST_NENTS) + return iommu_batch_flush(p, mask); + + return 0; +} + +/* Interrupts must be disabled. */ +static inline long iommu_batch_end(u64 mask) +{ + struct iommu_batch *p = this_cpu_ptr(&iommu_batch); + + BUG_ON(p->npages >= PGLIST_NENTS); + + return iommu_batch_flush(p, mask); +} + +static void *dma_4v_alloc_coherent(struct device *dev, size_t size, + dma_addr_t *dma_addrp, gfp_t gfp, + unsigned long attrs) +{ + u64 mask; + unsigned long flags, order, first_page, npages, n; + unsigned long prot = 0; + struct iommu *iommu; + struct iommu_map_table *tbl; + struct page *page; + void *ret; + long entry; + int nid; + + size = IO_PAGE_ALIGN(size); + order = get_order(size); + if (unlikely(order > MAX_PAGE_ORDER)) + return NULL; + + npages = size >> IO_PAGE_SHIFT; + + if (attrs & DMA_ATTR_WEAK_ORDERING) + prot = HV_PCI_MAP_ATTR_RELAXED_ORDER; + + nid = dev->archdata.numa_node; + page = alloc_pages_node(nid, gfp, order); + if (unlikely(!page)) + return NULL; + + first_page = (unsigned long) page_address(page); + memset((char *)first_page, 0, PAGE_SIZE << order); + + iommu = dev->archdata.iommu; + mask = dev->coherent_dma_mask; + if (!iommu_use_atu(iommu, mask)) + tbl = &iommu->tbl; + else + tbl = &iommu->atu->tbl; + + entry = iommu_tbl_range_alloc(dev, tbl, npages, NULL, + (unsigned long)(-1), 0); + + if (unlikely(entry == IOMMU_ERROR_CODE)) + goto range_alloc_fail; + + *dma_addrp = (tbl->table_map_base + (entry << IO_PAGE_SHIFT)); + ret = (void *) first_page; + first_page = __pa(first_page); + + local_irq_save(flags); + + iommu_batch_start(dev, + (HV_PCI_MAP_ATTR_READ | prot | + HV_PCI_MAP_ATTR_WRITE), + entry); + + for (n = 0; n < npages; n++) { + long err = iommu_batch_add(first_page + (n * PAGE_SIZE), mask); + if (unlikely(err < 0L)) + goto iommu_map_fail; + } + + if (unlikely(iommu_batch_end(mask) < 0L)) + goto iommu_map_fail; + + local_irq_restore(flags); + + return ret; + +iommu_map_fail: + local_irq_restore(flags); + iommu_tbl_range_free(tbl, *dma_addrp, npages, IOMMU_ERROR_CODE); + +range_alloc_fail: + free_pages(first_page, order); + return NULL; +} + +static unsigned long dma_4v_iotsb_bind(unsigned long devhandle, + unsigned long iotsb_num, + struct pci_bus *bus_dev) +{ + struct pci_dev *pdev; + unsigned long err; + unsigned int bus; + unsigned int device; + unsigned int fun; + + list_for_each_entry(pdev, &bus_dev->devices, bus_list) { + if (pdev->subordinate) { + /* No need to bind pci bridge */ + dma_4v_iotsb_bind(devhandle, iotsb_num, + pdev->subordinate); + } else { + bus = bus_dev->number; + device = PCI_SLOT(pdev->devfn); + fun = PCI_FUNC(pdev->devfn); + err = pci_sun4v_iotsb_bind(devhandle, iotsb_num, + HV_PCI_DEVICE_BUILD(bus, + device, + fun)); + + /* If bind fails for one device it is going to fail + * for rest of the devices because we are sharing + * IOTSB. So in case of failure simply return with + * error. + */ + if (err) + return err; + } + } + + return 0; +} + +static void dma_4v_iommu_demap(struct device *dev, unsigned long devhandle, + dma_addr_t dvma, unsigned long iotsb_num, + unsigned long entry, unsigned long npages) +{ + unsigned long num, flags; + unsigned long ret; + + local_irq_save(flags); + do { + if (dvma <= DMA_BIT_MASK(32)) { + num = pci_sun4v_iommu_demap(devhandle, + HV_PCI_TSBID(0, entry), + npages); + } else { + ret = pci_sun4v_iotsb_demap(devhandle, iotsb_num, + entry, npages, &num); + if (unlikely(ret != HV_EOK)) { + pr_err_ratelimited("pci_iotsb_demap() failed with error: %ld\n", + ret); + } + } + entry += num; + npages -= num; + } while (npages != 0); + local_irq_restore(flags); +} + +static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, + dma_addr_t dvma, unsigned long attrs) +{ + struct pci_pbm_info *pbm; + struct iommu *iommu; + struct atu *atu; + struct iommu_map_table *tbl; + unsigned long order, npages, entry; + unsigned long iotsb_num; + u32 devhandle; + + npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; + iommu = dev->archdata.iommu; + pbm = dev->archdata.host_controller; + atu = iommu->atu; + devhandle = pbm->devhandle; + + if (!iommu_use_atu(iommu, dvma)) { + tbl = &iommu->tbl; + iotsb_num = 0; /* we don't care for legacy iommu */ + } else { + tbl = &atu->tbl; + iotsb_num = atu->iotsb->iotsb_num; + } + entry = ((dvma - tbl->table_map_base) >> IO_PAGE_SHIFT); + dma_4v_iommu_demap(dev, devhandle, dvma, iotsb_num, entry, npages); + iommu_tbl_range_free(tbl, dvma, npages, IOMMU_ERROR_CODE); + order = get_order(size); + if (order < 10) + free_pages((unsigned long)cpu, order); +} + +static dma_addr_t dma_4v_map_phys(struct device *dev, phys_addr_t phys, + size_t sz, enum dma_data_direction direction, + unsigned long attrs) +{ + struct iommu *iommu; + struct atu *atu; + struct iommu_map_table *tbl; + u64 mask; + unsigned long flags, npages, oaddr; + unsigned long i, prot; + dma_addr_t bus_addr, ret; + long entry; + + if (unlikely(attrs & DMA_ATTR_MMIO)) + /* + * This check is included because older versions of the code + * lacked MMIO path support, and my ability to test this path + * is limited. However, from a software technical standpoint, + * there is no restriction, as the following code operates + * solely on physical addresses. + */ + goto bad; + + iommu = dev->archdata.iommu; + atu = iommu->atu; + + if (unlikely(direction == DMA_NONE)) + goto bad; + + oaddr = (unsigned long)(phys_to_virt(phys)); + npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); + npages >>= IO_PAGE_SHIFT; + + mask = *dev->dma_mask; + if (!iommu_use_atu(iommu, mask)) + tbl = &iommu->tbl; + else + tbl = &atu->tbl; + + entry = iommu_tbl_range_alloc(dev, tbl, npages, NULL, + (unsigned long)(-1), 0); + + if (unlikely(entry == IOMMU_ERROR_CODE)) + goto bad; + + bus_addr = (tbl->table_map_base + (entry << IO_PAGE_SHIFT)); + ret = bus_addr | (oaddr & ~IO_PAGE_MASK); + prot = HV_PCI_MAP_ATTR_READ; + if (direction != DMA_TO_DEVICE) + prot |= HV_PCI_MAP_ATTR_WRITE; + + if (attrs & DMA_ATTR_WEAK_ORDERING) + prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; + + local_irq_save(flags); + + iommu_batch_start(dev, prot, entry); + + phys &= IO_PAGE_MASK; + + for (i = 0; i < npages; i++, phys += IO_PAGE_SIZE) { + long err = iommu_batch_add(phys, mask); + if (unlikely(err < 0L)) + goto iommu_map_fail; + } + if (unlikely(iommu_batch_end(mask) < 0L)) + goto iommu_map_fail; + + local_irq_restore(flags); + + return ret; + +bad: + if (printk_ratelimit()) + WARN_ON(1); + return DMA_MAPPING_ERROR; + +iommu_map_fail: + local_irq_restore(flags); + iommu_tbl_range_free(tbl, bus_addr, npages, IOMMU_ERROR_CODE); + return DMA_MAPPING_ERROR; +} + +static void dma_4v_unmap_phys(struct device *dev, dma_addr_t bus_addr, + size_t sz, enum dma_data_direction direction, + unsigned long attrs) +{ + struct pci_pbm_info *pbm; + struct iommu *iommu; + struct atu *atu; + struct iommu_map_table *tbl; + unsigned long npages; + unsigned long iotsb_num; + long entry; + u32 devhandle; + + if (unlikely(direction == DMA_NONE)) { + if (printk_ratelimit()) + WARN_ON(1); + return; + } + + iommu = dev->archdata.iommu; + pbm = dev->archdata.host_controller; + atu = iommu->atu; + devhandle = pbm->devhandle; + + npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); + npages >>= IO_PAGE_SHIFT; + bus_addr &= IO_PAGE_MASK; + + if (bus_addr <= DMA_BIT_MASK(32)) { + iotsb_num = 0; /* we don't care for legacy iommu */ + tbl = &iommu->tbl; + } else { + iotsb_num = atu->iotsb->iotsb_num; + tbl = &atu->tbl; + } + entry = (bus_addr - tbl->table_map_base) >> IO_PAGE_SHIFT; + dma_4v_iommu_demap(dev, devhandle, bus_addr, iotsb_num, entry, npages); + iommu_tbl_range_free(tbl, bus_addr, npages, IOMMU_ERROR_CODE); +} + +static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction direction, + unsigned long attrs) +{ + struct scatterlist *s, *outs, *segstart; + unsigned long flags, handle, prot; + dma_addr_t dma_next = 0, dma_addr; + unsigned int max_seg_size; + unsigned long seg_boundary_size; + int outcount, incount, i; + struct iommu *iommu; + struct atu *atu; + struct iommu_map_table *tbl; + u64 mask; + unsigned long base_shift; + long err; + + BUG_ON(direction == DMA_NONE); + + iommu = dev->archdata.iommu; + if (nelems == 0 || !iommu) + return -EINVAL; + atu = iommu->atu; + + prot = HV_PCI_MAP_ATTR_READ; + if (direction != DMA_TO_DEVICE) + prot |= HV_PCI_MAP_ATTR_WRITE; + + if (attrs & DMA_ATTR_WEAK_ORDERING) + prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; + + outs = s = segstart = &sglist[0]; + outcount = 1; + incount = nelems; + handle = 0; + + /* Init first segment length for backout at failure */ + outs->dma_length = 0; + + local_irq_save(flags); + + iommu_batch_start(dev, prot, ~0UL); + + max_seg_size = dma_get_max_seg_size(dev); + seg_boundary_size = dma_get_seg_boundary_nr_pages(dev, IO_PAGE_SHIFT); + + mask = *dev->dma_mask; + if (!iommu_use_atu(iommu, mask)) + tbl = &iommu->tbl; + else + tbl = &atu->tbl; + + base_shift = tbl->table_map_base >> IO_PAGE_SHIFT; + + for_each_sg(sglist, s, nelems, i) { + unsigned long paddr, npages, entry, out_entry = 0, slen; + + slen = s->length; + /* Sanity check */ + if (slen == 0) { + dma_next = 0; + continue; + } + /* Allocate iommu entries for that segment */ + paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); + npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); + entry = iommu_tbl_range_alloc(dev, tbl, npages, + &handle, (unsigned long)(-1), 0); + + /* Handle failure */ + if (unlikely(entry == IOMMU_ERROR_CODE)) { + pr_err_ratelimited("iommu_alloc failed, iommu %p paddr %lx npages %lx\n", + tbl, paddr, npages); + goto iommu_map_failed; + } + + iommu_batch_new_entry(entry, mask); + + /* Convert entry to a dma_addr_t */ + dma_addr = tbl->table_map_base + (entry << IO_PAGE_SHIFT); + dma_addr |= (s->offset & ~IO_PAGE_MASK); + + /* Insert into HW table */ + paddr &= IO_PAGE_MASK; + while (npages--) { + err = iommu_batch_add(paddr, mask); + if (unlikely(err < 0L)) + goto iommu_map_failed; + paddr += IO_PAGE_SIZE; + } + + /* If we are in an open segment, try merging */ + if (segstart != s) { + /* We cannot merge if: + * - allocated dma_addr isn't contiguous to previous allocation + */ + if ((dma_addr != dma_next) || + (outs->dma_length + s->length > max_seg_size) || + (is_span_boundary(out_entry, base_shift, + seg_boundary_size, outs, s))) { + /* Can't merge: create a new segment */ + segstart = s; + outcount++; + outs = sg_next(outs); + } else { + outs->dma_length += s->length; + } + } + + if (segstart == s) { + /* This is a new segment, fill entries */ + outs->dma_address = dma_addr; + outs->dma_length = slen; + out_entry = entry; + } + + /* Calculate next page pointer for contiguous check */ + dma_next = dma_addr + slen; + } + + err = iommu_batch_end(mask); + + if (unlikely(err < 0L)) + goto iommu_map_failed; + + local_irq_restore(flags); + + if (outcount < incount) { + outs = sg_next(outs); + outs->dma_length = 0; + } + + return outcount; + +iommu_map_failed: + for_each_sg(sglist, s, nelems, i) { + if (s->dma_length != 0) { + unsigned long vaddr, npages; + + vaddr = s->dma_address & IO_PAGE_MASK; + npages = iommu_num_pages(s->dma_address, s->dma_length, + IO_PAGE_SIZE); + iommu_tbl_range_free(tbl, vaddr, npages, + IOMMU_ERROR_CODE); + /* XXX demap? XXX */ + s->dma_length = 0; + } + if (s == outs) + break; + } + local_irq_restore(flags); + + return -EINVAL; +} + +static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, + int nelems, enum dma_data_direction direction, + unsigned long attrs) +{ + struct pci_pbm_info *pbm; + struct scatterlist *sg; + struct iommu *iommu; + struct atu *atu; + unsigned long flags, entry; + unsigned long iotsb_num; + u32 devhandle; + + BUG_ON(direction == DMA_NONE); + + iommu = dev->archdata.iommu; + pbm = dev->archdata.host_controller; + atu = iommu->atu; + devhandle = pbm->devhandle; + + local_irq_save(flags); + + sg = sglist; + while (nelems--) { + dma_addr_t dma_handle = sg->dma_address; + unsigned int len = sg->dma_length; + unsigned long npages; + struct iommu_map_table *tbl; + unsigned long shift = IO_PAGE_SHIFT; + + if (!len) + break; + npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); + + if (dma_handle <= DMA_BIT_MASK(32)) { + iotsb_num = 0; /* we don't care for legacy iommu */ + tbl = &iommu->tbl; + } else { + iotsb_num = atu->iotsb->iotsb_num; + tbl = &atu->tbl; + } + entry = ((dma_handle - tbl->table_map_base) >> shift); + dma_4v_iommu_demap(dev, devhandle, dma_handle, iotsb_num, + entry, npages); + iommu_tbl_range_free(tbl, dma_handle, npages, + IOMMU_ERROR_CODE); + sg = sg_next(sg); + } + + local_irq_restore(flags); +} + +static int dma_4v_supported(struct device *dev, u64 device_mask) +{ + struct iommu *iommu = dev->archdata.iommu; + + if (ali_sound_dma_hack(dev, device_mask)) + return 1; + if (device_mask < iommu->dma_addr_mask) + return 0; + return 1; +} + +static const struct dma_map_ops sun4v_dma_ops = { + .alloc = dma_4v_alloc_coherent, + .free = dma_4v_free_coherent, + .map_phys = dma_4v_map_phys, + .unmap_phys = dma_4v_unmap_phys, + .map_sg = dma_4v_map_sg, + .unmap_sg = dma_4v_unmap_sg, + .dma_supported = dma_4v_supported, +}; + +static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) +{ + struct property *prop; + struct device_node *dp; + + dp = pbm->op->dev.of_node; + prop = of_find_property(dp, "66mhz-capable", NULL); + pbm->is_66mhz_capable = (prop != NULL); + pbm->pci_bus = pci_scan_one_pbm(pbm, parent); + + /* XXX register error interrupt handlers XXX */ +} + +static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, + struct iommu_map_table *iommu) +{ + struct iommu_pool *pool; + unsigned long i, pool_nr, cnt = 0; + u32 devhandle; + + devhandle = pbm->devhandle; + for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) { + pool = &(iommu->pools[pool_nr]); + for (i = pool->start; i <= pool->end; i++) { + unsigned long ret, io_attrs, ra; + + ret = pci_sun4v_iommu_getmap(devhandle, + HV_PCI_TSBID(0, i), + &io_attrs, &ra); + if (ret == HV_EOK) { + if (page_in_phys_avail(ra)) { + pci_sun4v_iommu_demap(devhandle, + HV_PCI_TSBID(0, + i), 1); + } else { + cnt++; + __set_bit(i, iommu->map); + } + } + } + } + return cnt; +} + +static int pci_sun4v_atu_alloc_iotsb(struct pci_pbm_info *pbm) +{ + struct atu *atu = pbm->iommu->atu; + struct atu_iotsb *iotsb; + void *table; + u64 table_size; + u64 iotsb_num; + unsigned long order; + unsigned long err; + + iotsb = kzalloc_obj(*iotsb); + if (!iotsb) { + err = -ENOMEM; + goto out_err; + } + atu->iotsb = iotsb; + + /* calculate size of IOTSB */ + table_size = (atu->size / IO_PAGE_SIZE) * 8; + order = get_order(table_size); + table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); + if (!table) { + err = -ENOMEM; + goto table_failed; + } + iotsb->table = table; + iotsb->ra = __pa(table); + iotsb->dvma_size = atu->size; + iotsb->dvma_base = atu->base; + iotsb->table_size = table_size; + iotsb->page_size = IO_PAGE_SIZE; + + /* configure and register IOTSB with HV */ + err = pci_sun4v_iotsb_conf(pbm->devhandle, + iotsb->ra, + iotsb->table_size, + iotsb->page_size, + iotsb->dvma_base, + &iotsb_num); + if (err) { + pr_err(PFX "pci_iotsb_conf failed error: %ld\n", err); + goto iotsb_conf_failed; + } + iotsb->iotsb_num = iotsb_num; + + err = dma_4v_iotsb_bind(pbm->devhandle, iotsb_num, pbm->pci_bus); + if (err) { + pr_err(PFX "pci_iotsb_bind failed error: %ld\n", err); + goto iotsb_conf_failed; + } + + return 0; + +iotsb_conf_failed: + free_pages((unsigned long)table, order); +table_failed: + kfree(iotsb); +out_err: + return err; +} + +static int pci_sun4v_atu_init(struct pci_pbm_info *pbm) +{ + struct atu *atu = pbm->iommu->atu; + unsigned long err; + const u64 *ranges; + u64 map_size, num_iotte; + u64 dma_mask; + const u32 *page_size; + int len; + + ranges = of_get_property(pbm->op->dev.of_node, "iommu-address-ranges", + &len); + if (!ranges) { + pr_err(PFX "No iommu-address-ranges\n"); + return -EINVAL; + } + + page_size = of_get_property(pbm->op->dev.of_node, "iommu-pagesizes", + NULL); + if (!page_size) { + pr_err(PFX "No iommu-pagesizes\n"); + return -EINVAL; + } + + /* There are 4 iommu-address-ranges supported. Each range is pair of + * {base, size}. The ranges[0] and ranges[1] are 32bit address space + * while ranges[2] and ranges[3] are 64bit space. We want to use 64bit + * address ranges to support 64bit addressing. Because 'size' for + * address ranges[2] and ranges[3] are same we can select either of + * ranges[2] or ranges[3] for mapping. However due to 'size' is too + * large for OS to allocate IOTSB we are using fix size 32G + * (ATU_64_SPACE_SIZE) which is more than enough for all PCIe devices + * to share. + */ + atu->ranges = (struct atu_ranges *)ranges; + atu->base = atu->ranges[3].base; + atu->size = ATU_64_SPACE_SIZE; + + /* Create IOTSB */ + err = pci_sun4v_atu_alloc_iotsb(pbm); + if (err) { + pr_err(PFX "Error creating ATU IOTSB\n"); + return err; + } + + /* Create ATU iommu map. + * One bit represents one iotte in IOTSB table. + */ + dma_mask = (roundup_pow_of_two(atu->size) - 1UL); + num_iotte = atu->size / IO_PAGE_SIZE; + map_size = num_iotte / 8; + atu->tbl.table_map_base = atu->base; + atu->dma_addr_mask = dma_mask; + atu->tbl.map = kzalloc(map_size, GFP_KERNEL); + if (!atu->tbl.map) + return -ENOMEM; + + iommu_tbl_pool_init(&atu->tbl, num_iotte, IO_PAGE_SHIFT, + NULL, false /* no large_pool */, + 0 /* default npools */, + false /* want span boundary checking */); + + return 0; +} + +static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm) +{ + static const u32 vdma_default[] = { 0x80000000, 0x80000000 }; + struct iommu *iommu = pbm->iommu; + unsigned long num_tsb_entries, sz; + u32 dma_mask, dma_offset; + const u32 *vdma; + + vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma", NULL); + if (!vdma) + vdma = vdma_default; + + if ((vdma[0] | vdma[1]) & ~IO_PAGE_MASK) { + printk(KERN_ERR PFX "Strange virtual-dma[%08x:%08x].\n", + vdma[0], vdma[1]); + return -EINVAL; + } + + dma_mask = (roundup_pow_of_two(vdma[1]) - 1UL); + num_tsb_entries = vdma[1] / IO_PAGE_SIZE; + + dma_offset = vdma[0]; + + /* Setup initial software IOMMU state. */ + spin_lock_init(&iommu->lock); + iommu->ctx_lowest_free = 1; + iommu->tbl.table_map_base = dma_offset; + iommu->dma_addr_mask = dma_mask; + + /* Allocate and initialize the free area map. */ + sz = (num_tsb_entries + 7) / 8; + sz = (sz + 7UL) & ~7UL; + iommu->tbl.map = kzalloc(sz, GFP_KERNEL); + if (!iommu->tbl.map) { + printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); + return -ENOMEM; + } + iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, + NULL, false /* no large_pool */, + 0 /* default npools */, + false /* want span boundary checking */); + sz = probe_existing_entries(pbm, &iommu->tbl); + if (sz) + printk("%s: Imported %lu TSB entries from OBP\n", + pbm->name, sz); + + return 0; +} + +#ifdef CONFIG_PCI_MSI +struct pci_sun4v_msiq_entry { + u64 version_type; +#define MSIQ_VERSION_MASK 0xffffffff00000000UL +#define MSIQ_VERSION_SHIFT 32 +#define MSIQ_TYPE_MASK 0x00000000000000ffUL +#define MSIQ_TYPE_SHIFT 0 +#define MSIQ_TYPE_NONE 0x00 +#define MSIQ_TYPE_MSG 0x01 +#define MSIQ_TYPE_MSI32 0x02 +#define MSIQ_TYPE_MSI64 0x03 +#define MSIQ_TYPE_INTX 0x08 +#define MSIQ_TYPE_NONE2 0xff + + u64 intx_sysino; + u64 reserved1; + u64 stick; + u64 req_id; /* bus/device/func */ +#define MSIQ_REQID_BUS_MASK 0xff00UL +#define MSIQ_REQID_BUS_SHIFT 8 +#define MSIQ_REQID_DEVICE_MASK 0x00f8UL +#define MSIQ_REQID_DEVICE_SHIFT 3 +#define MSIQ_REQID_FUNC_MASK 0x0007UL +#define MSIQ_REQID_FUNC_SHIFT 0 + + u64 msi_address; + + /* The format of this value is message type dependent. + * For MSI bits 15:0 are the data from the MSI packet. + * For MSI-X bits 31:0 are the data from the MSI packet. + * For MSG, the message code and message routing code where: + * bits 39:32 is the bus/device/fn of the msg target-id + * bits 18:16 is the message routing code + * bits 7:0 is the message code + * For INTx the low order 2-bits are: + * 00 - INTA + * 01 - INTB + * 10 - INTC + * 11 - INTD + */ + u64 msi_data; + + u64 reserved2; +}; + +static int pci_sun4v_get_head(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long *head) +{ + unsigned long err, limit; + + err = pci_sun4v_msiq_gethead(pbm->devhandle, msiqid, head); + if (unlikely(err)) + return -ENXIO; + + limit = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); + if (unlikely(*head >= limit)) + return -EFBIG; + + return 0; +} + +static int pci_sun4v_dequeue_msi(struct pci_pbm_info *pbm, + unsigned long msiqid, unsigned long *head, + unsigned long *msi) +{ + struct pci_sun4v_msiq_entry *ep; + unsigned long err, type; + + /* Note: void pointer arithmetic, 'head' is a byte offset */ + ep = (pbm->msi_queues + ((msiqid - pbm->msiq_first) * + (pbm->msiq_ent_count * + sizeof(struct pci_sun4v_msiq_entry))) + + *head); + + if ((ep->version_type & MSIQ_TYPE_MASK) == 0) + return 0; + + type = (ep->version_type & MSIQ_TYPE_MASK) >> MSIQ_TYPE_SHIFT; + if (unlikely(type != MSIQ_TYPE_MSI32 && + type != MSIQ_TYPE_MSI64)) + return -EINVAL; + + *msi = ep->msi_data; + + err = pci_sun4v_msi_setstate(pbm->devhandle, + ep->msi_data /* msi_num */, + HV_MSISTATE_IDLE); + if (unlikely(err)) + return -ENXIO; + + /* Clear the entry. */ + ep->version_type &= ~MSIQ_TYPE_MASK; + + (*head) += sizeof(struct pci_sun4v_msiq_entry); + if (*head >= + (pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry))) + *head = 0; + + return 1; +} + +static int pci_sun4v_set_head(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long head) +{ + unsigned long err; + + err = pci_sun4v_msiq_sethead(pbm->devhandle, msiqid, head); + if (unlikely(err)) + return -EINVAL; + + return 0; +} + +static int pci_sun4v_msi_setup(struct pci_pbm_info *pbm, unsigned long msiqid, + unsigned long msi, int is_msi64) +{ + if (pci_sun4v_msi_setmsiq(pbm->devhandle, msi, msiqid, + (is_msi64 ? + HV_MSITYPE_MSI64 : HV_MSITYPE_MSI32))) + return -ENXIO; + if (pci_sun4v_msi_setstate(pbm->devhandle, msi, HV_MSISTATE_IDLE)) + return -ENXIO; + if (pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_VALID)) + return -ENXIO; + return 0; +} + +static int pci_sun4v_msi_teardown(struct pci_pbm_info *pbm, unsigned long msi) +{ + unsigned long err, msiqid; + + err = pci_sun4v_msi_getmsiq(pbm->devhandle, msi, &msiqid); + if (err) + return -ENXIO; + + pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_INVALID); + + return 0; +} + +static int pci_sun4v_msiq_alloc(struct pci_pbm_info *pbm) +{ + unsigned long q_size, alloc_size, pages, order; + int i; + + q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); + alloc_size = (pbm->msiq_num * q_size); + order = get_order(alloc_size); + pages = __get_free_pages(GFP_KERNEL | __GFP_COMP, order); + if (pages == 0UL) { + printk(KERN_ERR "MSI: Cannot allocate MSI queues (o=%lu).\n", + order); + return -ENOMEM; + } + memset((char *)pages, 0, PAGE_SIZE << order); + pbm->msi_queues = (void *) pages; + + for (i = 0; i < pbm->msiq_num; i++) { + unsigned long err, base = __pa(pages + (i * q_size)); + unsigned long ret1, ret2; + + err = pci_sun4v_msiq_conf(pbm->devhandle, + pbm->msiq_first + i, + base, pbm->msiq_ent_count); + if (err) { + printk(KERN_ERR "MSI: msiq register fails (err=%lu)\n", + err); + goto h_error; + } + + err = pci_sun4v_msiq_info(pbm->devhandle, + pbm->msiq_first + i, + &ret1, &ret2); + if (err) { + printk(KERN_ERR "MSI: Cannot read msiq (err=%lu)\n", + err); + goto h_error; + } + if (ret1 != base || ret2 != pbm->msiq_ent_count) { + printk(KERN_ERR "MSI: Bogus qconf " + "expected[%lx:%x] got[%lx:%lx]\n", + base, pbm->msiq_ent_count, + ret1, ret2); + goto h_error; + } + } + + return 0; + +h_error: + free_pages(pages, order); + return -EINVAL; +} + +static void pci_sun4v_msiq_free(struct pci_pbm_info *pbm) +{ + unsigned long q_size, alloc_size, pages, order; + int i; + + for (i = 0; i < pbm->msiq_num; i++) { + unsigned long msiqid = pbm->msiq_first + i; + + (void) pci_sun4v_msiq_conf(pbm->devhandle, msiqid, 0UL, 0); + } + + q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); + alloc_size = (pbm->msiq_num * q_size); + order = get_order(alloc_size); + + pages = (unsigned long) pbm->msi_queues; + + free_pages(pages, order); + + pbm->msi_queues = NULL; +} + +static int pci_sun4v_msiq_build_irq(struct pci_pbm_info *pbm, + unsigned long msiqid, + unsigned long devino) +{ + unsigned int irq = sun4v_build_irq(pbm->devhandle, devino); + + if (!irq) + return -ENOMEM; + + if (pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_VALID)) + return -EINVAL; + if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE)) + return -EINVAL; + + return irq; +} + +static const struct sparc64_msiq_ops pci_sun4v_msiq_ops = { + .get_head = pci_sun4v_get_head, + .dequeue_msi = pci_sun4v_dequeue_msi, + .set_head = pci_sun4v_set_head, + .msi_setup = pci_sun4v_msi_setup, + .msi_teardown = pci_sun4v_msi_teardown, + .msiq_alloc = pci_sun4v_msiq_alloc, + .msiq_free = pci_sun4v_msiq_free, + .msiq_build_irq = pci_sun4v_msiq_build_irq, +}; + +static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) +{ + sparc64_pbm_msi_init(pbm, &pci_sun4v_msiq_ops); +} +#else /* CONFIG_PCI_MSI */ +static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) +{ +} +#endif /* !(CONFIG_PCI_MSI) */ + +static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm, + struct platform_device *op, u32 devhandle) +{ + struct device_node *dp = op->dev.of_node; + int err; + + pbm->numa_node = of_node_to_nid(dp); + + pbm->pci_ops = &sun4v_pci_ops; + pbm->config_space_reg_bits = 12; + + pbm->index = pci_num_pbms++; + + pbm->op = op; + + pbm->devhandle = devhandle; + + pbm->name = dp->full_name; + + printk("%s: SUN4V PCI Bus Module\n", pbm->name); + printk("%s: On NUMA node %d\n", pbm->name, pbm->numa_node); + + pci_determine_mem_io_space(pbm); + + pci_get_pbm_props(pbm); + + err = pci_sun4v_iommu_init(pbm); + if (err) + return err; + + pci_sun4v_msi_init(pbm); + + pci_sun4v_scan_bus(pbm, &op->dev); + + /* if atu_init fails its not complete failure. + * we can still continue using legacy iommu. + */ + if (pbm->iommu->atu) { + err = pci_sun4v_atu_init(pbm); + if (err) { + kfree(pbm->iommu->atu); + pbm->iommu->atu = NULL; + pr_err(PFX "ATU init failed, err=%d\n", err); + } + } + + pbm->next = pci_pbm_root; + pci_pbm_root = pbm; + + return 0; +} + +static int pci_sun4v_probe(struct platform_device *op) +{ + const struct linux_prom64_registers *regs; + static int hvapi_negotiated = 0; + struct pci_pbm_info *pbm; + struct device_node *dp; + struct iommu *iommu; + struct atu *atu; + u32 devhandle; + int i, err = -ENODEV; + static bool hv_atu = true; + + dp = op->dev.of_node; + + if (!hvapi_negotiated++) { + for (i = 0; i < ARRAY_SIZE(vpci_versions); i++) { + vpci_major = vpci_versions[i].major; + vpci_minor = vpci_versions[i].minor; + + err = sun4v_hvapi_register(HV_GRP_PCI, vpci_major, + &vpci_minor); + if (!err) + break; + } + + if (err) { + pr_err(PFX "Could not register hvapi, err=%d\n", err); + return err; + } + pr_info(PFX "Registered hvapi major[%lu] minor[%lu]\n", + vpci_major, vpci_minor); + + err = sun4v_hvapi_register(HV_GRP_ATU, vatu_major, &vatu_minor); + if (err) { + /* don't return an error if we fail to register the + * ATU group, but ATU hcalls won't be available. + */ + hv_atu = false; + } else { + pr_info(PFX "Registered hvapi ATU major[%lu] minor[%lu]\n", + vatu_major, vatu_minor); + } + + dma_ops = &sun4v_dma_ops; + } + + regs = of_get_property(dp, "reg", NULL); + err = -ENODEV; + if (!regs) { + printk(KERN_ERR PFX "Could not find config registers\n"); + goto out_err; + } + devhandle = (regs->phys_addr >> 32UL) & 0x0fffffff; + + err = -ENOMEM; + if (!iommu_batch_initialized) { + for_each_possible_cpu(i) { + unsigned long page = get_zeroed_page(GFP_KERNEL); + + if (!page) + goto out_err; + + per_cpu(iommu_batch, i).pglist = (u64 *) page; + } + iommu_batch_initialized = 1; + } + + pbm = kzalloc_obj(*pbm); + if (!pbm) { + printk(KERN_ERR PFX "Could not allocate pci_pbm_info\n"); + goto out_err; + } + + iommu = kzalloc_obj(struct iommu); + if (!iommu) { + printk(KERN_ERR PFX "Could not allocate pbm iommu\n"); + goto out_free_controller; + } + + pbm->iommu = iommu; + iommu->atu = NULL; + if (hv_atu) { + atu = kzalloc_obj(*atu); + if (!atu) + pr_err(PFX "Could not allocate atu\n"); + else + iommu->atu = atu; + } + + err = pci_sun4v_pbm_init(pbm, op, devhandle); + if (err) + goto out_free_iommu; + + dev_set_drvdata(&op->dev, pbm); + + return 0; + +out_free_iommu: + kfree(iommu->atu); + kfree(pbm->iommu); + +out_free_controller: + kfree(pbm); + +out_err: + return err; +} + +static const struct of_device_id pci_sun4v_match[] = { + { + .name = "pci", + .compatible = "SUNW,sun4v-pci", + }, + {}, +}; + +static struct platform_driver pci_sun4v_driver = { + .driver = { + .name = DRIVER_NAME, + .of_match_table = pci_sun4v_match, + }, + .probe = pci_sun4v_probe, +}; + +static int __init pci_sun4v_init(void) +{ + return platform_driver_register(&pci_sun4v_driver); +} + +subsys_initcall(pci_sun4v_init); diff --git a/arch/sparc/kernel/pci_sun4v.h b/arch/sparc/kernel/pci_sun4v.h new file mode 100644 index 000000000..d47263a99 --- /dev/null +++ b/arch/sparc/kernel/pci_sun4v.h @@ -0,0 +1,114 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* pci_sun4v.h: SUN4V specific PCI controller support. + * + * Copyright (C) 2006 David S. Miller (davem@davemloft.net) + */ + +#ifndef _PCI_SUN4V_H +#define _PCI_SUN4V_H + +long pci_sun4v_iommu_map(unsigned long devhandle, + unsigned long tsbid, + unsigned long num_ttes, + unsigned long io_attributes, + unsigned long io_page_list_pa); +unsigned long pci_sun4v_iommu_demap(unsigned long devhandle, + unsigned long tsbid, + unsigned long num_ttes); +unsigned long pci_sun4v_iommu_getmap(unsigned long devhandle, + unsigned long tsbid, + unsigned long *io_attributes, + unsigned long *real_address); +unsigned long pci_sun4v_config_get(unsigned long devhandle, + unsigned long pci_device, + unsigned long config_offset, + unsigned long size); +int pci_sun4v_config_put(unsigned long devhandle, + unsigned long pci_device, + unsigned long config_offset, + unsigned long size, + unsigned long data); + +unsigned long pci_sun4v_msiq_conf(unsigned long devhandle, + unsigned long msiqid, + unsigned long msiq_paddr, + unsigned long num_entries); +unsigned long pci_sun4v_msiq_info(unsigned long devhandle, + unsigned long msiqid, + unsigned long *msiq_paddr, + unsigned long *num_entries); +unsigned long pci_sun4v_msiq_getvalid(unsigned long devhandle, + unsigned long msiqid, + unsigned long *valid); +unsigned long pci_sun4v_msiq_setvalid(unsigned long devhandle, + unsigned long msiqid, + unsigned long valid); +unsigned long pci_sun4v_msiq_getstate(unsigned long devhandle, + unsigned long msiqid, + unsigned long *state); +unsigned long pci_sun4v_msiq_setstate(unsigned long devhandle, + unsigned long msiqid, + unsigned long state); +unsigned long pci_sun4v_msiq_gethead(unsigned long devhandle, + unsigned long msiqid, + unsigned long *head); +unsigned long pci_sun4v_msiq_sethead(unsigned long devhandle, + unsigned long msiqid, + unsigned long head); +unsigned long pci_sun4v_msiq_gettail(unsigned long devhandle, + unsigned long msiqid, + unsigned long *head); +unsigned long pci_sun4v_msi_getvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long *valid); +unsigned long pci_sun4v_msi_setvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long valid); +unsigned long pci_sun4v_msi_getmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long *msiq); +unsigned long pci_sun4v_msi_setmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long msiq, + unsigned long msitype); +unsigned long pci_sun4v_msi_getstate(unsigned long devhandle, + unsigned long msinum, + unsigned long *state); +unsigned long pci_sun4v_msi_setstate(unsigned long devhandle, + unsigned long msinum, + unsigned long state); +unsigned long pci_sun4v_msg_getmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long *msiq); +unsigned long pci_sun4v_msg_setmsiq(unsigned long devhandle, + unsigned long msinum, + unsigned long msiq); +unsigned long pci_sun4v_msg_getvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long *valid); +unsigned long pci_sun4v_msg_setvalid(unsigned long devhandle, + unsigned long msinum, + unsigned long valid); + +/* Sun4v HV IOMMU v2 APIs */ +unsigned long pci_sun4v_iotsb_conf(unsigned long devhandle, + unsigned long ra, + unsigned long table_size, + unsigned long page_size, + unsigned long dvma_base, + u64 *iotsb_num); +unsigned long pci_sun4v_iotsb_bind(unsigned long devhandle, + unsigned long iotsb_num, + unsigned int pci_device); +unsigned long pci_sun4v_iotsb_map(unsigned long devhandle, + unsigned long iotsb_num, + unsigned long iotsb_index_iottes, + unsigned long io_attributes, + unsigned long io_page_list_pa, + long *mapped); +unsigned long pci_sun4v_iotsb_demap(unsigned long devhandle, + unsigned long iotsb_num, + unsigned long iotsb_index, + unsigned long iottes, + unsigned long *demapped); +#endif /* !(_PCI_SUN4V_H) */ diff --git a/arch/sparc/kernel/pci_sun4v_asm.S b/arch/sparc/kernel/pci_sun4v_asm.S new file mode 100644 index 000000000..2b8051871 --- /dev/null +++ b/arch/sparc/kernel/pci_sun4v_asm.S @@ -0,0 +1,431 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* pci_sun4v_asm: Hypervisor calls for PCI support. + * + * Copyright (C) 2006, 2008 David S. Miller + */ + +#include +#include + + /* %o0: devhandle + * %o1: tsbid + * %o2: num ttes + * %o3: io_attributes + * %o4: io_page_list phys address + * + * returns %o0: -status if status was non-zero, else + * %o0: num pages mapped + */ +ENTRY(pci_sun4v_iommu_map) + mov %o5, %g1 + mov HV_FAST_PCI_IOMMU_MAP, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 1f + sub %g0, %o0, %o0 + mov %o1, %o0 +1: retl + nop +ENDPROC(pci_sun4v_iommu_map) + + /* %o0: devhandle + * %o1: tsbid + * %o2: num ttes + * + * returns %o0: num ttes demapped + */ +ENTRY(pci_sun4v_iommu_demap) + mov HV_FAST_PCI_IOMMU_DEMAP, %o5 + ta HV_FAST_TRAP + retl + mov %o1, %o0 +ENDPROC(pci_sun4v_iommu_demap) + + /* %o0: devhandle + * %o1: tsbid + * %o2: &io_attributes + * %o3: &real_address + * + * returns %o0: status + */ +ENTRY(pci_sun4v_iommu_getmap) + mov %o2, %o4 + mov HV_FAST_PCI_IOMMU_GETMAP, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + stx %o2, [%o3] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_iommu_getmap) + + /* %o0: devhandle + * %o1: pci_device + * %o2: pci_config_offset + * %o3: size + * + * returns %o0: data + * + * If there is an error, the data will be returned + * as all 1's. + */ +ENTRY(pci_sun4v_config_get) + mov HV_FAST_PCI_CONFIG_GET, %o5 + ta HV_FAST_TRAP + brnz,a,pn %o1, 1f + mov -1, %o2 +1: retl + mov %o2, %o0 +ENDPROC(pci_sun4v_config_get) + + /* %o0: devhandle + * %o1: pci_device + * %o2: pci_config_offset + * %o3: size + * %o4: data + * + * returns %o0: status + * + * status will be zero if the operation completed + * successfully, else -1 if not + */ +ENTRY(pci_sun4v_config_put) + mov HV_FAST_PCI_CONFIG_PUT, %o5 + ta HV_FAST_TRAP + brnz,a,pn %o1, 1f + mov -1, %o1 +1: retl + mov %o1, %o0 +ENDPROC(pci_sun4v_config_put) + + /* %o0: devhandle + * %o1: msiqid + * %o2: msiq phys address + * %o3: num entries + * + * returns %o0: status + * + * status will be zero if the operation completed + * successfully, else -1 if not + */ +ENTRY(pci_sun4v_msiq_conf) + mov HV_FAST_PCI_MSIQ_CONF, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msiq_conf) + + /* %o0: devhandle + * %o1: msiqid + * %o2: &msiq_phys_addr + * %o3: &msiq_num_entries + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msiq_info) + mov %o2, %o4 + mov HV_FAST_PCI_MSIQ_INFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%o4] + stx %o2, [%o3] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msiq_info) + + /* %o0: devhandle + * %o1: msiqid + * %o2: &valid + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msiq_getvalid) + mov HV_FAST_PCI_MSIQ_GETVALID, %o5 + ta HV_FAST_TRAP + stx %o1, [%o2] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msiq_getvalid) + + /* %o0: devhandle + * %o1: msiqid + * %o2: valid + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msiq_setvalid) + mov HV_FAST_PCI_MSIQ_SETVALID, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msiq_setvalid) + + /* %o0: devhandle + * %o1: msiqid + * %o2: &state + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msiq_getstate) + mov HV_FAST_PCI_MSIQ_GETSTATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%o2] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msiq_getstate) + + /* %o0: devhandle + * %o1: msiqid + * %o2: state + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msiq_setstate) + mov HV_FAST_PCI_MSIQ_SETSTATE, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msiq_setstate) + + /* %o0: devhandle + * %o1: msiqid + * %o2: &head + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msiq_gethead) + mov HV_FAST_PCI_MSIQ_GETHEAD, %o5 + ta HV_FAST_TRAP + stx %o1, [%o2] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msiq_gethead) + + /* %o0: devhandle + * %o1: msiqid + * %o2: head + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msiq_sethead) + mov HV_FAST_PCI_MSIQ_SETHEAD, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msiq_sethead) + + /* %o0: devhandle + * %o1: msiqid + * %o2: &tail + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msiq_gettail) + mov HV_FAST_PCI_MSIQ_GETTAIL, %o5 + ta HV_FAST_TRAP + stx %o1, [%o2] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msiq_gettail) + + /* %o0: devhandle + * %o1: msinum + * %o2: &valid + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msi_getvalid) + mov HV_FAST_PCI_MSI_GETVALID, %o5 + ta HV_FAST_TRAP + stx %o1, [%o2] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msi_getvalid) + + /* %o0: devhandle + * %o1: msinum + * %o2: valid + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msi_setvalid) + mov HV_FAST_PCI_MSI_SETVALID, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msi_setvalid) + + /* %o0: devhandle + * %o1: msinum + * %o2: &msiq + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msi_getmsiq) + mov HV_FAST_PCI_MSI_GETMSIQ, %o5 + ta HV_FAST_TRAP + stx %o1, [%o2] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msi_getmsiq) + + /* %o0: devhandle + * %o1: msinum + * %o2: msitype + * %o3: msiq + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msi_setmsiq) + mov HV_FAST_PCI_MSI_SETMSIQ, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msi_setmsiq) + + /* %o0: devhandle + * %o1: msinum + * %o2: &state + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msi_getstate) + mov HV_FAST_PCI_MSI_GETSTATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%o2] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msi_getstate) + + /* %o0: devhandle + * %o1: msinum + * %o2: state + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msi_setstate) + mov HV_FAST_PCI_MSI_SETSTATE, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msi_setstate) + + /* %o0: devhandle + * %o1: msinum + * %o2: &msiq + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msg_getmsiq) + mov HV_FAST_PCI_MSG_GETMSIQ, %o5 + ta HV_FAST_TRAP + stx %o1, [%o2] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msg_getmsiq) + + /* %o0: devhandle + * %o1: msinum + * %o2: msiq + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msg_setmsiq) + mov HV_FAST_PCI_MSG_SETMSIQ, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msg_setmsiq) + + /* %o0: devhandle + * %o1: msinum + * %o2: &valid + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msg_getvalid) + mov HV_FAST_PCI_MSG_GETVALID, %o5 + ta HV_FAST_TRAP + stx %o1, [%o2] + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msg_getvalid) + + /* %o0: devhandle + * %o1: msinum + * %o2: valid + * + * returns %o0: status + */ +ENTRY(pci_sun4v_msg_setvalid) + mov HV_FAST_PCI_MSG_SETVALID, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_msg_setvalid) + + /* + * %o0: devhandle + * %o1: r_addr + * %o2: size + * %o3: pagesize + * %o4: virt + * %o5: &iotsb_num/&iotsb_handle + * + * returns %o0: status + * %o1: iotsb_num/iotsb_handle + */ +ENTRY(pci_sun4v_iotsb_conf) + mov %o5, %g1 + mov HV_FAST_PCI_IOTSB_CONF, %o5 + ta HV_FAST_TRAP + retl + stx %o1, [%g1] +ENDPROC(pci_sun4v_iotsb_conf) + + /* + * %o0: devhandle + * %o1: iotsb_num/iotsb_handle + * %o2: pci_device + * + * returns %o0: status + */ +ENTRY(pci_sun4v_iotsb_bind) + mov HV_FAST_PCI_IOTSB_BIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_sun4v_iotsb_bind) + + /* + * %o0: devhandle + * %o1: iotsb_num/iotsb_handle + * %o2: index_count + * %o3: iotte_attributes + * %o4: io_page_list_p + * %o5: &mapped + * + * returns %o0: status + * %o1: #mapped + */ +ENTRY(pci_sun4v_iotsb_map) + mov %o5, %g1 + mov HV_FAST_PCI_IOTSB_MAP, %o5 + ta HV_FAST_TRAP + retl + stx %o1, [%g1] +ENDPROC(pci_sun4v_iotsb_map) + + /* + * %o0: devhandle + * %o1: iotsb_num/iotsb_handle + * %o2: iotsb_index + * %o3: #iottes + * %o4: &demapped + * + * returns %o0: status + * %o1: #demapped + */ +ENTRY(pci_sun4v_iotsb_demap) + mov HV_FAST_PCI_IOTSB_DEMAP, %o5 + ta HV_FAST_TRAP + retl + stx %o1, [%o4] +ENDPROC(pci_sun4v_iotsb_demap) diff --git a/arch/sparc/kernel/pcic.c b/arch/sparc/kernel/pcic.c new file mode 100644 index 000000000..4cd32ccc5 --- /dev/null +++ b/arch/sparc/kernel/pcic.c @@ -0,0 +1,814 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * pcic.c: MicroSPARC-IIep PCI controller support + * + * Copyright (C) 1998 V. Roganov and G. Raiko + * + * Code is derived from Ultra/PCI PSYCHO controller support, see that + * for author info. + * + * Support for diverse IIep based platforms by Pete Zaitcev. + * CP-1200 by Eric Brower. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include /* for cache flushing. */ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" +#include "irq.h" + +/* + * I studied different documents and many live PROMs both from 2.30 + * family and 3.xx versions. I came to the amazing conclusion: there is + * absolutely no way to route interrupts in IIep systems relying on + * information which PROM presents. We must hardcode interrupt routing + * schematics. And this actually sucks. -- zaitcev 1999/05/12 + * + * To find irq for a device we determine which routing map + * is in effect or, in other words, on which machine we are running. + * We use PROM name for this although other techniques may be used + * in special cases (Gleb reports a PROMless IIep based system). + * Once we know the map we take device configuration address and + * find PCIC pin number where INT line goes. Then we may either program + * preferred irq into the PCIC or supply the preexisting irq to the device. + */ +struct pcic_ca2irq { + unsigned char busno; /* PCI bus number */ + unsigned char devfn; /* Configuration address */ + unsigned char pin; /* PCIC external interrupt pin */ + unsigned char irq; /* Preferred IRQ (mappable in PCIC) */ + unsigned int force; /* Enforce preferred IRQ */ +}; + +struct pcic_sn2list { + char *sysname; + struct pcic_ca2irq *intmap; + int mapdim; +}; + +/* + * JavaEngine-1 apparently has different versions. + * + * According to communications with Sun folks, for P2 build 501-4628-03: + * pin 0 - parallel, audio; + * pin 1 - Ethernet; + * pin 2 - su; + * pin 3 - PS/2 kbd and mouse. + * + * OEM manual (805-1486): + * pin 0: Ethernet + * pin 1: All EBus + * pin 2: IGA (unused) + * pin 3: Not connected + * OEM manual says that 501-4628 & 501-4811 are the same thing, + * only the latter has NAND flash in place. + * + * So far unofficial Sun wins over the OEM manual. Poor OEMs... + */ +static struct pcic_ca2irq pcic_i_je1a[] = { /* 501-4811-03 */ + { 0, 0x00, 2, 12, 0 }, /* EBus: hogs all */ + { 0, 0x01, 1, 6, 1 }, /* Happy Meal */ + { 0, 0x80, 0, 7, 0 }, /* IGA (unused) */ +}; + +/* XXX JS-E entry is incomplete - PCI Slot 2 address (pin 7)? */ +static struct pcic_ca2irq pcic_i_jse[] = { + { 0, 0x00, 0, 13, 0 }, /* Ebus - serial and keyboard */ + { 0, 0x01, 1, 6, 0 }, /* hme */ + { 0, 0x08, 2, 9, 0 }, /* VGA - we hope not used :) */ + { 0, 0x10, 6, 8, 0 }, /* PCI INTA# in Slot 1 */ + { 0, 0x18, 7, 12, 0 }, /* PCI INTA# in Slot 2, shared w. RTC */ + { 0, 0x38, 4, 9, 0 }, /* All ISA devices. Read 8259. */ + { 0, 0x80, 5, 11, 0 }, /* EIDE */ + /* {0,0x88, 0,0,0} - unknown device... PMU? Probably no interrupt. */ + { 0, 0xA0, 4, 9, 0 }, /* USB */ + /* + * Some pins belong to non-PCI devices, we hardcode them in drivers. + * sun4m timers - irq 10, 14 + * PC style RTC - pin 7, irq 4 ? + * Smart card, Parallel - pin 4 shared with USB, ISA + * audio - pin 3, irq 5 ? + */ +}; + +/* SPARCengine-6 was the original release name of CP1200. + * The documentation differs between the two versions + */ +static struct pcic_ca2irq pcic_i_se6[] = { + { 0, 0x08, 0, 2, 0 }, /* SCSI */ + { 0, 0x01, 1, 6, 0 }, /* HME */ + { 0, 0x00, 3, 13, 0 }, /* EBus */ +}; + +/* + * Krups (courtesy of Varol Kaptan) + * No documentation available, but it was easy to guess + * because it was very similar to Espresso. + * + * pin 0 - kbd, mouse, serial; + * pin 1 - Ethernet; + * pin 2 - igs (we do not use it); + * pin 3 - audio; + * pin 4,5,6 - unused; + * pin 7 - RTC (from P2 onwards as David B. says). + */ +static struct pcic_ca2irq pcic_i_jk[] = { + { 0, 0x00, 0, 13, 0 }, /* Ebus - serial and keyboard */ + { 0, 0x01, 1, 6, 0 }, /* hme */ +}; + +/* + * Several entries in this list may point to the same routing map + * as several PROMs may be installed on the same physical board. + */ +#define SN2L_INIT(name, map) \ + { name, map, ARRAY_SIZE(map) } + +static struct pcic_sn2list pcic_known_sysnames[] = { + SN2L_INIT("SUNW,JavaEngine1", pcic_i_je1a), /* JE1, PROM 2.32 */ + SN2L_INIT("SUNW,JS-E", pcic_i_jse), /* PROLL JavaStation-E */ + SN2L_INIT("SUNW,SPARCengine-6", pcic_i_se6), /* SPARCengine-6/CP-1200 */ + SN2L_INIT("SUNW,JS-NC", pcic_i_jk), /* PROLL JavaStation-NC */ + SN2L_INIT("SUNW,JSIIep", pcic_i_jk), /* OBP JavaStation-NC */ + { NULL, NULL, 0 } +}; + +/* + * Only one PCIC per IIep, + * and since we have no SMP IIep, only one per system. + */ +static int pcic0_up; +static struct linux_pcic pcic0; + +void __iomem *pcic_regs; +static volatile int pcic_speculative; +static volatile int pcic_trapped; + +/* forward */ +unsigned int pcic_build_device_irq(struct platform_device *op, + unsigned int real_irq); + +#define CONFIG_CMD(bus, device_fn, where) (0x80000000 | (((unsigned int)bus) << 16) | (((unsigned int)device_fn) << 8) | (where & ~3)) + +static int pcic_read_config_dword(unsigned int busno, unsigned int devfn, + int where, u32 *value) +{ + struct linux_pcic *pcic; + unsigned long flags; + + pcic = &pcic0; + + local_irq_save(flags); +#if 0 /* does not fail here */ + pcic_speculative = 1; + pcic_trapped = 0; +#endif + writel(CONFIG_CMD(busno, devfn, where), pcic->pcic_config_space_addr); +#if 0 /* does not fail here */ + nop(); + if (pcic_trapped) { + local_irq_restore(flags); + *value = ~0; + return 0; + } +#endif + pcic_speculative = 2; + pcic_trapped = 0; + *value = readl(pcic->pcic_config_space_data + (where&4)); + nop(); + if (pcic_trapped) { + pcic_speculative = 0; + local_irq_restore(flags); + *value = ~0; + return 0; + } + pcic_speculative = 0; + local_irq_restore(flags); + return 0; +} + +static int pcic_read_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *val) +{ + unsigned int v; + + if (bus->number != 0) return -EINVAL; + switch (size) { + case 1: + pcic_read_config_dword(bus->number, devfn, where&~3, &v); + *val = 0xff & (v >> (8*(where & 3))); + return 0; + case 2: + if (where&1) return -EINVAL; + pcic_read_config_dword(bus->number, devfn, where&~3, &v); + *val = 0xffff & (v >> (8*(where & 3))); + return 0; + case 4: + if (where&3) return -EINVAL; + pcic_read_config_dword(bus->number, devfn, where&~3, val); + return 0; + } + return -EINVAL; +} + +static int pcic_write_config_dword(unsigned int busno, unsigned int devfn, + int where, u32 value) +{ + struct linux_pcic *pcic; + unsigned long flags; + + pcic = &pcic0; + + local_irq_save(flags); + writel(CONFIG_CMD(busno, devfn, where), pcic->pcic_config_space_addr); + writel(value, pcic->pcic_config_space_data + (where&4)); + local_irq_restore(flags); + return 0; +} + +static int pcic_write_config(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 val) +{ + unsigned int v; + + if (bus->number != 0) return -EINVAL; + switch (size) { + case 1: + pcic_read_config_dword(bus->number, devfn, where&~3, &v); + v = (v & ~(0xff << (8*(where&3)))) | + ((0xff&val) << (8*(where&3))); + return pcic_write_config_dword(bus->number, devfn, where&~3, v); + case 2: + if (where&1) return -EINVAL; + pcic_read_config_dword(bus->number, devfn, where&~3, &v); + v = (v & ~(0xffff << (8*(where&3)))) | + ((0xffff&val) << (8*(where&3))); + return pcic_write_config_dword(bus->number, devfn, where&~3, v); + case 4: + if (where&3) return -EINVAL; + return pcic_write_config_dword(bus->number, devfn, where, val); + } + return -EINVAL; +} + +static struct pci_ops pcic_ops = { + .read = pcic_read_config, + .write = pcic_write_config, +}; + +/* + * On sparc64 pcibios_init() calls pci_controller_probe(). + * We want PCIC probed little ahead so that interrupt controller + * would be operational. + */ +int __init pcic_probe(void) +{ + struct linux_pcic *pcic; + struct linux_prom_registers regs[PROMREG_MAX]; + struct linux_pbm_info* pbm; + char namebuf[64]; + phandle node; + int err; + + if (pcic0_up) { + prom_printf("PCIC: called twice!\n"); + prom_halt(); + } + pcic = &pcic0; + + node = prom_getchild (prom_root_node); + node = prom_searchsiblings (node, "pci"); + if (node == 0) + return -ENODEV; + /* + * Map in PCIC register set, config space, and IO base + */ + err = prom_getproperty(node, "reg", (char*)regs, sizeof(regs)); + if (err == 0 || err == -1) { + prom_printf("PCIC: Error, cannot get PCIC registers " + "from PROM.\n"); + prom_halt(); + } + + pcic0_up = 1; + + pcic->pcic_res_regs.name = "pcic_registers"; + pcic->pcic_regs = ioremap(regs[0].phys_addr, regs[0].reg_size); + if (!pcic->pcic_regs) { + prom_printf("PCIC: Error, cannot map PCIC registers.\n"); + prom_halt(); + } + + pcic->pcic_res_io.name = "pcic_io"; + if ((pcic->pcic_io = (unsigned long) + ioremap(regs[1].phys_addr, 0x10000)) == 0) { + prom_printf("PCIC: Error, cannot map PCIC IO Base.\n"); + prom_halt(); + } + + pcic->pcic_res_cfg_addr.name = "pcic_cfg_addr"; + if ((pcic->pcic_config_space_addr = + ioremap(regs[2].phys_addr, regs[2].reg_size * 2)) == NULL) { + prom_printf("PCIC: Error, cannot map " + "PCI Configuration Space Address.\n"); + prom_halt(); + } + + /* + * Docs say three least significant bits in address and data + * must be the same. Thus, we need adjust size of data. + */ + pcic->pcic_res_cfg_data.name = "pcic_cfg_data"; + if ((pcic->pcic_config_space_data = + ioremap(regs[3].phys_addr, regs[3].reg_size * 2)) == NULL) { + prom_printf("PCIC: Error, cannot map " + "PCI Configuration Space Data.\n"); + prom_halt(); + } + + pbm = &pcic->pbm; + pbm->prom_node = node; + prom_getstring(node, "name", namebuf, 63); namebuf[63] = 0; + strscpy(pbm->prom_name, namebuf); + + { + extern int pcic_nmi_trap_patch[4]; + + t_nmi[0] = pcic_nmi_trap_patch[0]; + t_nmi[1] = pcic_nmi_trap_patch[1]; + t_nmi[2] = pcic_nmi_trap_patch[2]; + t_nmi[3] = pcic_nmi_trap_patch[3]; + swift_flush_dcache(); + pcic_regs = pcic->pcic_regs; + } + + prom_getstring(prom_root_node, "name", namebuf, 63); namebuf[63] = 0; + { + struct pcic_sn2list *p; + + for (p = pcic_known_sysnames; p->sysname != NULL; p++) { + if (strcmp(namebuf, p->sysname) == 0) + break; + } + pcic->pcic_imap = p->intmap; + pcic->pcic_imdim = p->mapdim; + } + if (pcic->pcic_imap == NULL) { + /* + * We do not panic here for the sake of embedded systems. + */ + printk("PCIC: System %s is unknown, cannot route interrupts\n", + namebuf); + } + + return 0; +} + +static void __init pcic_pbm_scan_bus(struct linux_pcic *pcic) +{ + struct linux_pbm_info *pbm = &pcic->pbm; + + pbm->pci_bus = pci_scan_bus(pbm->pci_first_busno, &pcic_ops, pbm); + if (!pbm->pci_bus) + return; + +#if 0 /* deadwood transplanted from sparc64 */ + pci_fill_in_pbm_cookies(pbm->pci_bus, pbm, pbm->prom_node); + pci_record_assignments(pbm, pbm->pci_bus); + pci_assign_unassigned(pbm, pbm->pci_bus); + pci_fixup_irq(pbm, pbm->pci_bus); +#endif + pci_bus_add_devices(pbm->pci_bus); +} + +/* + * Main entry point from the PCI subsystem. + */ +static int __init pcic_init(void) +{ + struct linux_pcic *pcic; + + /* + * PCIC should be initialized at start of the timer. + * So, here we report the presence of PCIC and do some magic passes. + */ + if(!pcic0_up) + return 0; + pcic = &pcic0; + + /* + * Switch off IOTLB translation. + */ + writeb(PCI_DVMA_CONTROL_IOTLB_DISABLE, + pcic->pcic_regs+PCI_DVMA_CONTROL); + + /* + * Increase mapped size for PCI memory space (DMA access). + * Should be done in that order (size first, address second). + * Why we couldn't set up 4GB and forget about it? XXX + */ + writel(0xF0000000UL, pcic->pcic_regs+PCI_SIZE_0); + writel(0+PCI_BASE_ADDRESS_SPACE_MEMORY, + pcic->pcic_regs+PCI_BASE_ADDRESS_0); + + pcic_pbm_scan_bus(pcic); + + return 0; +} + +int pcic_present(void) +{ + return pcic0_up; +} + +static int pdev_to_pnode(struct linux_pbm_info *pbm, struct pci_dev *pdev) +{ + struct linux_prom_pci_registers regs[PROMREG_MAX]; + int err; + phandle node = prom_getchild(pbm->prom_node); + + while(node) { + err = prom_getproperty(node, "reg", + (char *)®s[0], sizeof(regs)); + if(err != 0 && err != -1) { + unsigned long devfn = (regs[0].which_io >> 8) & 0xff; + if(devfn == pdev->devfn) + return node; + } + node = prom_getsibling(node); + } + return 0; +} + +static inline struct pcidev_cookie *pci_devcookie_alloc(void) +{ + return kmalloc_obj(struct pcidev_cookie, GFP_ATOMIC); +} + +static void pcic_map_pci_device(struct linux_pcic *pcic, + struct pci_dev *dev, int node) +{ + char namebuf[64]; + unsigned long address; + unsigned long flags; + int j; + + if (node == 0 || node == -1) { + strscpy(namebuf, "???"); + } else { + prom_getstring(node, "name", namebuf, 63); namebuf[63] = 0; + } + + for (j = 0; j < 6; j++) { + address = dev->resource[j].start; + if (address == 0) break; /* are sequential */ + flags = dev->resource[j].flags; + if ((flags & IORESOURCE_IO) != 0) { + if (address < 0x10000) { + /* + * A device responds to I/O cycles on PCI. + * We generate these cycles with memory + * access into the fixed map (phys 0x30000000). + * + * Since a device driver does not want to + * do ioremap() before accessing PC-style I/O, + * we supply virtual, ready to access address. + * + * Note that request_region() + * works for these devices. + * + * XXX Neat trick, but it's a *bad* idea + * to shit into regions like that. + * What if we want to allocate one more + * PCI base address... + */ + dev->resource[j].start = + pcic->pcic_io + address; + dev->resource[j].end = 1; /* XXX */ + dev->resource[j].flags = + (flags & ~IORESOURCE_IO) | IORESOURCE_MEM; + } else { + /* + * OOPS... PCI Spec allows this. Sun does + * not have any devices getting above 64K + * so it must be user with a weird I/O + * board in a PCI slot. We must remap it + * under 64K but it is not done yet. XXX + */ + pci_info(dev, "PCIC: Skipping I/O space at " + "0x%lx, this will Oops if a driver " + "attaches device '%s'\n", address, + namebuf); + } + } + } +} + +static void +pcic_fill_irq(struct linux_pcic *pcic, struct pci_dev *dev, int node) +{ + struct pcic_ca2irq *p; + unsigned int real_irq; + int i, ivec; + char namebuf[64]; + + if (node == 0 || node == -1) { + strscpy(namebuf, "???"); + } else { + prom_getstring(node, "name", namebuf, sizeof(namebuf)); + } + + if ((p = pcic->pcic_imap) == NULL) { + dev->irq = 0; + return; + } + for (i = 0; i < pcic->pcic_imdim; i++) { + if (p->busno == dev->bus->number && p->devfn == dev->devfn) + break; + p++; + } + if (i >= pcic->pcic_imdim) { + pci_info(dev, "PCIC: device %s not found in %d\n", namebuf, + pcic->pcic_imdim); + dev->irq = 0; + return; + } + + i = p->pin; + if (i >= 0 && i < 4) { + ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_LO); + real_irq = ivec >> (i << 2) & 0xF; + } else if (i >= 4 && i < 8) { + ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_HI); + real_irq = ivec >> ((i-4) << 2) & 0xF; + } else { /* Corrupted map */ + pci_info(dev, "PCIC: BAD PIN %d\n", i); for (;;) {} + } +/* P3 */ /* printk("PCIC: device %s pin %d ivec 0x%x irq %x\n", namebuf, i, ivec, dev->irq); */ + + /* real_irq means PROM did not bother to program the upper + * half of PCIC. This happens on JS-E with PROM 3.11, for instance. + */ + if (real_irq == 0 || p->force) { + if (p->irq == 0 || p->irq >= 15) { /* Corrupted map */ + pci_info(dev, "PCIC: BAD IRQ %d\n", p->irq); for (;;) {} + } + pci_info(dev, "PCIC: setting irq %d at pin %d\n", p->irq, + p->pin); + real_irq = p->irq; + + i = p->pin; + if (i >= 4) { + ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_HI); + ivec &= ~(0xF << ((i - 4) << 2)); + ivec |= p->irq << ((i - 4) << 2); + writew(ivec, pcic->pcic_regs+PCI_INT_SELECT_HI); + } else { + ivec = readw(pcic->pcic_regs+PCI_INT_SELECT_LO); + ivec &= ~(0xF << (i << 2)); + ivec |= p->irq << (i << 2); + writew(ivec, pcic->pcic_regs+PCI_INT_SELECT_LO); + } + } + dev->irq = pcic_build_device_irq(NULL, real_irq); +} + +/* + * Normally called from {do_}pci_scan_bus... + */ +void pcibios_fixup_bus(struct pci_bus *bus) +{ + struct pci_dev *dev; + struct linux_pcic *pcic; + /* struct linux_pbm_info* pbm = &pcic->pbm; */ + int node; + struct pcidev_cookie *pcp; + + if (!pcic0_up) { + pci_info(bus, "pcibios_fixup_bus: no PCIC\n"); + return; + } + pcic = &pcic0; + + /* + * Next crud is an equivalent of pbm = pcic_bus_to_pbm(bus); + */ + if (bus->number != 0) { + pci_info(bus, "pcibios_fixup_bus: nonzero bus 0x%x\n", + bus->number); + return; + } + + list_for_each_entry(dev, &bus->devices, bus_list) { + node = pdev_to_pnode(&pcic->pbm, dev); + if(node == 0) + node = -1; + + /* cookies */ + pcp = pci_devcookie_alloc(); + pcp->pbm = &pcic->pbm; + pcp->prom_node = of_find_node_by_phandle(node); + dev->sysdata = pcp; + + /* fixing I/O to look like memory */ + if ((dev->class>>16) != PCI_BASE_CLASS_BRIDGE) + pcic_map_pci_device(pcic, dev, node); + + pcic_fill_irq(pcic, dev, node); + } +} + +/* Makes compiler happy */ +static volatile int pcic_timer_dummy; + +static void pcic_clear_clock_irq(void) +{ + pcic_timer_dummy = readl(pcic0.pcic_regs+PCI_SYS_LIMIT); +} + +/* CPU frequency is 100 MHz, timer increments every 4 CPU clocks */ +#define USECS_PER_JIFFY (1000000 / HZ) +#define TICK_TIMER_LIMIT ((100 * 1000000 / 4) / HZ) + +static unsigned int pcic_cycles_offset(void) +{ + u32 value, count; + + value = readl(pcic0.pcic_regs + PCI_SYS_COUNTER); + count = value & ~PCI_SYS_COUNTER_OVERFLOW; + + if (value & PCI_SYS_COUNTER_OVERFLOW) + count += TICK_TIMER_LIMIT; + /* + * We divide all by HZ + * to have microsecond resolution and to avoid overflow + */ + count = ((count / HZ) * USECS_PER_JIFFY) / (TICK_TIMER_LIMIT / HZ); + + /* Coordinate with the sparc_config.clock_rate setting */ + return count * 2; +} + +void __init pci_time_init(void) +{ + struct linux_pcic *pcic = &pcic0; + unsigned long v; + int timer_irq, irq; + int err; + +#ifndef CONFIG_SMP + /* + * The clock_rate is in SBUS dimension. + * We take into account this in pcic_cycles_offset() + */ + sparc_config.clock_rate = SBUS_CLOCK_RATE / HZ; + sparc_config.features |= FEAT_L10_CLOCKEVENT; +#endif + sparc_config.features |= FEAT_L10_CLOCKSOURCE; + sparc_config.get_cycles_offset = pcic_cycles_offset; + + writel (TICK_TIMER_LIMIT, pcic->pcic_regs+PCI_SYS_LIMIT); + /* PROM should set appropriate irq */ + v = readb(pcic->pcic_regs+PCI_COUNTER_IRQ); + timer_irq = PCI_COUNTER_IRQ_SYS(v); + writel (PCI_COUNTER_IRQ_SET(timer_irq, 0), + pcic->pcic_regs+PCI_COUNTER_IRQ); + irq = pcic_build_device_irq(NULL, timer_irq); + err = request_irq(irq, timer_interrupt, + IRQF_TIMER, "timer", NULL); + if (err) { + prom_printf("time_init: unable to attach IRQ%d\n", timer_irq); + prom_halt(); + } + local_irq_enable(); +} + + +#if 0 +static void watchdog_reset() { + writeb(0, pcic->pcic_regs+PCI_SYS_STATUS); +} +#endif + +/* + * NMI + */ +void pcic_nmi(unsigned int pend, struct pt_regs *regs) +{ + pend = swab32(pend); + + if (!pcic_speculative || (pend & PCI_SYS_INT_PENDING_PIO) == 0) { + /* + * XXX On CP-1200 PCI #SERR may happen, we do not know + * what to do about it yet. + */ + printk("Aiee, NMI pend 0x%x pc 0x%x spec %d, hanging\n", + pend, (int)regs->pc, pcic_speculative); + for (;;) { } + } + pcic_speculative = 0; + pcic_trapped = 1; + regs->pc = regs->npc; + regs->npc += 4; +} + +static inline unsigned long get_irqmask(int irq_nr) +{ + return 1 << irq_nr; +} + +static void pcic_mask_irq(struct irq_data *data) +{ + unsigned long mask, flags; + + mask = (unsigned long)data->chip_data; + local_irq_save(flags); + writel(mask, pcic0.pcic_regs+PCI_SYS_INT_TARGET_MASK_SET); + local_irq_restore(flags); +} + +static void pcic_unmask_irq(struct irq_data *data) +{ + unsigned long mask, flags; + + mask = (unsigned long)data->chip_data; + local_irq_save(flags); + writel(mask, pcic0.pcic_regs+PCI_SYS_INT_TARGET_MASK_CLEAR); + local_irq_restore(flags); +} + +static unsigned int pcic_startup_irq(struct irq_data *data) +{ + irq_link(data->irq); + pcic_unmask_irq(data); + return 0; +} + +static struct irq_chip pcic_irq = { + .name = "pcic", + .irq_startup = pcic_startup_irq, + .irq_mask = pcic_mask_irq, + .irq_unmask = pcic_unmask_irq, +}; + +unsigned int pcic_build_device_irq(struct platform_device *op, + unsigned int real_irq) +{ + unsigned int irq; + unsigned long mask; + + irq = 0; + mask = get_irqmask(real_irq); + if (mask == 0) + goto out; + + irq = irq_alloc(real_irq, real_irq); + if (irq == 0) + goto out; + + irq_set_chip_and_handler_name(irq, &pcic_irq, + handle_level_irq, "PCIC"); + irq_set_chip_data(irq, (void *)mask); + +out: + return irq; +} + + +static void pcic_load_profile_irq(int cpu, unsigned int limit) +{ + printk("PCIC: unimplemented code: FILE=%s LINE=%d", __FILE__, __LINE__); +} + +void __init sun4m_pci_init_IRQ(void) +{ + sparc_config.build_device_irq = pcic_build_device_irq; + sparc_config.clear_clock_irq = pcic_clear_clock_irq; + sparc_config.load_profile_irq = pcic_load_profile_irq; +} + +subsys_initcall(pcic_init); diff --git a/arch/sparc/kernel/pcr.c b/arch/sparc/kernel/pcr.c new file mode 100644 index 000000000..2a12c86af --- /dev/null +++ b/arch/sparc/kernel/pcr.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* pcr.c: Generic sparc64 performance counter infrastructure. + * + * Copyright (C) 2009 David S. Miller (davem@davemloft.net) + */ +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include + +/* This code is shared between various users of the performance + * counters. Users will be oprofile, pseudo-NMI watchdog, and the + * perf_event support layer. + */ + +/* Performance counter interrupts run unmasked at PIL level 15. + * Therefore we can't do things like wakeups and other work + * that expects IRQ disabling to be adhered to in locking etc. + * + * Therefore in such situations we defer the work by signalling + * a lower level cpu IRQ. + */ +void __irq_entry deferred_pcr_work_irq(int irq, struct pt_regs *regs) +{ + struct pt_regs *old_regs; + + clear_softint(1 << PIL_DEFERRED_PCR_WORK); + + old_regs = set_irq_regs(regs); + irq_enter(); +#ifdef CONFIG_IRQ_WORK + irq_work_run(); +#endif + irq_exit(); + set_irq_regs(old_regs); +} + +void arch_irq_work_raise(void) +{ + set_softint(1 << PIL_DEFERRED_PCR_WORK); +} + +const struct pcr_ops *pcr_ops; +EXPORT_SYMBOL_GPL(pcr_ops); + +static u64 direct_pcr_read(unsigned long reg_num) +{ + u64 val; + + WARN_ON_ONCE(reg_num != 0); + __asm__ __volatile__("rd %%pcr, %0" : "=r" (val)); + return val; +} + +static void direct_pcr_write(unsigned long reg_num, u64 val) +{ + WARN_ON_ONCE(reg_num != 0); + __asm__ __volatile__("wr %0, 0x0, %%pcr" : : "r" (val)); +} + +static u64 direct_pic_read(unsigned long reg_num) +{ + u64 val; + + WARN_ON_ONCE(reg_num != 0); + __asm__ __volatile__("rd %%pic, %0" : "=r" (val)); + return val; +} + +static void direct_pic_write(unsigned long reg_num, u64 val) +{ + WARN_ON_ONCE(reg_num != 0); + + /* Blackbird errata workaround. See commentary in + * arch/sparc64/kernel/smp.c:smp_percpu_timer_interrupt() + * for more information. + */ + __asm__ __volatile__("ba,pt %%xcc, 99f\n\t" + " nop\n\t" + ".align 64\n" + "99:wr %0, 0x0, %%pic\n\t" + "rd %%pic, %%g0" : : "r" (val)); +} + +static u64 direct_picl_value(unsigned int nmi_hz) +{ + u32 delta = local_cpu_data().clock_tick / nmi_hz; + + return ((u64)((0 - delta) & 0xffffffff)) << 32; +} + +static const struct pcr_ops direct_pcr_ops = { + .read_pcr = direct_pcr_read, + .write_pcr = direct_pcr_write, + .read_pic = direct_pic_read, + .write_pic = direct_pic_write, + .nmi_picl_value = direct_picl_value, + .pcr_nmi_enable = (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE), + .pcr_nmi_disable = PCR_PIC_PRIV, +}; + +static void n2_pcr_write(unsigned long reg_num, u64 val) +{ + unsigned long ret; + + WARN_ON_ONCE(reg_num != 0); + if (val & PCR_N2_HTRACE) { + ret = sun4v_niagara2_setperf(HV_N2_PERF_SPARC_CTL, val); + if (ret != HV_EOK) + direct_pcr_write(reg_num, val); + } else + direct_pcr_write(reg_num, val); +} + +static u64 n2_picl_value(unsigned int nmi_hz) +{ + u32 delta = local_cpu_data().clock_tick / (nmi_hz << 2); + + return ((u64)((0 - delta) & 0xffffffff)) << 32; +} + +static const struct pcr_ops n2_pcr_ops = { + .read_pcr = direct_pcr_read, + .write_pcr = n2_pcr_write, + .read_pic = direct_pic_read, + .write_pic = direct_pic_write, + .nmi_picl_value = n2_picl_value, + .pcr_nmi_enable = (PCR_PIC_PRIV | PCR_STRACE | PCR_UTRACE | + PCR_N2_TOE_OV1 | + (2 << PCR_N2_SL1_SHIFT) | + (0xff << PCR_N2_MASK1_SHIFT)), + .pcr_nmi_disable = PCR_PIC_PRIV, +}; + +static u64 n4_pcr_read(unsigned long reg_num) +{ + unsigned long val; + + (void) sun4v_vt_get_perfreg(reg_num, &val); + + return val; +} + +static void n4_pcr_write(unsigned long reg_num, u64 val) +{ + (void) sun4v_vt_set_perfreg(reg_num, val); +} + +static u64 n4_pic_read(unsigned long reg_num) +{ + unsigned long val; + + __asm__ __volatile__("ldxa [%1] %2, %0" + : "=r" (val) + : "r" (reg_num * 0x8UL), "i" (ASI_PIC)); + + return val; +} + +static void n4_pic_write(unsigned long reg_num, u64 val) +{ + __asm__ __volatile__("stxa %0, [%1] %2" + : /* no outputs */ + : "r" (val), "r" (reg_num * 0x8UL), "i" (ASI_PIC)); +} + +static u64 n4_picl_value(unsigned int nmi_hz) +{ + u32 delta = local_cpu_data().clock_tick / (nmi_hz << 2); + + return ((u64)((0 - delta) & 0xffffffff)); +} + +static const struct pcr_ops n4_pcr_ops = { + .read_pcr = n4_pcr_read, + .write_pcr = n4_pcr_write, + .read_pic = n4_pic_read, + .write_pic = n4_pic_write, + .nmi_picl_value = n4_picl_value, + .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE | + PCR_N4_UTRACE | PCR_N4_TOE | + (26 << PCR_N4_SL_SHIFT)), + .pcr_nmi_disable = PCR_N4_PICNPT, +}; + +static u64 n5_pcr_read(unsigned long reg_num) +{ + unsigned long val; + + (void) sun4v_t5_get_perfreg(reg_num, &val); + + return val; +} + +static void n5_pcr_write(unsigned long reg_num, u64 val) +{ + (void) sun4v_t5_set_perfreg(reg_num, val); +} + +static const struct pcr_ops n5_pcr_ops = { + .read_pcr = n5_pcr_read, + .write_pcr = n5_pcr_write, + .read_pic = n4_pic_read, + .write_pic = n4_pic_write, + .nmi_picl_value = n4_picl_value, + .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE | + PCR_N4_UTRACE | PCR_N4_TOE | + (26 << PCR_N4_SL_SHIFT)), + .pcr_nmi_disable = PCR_N4_PICNPT, +}; + +static u64 m7_pcr_read(unsigned long reg_num) +{ + unsigned long val; + + (void) sun4v_m7_get_perfreg(reg_num, &val); + + return val; +} + +static void m7_pcr_write(unsigned long reg_num, u64 val) +{ + (void) sun4v_m7_set_perfreg(reg_num, val); +} + +static const struct pcr_ops m7_pcr_ops = { + .read_pcr = m7_pcr_read, + .write_pcr = m7_pcr_write, + .read_pic = n4_pic_read, + .write_pic = n4_pic_write, + .nmi_picl_value = n4_picl_value, + .pcr_nmi_enable = (PCR_N4_PICNPT | PCR_N4_STRACE | + PCR_N4_UTRACE | PCR_N4_TOE | + (26 << PCR_N4_SL_SHIFT)), + .pcr_nmi_disable = PCR_N4_PICNPT, +}; + +static unsigned long perf_hsvc_group; +static unsigned long perf_hsvc_major; +static unsigned long perf_hsvc_minor; + +static int __init register_perf_hsvc(void) +{ + unsigned long hverror; + + if (tlb_type == hypervisor) { + switch (sun4v_chip_type) { + case SUN4V_CHIP_NIAGARA1: + perf_hsvc_group = HV_GRP_NIAG_PERF; + break; + + case SUN4V_CHIP_NIAGARA2: + perf_hsvc_group = HV_GRP_N2_CPU; + break; + + case SUN4V_CHIP_NIAGARA3: + perf_hsvc_group = HV_GRP_KT_CPU; + break; + + case SUN4V_CHIP_NIAGARA4: + perf_hsvc_group = HV_GRP_VT_CPU; + break; + + case SUN4V_CHIP_NIAGARA5: + perf_hsvc_group = HV_GRP_T5_CPU; + break; + + case SUN4V_CHIP_SPARC_M7: + perf_hsvc_group = HV_GRP_M7_PERF; + break; + + default: + return -ENODEV; + } + + + perf_hsvc_major = 1; + perf_hsvc_minor = 0; + hverror = sun4v_hvapi_register(perf_hsvc_group, + perf_hsvc_major, + &perf_hsvc_minor); + if (hverror) { + pr_err("perfmon: Could not register hvapi(0x%lx).\n", + hverror); + return -ENODEV; + } + } + return 0; +} + +static void __init unregister_perf_hsvc(void) +{ + if (tlb_type != hypervisor) + return; + sun4v_hvapi_unregister(perf_hsvc_group); +} + +static int __init setup_sun4v_pcr_ops(void) +{ + int ret = 0; + + switch (sun4v_chip_type) { + case SUN4V_CHIP_NIAGARA1: + case SUN4V_CHIP_NIAGARA2: + case SUN4V_CHIP_NIAGARA3: + pcr_ops = &n2_pcr_ops; + break; + + case SUN4V_CHIP_NIAGARA4: + pcr_ops = &n4_pcr_ops; + break; + + case SUN4V_CHIP_NIAGARA5: + pcr_ops = &n5_pcr_ops; + break; + + case SUN4V_CHIP_SPARC_M7: + pcr_ops = &m7_pcr_ops; + break; + + default: + ret = -ENODEV; + break; + } + + return ret; +} + +int __init pcr_arch_init(void) +{ + int err = register_perf_hsvc(); + + if (err) + return err; + + switch (tlb_type) { + case hypervisor: + err = setup_sun4v_pcr_ops(); + if (err) + goto out_unregister; + break; + + case cheetah: + case cheetah_plus: + pcr_ops = &direct_pcr_ops; + break; + + case spitfire: + /* UltraSPARC-I/II and derivatives lack a profile + * counter overflow interrupt so we can't make use of + * their hardware currently. + */ + fallthrough; + default: + err = -ENODEV; + goto out_unregister; + } + + return nmi_init(); + +out_unregister: + unregister_perf_hsvc(); + return err; +} diff --git a/arch/sparc/kernel/perf_event.c b/arch/sparc/kernel/perf_event.c new file mode 100644 index 000000000..0ce4ae343 --- /dev/null +++ b/arch/sparc/kernel/perf_event.c @@ -0,0 +1,1876 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Performance event support for sparc64. + * + * Copyright (C) 2009, 2010 David S. Miller + * + * This code is based almost entirely upon the x86 perf event + * code, which is: + * + * Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner + * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar + * Copyright (C) 2009 Jaswinder Singh Rajput + * Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter + * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" +#include "kstack.h" + +/* Two classes of sparc64 chips currently exist. All of which have + * 32-bit counters which can generate overflow interrupts on the + * transition from 0xffffffff to 0. + * + * All chips upto and including SPARC-T3 have two performance + * counters. The two 32-bit counters are accessed in one go using a + * single 64-bit register. + * + * On these older chips both counters are controlled using a single + * control register. The only way to stop all sampling is to clear + * all of the context (user, supervisor, hypervisor) sampling enable + * bits. But these bits apply to both counters, thus the two counters + * can't be enabled/disabled individually. + * + * Furthermore, the control register on these older chips have two + * event fields, one for each of the two counters. It's thus nearly + * impossible to have one counter going while keeping the other one + * stopped. Therefore it is possible to get overflow interrupts for + * counters not currently "in use" and that condition must be checked + * in the overflow interrupt handler. + * + * So we use a hack, in that we program inactive counters with the + * "sw_count0" and "sw_count1" events. These count how many times + * the instruction "sethi %hi(0xfc000), %g0" is executed. It's an + * unusual way to encode a NOP and therefore will not trigger in + * normal code. + * + * Starting with SPARC-T4 we have one control register per counter. + * And the counters are stored in individual registers. The registers + * for the counters are 64-bit but only a 32-bit counter is + * implemented. The event selections on SPARC-T4 lack any + * restrictions, therefore we can elide all of the complicated + * conflict resolution code we have for SPARC-T3 and earlier chips. + */ + +#define MAX_HWEVENTS 4 +#define MAX_PCRS 4 +#define MAX_PERIOD ((1UL << 32) - 1) + +#define PIC_UPPER_INDEX 0 +#define PIC_LOWER_INDEX 1 +#define PIC_NO_INDEX -1 + +struct cpu_hw_events { + /* Number of events currently scheduled onto this cpu. + * This tells how many entries in the arrays below + * are valid. + */ + int n_events; + + /* Number of new events added since the last hw_perf_disable(). + * This works because the perf event layer always adds new + * events inside of a perf_{disable,enable}() sequence. + */ + int n_added; + + /* Array of events current scheduled on this cpu. */ + struct perf_event *event[MAX_HWEVENTS]; + + /* Array of encoded longs, specifying the %pcr register + * encoding and the mask of PIC counters this even can + * be scheduled on. See perf_event_encode() et al. + */ + unsigned long events[MAX_HWEVENTS]; + + /* The current counter index assigned to an event. When the + * event hasn't been programmed into the cpu yet, this will + * hold PIC_NO_INDEX. The event->hw.idx value tells us where + * we ought to schedule the event. + */ + int current_idx[MAX_HWEVENTS]; + + /* Software copy of %pcr register(s) on this cpu. */ + u64 pcr[MAX_HWEVENTS]; + + /* Enabled/disable state. */ + int enabled; + + unsigned int txn_flags; +}; +static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, }; + +/* An event map describes the characteristics of a performance + * counter event. In particular it gives the encoding as well as + * a mask telling which counters the event can be measured on. + * + * The mask is unused on SPARC-T4 and later. + */ +struct perf_event_map { + u16 encoding; + u8 pic_mask; +#define PIC_NONE 0x00 +#define PIC_UPPER 0x01 +#define PIC_LOWER 0x02 +}; + +/* Encode a perf_event_map entry into a long. */ +static unsigned long perf_event_encode(const struct perf_event_map *pmap) +{ + return ((unsigned long) pmap->encoding << 16) | pmap->pic_mask; +} + +static u8 perf_event_get_msk(unsigned long val) +{ + return val & 0xff; +} + +static u64 perf_event_get_enc(unsigned long val) +{ + return val >> 16; +} + +#define C(x) PERF_COUNT_HW_CACHE_##x + +#define CACHE_OP_UNSUPPORTED 0xfffe +#define CACHE_OP_NONSENSE 0xffff + +typedef struct perf_event_map cache_map_t + [PERF_COUNT_HW_CACHE_MAX] + [PERF_COUNT_HW_CACHE_OP_MAX] + [PERF_COUNT_HW_CACHE_RESULT_MAX]; + +struct sparc_pmu { + const struct perf_event_map *(*event_map)(int); + const cache_map_t *cache_map; + int max_events; + u32 (*read_pmc)(int); + void (*write_pmc)(int, u64); + int upper_shift; + int lower_shift; + int event_mask; + int user_bit; + int priv_bit; + int hv_bit; + int irq_bit; + int upper_nop; + int lower_nop; + unsigned int flags; +#define SPARC_PMU_ALL_EXCLUDES_SAME 0x00000001 +#define SPARC_PMU_HAS_CONFLICTS 0x00000002 + int max_hw_events; + int num_pcrs; + int num_pic_regs; +}; + +static u32 sparc_default_read_pmc(int idx) +{ + u64 val; + + val = pcr_ops->read_pic(0); + if (idx == PIC_UPPER_INDEX) + val >>= 32; + + return val & 0xffffffff; +} + +static void sparc_default_write_pmc(int idx, u64 val) +{ + u64 shift, mask, pic; + + shift = 0; + if (idx == PIC_UPPER_INDEX) + shift = 32; + + mask = ((u64) 0xffffffff) << shift; + val <<= shift; + + pic = pcr_ops->read_pic(0); + pic &= ~mask; + pic |= val; + pcr_ops->write_pic(0, pic); +} + +static const struct perf_event_map ultra3_perfmon_event_map[] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER }, +}; + +static const struct perf_event_map *ultra3_event_map(int event_id) +{ + return &ultra3_perfmon_event_map[event_id]; +} + +static const cache_map_t ultra3_cache_map = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, }, + [C(RESULT_MISS)] = { 0x09, PIC_UPPER, }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER }, + [C(RESULT_MISS)] = { 0x0a, PIC_UPPER }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, }, + [C(RESULT_MISS)] = { 0x09, PIC_UPPER, }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE }, + [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, }, + [C(RESULT_MISS)] = { 0x0c, PIC_UPPER, }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER }, + [C(RESULT_MISS)] = { 0x0c, PIC_UPPER }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0x12, PIC_UPPER, }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0x11, PIC_UPPER, }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +}; + +static const struct sparc_pmu ultra3_pmu = { + .event_map = ultra3_event_map, + .cache_map = &ultra3_cache_map, + .max_events = ARRAY_SIZE(ultra3_perfmon_event_map), + .read_pmc = sparc_default_read_pmc, + .write_pmc = sparc_default_write_pmc, + .upper_shift = 11, + .lower_shift = 4, + .event_mask = 0x3f, + .user_bit = PCR_UTRACE, + .priv_bit = PCR_STRACE, + .upper_nop = 0x1c, + .lower_nop = 0x14, + .flags = (SPARC_PMU_ALL_EXCLUDES_SAME | + SPARC_PMU_HAS_CONFLICTS), + .max_hw_events = 2, + .num_pcrs = 1, + .num_pic_regs = 1, +}; + +/* Niagara1 is very limited. The upper PIC is hard-locked to count + * only instructions, so it is free running which creates all kinds of + * problems. Some hardware designs make one wonder if the creator + * even looked at how this stuff gets used by software. + */ +static const struct perf_event_map niagara1_perfmon_event_map[] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x00, PIC_UPPER }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x00, PIC_UPPER }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0, PIC_NONE }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x03, PIC_LOWER }, +}; + +static const struct perf_event_map *niagara1_event_map(int event_id) +{ + return &niagara1_perfmon_event_map[event_id]; +} + +static const cache_map_t niagara1_cache_map = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0x03, PIC_LOWER, }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0x03, PIC_LOWER, }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x00, PIC_UPPER }, + [C(RESULT_MISS)] = { 0x02, PIC_LOWER, }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE }, + [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0x07, PIC_LOWER, }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0x07, PIC_LOWER, }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0x05, PIC_LOWER, }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0x04, PIC_LOWER, }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +}; + +static const struct sparc_pmu niagara1_pmu = { + .event_map = niagara1_event_map, + .cache_map = &niagara1_cache_map, + .max_events = ARRAY_SIZE(niagara1_perfmon_event_map), + .read_pmc = sparc_default_read_pmc, + .write_pmc = sparc_default_write_pmc, + .upper_shift = 0, + .lower_shift = 4, + .event_mask = 0x7, + .user_bit = PCR_UTRACE, + .priv_bit = PCR_STRACE, + .upper_nop = 0x0, + .lower_nop = 0x0, + .flags = (SPARC_PMU_ALL_EXCLUDES_SAME | + SPARC_PMU_HAS_CONFLICTS), + .max_hw_events = 2, + .num_pcrs = 1, + .num_pic_regs = 1, +}; + +static const struct perf_event_map niagara2_perfmon_event_map[] = { + [PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER | PIC_LOWER }, + [PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER | PIC_LOWER }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER | PIC_LOWER }, + [PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER | PIC_LOWER }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER | PIC_LOWER }, + [PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER | PIC_LOWER }, +}; + +static const struct perf_event_map *niagara2_event_map(int event_id) +{ + return &niagara2_perfmon_event_map[event_id]; +} + +static const cache_map_t niagara2_cache_map = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, }, + [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, }, + [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x02ff, PIC_UPPER | PIC_LOWER, }, + [C(RESULT_MISS)] = { 0x0301, PIC_UPPER | PIC_LOWER, }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE }, + [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, }, + [C(RESULT_MISS)] = { 0x0330, PIC_UPPER | PIC_LOWER, }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, }, + [C(RESULT_MISS)] = { 0x0320, PIC_UPPER | PIC_LOWER, }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0x0b08, PIC_UPPER | PIC_LOWER, }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { 0xb04, PIC_UPPER | PIC_LOWER, }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +}; + +static const struct sparc_pmu niagara2_pmu = { + .event_map = niagara2_event_map, + .cache_map = &niagara2_cache_map, + .max_events = ARRAY_SIZE(niagara2_perfmon_event_map), + .read_pmc = sparc_default_read_pmc, + .write_pmc = sparc_default_write_pmc, + .upper_shift = 19, + .lower_shift = 6, + .event_mask = 0xfff, + .user_bit = PCR_UTRACE, + .priv_bit = PCR_STRACE, + .hv_bit = PCR_N2_HTRACE, + .irq_bit = 0x30, + .upper_nop = 0x220, + .lower_nop = 0x220, + .flags = (SPARC_PMU_ALL_EXCLUDES_SAME | + SPARC_PMU_HAS_CONFLICTS), + .max_hw_events = 2, + .num_pcrs = 1, + .num_pic_regs = 1, +}; + +static const struct perf_event_map niagara4_perfmon_event_map[] = { + [PERF_COUNT_HW_CPU_CYCLES] = { (26 << 6) }, + [PERF_COUNT_HW_INSTRUCTIONS] = { (3 << 6) | 0x3f }, + [PERF_COUNT_HW_CACHE_REFERENCES] = { (3 << 6) | 0x04 }, + [PERF_COUNT_HW_CACHE_MISSES] = { (16 << 6) | 0x07 }, + [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { (4 << 6) | 0x01 }, + [PERF_COUNT_HW_BRANCH_MISSES] = { (25 << 6) | 0x0f }, +}; + +static const struct perf_event_map *niagara4_event_map(int event_id) +{ + return &niagara4_perfmon_event_map[event_id]; +} + +static const cache_map_t niagara4_cache_map = { +[C(L1D)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x04 }, + [C(RESULT_MISS)] = { (16 << 6) | 0x07 }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x08 }, + [C(RESULT_MISS)] = { (16 << 6) | 0x07 }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(L1I)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x3f }, + [C(RESULT_MISS)] = { (11 << 6) | 0x03 }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE }, + [ C(RESULT_MISS) ] = { CACHE_OP_NONSENSE }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(LL)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x04 }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, + [C(OP_WRITE)] = { + [C(RESULT_ACCESS)] = { (3 << 6) | 0x08 }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, + [C(OP_PREFETCH)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(DTLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { (17 << 6) | 0x3f }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(ITLB)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { (6 << 6) | 0x3f }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(BPU)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +[C(NODE)] = { + [C(OP_READ)] = { + [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED }, + [C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_WRITE) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, + [ C(OP_PREFETCH) ] = { + [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED }, + [ C(RESULT_MISS) ] = { CACHE_OP_UNSUPPORTED }, + }, +}, +}; + +static u32 sparc_vt_read_pmc(int idx) +{ + u64 val = pcr_ops->read_pic(idx); + + return val & 0xffffffff; +} + +static void sparc_vt_write_pmc(int idx, u64 val) +{ + u64 pcr; + + pcr = pcr_ops->read_pcr(idx); + /* ensure ov and ntc are reset */ + pcr &= ~(PCR_N4_OV | PCR_N4_NTC); + + pcr_ops->write_pic(idx, val & 0xffffffff); + + pcr_ops->write_pcr(idx, pcr); +} + +static const struct sparc_pmu niagara4_pmu = { + .event_map = niagara4_event_map, + .cache_map = &niagara4_cache_map, + .max_events = ARRAY_SIZE(niagara4_perfmon_event_map), + .read_pmc = sparc_vt_read_pmc, + .write_pmc = sparc_vt_write_pmc, + .upper_shift = 5, + .lower_shift = 5, + .event_mask = 0x7ff, + .user_bit = PCR_N4_UTRACE, + .priv_bit = PCR_N4_STRACE, + + /* We explicitly don't support hypervisor tracing. The T4 + * generates the overflow event for precise events via a trap + * which will not be generated (ie. it's completely lost) if + * we happen to be in the hypervisor when the event triggers. + * Essentially, the overflow event reporting is completely + * unusable when you have hypervisor mode tracing enabled. + */ + .hv_bit = 0, + + .irq_bit = PCR_N4_TOE, + .upper_nop = 0, + .lower_nop = 0, + .flags = 0, + .max_hw_events = 4, + .num_pcrs = 4, + .num_pic_regs = 4, +}; + +static const struct sparc_pmu sparc_m7_pmu = { + .event_map = niagara4_event_map, + .cache_map = &niagara4_cache_map, + .max_events = ARRAY_SIZE(niagara4_perfmon_event_map), + .read_pmc = sparc_vt_read_pmc, + .write_pmc = sparc_vt_write_pmc, + .upper_shift = 5, + .lower_shift = 5, + .event_mask = 0x7ff, + .user_bit = PCR_N4_UTRACE, + .priv_bit = PCR_N4_STRACE, + + /* We explicitly don't support hypervisor tracing. */ + .hv_bit = 0, + + .irq_bit = PCR_N4_TOE, + .upper_nop = 0, + .lower_nop = 0, + .flags = 0, + .max_hw_events = 4, + .num_pcrs = 4, + .num_pic_regs = 4, +}; +static const struct sparc_pmu *sparc_pmu __read_mostly; + +static u64 event_encoding(u64 event_id, int idx) +{ + if (idx == PIC_UPPER_INDEX) + event_id <<= sparc_pmu->upper_shift; + else + event_id <<= sparc_pmu->lower_shift; + return event_id; +} + +static u64 mask_for_index(int idx) +{ + return event_encoding(sparc_pmu->event_mask, idx); +} + +static u64 nop_for_index(int idx) +{ + return event_encoding(idx == PIC_UPPER_INDEX ? + sparc_pmu->upper_nop : + sparc_pmu->lower_nop, idx); +} + +static inline void sparc_pmu_enable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx) +{ + u64 enc, val, mask = mask_for_index(idx); + int pcr_index = 0; + + if (sparc_pmu->num_pcrs > 1) + pcr_index = idx; + + enc = perf_event_get_enc(cpuc->events[idx]); + + val = cpuc->pcr[pcr_index]; + val &= ~mask; + val |= event_encoding(enc, idx); + cpuc->pcr[pcr_index] = val; + + pcr_ops->write_pcr(pcr_index, cpuc->pcr[pcr_index]); +} + +static inline void sparc_pmu_disable_event(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc, int idx) +{ + u64 mask = mask_for_index(idx); + u64 nop = nop_for_index(idx); + int pcr_index = 0; + u64 val; + + if (sparc_pmu->num_pcrs > 1) + pcr_index = idx; + + val = cpuc->pcr[pcr_index]; + val &= ~mask; + val |= nop; + cpuc->pcr[pcr_index] = val; + + pcr_ops->write_pcr(pcr_index, cpuc->pcr[pcr_index]); +} + +static u64 sparc_perf_event_update(struct perf_event *event, + struct hw_perf_event *hwc, int idx) +{ + int shift = 64 - 32; + u64 prev_raw_count, new_raw_count; + s64 delta; + +again: + prev_raw_count = local64_read(&hwc->prev_count); + new_raw_count = sparc_pmu->read_pmc(idx); + + if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, + new_raw_count) != prev_raw_count) + goto again; + + delta = (new_raw_count << shift) - (prev_raw_count << shift); + delta >>= shift; + + local64_add(delta, &event->count); + local64_sub(delta, &hwc->period_left); + + return new_raw_count; +} + +static int sparc_perf_event_set_period(struct perf_event *event, + struct hw_perf_event *hwc, int idx) +{ + s64 left = local64_read(&hwc->period_left); + s64 period = hwc->sample_period; + int ret = 0; + + /* The period may have been changed by PERF_EVENT_IOC_PERIOD */ + if (unlikely(period != hwc->last_period)) + left = period - (hwc->last_period - left); + + if (unlikely(left <= -period)) { + left = period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } + + if (unlikely(left <= 0)) { + left += period; + local64_set(&hwc->period_left, left); + hwc->last_period = period; + ret = 1; + } + if (left > MAX_PERIOD) + left = MAX_PERIOD; + + local64_set(&hwc->prev_count, (u64)-left); + + sparc_pmu->write_pmc(idx, (u64)(-left) & 0xffffffff); + + perf_event_update_userpage(event); + + return ret; +} + +static void read_in_all_counters(struct cpu_hw_events *cpuc) +{ + int i; + + for (i = 0; i < cpuc->n_events; i++) { + struct perf_event *cp = cpuc->event[i]; + + if (cpuc->current_idx[i] != PIC_NO_INDEX && + cpuc->current_idx[i] != cp->hw.idx) { + sparc_perf_event_update(cp, &cp->hw, + cpuc->current_idx[i]); + cpuc->current_idx[i] = PIC_NO_INDEX; + if (cp->hw.state & PERF_HES_STOPPED) + cp->hw.state |= PERF_HES_ARCH; + } + } +} + +/* On this PMU all PICs are programmed using a single PCR. Calculate + * the combined control register value. + * + * For such chips we require that all of the events have the same + * configuration, so just fetch the settings from the first entry. + */ +static void calculate_single_pcr(struct cpu_hw_events *cpuc) +{ + int i; + + if (!cpuc->n_added) + goto out; + + /* Assign to counters all unassigned events. */ + for (i = 0; i < cpuc->n_events; i++) { + struct perf_event *cp = cpuc->event[i]; + struct hw_perf_event *hwc = &cp->hw; + int idx = hwc->idx; + u64 enc; + + if (cpuc->current_idx[i] != PIC_NO_INDEX) + continue; + + sparc_perf_event_set_period(cp, hwc, idx); + cpuc->current_idx[i] = idx; + + enc = perf_event_get_enc(cpuc->events[i]); + cpuc->pcr[0] &= ~mask_for_index(idx); + if (hwc->state & PERF_HES_ARCH) { + cpuc->pcr[0] |= nop_for_index(idx); + } else { + cpuc->pcr[0] |= event_encoding(enc, idx); + hwc->state = 0; + } + } +out: + cpuc->pcr[0] |= cpuc->event[0]->hw.config_base; +} + +static void sparc_pmu_start(struct perf_event *event, int flags); + +/* On this PMU each PIC has its own PCR control register. */ +static void calculate_multiple_pcrs(struct cpu_hw_events *cpuc) +{ + int i; + + if (!cpuc->n_added) + goto out; + + for (i = 0; i < cpuc->n_events; i++) { + struct perf_event *cp = cpuc->event[i]; + struct hw_perf_event *hwc = &cp->hw; + int idx = hwc->idx; + + if (cpuc->current_idx[i] != PIC_NO_INDEX) + continue; + + cpuc->current_idx[i] = idx; + + if (cp->hw.state & PERF_HES_ARCH) + continue; + + sparc_pmu_start(cp, PERF_EF_RELOAD); + } +out: + for (i = 0; i < cpuc->n_events; i++) { + struct perf_event *cp = cpuc->event[i]; + int idx = cp->hw.idx; + + cpuc->pcr[idx] |= cp->hw.config_base; + } +} + +/* If performance event entries have been added, move existing events + * around (if necessary) and then assign new entries to counters. + */ +static void update_pcrs_for_enable(struct cpu_hw_events *cpuc) +{ + if (cpuc->n_added) + read_in_all_counters(cpuc); + + if (sparc_pmu->num_pcrs == 1) { + calculate_single_pcr(cpuc); + } else { + calculate_multiple_pcrs(cpuc); + } +} + +static void sparc_pmu_enable(struct pmu *pmu) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int i; + + if (cpuc->enabled) + return; + + cpuc->enabled = 1; + barrier(); + + if (cpuc->n_events) + update_pcrs_for_enable(cpuc); + + for (i = 0; i < sparc_pmu->num_pcrs; i++) + pcr_ops->write_pcr(i, cpuc->pcr[i]); +} + +static void sparc_pmu_disable(struct pmu *pmu) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int i; + + if (!cpuc->enabled) + return; + + cpuc->enabled = 0; + cpuc->n_added = 0; + + for (i = 0; i < sparc_pmu->num_pcrs; i++) { + u64 val = cpuc->pcr[i]; + + val &= ~(sparc_pmu->user_bit | sparc_pmu->priv_bit | + sparc_pmu->hv_bit | sparc_pmu->irq_bit); + cpuc->pcr[i] = val; + pcr_ops->write_pcr(i, cpuc->pcr[i]); + } +} + +static int active_event_index(struct cpu_hw_events *cpuc, + struct perf_event *event) +{ + int i; + + for (i = 0; i < cpuc->n_events; i++) { + if (cpuc->event[i] == event) + break; + } + BUG_ON(i == cpuc->n_events); + return cpuc->current_idx[i]; +} + +static void sparc_pmu_start(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int idx = active_event_index(cpuc, event); + + if (flags & PERF_EF_RELOAD) { + WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); + sparc_perf_event_set_period(event, &event->hw, idx); + } + + event->hw.state = 0; + + sparc_pmu_enable_event(cpuc, &event->hw, idx); + + perf_event_update_userpage(event); +} + +static void sparc_pmu_stop(struct perf_event *event, int flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int idx = active_event_index(cpuc, event); + + if (!(event->hw.state & PERF_HES_STOPPED)) { + sparc_pmu_disable_event(cpuc, &event->hw, idx); + event->hw.state |= PERF_HES_STOPPED; + } + + if (!(event->hw.state & PERF_HES_UPTODATE) && (flags & PERF_EF_UPDATE)) { + sparc_perf_event_update(event, &event->hw, idx); + event->hw.state |= PERF_HES_UPTODATE; + } +} + +static void sparc_pmu_del(struct perf_event *event, int _flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + unsigned long flags; + int i; + + local_irq_save(flags); + + for (i = 0; i < cpuc->n_events; i++) { + if (event == cpuc->event[i]) { + /* Absorb the final count and turn off the + * event. + */ + sparc_pmu_stop(event, PERF_EF_UPDATE); + + /* Shift remaining entries down into + * the existing slot. + */ + while (++i < cpuc->n_events) { + cpuc->event[i - 1] = cpuc->event[i]; + cpuc->events[i - 1] = cpuc->events[i]; + cpuc->current_idx[i - 1] = + cpuc->current_idx[i]; + } + + perf_event_update_userpage(event); + + cpuc->n_events--; + break; + } + } + + local_irq_restore(flags); +} + +static void sparc_pmu_read(struct perf_event *event) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int idx = active_event_index(cpuc, event); + struct hw_perf_event *hwc = &event->hw; + + sparc_perf_event_update(event, hwc, idx); +} + +static atomic_t active_events = ATOMIC_INIT(0); +static DEFINE_MUTEX(pmc_grab_mutex); + +static void perf_stop_nmi_watchdog(void *unused) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int i; + + stop_nmi_watchdog(NULL); + for (i = 0; i < sparc_pmu->num_pcrs; i++) + cpuc->pcr[i] = pcr_ops->read_pcr(i); +} + +static void perf_event_grab_pmc(void) +{ + if (atomic_inc_not_zero(&active_events)) + return; + + mutex_lock(&pmc_grab_mutex); + if (atomic_read(&active_events) == 0) { + if (atomic_read(&nmi_active) > 0) { + on_each_cpu(perf_stop_nmi_watchdog, NULL, 1); + BUG_ON(atomic_read(&nmi_active) != 0); + } + atomic_inc(&active_events); + } + mutex_unlock(&pmc_grab_mutex); +} + +static void perf_event_release_pmc(void) +{ + if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) { + if (atomic_read(&nmi_active) == 0) + on_each_cpu(start_nmi_watchdog, NULL, 1); + mutex_unlock(&pmc_grab_mutex); + } +} + +static const struct perf_event_map *sparc_map_cache_event(u64 config) +{ + unsigned int cache_type, cache_op, cache_result; + const struct perf_event_map *pmap; + + if (!sparc_pmu->cache_map) + return ERR_PTR(-ENOENT); + + cache_type = (config >> 0) & 0xff; + if (cache_type >= PERF_COUNT_HW_CACHE_MAX) + return ERR_PTR(-EINVAL); + + cache_op = (config >> 8) & 0xff; + if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) + return ERR_PTR(-EINVAL); + + cache_result = (config >> 16) & 0xff; + if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + return ERR_PTR(-EINVAL); + + pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]); + + if (pmap->encoding == CACHE_OP_UNSUPPORTED) + return ERR_PTR(-ENOENT); + + if (pmap->encoding == CACHE_OP_NONSENSE) + return ERR_PTR(-EINVAL); + + return pmap; +} + +static void hw_perf_event_destroy(struct perf_event *event) +{ + perf_event_release_pmc(); +} + +/* Make sure all events can be scheduled into the hardware at + * the same time. This is simplified by the fact that we only + * need to support 2 simultaneous HW events. + * + * As a side effect, the evts[]->hw.idx values will be assigned + * on success. These are pending indexes. When the events are + * actually programmed into the chip, these values will propagate + * to the per-cpu cpuc->current_idx[] slots, see the code in + * maybe_change_configuration() for details. + */ +static int sparc_check_constraints(struct perf_event **evts, + unsigned long *events, int n_ev) +{ + u8 msk0 = 0, msk1 = 0; + int idx0 = 0; + + /* This case is possible when we are invoked from + * hw_perf_group_sched_in(). + */ + if (!n_ev) + return 0; + + if (n_ev > sparc_pmu->max_hw_events) + return -1; + + if (!(sparc_pmu->flags & SPARC_PMU_HAS_CONFLICTS)) { + int i; + + for (i = 0; i < n_ev; i++) + evts[i]->hw.idx = i; + return 0; + } + + msk0 = perf_event_get_msk(events[0]); + if (n_ev == 1) { + if (msk0 & PIC_LOWER) + idx0 = 1; + goto success; + } + BUG_ON(n_ev != 2); + msk1 = perf_event_get_msk(events[1]); + + /* If both events can go on any counter, OK. */ + if (msk0 == (PIC_UPPER | PIC_LOWER) && + msk1 == (PIC_UPPER | PIC_LOWER)) + goto success; + + /* If one event is limited to a specific counter, + * and the other can go on both, OK. + */ + if ((msk0 == PIC_UPPER || msk0 == PIC_LOWER) && + msk1 == (PIC_UPPER | PIC_LOWER)) { + if (msk0 & PIC_LOWER) + idx0 = 1; + goto success; + } + + if ((msk1 == PIC_UPPER || msk1 == PIC_LOWER) && + msk0 == (PIC_UPPER | PIC_LOWER)) { + if (msk1 & PIC_UPPER) + idx0 = 1; + goto success; + } + + /* If the events are fixed to different counters, OK. */ + if ((msk0 == PIC_UPPER && msk1 == PIC_LOWER) || + (msk0 == PIC_LOWER && msk1 == PIC_UPPER)) { + if (msk0 & PIC_LOWER) + idx0 = 1; + goto success; + } + + /* Otherwise, there is a conflict. */ + return -1; + +success: + evts[0]->hw.idx = idx0; + if (n_ev == 2) + evts[1]->hw.idx = idx0 ^ 1; + return 0; +} + +static int check_excludes(struct perf_event **evts, int n_prev, int n_new) +{ + int eu = 0, ek = 0, eh = 0; + struct perf_event *event; + int i, n, first; + + if (!(sparc_pmu->flags & SPARC_PMU_ALL_EXCLUDES_SAME)) + return 0; + + n = n_prev + n_new; + if (n <= 1) + return 0; + + first = 1; + for (i = 0; i < n; i++) { + event = evts[i]; + if (first) { + eu = event->attr.exclude_user; + ek = event->attr.exclude_kernel; + eh = event->attr.exclude_hv; + first = 0; + } else if (event->attr.exclude_user != eu || + event->attr.exclude_kernel != ek || + event->attr.exclude_hv != eh) { + return -EAGAIN; + } + } + + return 0; +} + +static int collect_events(struct perf_event *group, int max_count, + struct perf_event *evts[], unsigned long *events, + int *current_idx) +{ + struct perf_event *event; + int n = 0; + + if (!is_software_event(group)) { + if (n >= max_count) + return -1; + evts[n] = group; + events[n] = group->hw.event_base; + current_idx[n++] = PIC_NO_INDEX; + } + for_each_sibling_event(event, group) { + if (!is_software_event(event) && + event->state != PERF_EVENT_STATE_OFF) { + if (n >= max_count) + return -1; + evts[n] = event; + events[n] = event->hw.event_base; + current_idx[n++] = PIC_NO_INDEX; + } + } + return n; +} + +static int sparc_pmu_add(struct perf_event *event, int ef_flags) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int n0, ret = -EAGAIN; + unsigned long flags; + + local_irq_save(flags); + + n0 = cpuc->n_events; + if (n0 >= sparc_pmu->max_hw_events) + goto out; + + cpuc->event[n0] = event; + cpuc->events[n0] = event->hw.event_base; + cpuc->current_idx[n0] = PIC_NO_INDEX; + + event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; + if (!(ef_flags & PERF_EF_START)) + event->hw.state |= PERF_HES_ARCH; + + /* + * If group events scheduling transaction was started, + * skip the schedulability test here, it will be performed + * at commit time(->commit_txn) as a whole + */ + if (cpuc->txn_flags & PERF_PMU_TXN_ADD) + goto nocheck; + + if (check_excludes(cpuc->event, n0, 1)) + goto out; + if (sparc_check_constraints(cpuc->event, cpuc->events, n0 + 1)) + goto out; + +nocheck: + cpuc->n_events++; + cpuc->n_added++; + + ret = 0; +out: + local_irq_restore(flags); + return ret; +} + +static int sparc_pmu_event_init(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + struct perf_event *evts[MAX_HWEVENTS]; + struct hw_perf_event *hwc = &event->hw; + unsigned long events[MAX_HWEVENTS]; + int current_idx_dmy[MAX_HWEVENTS]; + const struct perf_event_map *pmap; + int n; + + if (atomic_read(&nmi_active) < 0) + return -ENODEV; + + /* does not support taken branch sampling */ + if (has_branch_stack(event)) + return -EOPNOTSUPP; + + switch (attr->type) { + case PERF_TYPE_HARDWARE: + if (attr->config >= sparc_pmu->max_events) + return -EINVAL; + pmap = sparc_pmu->event_map(attr->config); + break; + + case PERF_TYPE_HW_CACHE: + pmap = sparc_map_cache_event(attr->config); + if (IS_ERR(pmap)) + return PTR_ERR(pmap); + break; + + case PERF_TYPE_RAW: + pmap = NULL; + break; + + default: + return -ENOENT; + + } + + if (pmap) { + hwc->event_base = perf_event_encode(pmap); + } else { + /* + * User gives us "(encoding << 16) | pic_mask" for + * PERF_TYPE_RAW events. + */ + hwc->event_base = attr->config; + } + + /* We save the enable bits in the config_base. */ + hwc->config_base = sparc_pmu->irq_bit; + if (!attr->exclude_user) + hwc->config_base |= sparc_pmu->user_bit; + if (!attr->exclude_kernel) + hwc->config_base |= sparc_pmu->priv_bit; + if (!attr->exclude_hv) + hwc->config_base |= sparc_pmu->hv_bit; + + n = 0; + if (event->group_leader != event) { + n = collect_events(event->group_leader, + sparc_pmu->max_hw_events - 1, + evts, events, current_idx_dmy); + if (n < 0) + return -EINVAL; + } + events[n] = hwc->event_base; + evts[n] = event; + + if (check_excludes(evts, n, 1)) + return -EINVAL; + + if (sparc_check_constraints(evts, events, n + 1)) + return -EINVAL; + + hwc->idx = PIC_NO_INDEX; + + /* Try to do all error checking before this point, as unwinding + * state after grabbing the PMC is difficult. + */ + perf_event_grab_pmc(); + event->destroy = hw_perf_event_destroy; + + if (!hwc->sample_period) { + hwc->sample_period = MAX_PERIOD; + hwc->last_period = hwc->sample_period; + local64_set(&hwc->period_left, hwc->sample_period); + } + + return 0; +} + +/* + * Start group events scheduling transaction + * Set the flag to make pmu::enable() not perform the + * schedulability test, it will be performed at commit time + */ +static void sparc_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags) +{ + struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); + + WARN_ON_ONCE(cpuhw->txn_flags); /* txn already in flight */ + + cpuhw->txn_flags = txn_flags; + if (txn_flags & ~PERF_PMU_TXN_ADD) + return; + + perf_pmu_disable(pmu); +} + +/* + * Stop group events scheduling transaction + * Clear the flag and pmu::enable() will perform the + * schedulability test. + */ +static void sparc_pmu_cancel_txn(struct pmu *pmu) +{ + struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); + unsigned int txn_flags; + + WARN_ON_ONCE(!cpuhw->txn_flags); /* no txn in flight */ + + txn_flags = cpuhw->txn_flags; + cpuhw->txn_flags = 0; + if (txn_flags & ~PERF_PMU_TXN_ADD) + return; + + perf_pmu_enable(pmu); +} + +/* + * Commit group events scheduling transaction + * Perform the group schedulability test as a whole + * Return 0 if success + */ +static int sparc_pmu_commit_txn(struct pmu *pmu) +{ + struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); + int n; + + if (!sparc_pmu) + return -EINVAL; + + WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ + + if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { + cpuc->txn_flags = 0; + return 0; + } + + n = cpuc->n_events; + if (check_excludes(cpuc->event, 0, n)) + return -EINVAL; + if (sparc_check_constraints(cpuc->event, cpuc->events, n)) + return -EAGAIN; + + cpuc->txn_flags = 0; + perf_pmu_enable(pmu); + return 0; +} + +static struct pmu pmu = { + .pmu_enable = sparc_pmu_enable, + .pmu_disable = sparc_pmu_disable, + .event_init = sparc_pmu_event_init, + .add = sparc_pmu_add, + .del = sparc_pmu_del, + .start = sparc_pmu_start, + .stop = sparc_pmu_stop, + .read = sparc_pmu_read, + .start_txn = sparc_pmu_start_txn, + .cancel_txn = sparc_pmu_cancel_txn, + .commit_txn = sparc_pmu_commit_txn, +}; + +void perf_event_print_debug(void) +{ + unsigned long flags; + int cpu, i; + + if (!sparc_pmu) + return; + + local_irq_save(flags); + + cpu = smp_processor_id(); + + pr_info("\n"); + for (i = 0; i < sparc_pmu->num_pcrs; i++) + pr_info("CPU#%d: PCR%d[%016llx]\n", + cpu, i, pcr_ops->read_pcr(i)); + for (i = 0; i < sparc_pmu->num_pic_regs; i++) + pr_info("CPU#%d: PIC%d[%016llx]\n", + cpu, i, pcr_ops->read_pic(i)); + + local_irq_restore(flags); +} + +static int __kprobes perf_event_nmi_handler(struct notifier_block *self, + unsigned long cmd, void *__args) +{ + struct die_args *args = __args; + struct perf_sample_data data; + struct cpu_hw_events *cpuc; + struct pt_regs *regs; + u64 finish_clock; + u64 start_clock; + int i; + + if (!atomic_read(&active_events)) + return NOTIFY_DONE; + + switch (cmd) { + case DIE_NMI: + break; + + default: + return NOTIFY_DONE; + } + + start_clock = sched_clock(); + + regs = args->regs; + + cpuc = this_cpu_ptr(&cpu_hw_events); + + /* If the PMU has the TOE IRQ enable bits, we need to do a + * dummy write to the %pcr to clear the overflow bits and thus + * the interrupt. + * + * Do this before we peek at the counters to determine + * overflow so we don't lose any events. + */ + if (sparc_pmu->irq_bit && + sparc_pmu->num_pcrs == 1) + pcr_ops->write_pcr(0, cpuc->pcr[0]); + + for (i = 0; i < cpuc->n_events; i++) { + struct perf_event *event = cpuc->event[i]; + int idx = cpuc->current_idx[i]; + struct hw_perf_event *hwc; + u64 val; + + if (sparc_pmu->irq_bit && + sparc_pmu->num_pcrs > 1) + pcr_ops->write_pcr(idx, cpuc->pcr[idx]); + + hwc = &event->hw; + val = sparc_perf_event_update(event, hwc, idx); + if (val & (1ULL << 31)) + continue; + + perf_sample_data_init(&data, 0, hwc->last_period); + if (!sparc_perf_event_set_period(event, hwc, idx)) + continue; + + perf_event_overflow(event, &data, regs); + } + + finish_clock = sched_clock(); + + perf_sample_event_took(finish_clock - start_clock); + + return NOTIFY_STOP; +} + +static __read_mostly struct notifier_block perf_event_nmi_notifier = { + .notifier_call = perf_event_nmi_handler, +}; + +static bool __init supported_pmu(void) +{ + if (!strcmp(sparc_pmu_type, "ultra3") || + !strcmp(sparc_pmu_type, "ultra3+") || + !strcmp(sparc_pmu_type, "ultra3i") || + !strcmp(sparc_pmu_type, "ultra4+")) { + sparc_pmu = &ultra3_pmu; + return true; + } + if (!strcmp(sparc_pmu_type, "niagara")) { + sparc_pmu = &niagara1_pmu; + return true; + } + if (!strcmp(sparc_pmu_type, "niagara2") || + !strcmp(sparc_pmu_type, "niagara3")) { + sparc_pmu = &niagara2_pmu; + return true; + } + if (!strcmp(sparc_pmu_type, "niagara4") || + !strcmp(sparc_pmu_type, "niagara5")) { + sparc_pmu = &niagara4_pmu; + return true; + } + if (!strcmp(sparc_pmu_type, "sparc-m7")) { + sparc_pmu = &sparc_m7_pmu; + return true; + } + return false; +} + +static int __init init_hw_perf_events(void) +{ + int err; + + pr_info("Performance events: "); + + err = pcr_arch_init(); + if (err || !supported_pmu()) { + pr_cont("No support for PMU type '%s'\n", sparc_pmu_type); + return 0; + } + + pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type); + + perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); + register_die_notifier(&perf_event_nmi_notifier); + + return 0; +} +pure_initcall(init_hw_perf_events); + +void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + unsigned long ksp, fp; +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + int graph = 0; +#endif + + stack_trace_flush(); + + perf_callchain_store(entry, regs->tpc); + + ksp = regs->u_regs[UREG_I6]; + fp = ksp + STACK_BIAS; + do { + struct sparc_stackf *sf; + struct pt_regs *regs; + unsigned long pc; + + if (!kstack_valid(current_thread_info(), fp)) + break; + + sf = (struct sparc_stackf *) fp; + regs = (struct pt_regs *) (sf + 1); + + if (kstack_is_trap_frame(current_thread_info(), regs)) { + if (user_mode(regs)) + break; + pc = regs->tpc; + fp = regs->u_regs[UREG_I6] + STACK_BIAS; + } else { + pc = sf->callers_pc; + fp = (unsigned long)sf->fp + STACK_BIAS; + } + perf_callchain_store(entry, pc); +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + if ((pc + 8UL) == (unsigned long) &return_to_handler) { + struct ftrace_ret_stack *ret_stack; + ret_stack = ftrace_graph_get_ret_stack(current, + graph); + if (ret_stack) { + pc = ret_stack->ret; + perf_callchain_store(entry, pc); + graph++; + } + } +#endif + } while (entry->nr < entry->max_stack); +} + +static inline int +valid_user_frame(const void __user *fp, unsigned long size) +{ + /* addresses should be at least 4-byte aligned */ + if (((unsigned long) fp) & 3) + return 0; + + return (__range_not_ok(fp, size, TASK_SIZE) == 0); +} + +static void perf_callchain_user_64(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + unsigned long ufp; + + ufp = regs->u_regs[UREG_FP] + STACK_BIAS; + do { + struct sparc_stackf __user *usf; + struct sparc_stackf sf; + unsigned long pc; + + usf = (struct sparc_stackf __user *)ufp; + if (!valid_user_frame(usf, sizeof(sf))) + break; + + if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) + break; + + pc = sf.callers_pc; + ufp = (unsigned long)sf.fp + STACK_BIAS; + perf_callchain_store(entry, pc); + } while (entry->nr < entry->max_stack); +} + +static void perf_callchain_user_32(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + unsigned long ufp; + + ufp = regs->u_regs[UREG_FP] & 0xffffffffUL; + do { + unsigned long pc; + + if (thread32_stack_is_64bit(ufp)) { + struct sparc_stackf __user *usf; + struct sparc_stackf sf; + + ufp += STACK_BIAS; + usf = (struct sparc_stackf __user *)ufp; + if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) + break; + pc = sf.callers_pc & 0xffffffff; + ufp = ((unsigned long) sf.fp) & 0xffffffff; + } else { + struct sparc_stackf32 __user *usf; + struct sparc_stackf32 sf; + usf = (struct sparc_stackf32 __user *)ufp; + if (__copy_from_user_inatomic(&sf, usf, sizeof(sf))) + break; + pc = sf.callers_pc; + ufp = (unsigned long)sf.fp; + } + perf_callchain_store(entry, pc); + } while (entry->nr < entry->max_stack); +} + +void +perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) +{ + u64 saved_fault_address = current_thread_info()->fault_address; + u8 saved_fault_code = get_thread_fault_code(); + + perf_callchain_store(entry, regs->tpc); + + if (!current->mm) + return; + + flushw_user(); + + pagefault_disable(); + + if (test_thread_flag(TIF_32BIT)) + perf_callchain_user_32(entry, regs); + else + perf_callchain_user_64(entry, regs); + + pagefault_enable(); + + set_thread_fault_code(saved_fault_code); + current_thread_info()->fault_address = saved_fault_address; +} diff --git a/arch/sparc/kernel/pmc.c b/arch/sparc/kernel/pmc.c new file mode 100644 index 000000000..69a0206e5 --- /dev/null +++ b/arch/sparc/kernel/pmc.c @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-2.0 +/* pmc - Driver implementation for power management functions + * of Power Management Controller (PMC) on SPARCstation-Voyager. + * + * Copyright (c) 2002 Eric Brower (ebrower@usa.net) + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/* Debug + * + * #define PMC_DEBUG_LED + * #define PMC_NO_IDLE + */ + +#define PMC_OBPNAME "SUNW,pmc" +#define PMC_DEVNAME "pmc" + +#define PMC_IDLE_REG 0x00 +#define PMC_IDLE_ON 0x01 + +static u8 __iomem *regs; + +#define pmc_readb(offs) (sbus_readb(regs+offs)) +#define pmc_writeb(val, offs) (sbus_writeb(val, regs+offs)) + +/* + * CPU idle callback function + * See .../arch/sparc/kernel/process.c + */ +static void pmc_swift_idle(void) +{ +#ifdef PMC_DEBUG_LED + set_auxio(0x00, AUXIO_LED); +#endif + + pmc_writeb(pmc_readb(PMC_IDLE_REG) | PMC_IDLE_ON, PMC_IDLE_REG); + +#ifdef PMC_DEBUG_LED + set_auxio(AUXIO_LED, 0x00); +#endif +} + +static int pmc_probe(struct platform_device *op) +{ + regs = of_ioremap(&op->resource[0], 0, + resource_size(&op->resource[0]), PMC_OBPNAME); + if (!regs) { + printk(KERN_ERR "%s: unable to map registers\n", PMC_DEVNAME); + return -ENODEV; + } + +#ifndef PMC_NO_IDLE + /* Assign power management IDLE handler */ + sparc_idle = pmc_swift_idle; +#endif + + printk(KERN_INFO "%s: power management initialized\n", PMC_DEVNAME); + return 0; +} + +static const struct of_device_id pmc_match[] = { + { + .name = PMC_OBPNAME, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, pmc_match); + +static struct platform_driver pmc_driver = { + .driver = { + .name = "pmc", + .of_match_table = pmc_match, + }, + .probe = pmc_probe, +}; + +static int __init pmc_init(void) +{ + return platform_driver_register(&pmc_driver); +} + +/* This driver is not critical to the boot process + * and is easiest to ioremap when SBus is already + * initialized, so we install ourselves thusly: + */ +__initcall(pmc_init); diff --git a/arch/sparc/kernel/power.c b/arch/sparc/kernel/power.c new file mode 100644 index 000000000..db8a3f9e3 --- /dev/null +++ b/arch/sparc/kernel/power.c @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: GPL-2.0 +/* power.c: Power management driver. + * + * Copyright (C) 1999, 2007, 2008 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static void __iomem *power_reg; + +static irqreturn_t power_handler(int irq, void *dev_id) +{ + orderly_poweroff(true); + + /* FIXME: Check registers for status... */ + return IRQ_HANDLED; +} + +static int has_button_interrupt(unsigned int irq, struct device_node *dp) +{ + if (irq == 0xffffffff) + return 0; + if (!of_property_read_bool(dp, "button")) + return 0; + + return 1; +} + +static int power_probe(struct platform_device *op) +{ + struct resource *res = &op->resource[0]; + unsigned int irq = op->archdata.irqs[0]; + + power_reg = of_ioremap(res, 0, 0x4, "power"); + + printk(KERN_INFO "%pOFn: Control reg at %llx\n", + op->dev.of_node, res->start); + + if (has_button_interrupt(irq, op->dev.of_node)) { + if (request_irq(irq, + power_handler, 0, "power", NULL) < 0) + printk(KERN_ERR "power: Cannot setup IRQ handler.\n"); + } + + return 0; +} + +static const struct of_device_id power_match[] = { + { + .name = "power", + }, + {}, +}; + +static struct platform_driver power_driver = { + .probe = power_probe, + .driver = { + .name = "power", + .of_match_table = power_match, + }, +}; + +builtin_platform_driver(power_driver); diff --git a/arch/sparc/kernel/process.c b/arch/sparc/kernel/process.c new file mode 100644 index 000000000..d72fa0665 --- /dev/null +++ b/arch/sparc/kernel/process.c @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * This file handles the architecture independent parts of process handling.. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" + +asmlinkage long sparc_fork(struct pt_regs *regs) +{ + unsigned long orig_i1; + long ret; + struct kernel_clone_args args = { + .exit_signal = SIGCHLD, + }; + + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + /* Reuse the parent's stack for the child. */ + args.stack = regs->u_regs[UREG_FP]; + + ret = kernel_clone(&args); + + /* If we get an error and potentially restart the system + * call, we're screwed because copy_thread() clobbered + * the parent's %o1. So detect that case and restore it + * here. + */ + if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK) + regs->u_regs[UREG_I1] = orig_i1; + + return ret; +} + +asmlinkage long sparc_vfork(struct pt_regs *regs) +{ + unsigned long orig_i1; + long ret; + struct kernel_clone_args args = { + .flags = CLONE_VFORK | CLONE_VM, + .exit_signal = SIGCHLD, + }; + + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + /* Reuse the parent's stack for the child. */ + args.stack = regs->u_regs[UREG_FP]; + + ret = kernel_clone(&args); + + /* If we get an error and potentially restart the system + * call, we're screwed because copy_thread() clobbered + * the parent's %o1. So detect that case and restore it + * here. + */ + if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK) + regs->u_regs[UREG_I1] = orig_i1; + + return ret; +} + +asmlinkage long sparc_clone(struct pt_regs *regs) +{ + unsigned long orig_i1; + unsigned int flags; + long ret; + struct kernel_clone_args args = {0}; + + synchronize_user_stack(); + + orig_i1 = regs->u_regs[UREG_I1]; + flags = lower_32_bits(regs->u_regs[UREG_I0]); + args.flags = (flags & ~CSIGNAL); + args.exit_signal = (flags & CSIGNAL); + args.tls = regs->u_regs[UREG_I3]; + +#ifdef CONFIG_COMPAT + if (test_thread_flag(TIF_32BIT)) { + args.pidfd = compat_ptr(regs->u_regs[UREG_I2]); + args.child_tid = compat_ptr(regs->u_regs[UREG_I4]); + args.parent_tid = compat_ptr(regs->u_regs[UREG_I2]); + } else +#endif + { + args.pidfd = (int __user *)regs->u_regs[UREG_I2]; + args.child_tid = (int __user *)regs->u_regs[UREG_I4]; + args.parent_tid = (int __user *)regs->u_regs[UREG_I2]; + } + + /* Did userspace give setup a separate stack for the child or are we + * reusing the parent's? + */ + if (regs->u_regs[UREG_I1]) + args.stack = regs->u_regs[UREG_I1]; + else + args.stack = regs->u_regs[UREG_FP]; + + ret = kernel_clone(&args); + + /* If we get an error and potentially restart the system + * call, we're screwed because copy_thread() clobbered + * the parent's %o1. So detect that case and restore it + * here. + */ + if ((unsigned long)ret >= -ERESTART_RESTARTBLOCK) + regs->u_regs[UREG_I1] = orig_i1; + + return ret; +} + +asmlinkage long sparc_clone3(struct pt_regs *regs) +{ + unsigned long sz; + struct clone_args __user *cl_args; + + synchronize_user_stack(); + + cl_args = (struct clone_args __user *)regs->u_regs[UREG_I0]; + sz = regs->u_regs[UREG_I1]; + + return sys_clone3(cl_args, sz); +} diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c new file mode 100644 index 000000000..dd8c6c02b --- /dev/null +++ b/arch/sparc/kernel/process_32.c @@ -0,0 +1,404 @@ +// SPDX-License-Identifier: GPL-2.0 +/* linux/arch/sparc/kernel/process.c + * + * Copyright (C) 1995, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + */ + +/* + * This file handles the architecture-dependent parts of process handling.. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" + +/* + * Power management idle function + * Set in pm platform drivers (apc.c and pmc.c) + */ +void (*sparc_idle)(void); + +/* + * Power-off handler instantiation for pm.h compliance + * This is done via auxio, but could be used as a fallback + * handler when auxio is not present-- unused for now... + */ +void (*pm_power_off)(void) = machine_power_off; +EXPORT_SYMBOL(pm_power_off); + +/* + * sysctl - toggle power-off restriction for serial console + * systems in machine_power_off() + */ +int scons_pwroff = 1; + +extern void fpsave(unsigned long *, unsigned long *, void *, unsigned long *); + +struct task_struct *last_task_used_math = NULL; +struct thread_info *current_set[NR_CPUS]; + +/* Idle loop support. */ +void arch_cpu_idle(void) +{ + if (sparc_idle) + (*sparc_idle)(); +} + +/* XXX cli/sti -> local_irq_xxx here, check this works once SMP is fixed. */ +void machine_halt(void) +{ + local_irq_enable(); + mdelay(8); + local_irq_disable(); + prom_halt(); + panic("Halt failed!"); +} + +void machine_restart(char * cmd) +{ + char *p; + + local_irq_enable(); + mdelay(8); + local_irq_disable(); + + p = strchr (reboot_command, '\n'); + if (p) *p = 0; + if (cmd) + prom_reboot(cmd); + if (*reboot_command) + prom_reboot(reboot_command); + prom_feval ("reset"); + panic("Reboot failed!"); +} + +void machine_power_off(void) +{ + if (auxio_power_register && + (!of_node_is_type(of_console_device, "serial") || scons_pwroff)) { + u8 power_register = sbus_readb(auxio_power_register); + power_register |= AUXIO_POWER_OFF; + sbus_writeb(power_register, auxio_power_register); + } + + machine_halt(); +} + +void show_regs(struct pt_regs *r) +{ + struct reg_window32 *rw = (struct reg_window32 *) r->u_regs[14]; + + show_regs_print_info(KERN_DEFAULT); + + printk("PSR: %08lx PC: %08lx NPC: %08lx Y: %08lx %s\n", + r->psr, r->pc, r->npc, r->y, print_tainted()); + printk("PC: <%pS>\n", (void *) r->pc); + printk("%%G: %08lx %08lx %08lx %08lx %08lx %08lx %08lx %08lx\n", + r->u_regs[0], r->u_regs[1], r->u_regs[2], r->u_regs[3], + r->u_regs[4], r->u_regs[5], r->u_regs[6], r->u_regs[7]); + printk("%%O: %08lx %08lx %08lx %08lx %08lx %08lx %08lx %08lx\n", + r->u_regs[8], r->u_regs[9], r->u_regs[10], r->u_regs[11], + r->u_regs[12], r->u_regs[13], r->u_regs[14], r->u_regs[15]); + printk("RPC: <%pS>\n", (void *) r->u_regs[15]); + + printk("%%L: %08lx %08lx %08lx %08lx %08lx %08lx %08lx %08lx\n", + rw->locals[0], rw->locals[1], rw->locals[2], rw->locals[3], + rw->locals[4], rw->locals[5], rw->locals[6], rw->locals[7]); + printk("%%I: %08lx %08lx %08lx %08lx %08lx %08lx %08lx %08lx\n", + rw->ins[0], rw->ins[1], rw->ins[2], rw->ins[3], + rw->ins[4], rw->ins[5], rw->ins[6], rw->ins[7]); +} + +/* + * The show_stack() is external API which we do not use ourselves. + * The oops is printed in die_if_kernel. + */ +void show_stack(struct task_struct *tsk, unsigned long *_ksp, const char *loglvl) +{ + unsigned long pc, fp; + unsigned long task_base; + struct reg_window32 *rw; + int count = 0; + + if (!tsk) + tsk = current; + + if (tsk == current && !_ksp) + __asm__ __volatile__("mov %%fp, %0" : "=r" (_ksp)); + + task_base = (unsigned long) task_stack_page(tsk); + fp = (unsigned long) _ksp; + do { + /* Bogus frame pointer? */ + if (fp < (task_base + sizeof(struct thread_info)) || + fp >= (task_base + (PAGE_SIZE << 1))) + break; + rw = (struct reg_window32 *) fp; + pc = rw->ins[7]; + printk("%s[%08lx : ", loglvl, pc); + printk("%s%pS ] ", loglvl, (void *) pc); + fp = rw->ins[6]; + } while (++count < 16); + printk("%s\n", loglvl); +} + +/* + * Free current thread data structures etc.. + */ +void exit_thread(struct task_struct *tsk) +{ +#ifndef CONFIG_SMP + if (last_task_used_math == tsk) { +#else + if (test_tsk_thread_flag(tsk, TIF_USEDFPU)) { +#endif + /* Keep process from leaving FPU in a bogon state. */ + put_psr(get_psr() | PSR_EF); + fpsave(&tsk->thread.float_regs[0], &tsk->thread.fsr, + &tsk->thread.fpqueue[0], &tsk->thread.fpqdepth); +#ifndef CONFIG_SMP + last_task_used_math = NULL; +#else + clear_ti_thread_flag(task_thread_info(tsk), TIF_USEDFPU); +#endif + } +} + +void flush_thread(void) +{ + current_thread_info()->w_saved = 0; + +#ifndef CONFIG_SMP + if(last_task_used_math == current) { +#else + if (test_thread_flag(TIF_USEDFPU)) { +#endif + /* Clean the fpu. */ + put_psr(get_psr() | PSR_EF); + fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, + ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); +#ifndef CONFIG_SMP + last_task_used_math = NULL; +#else + clear_thread_flag(TIF_USEDFPU); +#endif + } +} + +static inline struct sparc_stackf __user * +clone_stackframe(struct sparc_stackf __user *dst, + struct sparc_stackf __user *src) +{ + unsigned long size, fp; + struct sparc_stackf *tmp; + struct sparc_stackf __user *sp; + + if (get_user(tmp, &src->fp)) + return NULL; + + fp = (unsigned long) tmp; + size = (fp - ((unsigned long) src)); + fp = (unsigned long) dst; + sp = (struct sparc_stackf __user *)(fp - size); + + /* do_fork() grabs the parent semaphore, we must release it + * temporarily so we can build the child clone stack frame + * without deadlocking. + */ + if (__copy_user(sp, src, size)) + sp = NULL; + else if (put_user(fp, &sp->fp)) + sp = NULL; + + return sp; +} + +/* Copy a Sparc thread. The fork() return value conventions + * under SunOS are nothing short of bletcherous: + * Parent --> %o0 == childs pid, %o1 == 0 + * Child --> %o0 == parents pid, %o1 == 1 + * + * clone3() - Uses regular kernel return value conventions + * + * NOTE: We have a separate fork kpsr/kwim because + * the parent could change these values between + * sys_fork invocation and when we reach here + * if the parent should sleep while trying to + * allocate the task_struct and kernel stack in + * do_fork(). + * XXX See comment above sys_vfork in sparc64. todo. + */ +extern void ret_from_fork(void); +extern void ret_from_kernel_thread(void); + +int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) +{ + u64 clone_flags = args->flags; + unsigned long tls = args->tls; + struct thread_info *ti = task_thread_info(p); + struct pt_regs *childregs, *regs = current_pt_regs(); + char *new_stack; + unsigned long sp = args->stack ? args->stack : regs->u_regs[UREG_FP]; + +#ifndef CONFIG_SMP + if(last_task_used_math == current) { +#else + if (test_thread_flag(TIF_USEDFPU)) { +#endif + put_psr(get_psr() | PSR_EF); + fpsave(&p->thread.float_regs[0], &p->thread.fsr, + &p->thread.fpqueue[0], &p->thread.fpqdepth); + } + + /* + * p->thread_info new_stack childregs stack bottom + * ! ! ! ! + * V V (stk.fr.) V (pt_regs) V + * +----- - - - - - ------+===========+=============+ + */ + new_stack = task_stack_page(p) + THREAD_SIZE; + new_stack -= STACKFRAME_SZ + TRACEREG_SZ; + childregs = (struct pt_regs *) (new_stack + STACKFRAME_SZ); + + /* + * A new process must start with interrupts disabled, see schedule_tail() + * and finish_task_switch(). (If we do not do it and if a timer interrupt + * hits before we unlock and attempts to take the rq->lock, we deadlock.) + * + * Thus, kpsr |= PSR_PIL. + */ + ti->ksp = (unsigned long) new_stack; + p->thread.kregs = childregs; + + if (unlikely(args->fn)) { + extern int nwindows; + unsigned long psr; + memset(new_stack, 0, STACKFRAME_SZ + TRACEREG_SZ); + ti->kpc = (((unsigned long) ret_from_kernel_thread) - 0x8); + childregs->u_regs[UREG_G1] = (unsigned long) args->fn; + childregs->u_regs[UREG_G2] = (unsigned long) args->fn_arg; + psr = childregs->psr = get_psr(); + ti->kpsr = psr | PSR_PIL; + ti->kwim = 1 << (((psr & PSR_CWP) + 1) % nwindows); + return 0; + } + memcpy(new_stack, (char *)regs - STACKFRAME_SZ, STACKFRAME_SZ + TRACEREG_SZ); + childregs->u_regs[UREG_FP] = sp; + ti->kpc = (((unsigned long) ret_from_fork) - 0x8); + ti->kpsr = current->thread.fork_kpsr | PSR_PIL; + ti->kwim = current->thread.fork_kwim; + + if (sp != regs->u_regs[UREG_FP]) { + struct sparc_stackf __user *childstack; + struct sparc_stackf __user *parentstack; + + /* + * This is a clone() call with supplied user stack. + * Set some valid stack frames to give to the child. + */ + childstack = (struct sparc_stackf __user *) + (sp & ~0xfUL); + parentstack = (struct sparc_stackf __user *) + regs->u_regs[UREG_FP]; + +#if 0 + printk("clone: parent stack:\n"); + show_stackframe(parentstack); +#endif + + childstack = clone_stackframe(childstack, parentstack); + if (!childstack) + return -EFAULT; + +#if 0 + printk("clone: child stack:\n"); + show_stackframe(childstack); +#endif + + childregs->u_regs[UREG_FP] = (unsigned long)childstack; + } + +#ifdef CONFIG_SMP + /* FPU must be disabled on SMP. */ + childregs->psr &= ~PSR_EF; + clear_tsk_thread_flag(p, TIF_USEDFPU); +#endif + /* Handle return value conventions */ + if (regs->u_regs[UREG_G1] == __NR_clone3) { + /* clone3() - use regular kernel return value convention */ + + /* Set the return value for the child. */ + childregs->u_regs[UREG_I0] = 0; + } else { + /* clone()/fork() - use SunOS return value convention */ + + /* Set the return value for the child. */ + childregs->u_regs[UREG_I0] = current->pid; + childregs->u_regs[UREG_I1] = 1; + + /* Set the return value for the parent. */ + regs->u_regs[UREG_I1] = 0; + } + + if (clone_flags & CLONE_SETTLS) + childregs->u_regs[UREG_G7] = tls; + + return 0; +} + +unsigned long __get_wchan(struct task_struct *task) +{ + unsigned long pc, fp, bias = 0; + unsigned long task_base = (unsigned long) task; + unsigned long ret = 0; + struct reg_window32 *rw; + int count = 0; + + fp = task_thread_info(task)->ksp + bias; + do { + /* Bogus frame pointer? */ + if (fp < (task_base + sizeof(struct thread_info)) || + fp >= (task_base + (2 * PAGE_SIZE))) + break; + rw = (struct reg_window32 *) fp; + pc = rw->ins[7]; + if (!in_sched_functions(pc)) { + ret = pc; + goto out; + } + fp = rw->ins[6] + bias; + } while (++count < 16); + +out: + return ret; +} + diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c new file mode 100644 index 000000000..e889da8e4 --- /dev/null +++ b/arch/sparc/kernel/process_64.c @@ -0,0 +1,698 @@ +// SPDX-License-Identifier: GPL-2.0 +/* arch/sparc64/kernel/process.c + * + * Copyright (C) 1995, 1996, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1997, 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +/* + * This file handles the architecture-dependent parts of process handling.. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kstack.h" + +/* Idle loop support on sparc64. */ +void arch_cpu_idle(void) +{ + if (tlb_type != hypervisor) { + touch_nmi_watchdog(); + } else { + unsigned long pstate; + + raw_local_irq_enable(); + + /* The sun4v sleeping code requires that we have PSTATE.IE cleared over + * the cpu sleep hypervisor call. + */ + __asm__ __volatile__( + "rdpr %%pstate, %0\n\t" + "andn %0, %1, %0\n\t" + "wrpr %0, %%g0, %%pstate" + : "=&r" (pstate) + : "i" (PSTATE_IE)); + + if (!need_resched() && !cpu_is_offline(smp_processor_id())) { + sun4v_cpu_yield(); + /* If resumed by cpu_poke then we need to explicitly + * call scheduler_ipi(). + */ + scheduler_poke(); + } + + /* Re-enable interrupts. */ + __asm__ __volatile__( + "rdpr %%pstate, %0\n\t" + "or %0, %1, %0\n\t" + "wrpr %0, %%g0, %%pstate" + : "=&r" (pstate) + : "i" (PSTATE_IE)); + + raw_local_irq_disable(); + } +} + +#ifdef CONFIG_HOTPLUG_CPU +void __noreturn arch_cpu_idle_dead(void) +{ + sched_preempt_enable_no_resched(); + cpu_play_dead(); +} +#endif + +#ifdef CONFIG_COMPAT +static void show_regwindow32(struct pt_regs *regs) +{ + struct reg_window32 __user *rw; + struct reg_window32 r_w; + + __asm__ __volatile__ ("flushw"); + rw = compat_ptr((unsigned int)regs->u_regs[14]); + if (copy_from_user (&r_w, rw, sizeof(r_w))) { + return; + } + + printk("l0: %08x l1: %08x l2: %08x l3: %08x " + "l4: %08x l5: %08x l6: %08x l7: %08x\n", + r_w.locals[0], r_w.locals[1], r_w.locals[2], r_w.locals[3], + r_w.locals[4], r_w.locals[5], r_w.locals[6], r_w.locals[7]); + printk("i0: %08x i1: %08x i2: %08x i3: %08x " + "i4: %08x i5: %08x i6: %08x i7: %08x\n", + r_w.ins[0], r_w.ins[1], r_w.ins[2], r_w.ins[3], + r_w.ins[4], r_w.ins[5], r_w.ins[6], r_w.ins[7]); +} +#else +#define show_regwindow32(regs) do { } while (0) +#endif + +static void show_regwindow(struct pt_regs *regs) +{ + struct reg_window __user *rw; + struct reg_window *rwk; + struct reg_window r_w; + + if ((regs->tstate & TSTATE_PRIV) || !(test_thread_flag(TIF_32BIT))) { + __asm__ __volatile__ ("flushw"); + rw = (struct reg_window __user *) + (regs->u_regs[14] + STACK_BIAS); + rwk = (struct reg_window *) + (regs->u_regs[14] + STACK_BIAS); + if (!(regs->tstate & TSTATE_PRIV)) { + if (copy_from_user (&r_w, rw, sizeof(r_w))) { + return; + } + rwk = &r_w; + } + } else { + show_regwindow32(regs); + return; + } + printk("l0: %016lx l1: %016lx l2: %016lx l3: %016lx\n", + rwk->locals[0], rwk->locals[1], rwk->locals[2], rwk->locals[3]); + printk("l4: %016lx l5: %016lx l6: %016lx l7: %016lx\n", + rwk->locals[4], rwk->locals[5], rwk->locals[6], rwk->locals[7]); + printk("i0: %016lx i1: %016lx i2: %016lx i3: %016lx\n", + rwk->ins[0], rwk->ins[1], rwk->ins[2], rwk->ins[3]); + printk("i4: %016lx i5: %016lx i6: %016lx i7: %016lx\n", + rwk->ins[4], rwk->ins[5], rwk->ins[6], rwk->ins[7]); + if (regs->tstate & TSTATE_PRIV) + printk("I7: <%pS>\n", (void *) rwk->ins[7]); +} + +void show_regs(struct pt_regs *regs) +{ + show_regs_print_info(KERN_DEFAULT); + + printk("TSTATE: %016lx TPC: %016lx TNPC: %016lx Y: %08x %s\n", regs->tstate, + regs->tpc, regs->tnpc, regs->y, print_tainted()); + printk("TPC: <%pS>\n", (void *) regs->tpc); + printk("g0: %016lx g1: %016lx g2: %016lx g3: %016lx\n", + regs->u_regs[0], regs->u_regs[1], regs->u_regs[2], + regs->u_regs[3]); + printk("g4: %016lx g5: %016lx g6: %016lx g7: %016lx\n", + regs->u_regs[4], regs->u_regs[5], regs->u_regs[6], + regs->u_regs[7]); + printk("o0: %016lx o1: %016lx o2: %016lx o3: %016lx\n", + regs->u_regs[8], regs->u_regs[9], regs->u_regs[10], + regs->u_regs[11]); + printk("o4: %016lx o5: %016lx sp: %016lx ret_pc: %016lx\n", + regs->u_regs[12], regs->u_regs[13], regs->u_regs[14], + regs->u_regs[15]); + printk("RPC: <%pS>\n", (void *) regs->u_regs[15]); + show_regwindow(regs); + show_stack(current, (unsigned long *)regs->u_regs[UREG_FP], KERN_DEFAULT); +} + +union global_cpu_snapshot global_cpu_snapshot[NR_CPUS]; +static DEFINE_SPINLOCK(global_cpu_snapshot_lock); + +static void __global_reg_self(struct thread_info *tp, struct pt_regs *regs, + int this_cpu) +{ + struct global_reg_snapshot *rp; + + flushw_all(); + + rp = &global_cpu_snapshot[this_cpu].reg; + + rp->tstate = regs->tstate; + rp->tpc = regs->tpc; + rp->tnpc = regs->tnpc; + rp->o7 = regs->u_regs[UREG_I7]; + + if (regs->tstate & TSTATE_PRIV) { + struct reg_window *rw; + + rw = (struct reg_window *) + (regs->u_regs[UREG_FP] + STACK_BIAS); + if (kstack_valid(tp, (unsigned long) rw)) { + rp->i7 = rw->ins[7]; + rw = (struct reg_window *) + (rw->ins[6] + STACK_BIAS); + if (kstack_valid(tp, (unsigned long) rw)) + rp->rpc = rw->ins[7]; + } + } else { + rp->i7 = 0; + rp->rpc = 0; + } + rp->thread = tp; +} + +/* In order to avoid hangs we do not try to synchronize with the + * global register dump client cpus. The last store they make is to + * the thread pointer, so do a short poll waiting for that to become + * non-NULL. + */ +static void __global_reg_poll(struct global_reg_snapshot *gp) +{ + int limit = 0; + + while (!gp->thread && ++limit < 100) { + barrier(); + udelay(1); + } +} + +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu) +{ + struct thread_info *tp = current_thread_info(); + struct pt_regs *regs = get_irq_regs(); + unsigned long flags; + int this_cpu, cpu; + + if (!regs) + regs = tp->kregs; + + spin_lock_irqsave(&global_cpu_snapshot_lock, flags); + + this_cpu = raw_smp_processor_id(); + + memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); + + if (cpumask_test_cpu(this_cpu, mask) && this_cpu != exclude_cpu) + __global_reg_self(tp, regs, this_cpu); + + smp_fetch_global_regs(); + + for_each_cpu(cpu, mask) { + struct global_reg_snapshot *gp; + + if (cpu == exclude_cpu) + continue; + + gp = &global_cpu_snapshot[cpu].reg; + + __global_reg_poll(gp); + + tp = gp->thread; + printk("%c CPU[%3d]: TSTATE[%016lx] TPC[%016lx] TNPC[%016lx] TASK[%s:%d]\n", + (cpu == this_cpu ? '*' : ' '), cpu, + gp->tstate, gp->tpc, gp->tnpc, + ((tp && tp->task) ? tp->task->comm : "NULL"), + ((tp && tp->task) ? tp->task->pid : -1)); + + if (gp->tstate & TSTATE_PRIV) { + printk(" TPC[%pS] O7[%pS] I7[%pS] RPC[%pS]\n", + (void *) gp->tpc, + (void *) gp->o7, + (void *) gp->i7, + (void *) gp->rpc); + } else { + printk(" TPC[%lx] O7[%lx] I7[%lx] RPC[%lx]\n", + gp->tpc, gp->o7, gp->i7, gp->rpc); + } + + touch_nmi_watchdog(); + } + + memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); + + spin_unlock_irqrestore(&global_cpu_snapshot_lock, flags); +} + +#ifdef CONFIG_MAGIC_SYSRQ + +static void sysrq_handle_globreg(u8 key) +{ + trigger_all_cpu_backtrace(); +} + +static const struct sysrq_key_op sparc_globalreg_op = { + .handler = sysrq_handle_globreg, + .help_msg = "global-regs(y)", + .action_msg = "Show Global CPU Regs", +}; + +static void __global_pmu_self(int this_cpu) +{ + struct global_pmu_snapshot *pp; + int i, num; + + if (!pcr_ops) + return; + + pp = &global_cpu_snapshot[this_cpu].pmu; + + num = 1; + if (tlb_type == hypervisor && + sun4v_chip_type >= SUN4V_CHIP_NIAGARA4) + num = 4; + + for (i = 0; i < num; i++) { + pp->pcr[i] = pcr_ops->read_pcr(i); + pp->pic[i] = pcr_ops->read_pic(i); + } +} + +static void __global_pmu_poll(struct global_pmu_snapshot *pp) +{ + int limit = 0; + + while (!pp->pcr[0] && ++limit < 100) { + barrier(); + udelay(1); + } +} + +static void pmu_snapshot_all_cpus(void) +{ + unsigned long flags; + int this_cpu, cpu; + + spin_lock_irqsave(&global_cpu_snapshot_lock, flags); + + memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); + + this_cpu = raw_smp_processor_id(); + + __global_pmu_self(this_cpu); + + smp_fetch_global_pmu(); + + for_each_online_cpu(cpu) { + struct global_pmu_snapshot *pp = &global_cpu_snapshot[cpu].pmu; + + __global_pmu_poll(pp); + + printk("%c CPU[%3d]: PCR[%08lx:%08lx:%08lx:%08lx] PIC[%08lx:%08lx:%08lx:%08lx]\n", + (cpu == this_cpu ? '*' : ' '), cpu, + pp->pcr[0], pp->pcr[1], pp->pcr[2], pp->pcr[3], + pp->pic[0], pp->pic[1], pp->pic[2], pp->pic[3]); + + touch_nmi_watchdog(); + } + + memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); + + spin_unlock_irqrestore(&global_cpu_snapshot_lock, flags); +} + +static void sysrq_handle_globpmu(u8 key) +{ + pmu_snapshot_all_cpus(); +} + +static const struct sysrq_key_op sparc_globalpmu_op = { + .handler = sysrq_handle_globpmu, + .help_msg = "global-pmu(x)", + .action_msg = "Show Global PMU Regs", +}; + +static int __init sparc_sysrq_init(void) +{ + int ret = register_sysrq_key('y', &sparc_globalreg_op); + + if (!ret) + ret = register_sysrq_key('x', &sparc_globalpmu_op); + return ret; +} + +core_initcall(sparc_sysrq_init); + +#endif + +/* Free current thread data structures etc.. */ +void exit_thread(struct task_struct *tsk) +{ + struct thread_info *t = task_thread_info(tsk); + + if (t->utraps) { + if (t->utraps[0] < 2) + kfree (t->utraps); + else + t->utraps[0]--; + } +} + +void flush_thread(void) +{ + struct thread_info *t = current_thread_info(); + struct mm_struct *mm; + + mm = t->task->mm; + if (mm) + tsb_context_switch(mm); + + set_thread_wsaved(0); + + /* Clear FPU register state. */ + t->fpsaved[0] = 0; +} + +/* It's a bit more tricky when 64-bit tasks are involved... */ +static unsigned long clone_stackframe(unsigned long csp, unsigned long psp) +{ + bool stack_64bit = test_thread_64bit_stack(psp); + unsigned long fp, distance, rval; + + if (stack_64bit) { + csp += STACK_BIAS; + psp += STACK_BIAS; + __get_user(fp, &(((struct reg_window __user *)psp)->ins[6])); + fp += STACK_BIAS; + if (test_thread_flag(TIF_32BIT)) + fp &= 0xffffffff; + } else + __get_user(fp, &(((struct reg_window32 __user *)psp)->ins[6])); + + /* Now align the stack as this is mandatory in the Sparc ABI + * due to how register windows work. This hides the + * restriction from thread libraries etc. + */ + csp &= ~15UL; + + distance = fp - psp; + rval = (csp - distance); + if (raw_copy_in_user((void __user *)rval, (void __user *)psp, distance)) + rval = 0; + else if (!stack_64bit) { + if (put_user(((u32)csp), + &(((struct reg_window32 __user *)rval)->ins[6]))) + rval = 0; + } else { + if (put_user(((u64)csp - STACK_BIAS), + &(((struct reg_window __user *)rval)->ins[6]))) + rval = 0; + else + rval = rval - STACK_BIAS; + } + + return rval; +} + +/* Standard stuff. */ +static inline void shift_window_buffer(int first_win, int last_win, + struct thread_info *t) +{ + int i; + + for (i = first_win; i < last_win; i++) { + t->rwbuf_stkptrs[i] = t->rwbuf_stkptrs[i+1]; + memcpy(&t->reg_window[i], &t->reg_window[i+1], + sizeof(struct reg_window)); + } +} + +void synchronize_user_stack(void) +{ + struct thread_info *t = current_thread_info(); + unsigned long window; + + flush_user_windows(); + if ((window = get_thread_wsaved()) != 0) { + window -= 1; + do { + struct reg_window *rwin = &t->reg_window[window]; + int winsize = sizeof(struct reg_window); + unsigned long sp; + + sp = t->rwbuf_stkptrs[window]; + + if (test_thread_64bit_stack(sp)) + sp += STACK_BIAS; + else + winsize = sizeof(struct reg_window32); + + if (!copy_to_user((char __user *)sp, rwin, winsize)) { + shift_window_buffer(window, get_thread_wsaved() - 1, t); + set_thread_wsaved(get_thread_wsaved() - 1); + } + } while (window--); + } +} + +static void stack_unaligned(unsigned long sp) +{ + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) sp); +} + +static const char uwfault32[] = KERN_INFO \ + "%s[%d]: bad register window fault: SP %08lx (orig_sp %08lx) TPC %08lx O7 %08lx\n"; +static const char uwfault64[] = KERN_INFO \ + "%s[%d]: bad register window fault: SP %016lx (orig_sp %016lx) TPC %08lx O7 %016lx\n"; + +void fault_in_user_windows(struct pt_regs *regs) +{ + struct thread_info *t = current_thread_info(); + unsigned long window; + + flush_user_windows(); + window = get_thread_wsaved(); + + if (likely(window != 0)) { + window -= 1; + do { + struct reg_window *rwin = &t->reg_window[window]; + int winsize = sizeof(struct reg_window); + unsigned long sp, orig_sp; + + orig_sp = sp = t->rwbuf_stkptrs[window]; + + if (test_thread_64bit_stack(sp)) + sp += STACK_BIAS; + else + winsize = sizeof(struct reg_window32); + + if (unlikely(sp & 0x7UL)) + stack_unaligned(sp); + + if (unlikely(copy_to_user((char __user *)sp, + rwin, winsize))) { + if (show_unhandled_signals) + printk_ratelimited(is_compat_task() ? + uwfault32 : uwfault64, + current->comm, current->pid, + sp, orig_sp, + regs->tpc, + regs->u_regs[UREG_I7]); + goto barf; + } + } while (window--); + } + set_thread_wsaved(0); + return; + +barf: + set_thread_wsaved(window + 1); + force_sig(SIGSEGV); +} + +/* Copy a Sparc thread. The fork() return value conventions + * under SunOS are nothing short of bletcherous: + * Parent --> %o0 == childs pid, %o1 == 0 + * Child --> %o0 == parents pid, %o1 == 1 + * + * clone3() - Uses regular kernel return value conventions + */ +int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) +{ + u64 clone_flags = args->flags; + unsigned long tls = args->tls; + struct thread_info *t = task_thread_info(p); + struct pt_regs *regs = current_pt_regs(); + struct sparc_stackf *parent_sf; + unsigned long child_stack_sz; + char *child_trap_frame; + unsigned long sp = args->stack ? args->stack : regs->u_regs[UREG_FP]; + + /* Calculate offset to stack_frame & pt_regs */ + child_stack_sz = (STACKFRAME_SZ + TRACEREG_SZ); + child_trap_frame = (task_stack_page(p) + + (THREAD_SIZE - child_stack_sz)); + + t->new_child = 1; + t->ksp = ((unsigned long) child_trap_frame) - STACK_BIAS; + t->kregs = (struct pt_regs *) (child_trap_frame + + sizeof(struct sparc_stackf)); + t->fpsaved[0] = 0; + + if (unlikely(args->fn)) { + memset(child_trap_frame, 0, child_stack_sz); + __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = + (current_pt_regs()->tstate + 1) & TSTATE_CWP; + t->kregs->u_regs[UREG_G1] = (unsigned long) args->fn; + t->kregs->u_regs[UREG_G2] = (unsigned long) args->fn_arg; + return 0; + } + + parent_sf = ((struct sparc_stackf *) regs) - 1; + memcpy(child_trap_frame, parent_sf, child_stack_sz); + if (t->flags & _TIF_32BIT) { + sp &= 0x00000000ffffffffUL; + regs->u_regs[UREG_FP] &= 0x00000000ffffffffUL; + } + t->kregs->u_regs[UREG_FP] = sp; + __thread_flag_byte_ptr(t)[TI_FLAG_BYTE_CWP] = + (regs->tstate + 1) & TSTATE_CWP; + if (sp != regs->u_regs[UREG_FP]) { + unsigned long csp; + + csp = clone_stackframe(sp, regs->u_regs[UREG_FP]); + if (!csp) + return -EFAULT; + t->kregs->u_regs[UREG_FP] = csp; + } + if (t->utraps) + t->utraps[0]++; + + /* Handle return value conventions */ + if (regs->u_regs[UREG_G1] == __NR_clone3) { + /* clone3() - use regular kernel return value convention */ + + /* Set the return value for the child. */ + t->kregs->u_regs[UREG_I0] = 0; + + /* Clear g1 to indicate user thread */ + t->kregs->u_regs[UREG_G1] = 0; + } else { + /* clone()/fork() - use SunOS return value convention */ + + /* Set the return value for the child. */ + t->kregs->u_regs[UREG_I0] = current->pid; + t->kregs->u_regs[UREG_I1] = 1; + + /* Set the second return value for the parent. */ + regs->u_regs[UREG_I1] = 0; + } + + if (clone_flags & CLONE_SETTLS) + t->kregs->u_regs[UREG_G7] = tls; + + return 0; +} + +/* TIF_MCDPER in thread info flags for current task is updated lazily upon + * a context switch. Update this flag in current task's thread flags + * before dup so the dup'd task will inherit the current TIF_MCDPER flag. + */ +int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src) +{ + if (adi_capable()) { + register unsigned long tmp_mcdper; + + __asm__ __volatile__( + ".word 0x83438000\n\t" /* rd %mcdper, %g1 */ + "mov %%g1, %0\n\t" + : "=r" (tmp_mcdper) + : + : "g1"); + if (tmp_mcdper) + set_thread_flag(TIF_MCDPER); + else + clear_thread_flag(TIF_MCDPER); + } + + *dst = *src; + return 0; +} + +unsigned long __get_wchan(struct task_struct *task) +{ + unsigned long pc, fp, bias = 0; + struct thread_info *tp; + struct reg_window *rw; + unsigned long ret = 0; + int count = 0; + + tp = task_thread_info(task); + bias = STACK_BIAS; + fp = task_thread_info(task)->ksp + bias; + + do { + if (!kstack_valid(tp, fp)) + break; + rw = (struct reg_window *) fp; + pc = rw->ins[7]; + if (!in_sched_functions(pc)) { + ret = pc; + goto out; + } + fp = rw->ins[6] + bias; + } while (++count < 16); + +out: + return ret; +} diff --git a/arch/sparc/kernel/prom.h b/arch/sparc/kernel/prom.h new file mode 100644 index 000000000..26a1cca7c --- /dev/null +++ b/arch/sparc/kernel/prom.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PROM_H +#define __PROM_H + +#include +#include + +void of_console_init(void); + +extern unsigned int prom_early_allocated; + +#endif /* __PROM_H */ diff --git a/arch/sparc/kernel/prom_32.c b/arch/sparc/kernel/prom_32.c new file mode 100644 index 000000000..cd94f1e8d --- /dev/null +++ b/arch/sparc/kernel/prom_32.c @@ -0,0 +1,324 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Procedures for creating, accessing and interpreting the device tree. + * + * Paul Mackerras August 1996. + * Copyright (C) 1996-2005 Paul Mackerras. + * + * Adapted for 64bit PowerPC by Dave Engebretsen and Peter Bergner. + * {engebret|bergner}@us.ibm.com + * + * Adapted for sparc32 by David S. Miller davem@davemloft.net + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "prom.h" + +void * __init prom_early_alloc(unsigned long size) +{ + void *ret; + + ret = memblock_alloc_or_panic(size, SMP_CACHE_BYTES); + + prom_early_allocated += size; + + return ret; +} + +/* The following routines deal with the black magic of fully naming a + * node. + * + * Certain well known named nodes are just the simple name string. + * + * Actual devices have an address specifier appended to the base name + * string, like this "foo@addr". The "addr" can be in any number of + * formats, and the platform plus the type of the node determine the + * format and how it is constructed. + * + * For children of the ROOT node, the naming convention is fixed and + * determined by whether this is a sun4u or sun4v system. + * + * For children of other nodes, it is bus type specific. So + * we walk up the tree until we discover a "device_type" property + * we recognize and we go from there. + */ +static void __init sparc32_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom_registers *regs; + struct property *rprop; + + rprop = of_find_property(dp, "reg", NULL); + if (!rprop) + return; + + regs = rprop->value; + sprintf(tmp_buf, "%s@%x,%x", + name, + regs->which_io, regs->phys_addr); +} + +/* "name@slot,offset" */ +static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom_registers *regs; + struct property *prop; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + sprintf(tmp_buf, "%s@%x,%x", + name, + regs->which_io, + regs->phys_addr); +} + +/* "name@devnum[,func]" */ +static void __init pci_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom_pci_registers *regs; + struct property *prop; + unsigned int devfn; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + devfn = (regs->phys_hi >> 8) & 0xff; + if (devfn & 0x07) { + sprintf(tmp_buf, "%s@%x,%x", + name, + devfn >> 3, + devfn & 0x07); + } else { + sprintf(tmp_buf, "%s@%x", + name, + devfn >> 3); + } +} + +/* "name@addrhi,addrlo" */ +static void __init ebus_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom_registers *regs; + struct property *prop; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + + sprintf(tmp_buf, "%s@%x,%x", + name, + regs->which_io, regs->phys_addr); +} + +/* "name@irq,addrlo" */ +static void __init ambapp_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct amba_prom_registers *regs; + unsigned int *intr; + unsigned int reg0; + struct property *prop; + int interrupt = 0; + + /* In order to get a unique ID in the device tree (multiple AMBA devices + * may have the same name) the node number is printed + */ + prop = of_find_property(dp, "reg", NULL); + if (!prop) { + reg0 = (unsigned int)dp->phandle; + } else { + regs = prop->value; + reg0 = regs->phys_addr; + } + + /* Not all cores have Interrupt */ + prop = of_find_property(dp, "interrupts", NULL); + if (!prop) + intr = &interrupt; /* IRQ0 does not exist */ + else + intr = prop->value; + + sprintf(tmp_buf, "%s@%x,%x", name, *intr, reg0); +} + +static void __init __build_path_component(struct device_node *dp, char *tmp_buf) +{ + struct device_node *parent = dp->parent; + + if (parent != NULL) { + if (of_node_is_type(parent, "pci") || + of_node_is_type(parent, "pciex")) + return pci_path_component(dp, tmp_buf); + if (of_node_is_type(parent, "sbus")) + return sbus_path_component(dp, tmp_buf); + if (of_node_is_type(parent, "ebus")) + return ebus_path_component(dp, tmp_buf); + if (of_node_is_type(parent, "ambapp")) + return ambapp_path_component(dp, tmp_buf); + + /* "isa" is handled with platform naming */ + } + + /* Use platform naming convention. */ + return sparc32_path_component(dp, tmp_buf); +} + +char * __init build_path_component(struct device_node *dp) +{ + const char *name = of_get_property(dp, "name", NULL); + char tmp_buf[64], *n; + size_t n_sz; + + tmp_buf[0] = '\0'; + __build_path_component(dp, tmp_buf); + if (tmp_buf[0] == '\0') + strscpy(tmp_buf, name); + + n_sz = strlen(tmp_buf) + 1; + n = prom_early_alloc(n_sz); + strscpy(n, tmp_buf, n_sz); + + return n; +} + +extern void restore_current(void); + +void __init of_console_init(void) +{ + char *msg = "OF stdout device is: %s\n"; + const size_t of_console_path_sz = 256; + struct device_node *dp; + unsigned long flags; + const char *type; + phandle node; + int skip, tmp, fd; + + of_console_path = prom_early_alloc(of_console_path_sz); + + switch (prom_vers) { + case PROM_V0: + skip = 0; + switch (*romvec->pv_stdout) { + case PROMDEV_SCREEN: + type = "display"; + break; + + case PROMDEV_TTYB: + skip = 1; + fallthrough; + + case PROMDEV_TTYA: + type = "serial"; + break; + + default: + prom_printf("Invalid PROM_V0 stdout value %u\n", + *romvec->pv_stdout); + prom_halt(); + } + + tmp = skip; + for_each_node_by_type(dp, type) { + if (!tmp--) + break; + } + if (!dp) { + prom_printf("Cannot find PROM_V0 console node.\n"); + prom_halt(); + } + of_console_device = dp; + + sprintf(of_console_path, "%pOF", dp); + if (!strcmp(type, "serial")) { + strcat(of_console_path, + (skip ? ":b" : ":a")); + } + break; + + default: + case PROM_V2: + case PROM_V3: + fd = *romvec->pv_v2bootargs.fd_stdout; + + spin_lock_irqsave(&prom_lock, flags); + node = (*romvec->pv_v2devops.v2_inst2pkg)(fd); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + + if (!node) { + prom_printf("Cannot resolve stdout node from " + "instance %08x.\n", fd); + prom_halt(); + } + dp = of_find_node_by_phandle(node); + + if (!of_node_is_type(dp, "display") && + !of_node_is_type(dp, "serial")) { + prom_printf("Console device_type is neither display " + "nor serial.\n"); + prom_halt(); + } + + of_console_device = dp; + + if (prom_vers == PROM_V2) { + sprintf(of_console_path, "%pOF", dp); + switch (*romvec->pv_stdout) { + case PROMDEV_TTYA: + strcat(of_console_path, ":a"); + break; + case PROMDEV_TTYB: + strcat(of_console_path, ":b"); + break; + } + } else { + const char *path; + + dp = of_find_node_by_path("/"); + path = of_get_property(dp, "stdout-path", NULL); + if (!path) { + prom_printf("No stdout-path in root node.\n"); + prom_halt(); + } + strscpy(of_console_path, path, of_console_path_sz); + } + break; + } + + of_console_options = strrchr(of_console_path, ':'); + if (of_console_options) { + of_console_options++; + if (*of_console_options == '\0') + of_console_options = NULL; + } + + printk(msg, of_console_path); +} + +void __init of_fill_in_cpu_data(void) +{ +} + +void __init irq_trans_init(struct device_node *dp) +{ +} diff --git a/arch/sparc/kernel/prom_64.c b/arch/sparc/kernel/prom_64.c new file mode 100644 index 000000000..aa4799cbb --- /dev/null +++ b/arch/sparc/kernel/prom_64.c @@ -0,0 +1,637 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Procedures for creating, accessing and interpreting the device tree. + * + * Paul Mackerras August 1996. + * Copyright (C) 1996-2005 Paul Mackerras. + * + * Adapted for 64bit PowerPC by Dave Engebretsen and Peter Bergner. + * {engebret|bergner}@us.ibm.com + * + * Adapted for sparc64 by David S. Miller davem@davemloft.net + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "prom.h" + +void * __init prom_early_alloc(unsigned long size) +{ + void *ret = memblock_alloc(size, SMP_CACHE_BYTES); + + if (!ret) { + prom_printf("prom_early_alloc(%lu) failed\n", size); + prom_halt(); + } + + prom_early_allocated += size; + + return ret; +} + +/* The following routines deal with the black magic of fully naming a + * node. + * + * Certain well known named nodes are just the simple name string. + * + * Actual devices have an address specifier appended to the base name + * string, like this "foo@addr". The "addr" can be in any number of + * formats, and the platform plus the type of the node determine the + * format and how it is constructed. + * + * For children of the ROOT node, the naming convention is fixed and + * determined by whether this is a sun4u or sun4v system. + * + * For children of other nodes, it is bus type specific. So + * we walk up the tree until we discover a "device_type" property + * we recognize and we go from there. + * + * As an example, the boot device on my workstation has a full path: + * + * /pci@1e,600000/ide@d/disk@0,0:c + */ +static void __init sun4v_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom64_registers *regs; + struct property *rprop; + u32 high_bits, low_bits, type; + + rprop = of_find_property(dp, "reg", NULL); + if (!rprop) + return; + + regs = rprop->value; + if (!of_node_is_root(dp->parent)) { + sprintf(tmp_buf, "%s@%x,%x", + name, + (unsigned int) (regs->phys_addr >> 32UL), + (unsigned int) (regs->phys_addr & 0xffffffffUL)); + return; + } + + type = regs->phys_addr >> 60UL; + high_bits = (regs->phys_addr >> 32UL) & 0x0fffffffUL; + low_bits = (regs->phys_addr & 0xffffffffUL); + + if (type == 0 || type == 8) { + const char *prefix = (type == 0) ? "m" : "i"; + + if (low_bits) + sprintf(tmp_buf, "%s@%s%x,%x", + name, prefix, + high_bits, low_bits); + else + sprintf(tmp_buf, "%s@%s%x", + name, + prefix, + high_bits); + } else if (type == 12) { + sprintf(tmp_buf, "%s@%x", + name, high_bits); + } +} + +static void __init sun4u_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom64_registers *regs; + struct property *prop; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + if (!of_node_is_root(dp->parent)) { + sprintf(tmp_buf, "%s@%x,%x", + name, + (unsigned int) (regs->phys_addr >> 32UL), + (unsigned int) (regs->phys_addr & 0xffffffffUL)); + return; + } + + prop = of_find_property(dp, "upa-portid", NULL); + if (!prop) + prop = of_find_property(dp, "portid", NULL); + if (prop) { + unsigned long mask = 0xffffffffUL; + + if (tlb_type >= cheetah) + mask = 0x7fffff; + + sprintf(tmp_buf, "%s@%x,%x", + name, + *(u32 *)prop->value, + (unsigned int) (regs->phys_addr & mask)); + } +} + +/* "name@slot,offset" */ +static void __init sbus_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom_registers *regs; + struct property *prop; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + sprintf(tmp_buf, "%s@%x,%x", + name, + regs->which_io, + regs->phys_addr); +} + +/* "name@devnum[,func]" */ +static void __init pci_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom_pci_registers *regs; + struct property *prop; + unsigned int devfn; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + devfn = (regs->phys_hi >> 8) & 0xff; + if (devfn & 0x07) { + sprintf(tmp_buf, "%s@%x,%x", + name, + devfn >> 3, + devfn & 0x07); + } else { + sprintf(tmp_buf, "%s@%x", + name, + devfn >> 3); + } +} + +/* "name@UPA_PORTID,offset" */ +static void __init upa_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom64_registers *regs; + struct property *prop; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + + prop = of_find_property(dp, "upa-portid", NULL); + if (!prop) + return; + + sprintf(tmp_buf, "%s@%x,%x", + name, + *(u32 *) prop->value, + (unsigned int) (regs->phys_addr & 0xffffffffUL)); +} + +/* "name@reg" */ +static void __init vdev_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct property *prop; + u32 *regs; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + + sprintf(tmp_buf, "%s@%x", name, *regs); +} + +/* "name@addrhi,addrlo" */ +static void __init ebus_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct linux_prom64_registers *regs; + struct property *prop; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + + sprintf(tmp_buf, "%s@%x,%x", + name, + (unsigned int) (regs->phys_addr >> 32UL), + (unsigned int) (regs->phys_addr & 0xffffffffUL)); +} + +/* "name@bus,addr" */ +static void __init i2c_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct property *prop; + u32 *regs; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + + /* This actually isn't right... should look at the #address-cells + * property of the i2c bus node etc. etc. + */ + sprintf(tmp_buf, "%s@%x,%x", + name, regs[0], regs[1]); +} + +/* "name@reg0[,reg1]" */ +static void __init usb_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct property *prop; + u32 *regs; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + + if (prop->length == sizeof(u32) || regs[1] == 1) { + sprintf(tmp_buf, "%s@%x", + name, regs[0]); + } else { + sprintf(tmp_buf, "%s@%x,%x", + name, regs[0], regs[1]); + } +} + +/* "name@reg0reg1[,reg2reg3]" */ +static void __init ieee1394_path_component(struct device_node *dp, char *tmp_buf) +{ + const char *name = of_get_property(dp, "name", NULL); + struct property *prop; + u32 *regs; + + prop = of_find_property(dp, "reg", NULL); + if (!prop) + return; + + regs = prop->value; + + if (regs[2] || regs[3]) { + sprintf(tmp_buf, "%s@%08x%08x,%04x%08x", + name, regs[0], regs[1], regs[2], regs[3]); + } else { + sprintf(tmp_buf, "%s@%08x%08x", + name, regs[0], regs[1]); + } +} + +static void __init __build_path_component(struct device_node *dp, char *tmp_buf) +{ + struct device_node *parent = dp->parent; + + if (parent != NULL) { + if (of_node_is_type(parent, "pci") || + of_node_is_type(parent, "pciex")) { + pci_path_component(dp, tmp_buf); + return; + } + if (of_node_is_type(parent, "sbus")) { + sbus_path_component(dp, tmp_buf); + return; + } + if (of_node_is_type(parent, "upa")) { + upa_path_component(dp, tmp_buf); + return; + } + if (of_node_is_type(parent, "ebus")) { + ebus_path_component(dp, tmp_buf); + return; + } + if (of_node_name_eq(parent, "usb") || + of_node_name_eq(parent, "hub")) { + usb_path_component(dp, tmp_buf); + return; + } + if (of_node_is_type(parent, "i2c")) { + i2c_path_component(dp, tmp_buf); + return; + } + if (of_node_is_type(parent, "firewire")) { + ieee1394_path_component(dp, tmp_buf); + return; + } + if (of_node_is_type(parent, "virtual-devices")) { + vdev_path_component(dp, tmp_buf); + return; + } + /* "isa" is handled with platform naming */ + } + + /* Use platform naming convention. */ + if (tlb_type == hypervisor) { + sun4v_path_component(dp, tmp_buf); + return; + } else { + sun4u_path_component(dp, tmp_buf); + } +} + +char * __init build_path_component(struct device_node *dp) +{ + const char *name = of_get_property(dp, "name", NULL); + char tmp_buf[64], *n; + size_t n_sz; + + tmp_buf[0] = '\0'; + __build_path_component(dp, tmp_buf); + if (tmp_buf[0] == '\0') + strscpy(tmp_buf, name); + + n_sz = strlen(tmp_buf) + 1; + n = prom_early_alloc(n_sz); + strscpy(n, tmp_buf, n_sz); + + return n; +} + +static const char *get_mid_prop(void) +{ + return (tlb_type == spitfire ? "upa-portid" : "portid"); +} + +bool arch_find_n_match_cpu_physical_id(struct device_node *cpun, + int cpu, unsigned int *thread) +{ + const char *mid_prop = get_mid_prop(); + int this_cpu_id; + + /* On hypervisor based platforms we interrogate the 'reg' + * property. On everything else we look for a 'upa-portid', + * 'portid', or 'cpuid' property. + */ + + if (tlb_type == hypervisor) { + struct property *prop = of_find_property(cpun, "reg", NULL); + u32 *regs; + + if (!prop) { + pr_warn("CPU node missing reg property\n"); + return false; + } + regs = prop->value; + this_cpu_id = regs[0] & 0x0fffffff; + } else { + this_cpu_id = of_getintprop_default(cpun, mid_prop, -1); + + if (this_cpu_id < 0) { + mid_prop = "cpuid"; + this_cpu_id = of_getintprop_default(cpun, mid_prop, -1); + } + if (this_cpu_id < 0) { + pr_warn("CPU node missing cpu ID property\n"); + return false; + } + } + if (this_cpu_id == cpu) { + if (thread) { + int proc_id = cpu_data(cpu).proc_id; + + /* On sparc64, the cpu thread information is obtained + * either from OBP or the machine description. We've + * actually probed this information already long before + * this interface gets called so instead of interrogating + * both the OF node and the MDESC again, just use what + * we discovered already. + */ + if (proc_id < 0) + proc_id = 0; + *thread = proc_id; + } + return true; + } + return false; +} + +static void *of_iterate_over_cpus(void *(*func)(struct device_node *, int, int), int arg) +{ + struct device_node *dp; + const char *mid_prop; + + mid_prop = get_mid_prop(); + for_each_node_by_type(dp, "cpu") { + int cpuid = of_getintprop_default(dp, mid_prop, -1); + const char *this_mid_prop = mid_prop; + void *ret; + + if (cpuid < 0) { + this_mid_prop = "cpuid"; + cpuid = of_getintprop_default(dp, this_mid_prop, -1); + } + if (cpuid < 0) { + prom_printf("OF: Serious problem, cpu lacks " + "%s property", this_mid_prop); + prom_halt(); + } +#ifdef CONFIG_SMP + if (cpuid >= NR_CPUS) { + printk(KERN_WARNING "Ignoring CPU %d which is " + ">= NR_CPUS (%d)\n", + cpuid, NR_CPUS); + continue; + } +#endif + ret = func(dp, cpuid, arg); + if (ret) + return ret; + } + return NULL; +} + +static void *check_cpu_node(struct device_node *dp, int cpuid, int id) +{ + if (id == cpuid) + return dp; + return NULL; +} + +struct device_node *of_find_node_by_cpuid(int cpuid) +{ + return of_iterate_over_cpus(check_cpu_node, cpuid); +} + +static void *record_one_cpu(struct device_node *dp, int cpuid, int arg) +{ + ncpus_probed++; +#ifdef CONFIG_SMP + set_cpu_present(cpuid, true); + + if (num_possible_cpus() < nr_cpu_ids) + set_cpu_possible(cpuid, true); +#endif + return NULL; +} + +void __init of_populate_present_mask(void) +{ + if (tlb_type == hypervisor) + return; + + ncpus_probed = 0; + of_iterate_over_cpus(record_one_cpu, 0); +} + +static void *fill_in_one_cpu(struct device_node *dp, int cpuid, int arg) +{ + struct device_node *portid_parent = NULL; + int portid = -1; + + if (of_property_present(dp, "cpuid")) { + int limit = 2; + + portid_parent = dp; + while (limit--) { + portid_parent = portid_parent->parent; + if (!portid_parent) + break; + portid = of_getintprop_default(portid_parent, + "portid", -1); + if (portid >= 0) + break; + } + } + +#ifndef CONFIG_SMP + /* On uniprocessor we only want the values for the + * real physical cpu the kernel booted onto, however + * cpu_data() only has one entry at index 0. + */ + if (cpuid != real_hard_smp_processor_id()) + return NULL; + cpuid = 0; +#endif + + cpu_data(cpuid).clock_tick = + of_getintprop_default(dp, "clock-frequency", 0); + + if (portid_parent) { + cpu_data(cpuid).dcache_size = + of_getintprop_default(dp, "l1-dcache-size", + 16 * 1024); + cpu_data(cpuid).dcache_line_size = + of_getintprop_default(dp, "l1-dcache-line-size", + 32); + cpu_data(cpuid).icache_size = + of_getintprop_default(dp, "l1-icache-size", + 8 * 1024); + cpu_data(cpuid).icache_line_size = + of_getintprop_default(dp, "l1-icache-line-size", + 32); + cpu_data(cpuid).ecache_size = + of_getintprop_default(dp, "l2-cache-size", 0); + cpu_data(cpuid).ecache_line_size = + of_getintprop_default(dp, "l2-cache-line-size", 0); + if (!cpu_data(cpuid).ecache_size || + !cpu_data(cpuid).ecache_line_size) { + cpu_data(cpuid).ecache_size = + of_getintprop_default(portid_parent, + "l2-cache-size", + (4 * 1024 * 1024)); + cpu_data(cpuid).ecache_line_size = + of_getintprop_default(portid_parent, + "l2-cache-line-size", 64); + } + + cpu_data(cpuid).core_id = portid + 1; + cpu_data(cpuid).proc_id = portid; + } else { + cpu_data(cpuid).dcache_size = + of_getintprop_default(dp, "dcache-size", 16 * 1024); + cpu_data(cpuid).dcache_line_size = + of_getintprop_default(dp, "dcache-line-size", 32); + + cpu_data(cpuid).icache_size = + of_getintprop_default(dp, "icache-size", 16 * 1024); + cpu_data(cpuid).icache_line_size = + of_getintprop_default(dp, "icache-line-size", 32); + + cpu_data(cpuid).ecache_size = + of_getintprop_default(dp, "ecache-size", + (4 * 1024 * 1024)); + cpu_data(cpuid).ecache_line_size = + of_getintprop_default(dp, "ecache-line-size", 64); + + cpu_data(cpuid).core_id = 0; + cpu_data(cpuid).proc_id = -1; + } + + return NULL; +} + +void __init of_fill_in_cpu_data(void) +{ + if (tlb_type == hypervisor) + return; + + of_iterate_over_cpus(fill_in_one_cpu, 0); + + smp_fill_in_sib_core_maps(); +} + +void __init of_console_init(void) +{ + char *msg = "OF stdout device is: %s\n"; + struct device_node *dp; + phandle node; + + of_console_path = prom_early_alloc(256); + if (prom_ihandle2path(prom_stdout, of_console_path, 256) < 0) { + prom_printf("Cannot obtain path of stdout.\n"); + prom_halt(); + } + of_console_options = strrchr(of_console_path, ':'); + if (of_console_options) { + of_console_options++; + if (*of_console_options == '\0') + of_console_options = NULL; + } + + node = prom_inst2pkg(prom_stdout); + if (!node) { + prom_printf("Cannot resolve stdout node from " + "instance %08x.\n", prom_stdout); + prom_halt(); + } + + dp = of_find_node_by_phandle(node); + + if (!of_node_is_type(dp, "display") && !of_node_is_type(dp, "serial")) { + prom_printf("Console device_type is neither display " + "nor serial.\n"); + prom_halt(); + } + + of_console_device = dp; + + printk(msg, of_console_path); +} diff --git a/arch/sparc/kernel/prom_common.c b/arch/sparc/kernel/prom_common.c new file mode 100644 index 000000000..d258fd10d --- /dev/null +++ b/arch/sparc/kernel/prom_common.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* prom_common.c: OF device tree support common code. + * + * Paul Mackerras August 1996. + * Copyright (C) 1996-2005 Paul Mackerras. + * + * Adapted for 64bit PowerPC by Dave Engebretsen and Peter Bergner. + * {engebret|bergner}@us.ibm.com + * + * Adapted for sparc by David S. Miller davem@davemloft.net + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "prom.h" + +struct device_node *of_console_device; +EXPORT_SYMBOL(of_console_device); + +char *of_console_path; +EXPORT_SYMBOL(of_console_path); + +char *of_console_options; +EXPORT_SYMBOL(of_console_options); + +int of_getintprop_default(struct device_node *np, const char *name, int def) +{ + struct property *prop; + int len; + + prop = of_find_property(np, name, &len); + if (!prop || len != 4) + return def; + + return *(int *) prop->value; +} +EXPORT_SYMBOL(of_getintprop_default); + +DEFINE_MUTEX(of_set_property_mutex); +EXPORT_SYMBOL(of_set_property_mutex); + +int of_set_property(struct device_node *dp, const char *name, void *val, int len) +{ + struct property **prevp; + unsigned long flags; + void *new_val; + int err; + + new_val = kmemdup(val, len, GFP_KERNEL); + if (!new_val) + return -ENOMEM; + + err = -ENODEV; + + mutex_lock(&of_set_property_mutex); + raw_spin_lock_irqsave(&devtree_lock, flags); + prevp = &dp->properties; + while (*prevp) { + struct property *prop = *prevp; + + if (!strcasecmp(prop->name, name)) { + void *old_val = prop->value; + int ret; + + ret = prom_setprop(dp->phandle, name, val, len); + + err = -EINVAL; + if (ret >= 0) { + prop->value = new_val; + prop->length = len; + + if (OF_IS_DYNAMIC(prop)) + kfree(old_val); + + OF_MARK_DYNAMIC(prop); + + err = 0; + } + break; + } + prevp = &(*prevp)->next; + } + raw_spin_unlock_irqrestore(&devtree_lock, flags); + mutex_unlock(&of_set_property_mutex); + + /* XXX Upate procfs if necessary... */ + + return err; +} +EXPORT_SYMBOL(of_set_property); + +int of_find_in_proplist(const char *list, const char *match, int len) +{ + while (len > 0) { + int l; + + if (!strcmp(list, match)) + return 1; + l = strlen(list) + 1; + list += l; + len -= l; + } + return 0; +} +EXPORT_SYMBOL(of_find_in_proplist); + +/* + * SPARC32 and SPARC64's prom_nextprop() do things differently + * here, despite sharing the same interface. SPARC32 doesn't fill in 'buf', + * returning NULL on an error. SPARC64 fills in 'buf', but sets it to an + * empty string upon error. + */ +static int __init handle_nextprop_quirks(char *buf, const char *name) +{ + size_t name_len; + + name_len = name ? strlen(name) : 0; + if (name_len == 0) + return -1; + +#ifdef CONFIG_SPARC32 + strscpy(buf, name, name_len + 1); +#endif + return 0; +} + +static int __init prom_common_nextprop(phandle node, char *prev, char *buf) +{ + const char *name; + + buf[0] = '\0'; + name = prom_nextprop(node, prev, buf); + return handle_nextprop_quirks(buf, name); +} + +unsigned int prom_early_allocated __initdata; + +static struct of_pdt_ops prom_sparc_ops __initdata = { + .nextprop = prom_common_nextprop, + .getproplen = prom_getproplen, + .getproperty = prom_getproperty, + .getchild = prom_getchild, + .getsibling = prom_getsibling, +}; + +void __init prom_build_devicetree(void) +{ + of_pdt_build_devicetree(prom_root_node, &prom_sparc_ops); + of_console_init(); + + pr_info("PROM: Built device tree with %u bytes of memory.\n", + prom_early_allocated); +} diff --git a/arch/sparc/kernel/prom_irqtrans.c b/arch/sparc/kernel/prom_irqtrans.c new file mode 100644 index 000000000..5752bfd73 --- /dev/null +++ b/arch/sparc/kernel/prom_irqtrans.c @@ -0,0 +1,844 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "prom.h" + +#ifdef CONFIG_PCI +/* PSYCHO interrupt mapping support. */ +#define PSYCHO_IMAP_A_SLOT0 0x0c00UL +#define PSYCHO_IMAP_B_SLOT0 0x0c20UL +static unsigned long psycho_pcislot_imap_offset(unsigned long ino) +{ + unsigned int bus = (ino & 0x10) >> 4; + unsigned int slot = (ino & 0x0c) >> 2; + + if (bus == 0) + return PSYCHO_IMAP_A_SLOT0 + (slot * 8); + else + return PSYCHO_IMAP_B_SLOT0 + (slot * 8); +} + +#define PSYCHO_OBIO_IMAP_BASE 0x1000UL + +#define PSYCHO_ONBOARD_IRQ_BASE 0x20 +#define psycho_onboard_imap_offset(__ino) \ + (PSYCHO_OBIO_IMAP_BASE + (((__ino) & 0x1f) << 3)) + +#define PSYCHO_ICLR_A_SLOT0 0x1400UL +#define PSYCHO_ICLR_SCSI 0x1800UL + +#define psycho_iclr_offset(ino) \ + ((ino & 0x20) ? (PSYCHO_ICLR_SCSI + (((ino) & 0x1f) << 3)) : \ + (PSYCHO_ICLR_A_SLOT0 + (((ino) & 0x1f)<<3))) + +static unsigned int psycho_irq_build(struct device_node *dp, + unsigned int ino, + void *_data) +{ + unsigned long controller_regs = (unsigned long) _data; + unsigned long imap, iclr; + unsigned long imap_off, iclr_off; + int inofixup = 0; + + ino &= 0x3f; + if (ino < PSYCHO_ONBOARD_IRQ_BASE) { + /* PCI slot */ + imap_off = psycho_pcislot_imap_offset(ino); + } else { + /* Onboard device */ + imap_off = psycho_onboard_imap_offset(ino); + } + + /* Now build the IRQ bucket. */ + imap = controller_regs + imap_off; + + iclr_off = psycho_iclr_offset(ino); + iclr = controller_regs + iclr_off; + + if ((ino & 0x20) == 0) + inofixup = ino & 0x03; + + return build_irq(inofixup, iclr, imap); +} + +static void __init psycho_irq_trans_init(struct device_node *dp) +{ + const struct linux_prom64_registers *regs; + + dp->irq_trans = prom_early_alloc(sizeof(struct of_irq_controller)); + dp->irq_trans->irq_build = psycho_irq_build; + + regs = of_get_property(dp, "reg", NULL); + dp->irq_trans->data = (void *) regs[2].phys_addr; +} + +#define sabre_read(__reg) \ +({ u64 __ret; \ + __asm__ __volatile__("ldxa [%1] %2, %0" \ + : "=r" (__ret) \ + : "r" (__reg), "i" (ASI_PHYS_BYPASS_EC_E) \ + : "memory"); \ + __ret; \ +}) + +struct sabre_irq_data { + unsigned long controller_regs; + unsigned int pci_first_busno; +}; +#define SABRE_CONFIGSPACE 0x001000000UL +#define SABRE_WRSYNC 0x1c20UL + +#define SABRE_CONFIG_BASE(CONFIG_SPACE) \ + (CONFIG_SPACE | (1UL << 24)) +#define SABRE_CONFIG_ENCODE(BUS, DEVFN, REG) \ + (((unsigned long)(BUS) << 16) | \ + ((unsigned long)(DEVFN) << 8) | \ + ((unsigned long)(REG))) + +/* When a device lives behind a bridge deeper in the PCI bus topology + * than APB, a special sequence must run to make sure all pending DMA + * transfers at the time of IRQ delivery are visible in the coherency + * domain by the cpu. This sequence is to perform a read on the far + * side of the non-APB bridge, then perform a read of Sabre's DMA + * write-sync register. + */ +static void sabre_wsync_handler(unsigned int ino, void *_arg1, void *_arg2) +{ + unsigned int phys_hi = (unsigned int) (unsigned long) _arg1; + struct sabre_irq_data *irq_data = _arg2; + unsigned long controller_regs = irq_data->controller_regs; + unsigned long sync_reg = controller_regs + SABRE_WRSYNC; + unsigned long config_space = controller_regs + SABRE_CONFIGSPACE; + unsigned int bus, devfn; + u16 _unused; + + config_space = SABRE_CONFIG_BASE(config_space); + + bus = (phys_hi >> 16) & 0xff; + devfn = (phys_hi >> 8) & 0xff; + + config_space |= SABRE_CONFIG_ENCODE(bus, devfn, 0x00); + + __asm__ __volatile__("membar #Sync\n\t" + "lduha [%1] %2, %0\n\t" + "membar #Sync" + : "=r" (_unused) + : "r" ((u16 *) config_space), + "i" (ASI_PHYS_BYPASS_EC_E_L) + : "memory"); + + sabre_read(sync_reg); +} + +#define SABRE_IMAP_A_SLOT0 0x0c00UL +#define SABRE_IMAP_B_SLOT0 0x0c20UL +#define SABRE_ICLR_A_SLOT0 0x1400UL +#define SABRE_ICLR_B_SLOT0 0x1480UL +#define SABRE_ICLR_SCSI 0x1800UL +#define SABRE_ICLR_ETH 0x1808UL +#define SABRE_ICLR_BPP 0x1810UL +#define SABRE_ICLR_AU_REC 0x1818UL +#define SABRE_ICLR_AU_PLAY 0x1820UL +#define SABRE_ICLR_PFAIL 0x1828UL +#define SABRE_ICLR_KMS 0x1830UL +#define SABRE_ICLR_FLPY 0x1838UL +#define SABRE_ICLR_SHW 0x1840UL +#define SABRE_ICLR_KBD 0x1848UL +#define SABRE_ICLR_MS 0x1850UL +#define SABRE_ICLR_SER 0x1858UL +#define SABRE_ICLR_UE 0x1870UL +#define SABRE_ICLR_CE 0x1878UL +#define SABRE_ICLR_PCIERR 0x1880UL + +static unsigned long sabre_pcislot_imap_offset(unsigned long ino) +{ + unsigned int bus = (ino & 0x10) >> 4; + unsigned int slot = (ino & 0x0c) >> 2; + + if (bus == 0) + return SABRE_IMAP_A_SLOT0 + (slot * 8); + else + return SABRE_IMAP_B_SLOT0 + (slot * 8); +} + +#define SABRE_OBIO_IMAP_BASE 0x1000UL +#define SABRE_ONBOARD_IRQ_BASE 0x20 +#define sabre_onboard_imap_offset(__ino) \ + (SABRE_OBIO_IMAP_BASE + (((__ino) & 0x1f) << 3)) + +#define sabre_iclr_offset(ino) \ + ((ino & 0x20) ? (SABRE_ICLR_SCSI + (((ino) & 0x1f) << 3)) : \ + (SABRE_ICLR_A_SLOT0 + (((ino) & 0x1f)<<3))) + +static int sabre_device_needs_wsync(struct device_node *dp) +{ + struct device_node *parent = dp->parent; + const char *parent_model, *parent_compat; + + /* This traversal up towards the root is meant to + * handle two cases: + * + * 1) non-PCI bus sitting under PCI, such as 'ebus' + * 2) the PCI controller interrupts themselves, which + * will use the sabre_irq_build but do not need + * the DMA synchronization handling + */ + while (parent) { + if (of_node_is_type(parent, "pci")) + break; + parent = parent->parent; + } + + if (!parent) + return 0; + + parent_model = of_get_property(parent, + "model", NULL); + if (parent_model && + (!strcmp(parent_model, "SUNW,sabre") || + !strcmp(parent_model, "SUNW,simba"))) + return 0; + + parent_compat = of_get_property(parent, + "compatible", NULL); + if (parent_compat && + (!strcmp(parent_compat, "pci108e,a000") || + !strcmp(parent_compat, "pci108e,a001"))) + return 0; + + return 1; +} + +static unsigned int sabre_irq_build(struct device_node *dp, + unsigned int ino, + void *_data) +{ + struct sabre_irq_data *irq_data = _data; + unsigned long controller_regs = irq_data->controller_regs; + const struct linux_prom_pci_registers *regs; + unsigned long imap, iclr; + unsigned long imap_off, iclr_off; + int inofixup = 0; + int irq; + + ino &= 0x3f; + if (ino < SABRE_ONBOARD_IRQ_BASE) { + /* PCI slot */ + imap_off = sabre_pcislot_imap_offset(ino); + } else { + /* onboard device */ + imap_off = sabre_onboard_imap_offset(ino); + } + + /* Now build the IRQ bucket. */ + imap = controller_regs + imap_off; + + iclr_off = sabre_iclr_offset(ino); + iclr = controller_regs + iclr_off; + + if ((ino & 0x20) == 0) + inofixup = ino & 0x03; + + irq = build_irq(inofixup, iclr, imap); + + /* If the parent device is a PCI<->PCI bridge other than + * APB, we have to install a pre-handler to ensure that + * all pending DMA is drained before the interrupt handler + * is run. + */ + regs = of_get_property(dp, "reg", NULL); + if (regs && sabre_device_needs_wsync(dp)) { + irq_install_pre_handler(irq, + sabre_wsync_handler, + (void *) (long) regs->phys_hi, + (void *) irq_data); + } + + return irq; +} + +static void __init sabre_irq_trans_init(struct device_node *dp) +{ + const struct linux_prom64_registers *regs; + struct sabre_irq_data *irq_data; + const u32 *busrange; + + dp->irq_trans = prom_early_alloc(sizeof(struct of_irq_controller)); + dp->irq_trans->irq_build = sabre_irq_build; + + irq_data = prom_early_alloc(sizeof(struct sabre_irq_data)); + + regs = of_get_property(dp, "reg", NULL); + irq_data->controller_regs = regs[0].phys_addr; + + busrange = of_get_property(dp, "bus-range", NULL); + irq_data->pci_first_busno = busrange[0]; + + dp->irq_trans->data = irq_data; +} + +/* SCHIZO interrupt mapping support. Unlike Psycho, for this controller the + * imap/iclr registers are per-PBM. + */ +#define SCHIZO_IMAP_BASE 0x1000UL +#define SCHIZO_ICLR_BASE 0x1400UL + +static unsigned long schizo_imap_offset(unsigned long ino) +{ + return SCHIZO_IMAP_BASE + (ino * 8UL); +} + +static unsigned long schizo_iclr_offset(unsigned long ino) +{ + return SCHIZO_ICLR_BASE + (ino * 8UL); +} + +static unsigned long schizo_ino_to_iclr(unsigned long pbm_regs, + unsigned int ino) +{ + + return pbm_regs + schizo_iclr_offset(ino); +} + +static unsigned long schizo_ino_to_imap(unsigned long pbm_regs, + unsigned int ino) +{ + return pbm_regs + schizo_imap_offset(ino); +} + +#define schizo_read(__reg) \ +({ u64 __ret; \ + __asm__ __volatile__("ldxa [%1] %2, %0" \ + : "=r" (__ret) \ + : "r" (__reg), "i" (ASI_PHYS_BYPASS_EC_E) \ + : "memory"); \ + __ret; \ +}) +#define schizo_write(__reg, __val) \ + __asm__ __volatile__("stxa %0, [%1] %2" \ + : /* no outputs */ \ + : "r" (__val), "r" (__reg), \ + "i" (ASI_PHYS_BYPASS_EC_E) \ + : "memory") + +static void tomatillo_wsync_handler(unsigned int ino, void *_arg1, void *_arg2) +{ + unsigned long sync_reg = (unsigned long) _arg2; + u64 mask = 1UL << (ino & IMAP_INO); + u64 val; + int limit; + + schizo_write(sync_reg, mask); + + limit = 100000; + val = 0; + while (--limit) { + val = schizo_read(sync_reg); + if (!(val & mask)) + break; + } + if (limit <= 0) { + printk("tomatillo_wsync_handler: DMA won't sync [%llx:%llx]\n", + val, mask); + } + + if (_arg1) { + static unsigned char cacheline[64] + __attribute__ ((aligned (64))); + + __asm__ __volatile__("rd %%fprs, %0\n\t" + "or %0, %4, %1\n\t" + "wr %1, 0x0, %%fprs\n\t" + "stda %%f0, [%5] %6\n\t" + "wr %0, 0x0, %%fprs\n\t" + "membar #Sync" + : "=&r" (mask), "=&r" (val) + : "0" (mask), "1" (val), + "i" (FPRS_FEF), "r" (&cacheline[0]), + "i" (ASI_BLK_COMMIT_P)); + } +} + +struct schizo_irq_data { + unsigned long pbm_regs; + unsigned long sync_reg; + u32 portid; + int chip_version; +}; + +static unsigned int schizo_irq_build(struct device_node *dp, + unsigned int ino, + void *_data) +{ + struct schizo_irq_data *irq_data = _data; + unsigned long pbm_regs = irq_data->pbm_regs; + unsigned long imap, iclr; + int ign_fixup; + int irq; + int is_tomatillo; + + ino &= 0x3f; + + /* Now build the IRQ bucket. */ + imap = schizo_ino_to_imap(pbm_regs, ino); + iclr = schizo_ino_to_iclr(pbm_regs, ino); + + /* On Schizo, no inofixup occurs. This is because each + * INO has its own IMAP register. On Psycho and Sabre + * there is only one IMAP register for each PCI slot even + * though four different INOs can be generated by each + * PCI slot. + * + * But, for JBUS variants (essentially, Tomatillo), we have + * to fixup the lowest bit of the interrupt group number. + */ + ign_fixup = 0; + + is_tomatillo = (irq_data->sync_reg != 0UL); + + if (is_tomatillo) { + if (irq_data->portid & 1) + ign_fixup = (1 << 6); + } + + irq = build_irq(ign_fixup, iclr, imap); + + if (is_tomatillo) { + irq_install_pre_handler(irq, + tomatillo_wsync_handler, + ((irq_data->chip_version <= 4) ? + (void *) 1 : (void *) 0), + (void *) irq_data->sync_reg); + } + + return irq; +} + +static void __init __schizo_irq_trans_init(struct device_node *dp, + int is_tomatillo) +{ + const struct linux_prom64_registers *regs; + struct schizo_irq_data *irq_data; + + dp->irq_trans = prom_early_alloc(sizeof(struct of_irq_controller)); + dp->irq_trans->irq_build = schizo_irq_build; + + irq_data = prom_early_alloc(sizeof(struct schizo_irq_data)); + + regs = of_get_property(dp, "reg", NULL); + dp->irq_trans->data = irq_data; + + irq_data->pbm_regs = regs[0].phys_addr; + if (is_tomatillo) + irq_data->sync_reg = regs[3].phys_addr + 0x1a18UL; + else + irq_data->sync_reg = 0UL; + irq_data->portid = of_getintprop_default(dp, "portid", 0); + irq_data->chip_version = of_getintprop_default(dp, "version#", 0); +} + +static void __init schizo_irq_trans_init(struct device_node *dp) +{ + __schizo_irq_trans_init(dp, 0); +} + +static void __init tomatillo_irq_trans_init(struct device_node *dp) +{ + __schizo_irq_trans_init(dp, 1); +} + +static unsigned int pci_sun4v_irq_build(struct device_node *dp, + unsigned int devino, + void *_data) +{ + u32 devhandle = (u32) (unsigned long) _data; + + return sun4v_build_irq(devhandle, devino); +} + +static void __init pci_sun4v_irq_trans_init(struct device_node *dp) +{ + const struct linux_prom64_registers *regs; + + dp->irq_trans = prom_early_alloc(sizeof(struct of_irq_controller)); + dp->irq_trans->irq_build = pci_sun4v_irq_build; + + regs = of_get_property(dp, "reg", NULL); + dp->irq_trans->data = (void *) (unsigned long) + ((regs->phys_addr >> 32UL) & 0x0fffffff); +} + +struct fire_irq_data { + unsigned long pbm_regs; + u32 portid; +}; + +#define FIRE_IMAP_BASE 0x001000 +#define FIRE_ICLR_BASE 0x001400 + +static unsigned long fire_imap_offset(unsigned long ino) +{ + return FIRE_IMAP_BASE + (ino * 8UL); +} + +static unsigned long fire_iclr_offset(unsigned long ino) +{ + return FIRE_ICLR_BASE + (ino * 8UL); +} + +static unsigned long fire_ino_to_iclr(unsigned long pbm_regs, + unsigned int ino) +{ + return pbm_regs + fire_iclr_offset(ino); +} + +static unsigned long fire_ino_to_imap(unsigned long pbm_regs, + unsigned int ino) +{ + return pbm_regs + fire_imap_offset(ino); +} + +static unsigned int fire_irq_build(struct device_node *dp, + unsigned int ino, + void *_data) +{ + struct fire_irq_data *irq_data = _data; + unsigned long pbm_regs = irq_data->pbm_regs; + unsigned long imap, iclr; + unsigned long int_ctrlr; + + ino &= 0x3f; + + /* Now build the IRQ bucket. */ + imap = fire_ino_to_imap(pbm_regs, ino); + iclr = fire_ino_to_iclr(pbm_regs, ino); + + /* Set the interrupt controller number. */ + int_ctrlr = 1 << 6; + upa_writeq(int_ctrlr, imap); + + /* The interrupt map registers do not have an INO field + * like other chips do. They return zero in the INO + * field, and the interrupt controller number is controlled + * in bits 6 to 9. So in order for build_irq() to get + * the INO right we pass it in as part of the fixup + * which will get added to the map register zero value + * read by build_irq(). + */ + ino |= (irq_data->portid << 6); + ino -= int_ctrlr; + return build_irq(ino, iclr, imap); +} + +static void __init fire_irq_trans_init(struct device_node *dp) +{ + const struct linux_prom64_registers *regs; + struct fire_irq_data *irq_data; + + dp->irq_trans = prom_early_alloc(sizeof(struct of_irq_controller)); + dp->irq_trans->irq_build = fire_irq_build; + + irq_data = prom_early_alloc(sizeof(struct fire_irq_data)); + + regs = of_get_property(dp, "reg", NULL); + dp->irq_trans->data = irq_data; + + irq_data->pbm_regs = regs[0].phys_addr; + irq_data->portid = of_getintprop_default(dp, "portid", 0); +} +#endif /* CONFIG_PCI */ + +#ifdef CONFIG_SBUS +/* INO number to IMAP register offset for SYSIO external IRQ's. + * This should conform to both Sunfire/Wildfire server and Fusion + * desktop designs. + */ +#define SYSIO_IMAP_SLOT0 0x2c00UL +#define SYSIO_IMAP_SLOT1 0x2c08UL +#define SYSIO_IMAP_SLOT2 0x2c10UL +#define SYSIO_IMAP_SLOT3 0x2c18UL +#define SYSIO_IMAP_SCSI 0x3000UL +#define SYSIO_IMAP_ETH 0x3008UL +#define SYSIO_IMAP_BPP 0x3010UL +#define SYSIO_IMAP_AUDIO 0x3018UL +#define SYSIO_IMAP_PFAIL 0x3020UL +#define SYSIO_IMAP_KMS 0x3028UL +#define SYSIO_IMAP_FLPY 0x3030UL +#define SYSIO_IMAP_SHW 0x3038UL +#define SYSIO_IMAP_KBD 0x3040UL +#define SYSIO_IMAP_MS 0x3048UL +#define SYSIO_IMAP_SER 0x3050UL +#define SYSIO_IMAP_TIM0 0x3060UL +#define SYSIO_IMAP_TIM1 0x3068UL +#define SYSIO_IMAP_UE 0x3070UL +#define SYSIO_IMAP_CE 0x3078UL +#define SYSIO_IMAP_SBERR 0x3080UL +#define SYSIO_IMAP_PMGMT 0x3088UL +#define SYSIO_IMAP_GFX 0x3090UL +#define SYSIO_IMAP_EUPA 0x3098UL + +#define bogon ((unsigned long) -1) +static unsigned long sysio_irq_offsets[] = { + /* SBUS Slot 0 --> 3, level 1 --> 7 */ + SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, + SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, + SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, + SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, + SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, + SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, + SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, + SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, + + /* Onboard devices (not relevant/used on SunFire). */ + SYSIO_IMAP_SCSI, + SYSIO_IMAP_ETH, + SYSIO_IMAP_BPP, + bogon, + SYSIO_IMAP_AUDIO, + SYSIO_IMAP_PFAIL, + bogon, + bogon, + SYSIO_IMAP_KMS, + SYSIO_IMAP_FLPY, + SYSIO_IMAP_SHW, + SYSIO_IMAP_KBD, + SYSIO_IMAP_MS, + SYSIO_IMAP_SER, + bogon, + bogon, + SYSIO_IMAP_TIM0, + SYSIO_IMAP_TIM1, + bogon, + bogon, + SYSIO_IMAP_UE, + SYSIO_IMAP_CE, + SYSIO_IMAP_SBERR, + SYSIO_IMAP_PMGMT, + SYSIO_IMAP_GFX, + SYSIO_IMAP_EUPA, +}; + +#undef bogon + +#define NUM_SYSIO_OFFSETS ARRAY_SIZE(sysio_irq_offsets) + +/* Convert Interrupt Mapping register pointer to associated + * Interrupt Clear register pointer, SYSIO specific version. + */ +#define SYSIO_ICLR_UNUSED0 0x3400UL +#define SYSIO_ICLR_SLOT0 0x3408UL +#define SYSIO_ICLR_SLOT1 0x3448UL +#define SYSIO_ICLR_SLOT2 0x3488UL +#define SYSIO_ICLR_SLOT3 0x34c8UL +static unsigned long sysio_imap_to_iclr(unsigned long imap) +{ + unsigned long diff = SYSIO_ICLR_UNUSED0 - SYSIO_IMAP_SLOT0; + return imap + diff; +} + +static unsigned int sbus_of_build_irq(struct device_node *dp, + unsigned int ino, + void *_data) +{ + unsigned long reg_base = (unsigned long) _data; + const struct linux_prom_registers *regs; + unsigned long imap, iclr; + int sbus_slot = 0; + int sbus_level = 0; + + ino &= 0x3f; + + regs = of_get_property(dp, "reg", NULL); + if (regs) + sbus_slot = regs->which_io; + + if (ino < 0x20) + ino += (sbus_slot * 8); + + imap = sysio_irq_offsets[ino]; + if (imap == ((unsigned long)-1)) { + prom_printf("get_irq_translations: Bad SYSIO INO[%x]\n", + ino); + prom_halt(); + } + imap += reg_base; + + /* SYSIO inconsistency. For external SLOTS, we have to select + * the right ICLR register based upon the lower SBUS irq level + * bits. + */ + if (ino >= 0x20) { + iclr = sysio_imap_to_iclr(imap); + } else { + sbus_level = ino & 0x7; + + switch(sbus_slot) { + case 0: + iclr = reg_base + SYSIO_ICLR_SLOT0; + break; + case 1: + iclr = reg_base + SYSIO_ICLR_SLOT1; + break; + case 2: + iclr = reg_base + SYSIO_ICLR_SLOT2; + break; + default: + case 3: + iclr = reg_base + SYSIO_ICLR_SLOT3; + break; + } + + iclr += ((unsigned long)sbus_level - 1UL) * 8UL; + } + return build_irq(sbus_level, iclr, imap); +} + +static void __init sbus_irq_trans_init(struct device_node *dp) +{ + const struct linux_prom64_registers *regs; + + dp->irq_trans = prom_early_alloc(sizeof(struct of_irq_controller)); + dp->irq_trans->irq_build = sbus_of_build_irq; + + regs = of_get_property(dp, "reg", NULL); + dp->irq_trans->data = (void *) (unsigned long) regs->phys_addr; +} +#endif /* CONFIG_SBUS */ + + +static unsigned int central_build_irq(struct device_node *dp, + unsigned int ino, + void *_data) +{ + struct device_node *central_dp = _data; + struct platform_device *central_op = of_find_device_by_node(central_dp); + struct resource *res; + unsigned long imap, iclr; + u32 tmp; + + if (of_node_name_eq(dp, "eeprom")) { + res = ¢ral_op->resource[5]; + } else if (of_node_name_eq(dp, "zs")) { + res = ¢ral_op->resource[4]; + } else if (of_node_name_eq(dp, "clock-board")) { + res = ¢ral_op->resource[3]; + } else { + return ino; + } + + imap = res->start + 0x00UL; + iclr = res->start + 0x10UL; + + /* Set the INO state to idle, and disable. */ + upa_writel(0, iclr); + upa_readl(iclr); + + tmp = upa_readl(imap); + tmp &= ~0x80000000; + upa_writel(tmp, imap); + + return build_irq(0, iclr, imap); +} + +static void __init central_irq_trans_init(struct device_node *dp) +{ + dp->irq_trans = prom_early_alloc(sizeof(struct of_irq_controller)); + dp->irq_trans->irq_build = central_build_irq; + + dp->irq_trans->data = dp; +} + +struct irq_trans { + const char *name; + void (*init)(struct device_node *); +}; + +#ifdef CONFIG_PCI +static struct irq_trans __initdata pci_irq_trans_table[] = { + { "SUNW,sabre", sabre_irq_trans_init }, + { "pci108e,a000", sabre_irq_trans_init }, + { "pci108e,a001", sabre_irq_trans_init }, + { "SUNW,psycho", psycho_irq_trans_init }, + { "pci108e,8000", psycho_irq_trans_init }, + { "SUNW,schizo", schizo_irq_trans_init }, + { "pci108e,8001", schizo_irq_trans_init }, + { "SUNW,schizo+", schizo_irq_trans_init }, + { "pci108e,8002", schizo_irq_trans_init }, + { "SUNW,tomatillo", tomatillo_irq_trans_init }, + { "pci108e,a801", tomatillo_irq_trans_init }, + { "SUNW,sun4v-pci", pci_sun4v_irq_trans_init }, + { "pciex108e,80f0", fire_irq_trans_init }, +}; +#endif + +static unsigned int sun4v_vdev_irq_build(struct device_node *dp, + unsigned int devino, + void *_data) +{ + u32 devhandle = (u32) (unsigned long) _data; + + return sun4v_build_irq(devhandle, devino); +} + +static void __init sun4v_vdev_irq_trans_init(struct device_node *dp) +{ + const struct linux_prom64_registers *regs; + + dp->irq_trans = prom_early_alloc(sizeof(struct of_irq_controller)); + dp->irq_trans->irq_build = sun4v_vdev_irq_build; + + regs = of_get_property(dp, "reg", NULL); + dp->irq_trans->data = (void *) (unsigned long) + ((regs->phys_addr >> 32UL) & 0x0fffffff); +} + +void __init irq_trans_init(struct device_node *dp) +{ +#ifdef CONFIG_PCI + const char *model; + int i; +#endif + +#ifdef CONFIG_PCI + model = of_get_property(dp, "model", NULL); + if (!model) + model = of_get_property(dp, "compatible", NULL); + if (model) { + for (i = 0; i < ARRAY_SIZE(pci_irq_trans_table); i++) { + struct irq_trans *t = &pci_irq_trans_table[i]; + + if (!strcmp(model, t->name)) { + t->init(dp); + return; + } + } + } +#endif +#ifdef CONFIG_SBUS + if (of_node_name_eq(dp, "sbus") || + of_node_name_eq(dp, "sbi")) { + sbus_irq_trans_init(dp); + return; + } +#endif + if (of_node_name_eq(dp, "fhc") && + of_node_name_eq(dp->parent, "central")) { + central_irq_trans_init(dp); + return; + } + if (of_node_name_eq(dp, "virtual-devices") || + of_node_name_eq(dp, "niu")) { + sun4v_vdev_irq_trans_init(dp); + return; + } +} diff --git a/arch/sparc/kernel/psycho_common.c b/arch/sparc/kernel/psycho_common.c new file mode 100644 index 000000000..4557ef18f --- /dev/null +++ b/arch/sparc/kernel/psycho_common.c @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-2.0 +/* psycho_common.c: Code common to PSYCHO and derivative PCI controllers. + * + * Copyright (C) 2008 David S. Miller + */ +#include +#include +#include +#include + +#include + +#include "pci_impl.h" +#include "iommu_common.h" +#include "psycho_common.h" + +#define PSYCHO_STRBUF_CTRL_DENAB 0x0000000000000002ULL +#define PSYCHO_STCERR_WRITE 0x0000000000000002ULL +#define PSYCHO_STCERR_READ 0x0000000000000001ULL +#define PSYCHO_STCTAG_PPN 0x0fffffff00000000ULL +#define PSYCHO_STCTAG_VPN 0x00000000ffffe000ULL +#define PSYCHO_STCTAG_VALID 0x0000000000000002ULL +#define PSYCHO_STCTAG_WRITE 0x0000000000000001ULL +#define PSYCHO_STCLINE_LINDX 0x0000000001e00000ULL +#define PSYCHO_STCLINE_SPTR 0x00000000001f8000ULL +#define PSYCHO_STCLINE_LADDR 0x0000000000007f00ULL +#define PSYCHO_STCLINE_EPTR 0x00000000000000fcULL +#define PSYCHO_STCLINE_VALID 0x0000000000000002ULL +#define PSYCHO_STCLINE_FOFN 0x0000000000000001ULL + +static DEFINE_SPINLOCK(stc_buf_lock); +static unsigned long stc_error_buf[128]; +static unsigned long stc_tag_buf[16]; +static unsigned long stc_line_buf[16]; + +static void psycho_check_stc_error(struct pci_pbm_info *pbm) +{ + unsigned long err_base, tag_base, line_base; + struct strbuf *strbuf = &pbm->stc; + u64 control; + int i; + + if (!strbuf->strbuf_control) + return; + + err_base = strbuf->strbuf_err_stat; + tag_base = strbuf->strbuf_tag_diag; + line_base = strbuf->strbuf_line_diag; + + spin_lock(&stc_buf_lock); + + /* This is __REALLY__ dangerous. When we put the streaming + * buffer into diagnostic mode to probe its tags and error + * status, we _must_ clear all of the line tag valid bits + * before re-enabling the streaming buffer. If any dirty data + * lives in the STC when we do this, we will end up + * invalidating it before it has a chance to reach main + * memory. + */ + control = upa_readq(strbuf->strbuf_control); + upa_writeq(control | PSYCHO_STRBUF_CTRL_DENAB, strbuf->strbuf_control); + for (i = 0; i < 128; i++) { + u64 val; + + val = upa_readq(err_base + (i * 8UL)); + upa_writeq(0UL, err_base + (i * 8UL)); + stc_error_buf[i] = val; + } + for (i = 0; i < 16; i++) { + stc_tag_buf[i] = upa_readq(tag_base + (i * 8UL)); + stc_line_buf[i] = upa_readq(line_base + (i * 8UL)); + upa_writeq(0UL, tag_base + (i * 8UL)); + upa_writeq(0UL, line_base + (i * 8UL)); + } + + /* OK, state is logged, exit diagnostic mode. */ + upa_writeq(control, strbuf->strbuf_control); + + for (i = 0; i < 16; i++) { + int j, saw_error, first, last; + + saw_error = 0; + first = i * 8; + last = first + 8; + for (j = first; j < last; j++) { + u64 errval = stc_error_buf[j]; + if (errval != 0) { + saw_error++; + printk(KERN_ERR "%s: STC_ERR(%d)[wr(%d)" + "rd(%d)]\n", + pbm->name, + j, + (errval & PSYCHO_STCERR_WRITE) ? 1 : 0, + (errval & PSYCHO_STCERR_READ) ? 1 : 0); + } + } + if (saw_error != 0) { + u64 tagval = stc_tag_buf[i]; + u64 lineval = stc_line_buf[i]; + printk(KERN_ERR "%s: STC_TAG(%d)[PA(%016llx)VA(%08llx)" + "V(%d)W(%d)]\n", + pbm->name, + i, + ((tagval & PSYCHO_STCTAG_PPN) >> 19UL), + (tagval & PSYCHO_STCTAG_VPN), + ((tagval & PSYCHO_STCTAG_VALID) ? 1 : 0), + ((tagval & PSYCHO_STCTAG_WRITE) ? 1 : 0)); + printk(KERN_ERR "%s: STC_LINE(%d)[LIDX(%llx)SP(%llx)" + "LADDR(%llx)EP(%llx)V(%d)FOFN(%d)]\n", + pbm->name, + i, + ((lineval & PSYCHO_STCLINE_LINDX) >> 21UL), + ((lineval & PSYCHO_STCLINE_SPTR) >> 15UL), + ((lineval & PSYCHO_STCLINE_LADDR) >> 8UL), + ((lineval & PSYCHO_STCLINE_EPTR) >> 2UL), + ((lineval & PSYCHO_STCLINE_VALID) ? 1 : 0), + ((lineval & PSYCHO_STCLINE_FOFN) ? 1 : 0)); + } + } + + spin_unlock(&stc_buf_lock); +} + +#define PSYCHO_IOMMU_TAG 0xa580UL +#define PSYCHO_IOMMU_DATA 0xa600UL + +static void psycho_record_iommu_tags_and_data(struct pci_pbm_info *pbm, + u64 *tag, u64 *data) +{ + int i; + + for (i = 0; i < 16; i++) { + unsigned long base = pbm->controller_regs; + unsigned long off = i * 8UL; + + tag[i] = upa_readq(base + PSYCHO_IOMMU_TAG+off); + data[i] = upa_readq(base + PSYCHO_IOMMU_DATA+off); + + /* Now clear out the entry. */ + upa_writeq(0, base + PSYCHO_IOMMU_TAG + off); + upa_writeq(0, base + PSYCHO_IOMMU_DATA + off); + } +} + +#define PSYCHO_IOMMU_TAG_ERRSTS (0x3UL << 23UL) +#define PSYCHO_IOMMU_TAG_ERR (0x1UL << 22UL) +#define PSYCHO_IOMMU_TAG_WRITE (0x1UL << 21UL) +#define PSYCHO_IOMMU_TAG_STREAM (0x1UL << 20UL) +#define PSYCHO_IOMMU_TAG_SIZE (0x1UL << 19UL) +#define PSYCHO_IOMMU_TAG_VPAGE 0x7ffffULL +#define PSYCHO_IOMMU_DATA_VALID (1UL << 30UL) +#define PSYCHO_IOMMU_DATA_CACHE (1UL << 28UL) +#define PSYCHO_IOMMU_DATA_PPAGE 0xfffffffULL + +static void psycho_dump_iommu_tags_and_data(struct pci_pbm_info *pbm, + u64 *tag, u64 *data) +{ + int i; + + for (i = 0; i < 16; i++) { + u64 tag_val, data_val; + const char *type_str; + tag_val = tag[i]; + if (!(tag_val & PSYCHO_IOMMU_TAG_ERR)) + continue; + + data_val = data[i]; + switch((tag_val & PSYCHO_IOMMU_TAG_ERRSTS) >> 23UL) { + case 0: + type_str = "Protection Error"; + break; + case 1: + type_str = "Invalid Error"; + break; + case 2: + type_str = "TimeOut Error"; + break; + case 3: + default: + type_str = "ECC Error"; + break; + } + + printk(KERN_ERR "%s: IOMMU TAG(%d)[error(%s) wr(%d) " + "str(%d) sz(%dK) vpg(%08llx)]\n", + pbm->name, i, type_str, + ((tag_val & PSYCHO_IOMMU_TAG_WRITE) ? 1 : 0), + ((tag_val & PSYCHO_IOMMU_TAG_STREAM) ? 1 : 0), + ((tag_val & PSYCHO_IOMMU_TAG_SIZE) ? 64 : 8), + (tag_val & PSYCHO_IOMMU_TAG_VPAGE) << IOMMU_PAGE_SHIFT); + printk(KERN_ERR "%s: IOMMU DATA(%d)[valid(%d) cache(%d) " + "ppg(%016llx)]\n", + pbm->name, i, + ((data_val & PSYCHO_IOMMU_DATA_VALID) ? 1 : 0), + ((data_val & PSYCHO_IOMMU_DATA_CACHE) ? 1 : 0), + (data_val & PSYCHO_IOMMU_DATA_PPAGE) << IOMMU_PAGE_SHIFT); + } +} + +#define PSYCHO_IOMMU_CTRL_XLTESTAT 0x0000000006000000UL +#define PSYCHO_IOMMU_CTRL_XLTEERR 0x0000000001000000UL + +void psycho_check_iommu_error(struct pci_pbm_info *pbm, + unsigned long afsr, + unsigned long afar, + enum psycho_error_type type) +{ + u64 control, iommu_tag[16], iommu_data[16]; + struct iommu *iommu = pbm->iommu; + unsigned long flags; + + spin_lock_irqsave(&iommu->lock, flags); + control = upa_readq(iommu->iommu_control); + if (control & PSYCHO_IOMMU_CTRL_XLTEERR) { + const char *type_str; + + control &= ~PSYCHO_IOMMU_CTRL_XLTEERR; + upa_writeq(control, iommu->iommu_control); + + switch ((control & PSYCHO_IOMMU_CTRL_XLTESTAT) >> 25UL) { + case 0: + type_str = "Protection Error"; + break; + case 1: + type_str = "Invalid Error"; + break; + case 2: + type_str = "TimeOut Error"; + break; + case 3: + default: + type_str = "ECC Error"; + break; + } + printk(KERN_ERR "%s: IOMMU Error, type[%s]\n", + pbm->name, type_str); + + /* It is very possible for another DVMA to occur while + * we do this probe, and corrupt the system further. + * But we are so screwed at this point that we are + * likely to crash hard anyways, so get as much + * diagnostic information to the console as we can. + */ + psycho_record_iommu_tags_and_data(pbm, iommu_tag, iommu_data); + psycho_dump_iommu_tags_and_data(pbm, iommu_tag, iommu_data); + } + psycho_check_stc_error(pbm); + spin_unlock_irqrestore(&iommu->lock, flags); +} + +#define PSYCHO_PCICTRL_SBH_ERR 0x0000000800000000UL +#define PSYCHO_PCICTRL_SERR 0x0000000400000000UL + +static irqreturn_t psycho_pcierr_intr_other(struct pci_pbm_info *pbm) +{ + irqreturn_t ret = IRQ_NONE; + u64 csr, csr_error_bits; + u16 stat, *addr; + + csr = upa_readq(pbm->pci_csr); + csr_error_bits = csr & (PSYCHO_PCICTRL_SBH_ERR | PSYCHO_PCICTRL_SERR); + if (csr_error_bits) { + /* Clear the errors. */ + upa_writeq(csr, pbm->pci_csr); + + /* Log 'em. */ + if (csr_error_bits & PSYCHO_PCICTRL_SBH_ERR) + printk(KERN_ERR "%s: PCI streaming byte hole " + "error asserted.\n", pbm->name); + if (csr_error_bits & PSYCHO_PCICTRL_SERR) + printk(KERN_ERR "%s: PCI SERR signal asserted.\n", + pbm->name); + ret = IRQ_HANDLED; + } + addr = psycho_pci_config_mkaddr(pbm, pbm->pci_first_busno, + 0, PCI_STATUS); + pci_config_read16(addr, &stat); + if (stat & (PCI_STATUS_PARITY | + PCI_STATUS_SIG_TARGET_ABORT | + PCI_STATUS_REC_TARGET_ABORT | + PCI_STATUS_REC_MASTER_ABORT | + PCI_STATUS_SIG_SYSTEM_ERROR)) { + printk(KERN_ERR "%s: PCI bus error, PCI_STATUS[%04x]\n", + pbm->name, stat); + pci_config_write16(addr, 0xffff); + ret = IRQ_HANDLED; + } + return ret; +} + +#define PSYCHO_PCIAFSR_PMA 0x8000000000000000ULL +#define PSYCHO_PCIAFSR_PTA 0x4000000000000000ULL +#define PSYCHO_PCIAFSR_PRTRY 0x2000000000000000ULL +#define PSYCHO_PCIAFSR_PPERR 0x1000000000000000ULL +#define PSYCHO_PCIAFSR_SMA 0x0800000000000000ULL +#define PSYCHO_PCIAFSR_STA 0x0400000000000000ULL +#define PSYCHO_PCIAFSR_SRTRY 0x0200000000000000ULL +#define PSYCHO_PCIAFSR_SPERR 0x0100000000000000ULL +#define PSYCHO_PCIAFSR_RESV1 0x00ff000000000000ULL +#define PSYCHO_PCIAFSR_BMSK 0x0000ffff00000000ULL +#define PSYCHO_PCIAFSR_BLK 0x0000000080000000ULL +#define PSYCHO_PCIAFSR_RESV2 0x0000000040000000ULL +#define PSYCHO_PCIAFSR_MID 0x000000003e000000ULL +#define PSYCHO_PCIAFSR_RESV3 0x0000000001ffffffULL + +irqreturn_t psycho_pcierr_intr(int irq, void *dev_id) +{ + struct pci_pbm_info *pbm = dev_id; + u64 afsr, afar, error_bits; + int reported; + + afsr = upa_readq(pbm->pci_afsr); + afar = upa_readq(pbm->pci_afar); + error_bits = afsr & + (PSYCHO_PCIAFSR_PMA | PSYCHO_PCIAFSR_PTA | + PSYCHO_PCIAFSR_PRTRY | PSYCHO_PCIAFSR_PPERR | + PSYCHO_PCIAFSR_SMA | PSYCHO_PCIAFSR_STA | + PSYCHO_PCIAFSR_SRTRY | PSYCHO_PCIAFSR_SPERR); + if (!error_bits) + return psycho_pcierr_intr_other(pbm); + upa_writeq(error_bits, pbm->pci_afsr); + printk(KERN_ERR "%s: PCI Error, primary error type[%s]\n", + pbm->name, + (((error_bits & PSYCHO_PCIAFSR_PMA) ? + "Master Abort" : + ((error_bits & PSYCHO_PCIAFSR_PTA) ? + "Target Abort" : + ((error_bits & PSYCHO_PCIAFSR_PRTRY) ? + "Excessive Retries" : + ((error_bits & PSYCHO_PCIAFSR_PPERR) ? + "Parity Error" : "???")))))); + printk(KERN_ERR "%s: bytemask[%04llx] UPA_MID[%02llx] was_block(%d)\n", + pbm->name, + (afsr & PSYCHO_PCIAFSR_BMSK) >> 32UL, + (afsr & PSYCHO_PCIAFSR_MID) >> 25UL, + (afsr & PSYCHO_PCIAFSR_BLK) ? 1 : 0); + printk(KERN_ERR "%s: PCI AFAR [%016llx]\n", pbm->name, afar); + printk(KERN_ERR "%s: PCI Secondary errors [", pbm->name); + reported = 0; + if (afsr & PSYCHO_PCIAFSR_SMA) { + reported++; + printk("(Master Abort)"); + } + if (afsr & PSYCHO_PCIAFSR_STA) { + reported++; + printk("(Target Abort)"); + } + if (afsr & PSYCHO_PCIAFSR_SRTRY) { + reported++; + printk("(Excessive Retries)"); + } + if (afsr & PSYCHO_PCIAFSR_SPERR) { + reported++; + printk("(Parity Error)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + if (error_bits & (PSYCHO_PCIAFSR_PTA | PSYCHO_PCIAFSR_STA)) { + psycho_check_iommu_error(pbm, afsr, afar, PCI_ERR); + pci_scan_for_target_abort(pbm, pbm->pci_bus); + } + if (error_bits & (PSYCHO_PCIAFSR_PMA | PSYCHO_PCIAFSR_SMA)) + pci_scan_for_master_abort(pbm, pbm->pci_bus); + + if (error_bits & (PSYCHO_PCIAFSR_PPERR | PSYCHO_PCIAFSR_SPERR)) + pci_scan_for_parity_error(pbm, pbm->pci_bus); + + return IRQ_HANDLED; +} + +static void psycho_iommu_flush(struct pci_pbm_info *pbm) +{ + int i; + + for (i = 0; i < 16; i++) { + unsigned long off = i * 8; + + upa_writeq(0, pbm->controller_regs + PSYCHO_IOMMU_TAG + off); + upa_writeq(0, pbm->controller_regs + PSYCHO_IOMMU_DATA + off); + } +} + +#define PSYCHO_IOMMU_CONTROL 0x0200UL +#define PSYCHO_IOMMU_CTRL_TSBSZ 0x0000000000070000UL +#define PSYCHO_IOMMU_TSBSZ_1K 0x0000000000000000UL +#define PSYCHO_IOMMU_TSBSZ_2K 0x0000000000010000UL +#define PSYCHO_IOMMU_TSBSZ_4K 0x0000000000020000UL +#define PSYCHO_IOMMU_TSBSZ_8K 0x0000000000030000UL +#define PSYCHO_IOMMU_TSBSZ_16K 0x0000000000040000UL +#define PSYCHO_IOMMU_TSBSZ_32K 0x0000000000050000UL +#define PSYCHO_IOMMU_TSBSZ_64K 0x0000000000060000UL +#define PSYCHO_IOMMU_TSBSZ_128K 0x0000000000070000UL +#define PSYCHO_IOMMU_CTRL_TBWSZ 0x0000000000000004UL +#define PSYCHO_IOMMU_CTRL_DENAB 0x0000000000000002UL +#define PSYCHO_IOMMU_CTRL_ENAB 0x0000000000000001UL +#define PSYCHO_IOMMU_FLUSH 0x0210UL +#define PSYCHO_IOMMU_TSBBASE 0x0208UL + +int psycho_iommu_init(struct pci_pbm_info *pbm, int tsbsize, + u32 dvma_offset, u32 dma_mask, + unsigned long write_complete_offset) +{ + struct iommu *iommu = pbm->iommu; + u64 control; + int err; + + iommu->iommu_control = pbm->controller_regs + PSYCHO_IOMMU_CONTROL; + iommu->iommu_tsbbase = pbm->controller_regs + PSYCHO_IOMMU_TSBBASE; + iommu->iommu_flush = pbm->controller_regs + PSYCHO_IOMMU_FLUSH; + iommu->iommu_tags = pbm->controller_regs + PSYCHO_IOMMU_TAG; + iommu->write_complete_reg = (pbm->controller_regs + + write_complete_offset); + + iommu->iommu_ctxflush = 0; + + control = upa_readq(iommu->iommu_control); + control |= PSYCHO_IOMMU_CTRL_DENAB; + upa_writeq(control, iommu->iommu_control); + + psycho_iommu_flush(pbm); + + /* Leave diag mode enabled for full-flushing done in pci_iommu.c */ + err = iommu_table_init(iommu, tsbsize * 1024 * 8, + dvma_offset, dma_mask, pbm->numa_node); + if (err) + return err; + + upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase); + + control = upa_readq(iommu->iommu_control); + control &= ~(PSYCHO_IOMMU_CTRL_TSBSZ | PSYCHO_IOMMU_CTRL_TBWSZ); + control |= PSYCHO_IOMMU_CTRL_ENAB; + + switch (tsbsize) { + case 64: + control |= PSYCHO_IOMMU_TSBSZ_64K; + break; + case 128: + control |= PSYCHO_IOMMU_TSBSZ_128K; + break; + default: + return -EINVAL; + } + + upa_writeq(control, iommu->iommu_control); + + return 0; + +} + +void psycho_pbm_init_common(struct pci_pbm_info *pbm, struct platform_device *op, + const char *chip_name, int chip_type) +{ + struct device_node *dp = op->dev.of_node; + + pbm->name = dp->full_name; + pbm->numa_node = NUMA_NO_NODE; + pbm->chip_type = chip_type; + pbm->chip_version = of_getintprop_default(dp, "version#", 0); + pbm->chip_revision = of_getintprop_default(dp, "module-revision#", 0); + pbm->op = op; + pbm->pci_ops = &sun4u_pci_ops; + pbm->config_space_reg_bits = 8; + pbm->index = pci_num_pbms++; + pci_get_pbm_props(pbm); + pci_determine_mem_io_space(pbm); + + printk(KERN_INFO "%s: %s PCI Bus Module ver[%x:%x]\n", + pbm->name, chip_name, + pbm->chip_version, pbm->chip_revision); +} diff --git a/arch/sparc/kernel/psycho_common.h b/arch/sparc/kernel/psycho_common.h new file mode 100644 index 000000000..6925231c5 --- /dev/null +++ b/arch/sparc/kernel/psycho_common.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _PSYCHO_COMMON_H +#define _PSYCHO_COMMON_H + +/* U2P Programmer's Manual, page 13-55, configuration space + * address format: + * + * 32 24 23 16 15 11 10 8 7 2 1 0 + * --------------------------------------------------------- + * |0 0 0 0 0 0 0 0 1| bus | device | function | reg | 0 0 | + * --------------------------------------------------------- + */ +#define PSYCHO_CONFIG_BASE(PBM) \ + ((PBM)->config_space | (1UL << 24)) +#define PSYCHO_CONFIG_ENCODE(BUS, DEVFN, REG) \ + (((unsigned long)(BUS) << 16) | \ + ((unsigned long)(DEVFN) << 8) | \ + ((unsigned long)(REG))) + +static inline void *psycho_pci_config_mkaddr(struct pci_pbm_info *pbm, + unsigned char bus, + unsigned int devfn, + int where) +{ + return (void *) + (PSYCHO_CONFIG_BASE(pbm) | + PSYCHO_CONFIG_ENCODE(bus, devfn, where)); +} + +enum psycho_error_type { + UE_ERR, CE_ERR, PCI_ERR +}; + +void psycho_check_iommu_error(struct pci_pbm_info *pbm, + unsigned long afsr, + unsigned long afar, + enum psycho_error_type type); + +irqreturn_t psycho_pcierr_intr(int irq, void *dev_id); + +int psycho_iommu_init(struct pci_pbm_info *pbm, int tsbsize, + u32 dvma_offset, u32 dma_mask, + unsigned long write_complete_offset); + +void psycho_pbm_init_common(struct pci_pbm_info *pbm, + struct platform_device *op, + const char *chip_name, int chip_type); + +#endif /* _PSYCHO_COMMON_H */ diff --git a/arch/sparc/kernel/ptrace_32.c b/arch/sparc/kernel/ptrace_32.c new file mode 100644 index 000000000..5991b1731 --- /dev/null +++ b/arch/sparc/kernel/ptrace_32.c @@ -0,0 +1,448 @@ +// SPDX-License-Identifier: GPL-2.0 +/* ptrace.c: Sparc process tracing support. + * + * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net) + * + * Based upon code written by Ross Biro, Linus Torvalds, Bob Manson, + * and David Mosberger. + * + * Added Linux support -miguel (weird, eh?, the original code was meant + * to emulate SunOS). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "kernel.h" + +/* #define ALLOW_INIT_TRACING */ + +/* + * Called by kernel/ptrace.c when detaching.. + * + * Make sure single step bits etc are not set. + */ +void ptrace_disable(struct task_struct *child) +{ + /* nothing to do */ +} + +enum sparc_regset { + REGSET_GENERAL, + REGSET_FP, +}; + +static int regwindow32_get(struct task_struct *target, + const struct pt_regs *regs, + u32 *uregs) +{ + unsigned long reg_window = regs->u_regs[UREG_I6]; + int size = 16 * sizeof(u32); + + if (target == current) { + if (copy_from_user(uregs, (void __user *)reg_window, size)) + return -EFAULT; + } else { + if (access_process_vm(target, reg_window, uregs, size, + FOLL_FORCE) != size) + return -EFAULT; + } + return 0; +} + +static int regwindow32_set(struct task_struct *target, + const struct pt_regs *regs, + u32 *uregs) +{ + unsigned long reg_window = regs->u_regs[UREG_I6]; + int size = 16 * sizeof(u32); + + if (target == current) { + if (copy_to_user((void __user *)reg_window, uregs, size)) + return -EFAULT; + } else { + if (access_process_vm(target, reg_window, uregs, size, + FOLL_FORCE | FOLL_WRITE) != size) + return -EFAULT; + } + return 0; +} + +static int genregs32_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = target->thread.kregs; + u32 uregs[16]; + + if (target == current) + flush_user_windows(); + + membuf_write(&to, regs->u_regs, 16 * sizeof(u32)); + if (!to.left) + return 0; + if (regwindow32_get(target, regs, uregs)) + return -EFAULT; + membuf_write(&to, uregs, 16 * sizeof(u32)); + membuf_store(&to, regs->psr); + membuf_store(&to, regs->pc); + membuf_store(&to, regs->npc); + membuf_store(&to, regs->y); + return membuf_zero(&to, 2 * sizeof(u32)); +} + +static int genregs32_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = target->thread.kregs; + u32 uregs[16]; + u32 psr; + int ret; + + if (target == current) + flush_user_windows(); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + regs->u_regs, + 0, 16 * sizeof(u32)); + if (ret || !count) + return ret; + + if (regwindow32_get(target, regs, uregs)) + return -EFAULT; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + uregs, + 16 * sizeof(u32), 32 * sizeof(u32)); + if (ret) + return ret; + if (regwindow32_set(target, regs, uregs)) + return -EFAULT; + if (!count) + return 0; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &psr, + 32 * sizeof(u32), 33 * sizeof(u32)); + if (ret) + return ret; + regs->psr = (regs->psr & ~(PSR_ICC | PSR_SYSCALL)) | + (psr & (PSR_ICC | PSR_SYSCALL)); + if (!count) + return 0; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->pc, + 33 * sizeof(u32), 34 * sizeof(u32)); + if (ret || !count) + return ret; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->npc, + 34 * sizeof(u32), 35 * sizeof(u32)); + if (ret || !count) + return ret; + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->y, + 35 * sizeof(u32), 36 * sizeof(u32)); + if (ret || !count) + return ret; + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, 36 * sizeof(u32), + 38 * sizeof(u32)); + return 0; +} + +static int fpregs32_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ +#if 0 + if (target == current) + save_and_clear_fpu(); +#endif + + membuf_write(&to, target->thread.float_regs, 32 * sizeof(u32)); + membuf_zero(&to, sizeof(u32)); + membuf_write(&to, &target->thread.fsr, sizeof(u32)); + membuf_store(&to, (u32)((1 << 8) | (8 << 16))); + return membuf_zero(&to, 64 * sizeof(u32)); +} + +static int fpregs32_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned long *fpregs = target->thread.float_regs; + int ret; + +#if 0 + if (target == current) + save_and_clear_fpu(); +#endif + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + fpregs, + 0, 32 * sizeof(u32)); + if (!ret) + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 32 * sizeof(u32), + 33 * sizeof(u32)); + if (!ret) + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fsr, + 33 * sizeof(u32), + 34 * sizeof(u32)); + if (!ret) + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 34 * sizeof(u32), -1); + return ret; +} + +static const struct user_regset sparc32_regsets[] = { + /* Format is: + * G0 --> G7 + * O0 --> O7 + * L0 --> L7 + * I0 --> I7 + * PSR, PC, nPC, Y, WIM, TBR + */ + [REGSET_GENERAL] = { + USER_REGSET_NOTE_TYPE(PRSTATUS), + .n = 38, + .size = sizeof(u32), .align = sizeof(u32), + .regset_get = genregs32_get, .set = genregs32_set + }, + /* Format is: + * F0 --> F31 + * empty 32-bit word + * FSR (32--bit word) + * FPU QUEUE COUNT (8-bit char) + * FPU QUEUE ENTRYSIZE (8-bit char) + * FPU ENABLED (8-bit char) + * empty 8-bit char + * FPU QUEUE (64 32-bit ints) + */ + [REGSET_FP] = { + USER_REGSET_NOTE_TYPE(PRFPREG), + .n = 99, + .size = sizeof(u32), .align = sizeof(u32), + .regset_get = fpregs32_get, .set = fpregs32_set + }, +}; + +static int getregs_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = target->thread.kregs; + + if (target == current) + flush_user_windows(); + + membuf_store(&to, regs->psr); + membuf_store(&to, regs->pc); + membuf_store(&to, regs->npc); + membuf_store(&to, regs->y); + return membuf_write(&to, regs->u_regs + 1, 15 * sizeof(u32)); +} + +static int setregs_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = target->thread.kregs; + u32 v[4]; + int ret; + + if (target == current) + flush_user_windows(); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + v, + 0, 4 * sizeof(u32)); + if (ret) + return ret; + regs->psr = (regs->psr & ~(PSR_ICC | PSR_SYSCALL)) | + (v[0] & (PSR_ICC | PSR_SYSCALL)); + regs->pc = v[1]; + regs->npc = v[2]; + regs->y = v[3]; + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + regs->u_regs + 1, + 4 * sizeof(u32) , 19 * sizeof(u32)); +} + +static int getfpregs_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ +#if 0 + if (target == current) + save_and_clear_fpu(); +#endif + membuf_write(&to, &target->thread.float_regs, 32 * sizeof(u32)); + membuf_write(&to, &target->thread.fsr, sizeof(u32)); + return membuf_zero(&to, 35 * sizeof(u32)); +} + +static int setfpregs_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned long *fpregs = target->thread.float_regs; + int ret; + +#if 0 + if (target == current) + save_and_clear_fpu(); +#endif + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + fpregs, + 0, 32 * sizeof(u32)); + if (ret) + return ret; + return user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &target->thread.fsr, + 32 * sizeof(u32), + 33 * sizeof(u32)); +} + +static const struct user_regset ptrace32_regsets[] = { + [REGSET_GENERAL] = { + .n = 19, .size = sizeof(u32), + .regset_get = getregs_get, .set = setregs_set, + }, + [REGSET_FP] = { + .n = 68, .size = sizeof(u32), + .regset_get = getfpregs_get, .set = setfpregs_set, + }, +}; + +static const struct user_regset_view ptrace32_view = { + .regsets = ptrace32_regsets, .n = ARRAY_SIZE(ptrace32_regsets) +}; + +static const struct user_regset_view user_sparc32_view = { + .name = "sparc", .e_machine = EM_SPARC, + .regsets = sparc32_regsets, .n = ARRAY_SIZE(sparc32_regsets) +}; + +const struct user_regset_view *task_user_regset_view(struct task_struct *task) +{ + return &user_sparc32_view; +} + +struct fps { + unsigned long regs[32]; + unsigned long fsr; + unsigned long flags; + unsigned long extra; + unsigned long fpqd; + struct fq { + unsigned long *insnaddr; + unsigned long insn; + } fpq[16]; +}; + +long arch_ptrace(struct task_struct *child, long request, + unsigned long addr, unsigned long data) +{ + unsigned long addr2 = current->thread.kregs->u_regs[UREG_I4]; + void __user *addr2p; + struct pt_regs __user *pregs; + struct fps __user *fps; + int ret; + + addr2p = (void __user *) addr2; + pregs = (struct pt_regs __user *) addr; + fps = (struct fps __user *) addr; + + switch(request) { + case PTRACE_GETREGS: { + ret = copy_regset_to_user(child, &ptrace32_view, + REGSET_GENERAL, 0, + 19 * sizeof(u32), + pregs); + break; + } + + case PTRACE_SETREGS: { + ret = copy_regset_from_user(child, &ptrace32_view, + REGSET_GENERAL, 0, + 19 * sizeof(u32), + pregs); + break; + } + + case PTRACE_GETFPREGS: { + ret = copy_regset_to_user(child, &ptrace32_view, + REGSET_FP, 0, + 68 * sizeof(u32), + fps); + break; + } + + case PTRACE_SETFPREGS: { + ret = copy_regset_from_user(child, &ptrace32_view, + REGSET_FP, 0, + 33 * sizeof(u32), + fps); + break; + } + + case PTRACE_READTEXT: + case PTRACE_READDATA: + ret = ptrace_readdata(child, addr, addr2p, data); + + if (ret == data) + ret = 0; + else if (ret >= 0) + ret = -EIO; + break; + + case PTRACE_WRITETEXT: + case PTRACE_WRITEDATA: + ret = ptrace_writedata(child, addr2p, addr, data); + + if (ret == data) + ret = 0; + else if (ret >= 0) + ret = -EIO; + break; + + default: + if (request == PTRACE_SPARC_DETACH) + request = PTRACE_DETACH; + ret = ptrace_request(child, request, addr, data); + break; + } + + return ret; +} + +asmlinkage int syscall_trace(struct pt_regs *regs, int syscall_exit_p) +{ + int ret = 0; + + if (test_thread_flag(TIF_SYSCALL_TRACE)) { + if (syscall_exit_p) + ptrace_report_syscall_exit(regs, 0); + else + ret = !ptrace_report_syscall_permit_entry(regs); + } + + return ret; +} diff --git a/arch/sparc/kernel/ptrace_64.c b/arch/sparc/kernel/ptrace_64.c new file mode 100644 index 000000000..825ddf55f --- /dev/null +++ b/arch/sparc/kernel/ptrace_64.c @@ -0,0 +1,1176 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* ptrace.c: Sparc process tracing support. + * + * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * + * Based upon code written by Ross Biro, Linus Torvalds, Bob Manson, + * and David Mosberger. + * + * Added Linux support -miguel (weird, eh?, the original code was meant + * to emulate SunOS). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include + +#include "entry.h" + +/* #define ALLOW_INIT_TRACING */ + +struct pt_regs_offset { + const char *name; + int offset; +}; + +#define REG_OFFSET_NAME(n, r) \ + {.name = n, .offset = (PT_V9_##r)} +#define REG_OFFSET_END {.name = NULL, .offset = 0} + +static const struct pt_regs_offset regoffset_table[] = { + REG_OFFSET_NAME("g0", G0), + REG_OFFSET_NAME("g1", G1), + REG_OFFSET_NAME("g2", G2), + REG_OFFSET_NAME("g3", G3), + REG_OFFSET_NAME("g4", G4), + REG_OFFSET_NAME("g5", G5), + REG_OFFSET_NAME("g6", G6), + REG_OFFSET_NAME("g7", G7), + + REG_OFFSET_NAME("i0", I0), + REG_OFFSET_NAME("i1", I1), + REG_OFFSET_NAME("i2", I2), + REG_OFFSET_NAME("i3", I3), + REG_OFFSET_NAME("i4", I4), + REG_OFFSET_NAME("i5", I5), + REG_OFFSET_NAME("i6", I6), + REG_OFFSET_NAME("i7", I7), + + REG_OFFSET_NAME("tstate", TSTATE), + REG_OFFSET_NAME("pc", TPC), + REG_OFFSET_NAME("npc", TNPC), + REG_OFFSET_NAME("y", Y), + REG_OFFSET_NAME("lr", I7), + + REG_OFFSET_END, +}; + +/* + * Called by kernel/ptrace.c when detaching.. + * + * Make sure single step bits etc are not set. + */ +void ptrace_disable(struct task_struct *child) +{ + /* nothing to do */ +} + +/* To get the necessary page struct, access_process_vm() first calls + * get_user_pages(). This has done a flush_dcache_page() on the + * accessed page. Then our caller (copy_{to,from}_user_page()) did + * to memcpy to read/write the data from that page. + * + * Now, the only thing we have to do is: + * 1) flush the D-cache if it's possible than an illegal alias + * has been created + * 2) flush the I-cache if this is pre-cheetah and we did a write + */ +void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, + unsigned long uaddr, void *kaddr, + unsigned long len, int write) +{ + BUG_ON(len > PAGE_SIZE); + + if (tlb_type == hypervisor) + return; + + preempt_disable(); + +#ifdef DCACHE_ALIASING_POSSIBLE + /* If bit 13 of the kernel address we used to access the + * user page is the same as the virtual address that page + * is mapped to in the user's address space, we can skip the + * D-cache flush. + */ + if ((uaddr ^ (unsigned long) kaddr) & (1UL << 13)) { + unsigned long start = __pa(kaddr); + unsigned long end = start + len; + unsigned long dcache_line_size; + + dcache_line_size = local_cpu_data().dcache_line_size; + + if (tlb_type == spitfire) { + for (; start < end; start += dcache_line_size) + spitfire_put_dcache_tag(start & 0x3fe0, 0x0); + } else { + start &= ~(dcache_line_size - 1); + for (; start < end; start += dcache_line_size) + __asm__ __volatile__( + "stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (start), + "i" (ASI_DCACHE_INVALIDATE)); + } + } +#endif + if (write && tlb_type == spitfire) { + unsigned long start = (unsigned long) kaddr; + unsigned long end = start + len; + unsigned long icache_line_size; + + icache_line_size = local_cpu_data().icache_line_size; + + for (; start < end; start += icache_line_size) + flushi(start); + } + + preempt_enable(); +} +EXPORT_SYMBOL_GPL(flush_ptrace_access); + +static int get_from_target(struct task_struct *target, unsigned long uaddr, + void *kbuf, int len) +{ + if (target == current) { + if (copy_from_user(kbuf, (void __user *) uaddr, len)) + return -EFAULT; + } else { + int len2 = access_process_vm(target, uaddr, kbuf, len, + FOLL_FORCE); + if (len2 != len) + return -EFAULT; + } + return 0; +} + +static int set_to_target(struct task_struct *target, unsigned long uaddr, + void *kbuf, int len) +{ + if (target == current) { + if (copy_to_user((void __user *) uaddr, kbuf, len)) + return -EFAULT; + } else { + int len2 = access_process_vm(target, uaddr, kbuf, len, + FOLL_FORCE | FOLL_WRITE); + if (len2 != len) + return -EFAULT; + } + return 0; +} + +static int regwindow64_get(struct task_struct *target, + const struct pt_regs *regs, + struct reg_window *wbuf) +{ + unsigned long rw_addr = regs->u_regs[UREG_I6]; + + if (!test_thread_64bit_stack(rw_addr)) { + struct reg_window32 win32; + int i; + + if (get_from_target(target, rw_addr, &win32, sizeof(win32))) + return -EFAULT; + for (i = 0; i < 8; i++) + wbuf->locals[i] = win32.locals[i]; + for (i = 0; i < 8; i++) + wbuf->ins[i] = win32.ins[i]; + } else { + rw_addr += STACK_BIAS; + if (get_from_target(target, rw_addr, wbuf, sizeof(*wbuf))) + return -EFAULT; + } + + return 0; +} + +static int regwindow64_set(struct task_struct *target, + const struct pt_regs *regs, + struct reg_window *wbuf) +{ + unsigned long rw_addr = regs->u_regs[UREG_I6]; + + if (!test_thread_64bit_stack(rw_addr)) { + struct reg_window32 win32; + int i; + + for (i = 0; i < 8; i++) + win32.locals[i] = wbuf->locals[i]; + for (i = 0; i < 8; i++) + win32.ins[i] = wbuf->ins[i]; + + if (set_to_target(target, rw_addr, &win32, sizeof(win32))) + return -EFAULT; + } else { + rw_addr += STACK_BIAS; + if (set_to_target(target, rw_addr, wbuf, sizeof(*wbuf))) + return -EFAULT; + } + + return 0; +} + +enum sparc_regset { + REGSET_GENERAL, + REGSET_FP, +}; + +static int genregs64_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = task_pt_regs(target); + struct reg_window window; + + if (target == current) + flushw_user(); + + membuf_write(&to, regs->u_regs, 16 * sizeof(u64)); + if (!to.left) + return 0; + if (regwindow64_get(target, regs, &window)) + return -EFAULT; + membuf_write(&to, &window, 16 * sizeof(u64)); + /* TSTATE, TPC, TNPC */ + membuf_write(&to, ®s->tstate, 3 * sizeof(u64)); + return membuf_store(&to, (u64)regs->y); +} + +static int genregs64_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + int ret; + + if (target == current) + flushw_user(); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + regs->u_regs, + 0, 16 * sizeof(u64)); + if (!ret && count && pos < (32 * sizeof(u64))) { + struct reg_window window; + + if (regwindow64_get(target, regs, &window)) + return -EFAULT; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &window, + 16 * sizeof(u64), + 32 * sizeof(u64)); + + if (!ret && + regwindow64_set(target, regs, &window)) + return -EFAULT; + } + + if (!ret && count > 0) { + unsigned long tstate; + + /* TSTATE */ + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &tstate, + 32 * sizeof(u64), + 33 * sizeof(u64)); + if (!ret) { + /* Only the condition codes and the "in syscall" + * state can be modified in the %tstate register. + */ + tstate &= (TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL); + regs->tstate &= ~(TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL); + regs->tstate |= tstate; + } + } + + if (!ret) { + /* TPC, TNPC */ + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->tpc, + 33 * sizeof(u64), + 35 * sizeof(u64)); + } + + if (!ret) { + unsigned long y = regs->y; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &y, + 35 * sizeof(u64), + 36 * sizeof(u64)); + if (!ret) + regs->y = y; + } + + if (!ret) + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 36 * sizeof(u64), -1); + + return ret; +} + +static int fpregs64_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct thread_info *t = task_thread_info(target); + unsigned long fprs; + + if (target == current) + save_and_clear_fpu(); + + fprs = t->fpsaved[0]; + + if (fprs & FPRS_DL) + membuf_write(&to, t->fpregs, 16 * sizeof(u64)); + else + membuf_zero(&to, 16 * sizeof(u64)); + + if (fprs & FPRS_DU) + membuf_write(&to, t->fpregs + 16, 16 * sizeof(u64)); + else + membuf_zero(&to, 16 * sizeof(u64)); + if (fprs & FPRS_FEF) { + membuf_store(&to, t->xfsr[0]); + membuf_store(&to, t->gsr[0]); + } else { + membuf_zero(&to, 2 * sizeof(u64)); + } + return membuf_store(&to, fprs); +} + +static int fpregs64_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned long *fpregs = task_thread_info(target)->fpregs; + unsigned long fprs; + int ret; + + if (target == current) + save_and_clear_fpu(); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + fpregs, + 0, 32 * sizeof(u64)); + if (!ret) + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + task_thread_info(target)->xfsr, + 32 * sizeof(u64), + 33 * sizeof(u64)); + if (!ret) + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + task_thread_info(target)->gsr, + 33 * sizeof(u64), + 34 * sizeof(u64)); + + fprs = task_thread_info(target)->fpsaved[0]; + if (!ret && count > 0) { + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &fprs, + 34 * sizeof(u64), + 35 * sizeof(u64)); + } + + fprs |= (FPRS_FEF | FPRS_DL | FPRS_DU); + task_thread_info(target)->fpsaved[0] = fprs; + + if (!ret) + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 35 * sizeof(u64), -1); + return ret; +} + +static const struct user_regset sparc64_regsets[] = { + /* Format is: + * G0 --> G7 + * O0 --> O7 + * L0 --> L7 + * I0 --> I7 + * TSTATE, TPC, TNPC, Y + */ + [REGSET_GENERAL] = { + USER_REGSET_NOTE_TYPE(PRSTATUS), + .n = 36, + .size = sizeof(u64), .align = sizeof(u64), + .regset_get = genregs64_get, .set = genregs64_set + }, + /* Format is: + * F0 --> F63 + * FSR + * GSR + * FPRS + */ + [REGSET_FP] = { + USER_REGSET_NOTE_TYPE(PRFPREG), + .n = 35, + .size = sizeof(u64), .align = sizeof(u64), + .regset_get = fpregs64_get, .set = fpregs64_set + }, +}; + +static int getregs64_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = task_pt_regs(target); + + if (target == current) + flushw_user(); + + membuf_write(&to, regs->u_regs + 1, 15 * sizeof(u64)); + membuf_store(&to, (u64)0); + membuf_write(&to, ®s->tstate, 3 * sizeof(u64)); + return membuf_store(&to, (u64)regs->y); +} + +static int setregs64_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + unsigned long y = regs->y; + unsigned long tstate; + int ret; + + if (target == current) + flushw_user(); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + regs->u_regs + 1, + 0 * sizeof(u64), + 15 * sizeof(u64)); + if (ret) + return ret; + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 15 * sizeof(u64), 16 * sizeof(u64)); + /* TSTATE */ + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &tstate, + 16 * sizeof(u64), + 17 * sizeof(u64)); + if (ret) + return ret; + /* Only the condition codes and the "in syscall" + * state can be modified in the %tstate register. + */ + tstate &= (TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL); + regs->tstate &= ~(TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL); + regs->tstate |= tstate; + + /* TPC, TNPC */ + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + ®s->tpc, + 17 * sizeof(u64), + 19 * sizeof(u64)); + if (ret) + return ret; + /* Y */ + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &y, + 19 * sizeof(u64), + 20 * sizeof(u64)); + if (!ret) + regs->y = y; + return ret; +} + +static const struct user_regset ptrace64_regsets[] = { + /* Format is: + * G1 --> G7 + * O0 --> O7 + * 0 + * TSTATE, TPC, TNPC, Y + */ + [REGSET_GENERAL] = { + .n = 20, .size = sizeof(u64), + .regset_get = getregs64_get, .set = setregs64_set, + }, +}; + +static const struct user_regset_view ptrace64_view = { + .regsets = ptrace64_regsets, .n = ARRAY_SIZE(ptrace64_regsets) +}; + +static const struct user_regset_view user_sparc64_view = { + .name = "sparc64", .e_machine = EM_SPARCV9, + .regsets = sparc64_regsets, .n = ARRAY_SIZE(sparc64_regsets) +}; + +#ifdef CONFIG_COMPAT +static int genregs32_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = task_pt_regs(target); + u32 uregs[16]; + int i; + + if (target == current) + flushw_user(); + + for (i = 0; i < 16; i++) + membuf_store(&to, (u32)regs->u_regs[i]); + if (!to.left) + return 0; + if (get_from_target(target, regs->u_regs[UREG_I6], + uregs, sizeof(uregs))) + return -EFAULT; + membuf_write(&to, uregs, 16 * sizeof(u32)); + membuf_store(&to, (u32)tstate_to_psr(regs->tstate)); + membuf_store(&to, (u32)(regs->tpc)); + membuf_store(&to, (u32)(regs->tnpc)); + membuf_store(&to, (u32)(regs->y)); + return membuf_zero(&to, 2 * sizeof(u32)); +} + +static int genregs32_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + compat_ulong_t __user *reg_window; + const compat_ulong_t *k = kbuf; + const compat_ulong_t __user *u = ubuf; + compat_ulong_t reg; + + if (target == current) + flushw_user(); + + pos /= sizeof(reg); + count /= sizeof(reg); + + if (kbuf) { + for (; count > 0 && pos < 16; count--) + regs->u_regs[pos++] = *k++; + + reg_window = (compat_ulong_t __user *) regs->u_regs[UREG_I6]; + reg_window -= 16; + if (target == current) { + for (; count > 0 && pos < 32; count--) { + if (put_user(*k++, ®_window[pos++])) + return -EFAULT; + } + } else { + for (; count > 0 && pos < 32; count--) { + if (access_process_vm(target, + (unsigned long) + ®_window[pos], + (void *) k, + sizeof(*k), + FOLL_FORCE | FOLL_WRITE) + != sizeof(*k)) + return -EFAULT; + k++; + pos++; + } + } + } else { + for (; count > 0 && pos < 16; count--) { + if (get_user(reg, u++)) + return -EFAULT; + regs->u_regs[pos++] = reg; + } + + reg_window = (compat_ulong_t __user *) regs->u_regs[UREG_I6]; + reg_window -= 16; + if (target == current) { + for (; count > 0 && pos < 32; count--) { + if (get_user(reg, u++) || + put_user(reg, ®_window[pos++])) + return -EFAULT; + } + } else { + for (; count > 0 && pos < 32; count--) { + if (get_user(reg, u++)) + return -EFAULT; + if (access_process_vm(target, + (unsigned long) + ®_window[pos], + ®, sizeof(reg), + FOLL_FORCE | FOLL_WRITE) + != sizeof(reg)) + return -EFAULT; + pos++; + u++; + } + } + } + while (count > 0) { + unsigned long tstate; + + if (kbuf) + reg = *k++; + else if (get_user(reg, u++)) + return -EFAULT; + + switch (pos) { + case 32: /* PSR */ + tstate = regs->tstate; + tstate &= ~(TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL); + tstate |= psr_to_tstate_icc(reg); + if (reg & PSR_SYSCALL) + tstate |= TSTATE_SYSCALL; + regs->tstate = tstate; + break; + case 33: /* PC */ + regs->tpc = reg; + break; + case 34: /* NPC */ + regs->tnpc = reg; + break; + case 35: /* Y */ + regs->y = reg; + break; + case 36: /* WIM */ + case 37: /* TBR */ + break; + default: + goto finish; + } + + pos++; + count--; + } +finish: + pos *= sizeof(reg); + count *= sizeof(reg); + + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 38 * sizeof(reg), -1); + return 0; +} + +static int fpregs32_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct thread_info *t = task_thread_info(target); + bool enabled; + + if (target == current) + save_and_clear_fpu(); + + enabled = t->fpsaved[0] & FPRS_FEF; + + membuf_write(&to, t->fpregs, 32 * sizeof(u32)); + membuf_zero(&to, sizeof(u32)); + if (enabled) + membuf_store(&to, (u32)t->xfsr[0]); + else + membuf_zero(&to, sizeof(u32)); + membuf_store(&to, (u32)((enabled << 8) | (8 << 16))); + return membuf_zero(&to, 64 * sizeof(u32)); +} + +static int fpregs32_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned long *fpregs = task_thread_info(target)->fpregs; + unsigned long fprs; + int ret; + + if (target == current) + save_and_clear_fpu(); + + fprs = task_thread_info(target)->fpsaved[0]; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + fpregs, + 0, 32 * sizeof(u32)); + if (!ret) + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 32 * sizeof(u32), + 33 * sizeof(u32)); + if (!ret && count > 0) { + compat_ulong_t fsr; + unsigned long val; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &fsr, + 33 * sizeof(u32), + 34 * sizeof(u32)); + if (!ret) { + val = task_thread_info(target)->xfsr[0]; + val &= 0xffffffff00000000UL; + val |= fsr; + task_thread_info(target)->xfsr[0] = val; + } + } + + fprs |= (FPRS_FEF | FPRS_DL); + task_thread_info(target)->fpsaved[0] = fprs; + + if (!ret) + user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf, + 34 * sizeof(u32), -1); + return ret; +} + +static const struct user_regset sparc32_regsets[] = { + /* Format is: + * G0 --> G7 + * O0 --> O7 + * L0 --> L7 + * I0 --> I7 + * PSR, PC, nPC, Y, WIM, TBR + */ + [REGSET_GENERAL] = { + USER_REGSET_NOTE_TYPE(PRSTATUS), + .n = 38, + .size = sizeof(u32), .align = sizeof(u32), + .regset_get = genregs32_get, .set = genregs32_set + }, + /* Format is: + * F0 --> F31 + * empty 32-bit word + * FSR (32--bit word) + * FPU QUEUE COUNT (8-bit char) + * FPU QUEUE ENTRYSIZE (8-bit char) + * FPU ENABLED (8-bit char) + * empty 8-bit char + * FPU QUEUE (64 32-bit ints) + */ + [REGSET_FP] = { + USER_REGSET_NOTE_TYPE(PRFPREG), + .n = 99, + .size = sizeof(u32), .align = sizeof(u32), + .regset_get = fpregs32_get, .set = fpregs32_set + }, +}; + +static int getregs_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + const struct pt_regs *regs = task_pt_regs(target); + int i; + + if (target == current) + flushw_user(); + + membuf_store(&to, (u32)tstate_to_psr(regs->tstate)); + membuf_store(&to, (u32)(regs->tpc)); + membuf_store(&to, (u32)(regs->tnpc)); + membuf_store(&to, (u32)(regs->y)); + for (i = 1; i < 16; i++) + membuf_store(&to, (u32)regs->u_regs[i]); + return to.left; +} + +static int setregs_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + struct pt_regs *regs = task_pt_regs(target); + unsigned long tstate; + u32 uregs[19]; + int i, ret; + + if (target == current) + flushw_user(); + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + uregs, + 0, 19 * sizeof(u32)); + if (ret) + return ret; + + tstate = regs->tstate; + tstate &= ~(TSTATE_ICC | TSTATE_XCC | TSTATE_SYSCALL); + tstate |= psr_to_tstate_icc(uregs[0]); + if (uregs[0] & PSR_SYSCALL) + tstate |= TSTATE_SYSCALL; + regs->tstate = tstate; + regs->tpc = uregs[1]; + regs->tnpc = uregs[2]; + regs->y = uregs[3]; + + for (i = 1; i < 15; i++) + regs->u_regs[i] = uregs[3 + i]; + return 0; +} + +static int getfpregs_get(struct task_struct *target, + const struct user_regset *regset, + struct membuf to) +{ + struct thread_info *t = task_thread_info(target); + + if (target == current) + save_and_clear_fpu(); + + membuf_write(&to, t->fpregs, 32 * sizeof(u32)); + if (t->fpsaved[0] & FPRS_FEF) + membuf_store(&to, (u32)t->xfsr[0]); + else + membuf_zero(&to, sizeof(u32)); + return membuf_zero(&to, 35 * sizeof(u32)); +} + +static int setfpregs_set(struct task_struct *target, + const struct user_regset *regset, + unsigned int pos, unsigned int count, + const void *kbuf, const void __user *ubuf) +{ + unsigned long *fpregs = task_thread_info(target)->fpregs; + unsigned long fprs; + int ret; + + if (target == current) + save_and_clear_fpu(); + + fprs = task_thread_info(target)->fpsaved[0]; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + fpregs, + 0, 32 * sizeof(u32)); + if (!ret) { + compat_ulong_t fsr; + unsigned long val; + + ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, + &fsr, + 32 * sizeof(u32), + 33 * sizeof(u32)); + if (!ret) { + val = task_thread_info(target)->xfsr[0]; + val &= 0xffffffff00000000UL; + val |= fsr; + task_thread_info(target)->xfsr[0] = val; + } + } + + fprs |= (FPRS_FEF | FPRS_DL); + task_thread_info(target)->fpsaved[0] = fprs; + return ret; +} + +static const struct user_regset ptrace32_regsets[] = { + [REGSET_GENERAL] = { + .n = 19, .size = sizeof(u32), + .regset_get = getregs_get, .set = setregs_set, + }, + [REGSET_FP] = { + .n = 68, .size = sizeof(u32), + .regset_get = getfpregs_get, .set = setfpregs_set, + }, +}; + +static const struct user_regset_view ptrace32_view = { + .regsets = ptrace32_regsets, .n = ARRAY_SIZE(ptrace32_regsets) +}; + +static const struct user_regset_view user_sparc32_view = { + .name = "sparc", .e_machine = EM_SPARC, + .regsets = sparc32_regsets, .n = ARRAY_SIZE(sparc32_regsets) +}; +#endif /* CONFIG_COMPAT */ + +const struct user_regset_view *task_user_regset_view(struct task_struct *task) +{ +#ifdef CONFIG_COMPAT + if (test_tsk_thread_flag(task, TIF_32BIT)) + return &user_sparc32_view; +#endif + return &user_sparc64_view; +} + +#ifdef CONFIG_COMPAT +struct compat_fps { + unsigned int regs[32]; + unsigned int fsr; + unsigned int flags; + unsigned int extra; + unsigned int fpqd; + struct compat_fq { + unsigned int insnaddr; + unsigned int insn; + } fpq[16]; +}; + +long compat_arch_ptrace(struct task_struct *child, compat_long_t request, + compat_ulong_t caddr, compat_ulong_t cdata) +{ + compat_ulong_t caddr2 = task_pt_regs(current)->u_regs[UREG_I4]; + struct pt_regs32 __user *pregs; + struct compat_fps __user *fps; + unsigned long addr2 = caddr2; + unsigned long addr = caddr; + unsigned long data = cdata; + int ret; + + pregs = (struct pt_regs32 __user *) addr; + fps = (struct compat_fps __user *) addr; + + switch (request) { + case PTRACE_PEEKUSR: + ret = (addr != 0) ? -EIO : 0; + break; + + case PTRACE_GETREGS: + ret = copy_regset_to_user(child, &ptrace32_view, + REGSET_GENERAL, 0, + 19 * sizeof(u32), + pregs); + break; + + case PTRACE_SETREGS: + ret = copy_regset_from_user(child, &ptrace32_view, + REGSET_GENERAL, 0, + 19 * sizeof(u32), + pregs); + break; + + case PTRACE_GETFPREGS: + ret = copy_regset_to_user(child, &ptrace32_view, + REGSET_FP, 0, + 68 * sizeof(u32), + fps); + break; + + case PTRACE_SETFPREGS: + ret = copy_regset_from_user(child, &ptrace32_view, + REGSET_FP, 0, + 33 * sizeof(u32), + fps); + break; + + case PTRACE_READTEXT: + case PTRACE_READDATA: + ret = ptrace_readdata(child, addr, + (char __user *)addr2, data); + if (ret == data) + ret = 0; + else if (ret >= 0) + ret = -EIO; + break; + + case PTRACE_WRITETEXT: + case PTRACE_WRITEDATA: + ret = ptrace_writedata(child, (char __user *) addr2, + addr, data); + if (ret == data) + ret = 0; + else if (ret >= 0) + ret = -EIO; + break; + + default: + if (request == PTRACE_SPARC_DETACH) + request = PTRACE_DETACH; + ret = compat_ptrace_request(child, request, addr, data); + break; + } + + return ret; +} +#endif /* CONFIG_COMPAT */ + +struct fps { + unsigned int regs[64]; + unsigned long fsr; +}; + +long arch_ptrace(struct task_struct *child, long request, + unsigned long addr, unsigned long data) +{ + const struct user_regset_view *view = task_user_regset_view(current); + unsigned long addr2 = task_pt_regs(current)->u_regs[UREG_I4]; + struct pt_regs __user *pregs; + struct fps __user *fps; + void __user *addr2p; + int ret; + + pregs = (struct pt_regs __user *) addr; + fps = (struct fps __user *) addr; + addr2p = (void __user *) addr2; + + switch (request) { + case PTRACE_PEEKUSR: + ret = (addr != 0) ? -EIO : 0; + break; + + case PTRACE_GETREGS64: + ret = copy_regset_to_user(child, &ptrace64_view, + REGSET_GENERAL, 0, + 19 * sizeof(u64), + pregs); + break; + + case PTRACE_SETREGS64: + ret = copy_regset_from_user(child, &ptrace64_view, + REGSET_GENERAL, 0, + 19 * sizeof(u64), + pregs); + break; + + case PTRACE_GETFPREGS64: + ret = copy_regset_to_user(child, view, REGSET_FP, + 0 * sizeof(u64), + 33 * sizeof(u64), + fps); + break; + + case PTRACE_SETFPREGS64: + ret = copy_regset_from_user(child, view, REGSET_FP, + 0 * sizeof(u64), + 33 * sizeof(u64), + fps); + break; + + case PTRACE_READTEXT: + case PTRACE_READDATA: + ret = ptrace_readdata(child, addr, addr2p, data); + if (ret == data) + ret = 0; + else if (ret >= 0) + ret = -EIO; + break; + + case PTRACE_WRITETEXT: + case PTRACE_WRITEDATA: + ret = ptrace_writedata(child, addr2p, addr, data); + if (ret == data) + ret = 0; + else if (ret >= 0) + ret = -EIO; + break; + + default: + if (request == PTRACE_SPARC_DETACH) + request = PTRACE_DETACH; + ret = ptrace_request(child, request, addr, data); + break; + } + + return ret; +} + +asmlinkage int syscall_trace_enter(struct pt_regs *regs) +{ + int ret = 0; + + /* do the secure computing check first */ + secure_computing_strict(regs->u_regs[UREG_G1]); + + if (test_thread_flag(TIF_NOHZ)) + user_exit(); + + if (test_thread_flag(TIF_SYSCALL_TRACE)) + ret = !ptrace_report_syscall_permit_entry(regs); + + if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_enter(regs, regs->u_regs[UREG_G1]); + + audit_syscall_entry(regs->u_regs[UREG_G1], regs->u_regs[UREG_I0], + regs->u_regs[UREG_I1], regs->u_regs[UREG_I2], + regs->u_regs[UREG_I3]); + + return ret; +} + +asmlinkage void syscall_trace_leave(struct pt_regs *regs) +{ + if (test_thread_flag(TIF_NOHZ)) + user_exit(); + + audit_syscall_exit(regs); + + if (unlikely(test_thread_flag(TIF_SYSCALL_TRACEPOINT))) + trace_sys_exit(regs, regs->u_regs[UREG_I0]); + + if (test_thread_flag(TIF_SYSCALL_TRACE)) + ptrace_report_syscall_exit(regs, 0); + + if (test_thread_flag(TIF_NOHZ)) + user_enter(); +} + +/** + * regs_query_register_offset() - query register offset from its name + * @name: the name of a register + * + * regs_query_register_offset() returns the offset of a register in struct + * pt_regs from its name. If the name is invalid, this returns -EINVAL; + */ +int regs_query_register_offset(const char *name) +{ + const struct pt_regs_offset *roff; + + for (roff = regoffset_table; roff->name != NULL; roff++) + if (!strcmp(roff->name, name)) + return roff->offset; + return -EINVAL; +} + +/** + * regs_within_kernel_stack() - check the address in the stack + * @regs: pt_regs which contains kernel stack pointer. + * @addr: address which is checked. + * + * regs_within_kernel_stack() checks @addr is within the kernel stack page(s). + * If @addr is within the kernel stack, it returns true. If not, returns false. + */ +static inline int regs_within_kernel_stack(struct pt_regs *regs, + unsigned long addr) +{ + unsigned long ksp = kernel_stack_pointer(regs) + STACK_BIAS; + return ((addr & ~(THREAD_SIZE - 1)) == + (ksp & ~(THREAD_SIZE - 1))); +} + +/** + * regs_get_kernel_stack_nth() - get Nth entry of the stack + * @regs: pt_regs which contains kernel stack pointer. + * @n: stack entry number. + * + * regs_get_kernel_stack_nth() returns @n th entry of the kernel stack which + * is specified by @regs. If the @n th entry is NOT in the kernel stack, + * this returns 0. + */ +unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) +{ + unsigned long ksp = kernel_stack_pointer(regs) + STACK_BIAS; + unsigned long *addr = (unsigned long *)ksp; + addr += n; + if (regs_within_kernel_stack(regs, (unsigned long)addr)) + return *addr; + else + return 0; +} diff --git a/arch/sparc/kernel/reboot.c b/arch/sparc/kernel/reboot.c new file mode 100644 index 000000000..69c1b6c04 --- /dev/null +++ b/arch/sparc/kernel/reboot.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* reboot.c: reboot/shutdown/halt/poweroff handling + * + * Copyright (C) 2008 David S. Miller + */ +#include +#include +#include +#include +#include + +#include +#include +#include + +/* sysctl - toggle power-off restriction for serial console + * systems in machine_power_off() + */ +int scons_pwroff = 1; + +/* This isn't actually used, it exists merely to satisfy the + * reference in kernel/sys.c + */ +void (*pm_power_off)(void) = machine_power_off; +EXPORT_SYMBOL(pm_power_off); + +void machine_power_off(void) +{ + if (!of_node_is_type(of_console_device, "serial") || scons_pwroff) + prom_halt_power_off(); + + prom_halt(); +} + +void machine_halt(void) +{ + prom_halt(); + panic("Halt failed!"); +} + +void machine_restart(char *cmd) +{ + char *p; + + p = strchr(reboot_command, '\n'); + if (p) + *p = 0; + if (cmd) + prom_reboot(cmd); + if (*reboot_command) + prom_reboot(reboot_command); + prom_reboot(""); + panic("Reboot failed!"); +} + diff --git a/arch/sparc/kernel/rtrap_32.S b/arch/sparc/kernel/rtrap_32.S new file mode 100644 index 000000000..8931fe266 --- /dev/null +++ b/arch/sparc/kernel/rtrap_32.S @@ -0,0 +1,265 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * rtrap.S: Return from Sparc trap low-level code. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define t_psr l0 +#define t_pc l1 +#define t_npc l2 +#define t_wim l3 +#define twin_tmp1 l4 +#define glob_tmp g4 +#define curptr g6 + + /* 7 WINDOW SPARC PATCH INSTRUCTIONS */ + .globl rtrap_7win_patch1, rtrap_7win_patch2, rtrap_7win_patch3 + .globl rtrap_7win_patch4, rtrap_7win_patch5 +rtrap_7win_patch1: srl %t_wim, 0x6, %glob_tmp +rtrap_7win_patch2: and %glob_tmp, 0x7f, %glob_tmp +rtrap_7win_patch3: srl %g1, 7, %g2 +rtrap_7win_patch4: srl %g2, 6, %g2 +rtrap_7win_patch5: and %g1, 0x7f, %g1 + /* END OF PATCH INSTRUCTIONS */ + + /* We need to check for a few things which are: + * 1) The need to call schedule() because this + * processes quantum is up. + * 2) Pending signals for this process, if any + * exist we need to call do_signal() to do + * the needy. + * + * Else we just check if the rett would land us + * in an invalid window, if so we need to grab + * it off the user/kernel stack first. + */ + + .globl ret_trap_entry, rtrap_patch1, rtrap_patch2 + .globl rtrap_patch3, rtrap_patch4, rtrap_patch5 + .globl ret_trap_lockless_ipi +ret_trap_entry: +ret_trap_lockless_ipi: + andcc %t_psr, PSR_PS, %g0 + sethi %hi(PSR_SYSCALL), %g1 + be 1f + andn %t_psr, %g1, %t_psr + + wr %t_psr, 0x0, %psr + b ret_trap_kernel + nop + +1: + ld [%curptr + TI_FLAGS], %g2 + andcc %g2, (_TIF_NEED_RESCHED), %g0 + be signal_p + nop + + call schedule + nop + + ld [%curptr + TI_FLAGS], %g2 +signal_p: + andcc %g2, _TIF_DO_NOTIFY_RESUME_MASK, %g0 + bz,a ret_trap_continue + ld [%sp + STACKFRAME_SZ + PT_PSR], %t_psr + + mov %g2, %o2 + mov %l6, %o1 + call do_notify_resume + add %sp, STACKFRAME_SZ, %o0 ! pt_regs ptr + + b signal_p + ld [%curptr + TI_FLAGS], %g2 + +ret_trap_continue: + sethi %hi(PSR_SYSCALL), %g1 + andn %t_psr, %g1, %t_psr + wr %t_psr, 0x0, %psr + WRITE_PAUSE + + ld [%curptr + TI_W_SAVED], %twin_tmp1 + orcc %g0, %twin_tmp1, %g0 + be ret_trap_nobufwins + nop + + wr %t_psr, PSR_ET, %psr + WRITE_PAUSE + + mov 1, %o1 + call try_to_clear_window_buffer + add %sp, STACKFRAME_SZ, %o0 + + b signal_p + ld [%curptr + TI_FLAGS], %g2 + +ret_trap_nobufwins: + /* Load up the user's out registers so we can pull + * a window from the stack, if necessary. + */ + LOAD_PT_INS(sp) + + /* If there are already live user windows in the + * set we can return from trap safely. + */ + ld [%curptr + TI_UWINMASK], %twin_tmp1 + orcc %g0, %twin_tmp1, %g0 + bne ret_trap_userwins_ok + nop + + /* Calculate new %wim, we have to pull a register + * window from the users stack. + */ +ret_trap_pull_one_window: + rd %wim, %t_wim + sll %t_wim, 0x1, %twin_tmp1 +rtrap_patch1: srl %t_wim, 0x7, %glob_tmp + or %glob_tmp, %twin_tmp1, %glob_tmp +rtrap_patch2: and %glob_tmp, 0xff, %glob_tmp + + wr %glob_tmp, 0x0, %wim + + /* Here comes the architecture specific + * branch to the user stack checking routine + * for return from traps. + */ + b srmmu_rett_stackchk + andcc %fp, 0x7, %g0 + +ret_trap_userwins_ok: + LOAD_PT_PRIV(sp, t_psr, t_pc, t_npc) + or %t_pc, %t_npc, %g2 + andcc %g2, 0x3, %g0 + sethi %hi(PSR_SYSCALL), %g2 + be 1f + andn %t_psr, %g2, %t_psr + + b ret_trap_unaligned_pc + add %sp, STACKFRAME_SZ, %o0 + +1: + LOAD_PT_YREG(sp, g1) + LOAD_PT_GLOBALS(sp) + + wr %t_psr, 0x0, %psr + WRITE_PAUSE + + jmp %t_pc + rett %t_npc + +ret_trap_unaligned_pc: + ld [%sp + STACKFRAME_SZ + PT_PC], %o1 + ld [%sp + STACKFRAME_SZ + PT_NPC], %o2 + ld [%sp + STACKFRAME_SZ + PT_PSR], %o3 + + wr %t_wim, 0x0, %wim ! or else... + + wr %t_psr, PSR_ET, %psr + WRITE_PAUSE + + call do_memaccess_unaligned + nop + + b signal_p + ld [%curptr + TI_FLAGS], %g2 + +ret_trap_kernel: + /* Will the rett land us in the invalid window? */ + mov 2, %g1 + sll %g1, %t_psr, %g1 +rtrap_patch3: srl %g1, 8, %g2 + or %g1, %g2, %g1 + rd %wim, %g2 + andcc %g2, %g1, %g0 + be 1f ! Nope, just return from the trap + sll %g2, 0x1, %g1 + + /* We have to grab a window before returning. */ +rtrap_patch4: srl %g2, 7, %g2 + or %g1, %g2, %g1 +rtrap_patch5: and %g1, 0xff, %g1 + + wr %g1, 0x0, %wim + + /* Grrr, make sure we load from the right %sp... */ + LOAD_PT_ALL(sp, t_psr, t_pc, t_npc, g1) + + restore %g0, %g0, %g0 + LOAD_WINDOW(sp) + b 2f + save %g0, %g0, %g0 + + /* Reload the entire frame in case this is from a + * kernel system call or whatever... + */ +1: + LOAD_PT_ALL(sp, t_psr, t_pc, t_npc, g1) +2: + sethi %hi(PSR_SYSCALL), %twin_tmp1 + andn %t_psr, %twin_tmp1, %t_psr + wr %t_psr, 0x0, %psr + WRITE_PAUSE + + jmp %t_pc + rett %t_npc + +ret_trap_user_stack_is_bolixed: + wr %t_wim, 0x0, %wim + + wr %t_psr, PSR_ET, %psr + WRITE_PAUSE + + call window_ret_fault + add %sp, STACKFRAME_SZ, %o0 + + b signal_p + ld [%curptr + TI_FLAGS], %g2 + + .globl srmmu_rett_stackchk +srmmu_rett_stackchk: + bne ret_trap_user_stack_is_bolixed + sethi %hi(PAGE_OFFSET), %g1 + cmp %g1, %fp + bleu ret_trap_user_stack_is_bolixed + mov AC_M_SFSR, %g1 +LEON_PI(lda [%g1] ASI_LEON_MMUREGS, %g0) +SUN_PI_(lda [%g1] ASI_M_MMUREGS, %g0) + +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %g1) +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %g1) + or %g1, 0x2, %g1 +LEON_PI(sta %g1, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %g1, [%g0] ASI_M_MMUREGS) + + restore %g0, %g0, %g0 + + LOAD_WINDOW(sp) + + save %g0, %g0, %g0 + + andn %g1, 0x2, %g1 +LEON_PI(sta %g1, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %g1, [%g0] ASI_M_MMUREGS) + + mov AC_M_SFAR, %g2 +LEON_PI(lda [%g2] ASI_LEON_MMUREGS, %g2) +SUN_PI_(lda [%g2] ASI_M_MMUREGS, %g2) + + mov AC_M_SFSR, %g1 +LEON_PI(lda [%g1] ASI_LEON_MMUREGS, %g1) +SUN_PI_(lda [%g1] ASI_M_MMUREGS, %g1) + andcc %g1, 0x2, %g0 + be ret_trap_userwins_ok + nop + + b,a ret_trap_user_stack_is_bolixed diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S new file mode 100644 index 000000000..eef102765 --- /dev/null +++ b/arch/sparc/kernel/rtrap_64.S @@ -0,0 +1,378 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * rtrap.S: Preparing for return from trap on Sparc V9. + * + * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + + +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_CONTEXT_TRACKING_USER +# define SCHEDULE_USER schedule_user +#else +# define SCHEDULE_USER schedule +#endif + + .text + .align 32 +__handle_preemption: + call SCHEDULE_USER +661: wrpr %g0, RTRAP_PSTATE, %pstate + /* If userspace is using ADI, it could potentially pass + * a pointer with version tag embedded in it. To maintain + * the ADI security, we must re-enable PSTATE.mcde before + * we continue execution in the kernel for another thread. + */ + .section .sun_m7_1insn_patch, "ax" + .word 661b + wrpr %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate + .previous + ba,pt %xcc, __handle_preemption_continue + wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate + +__handle_user_windows: + add %sp, PTREGS_OFF, %o0 + call fault_in_user_windows +661: wrpr %g0, RTRAP_PSTATE, %pstate + /* If userspace is using ADI, it could potentially pass + * a pointer with version tag embedded in it. To maintain + * the ADI security, we must re-enable PSTATE.mcde before + * we continue execution in the kernel for another thread. + */ + .section .sun_m7_1insn_patch, "ax" + .word 661b + wrpr %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate + .previous + ba,pt %xcc, __handle_preemption_continue + wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate + +__handle_userfpu: + rd %fprs, %l5 + andcc %l5, FPRS_FEF, %g0 + sethi %hi(TSTATE_PEF), %o0 + be,a,pn %icc, __handle_userfpu_continue + andn %l1, %o0, %l1 + ba,a,pt %xcc, __handle_userfpu_continue + +__handle_signal: + mov %l5, %o1 + add %sp, PTREGS_OFF, %o0 + mov %l0, %o2 + call do_notify_resume +661: wrpr %g0, RTRAP_PSTATE, %pstate + /* If userspace is using ADI, it could potentially pass + * a pointer with version tag embedded in it. To maintain + * the ADI security, we must re-enable PSTATE.mcde before + * we continue execution in the kernel for another thread. + */ + .section .sun_m7_1insn_patch, "ax" + .word 661b + wrpr %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate + .previous + wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate + + /* Signal delivery can modify pt_regs tstate, so we must + * reload it. + */ + ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 + sethi %hi(0xf << 20), %l4 + and %l1, %l4, %l4 + andn %l1, %l4, %l1 + ba,pt %xcc, __handle_preemption_continue + srl %l4, 20, %l4 + + /* When returning from a NMI (%pil==15) interrupt we want to + * avoid running softirqs, doing IRQ tracing, preempting, etc. + */ + .globl rtrap_nmi +rtrap_nmi: ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 + sethi %hi(0xf << 20), %l4 + and %l1, %l4, %l4 + andn %l1, %l4, %l1 + srl %l4, 20, %l4 + ba,pt %xcc, rtrap_no_irq_enable + nop + /* Do not actually set the %pil here. We will do that + * below after we clear PSTATE_IE in the %pstate register. + * If we re-enable interrupts here, we can recurse down + * the hardirq stack potentially endlessly, causing a + * stack overflow. + */ + + .align 64 + .globl rtrap_irq, rtrap, irqsz_patchme, rtrap_xcall +rtrap_irq: +rtrap: + /* mm/ultra.S:xcall_report_regs KNOWS about this load. */ + ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 +rtrap_xcall: + sethi %hi(0xf << 20), %l4 + and %l1, %l4, %l4 + andn %l1, %l4, %l1 + srl %l4, 20, %l4 +#ifdef CONFIG_TRACE_IRQFLAGS + brnz,pn %l4, rtrap_no_irq_enable + nop + call trace_hardirqs_on + nop + /* Do not actually set the %pil here. We will do that + * below after we clear PSTATE_IE in the %pstate register. + * If we re-enable interrupts here, we can recurse down + * the hardirq stack potentially endlessly, causing a + * stack overflow. + * + * It is tempting to put this test and trace_hardirqs_on + * call at the 'rt_continue' label, but that will not work + * as that path hits unconditionally and we do not want to + * execute this in NMI return paths, for example. + */ +#endif +rtrap_no_irq_enable: + andcc %l1, TSTATE_PRIV, %l3 + bne,pn %icc, to_kernel + nop + + /* We must hold IRQs off and atomically test schedule+signal + * state, then hold them off all the way back to userspace. + * If we are returning to kernel, none of this matters. Note + * that we are disabling interrupts via PSTATE_IE, not using + * %pil. + * + * If we do not do this, there is a window where we would do + * the tests, later the signal/resched event arrives but we do + * not process it since we are still in kernel mode. It would + * take until the next local IRQ before the signal/resched + * event would be handled. + * + * This also means that if we have to deal with user + * windows, we have to redo all of these sched+signal checks + * with IRQs disabled. + */ +to_user: wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate + wrpr 0, %pil +__handle_preemption_continue: + ldx [%g6 + TI_FLAGS], %l0 + sethi %hi(_TIF_USER_WORK_MASK), %o0 + or %o0, %lo(_TIF_USER_WORK_MASK), %o0 + andcc %l0, %o0, %g0 + sethi %hi(TSTATE_PEF), %o0 + be,pt %xcc, user_nowork + andcc %l1, %o0, %g0 + andcc %l0, _TIF_NEED_RESCHED, %g0 + bne,pn %xcc, __handle_preemption + andcc %l0, _TIF_DO_NOTIFY_RESUME_MASK, %g0 + bne,pn %xcc, __handle_signal + ldub [%g6 + TI_WSAVED], %o2 + brnz,pn %o2, __handle_user_windows + nop + sethi %hi(TSTATE_PEF), %o0 + andcc %l1, %o0, %g0 + + /* This fpdepth clear is necessary for non-syscall rtraps only */ +user_nowork: + bne,pn %xcc, __handle_userfpu + stb %g0, [%g6 + TI_FPDEPTH] +__handle_userfpu_continue: + +rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 + ldx [%sp + PTREGS_OFF + PT_V9_G2], %g2 + + ldx [%sp + PTREGS_OFF + PT_V9_G3], %g3 + ldx [%sp + PTREGS_OFF + PT_V9_G4], %g4 + ldx [%sp + PTREGS_OFF + PT_V9_G5], %g5 + brz,pt %l3, 1f + mov %g6, %l2 + + /* Must do this before thread reg is clobbered below. */ + LOAD_PER_CPU_BASE(%g5, %g6, %i0, %i1, %i2) +1: + ldx [%sp + PTREGS_OFF + PT_V9_G6], %g6 + ldx [%sp + PTREGS_OFF + PT_V9_G7], %g7 + + /* Normal globals are restored, go to trap globals. */ +661: wrpr %g0, RTRAP_PSTATE_AG_IRQOFF, %pstate + nop + .section .sun4v_2insn_patch, "ax" + .word 661b + wrpr %g0, RTRAP_PSTATE_IRQOFF, %pstate + SET_GL(1) + .previous + + mov %l2, %g6 + + ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 + ldx [%sp + PTREGS_OFF + PT_V9_I1], %i1 + + ldx [%sp + PTREGS_OFF + PT_V9_I2], %i2 + ldx [%sp + PTREGS_OFF + PT_V9_I3], %i3 + ldx [%sp + PTREGS_OFF + PT_V9_I4], %i4 + ldx [%sp + PTREGS_OFF + PT_V9_I5], %i5 + ldx [%sp + PTREGS_OFF + PT_V9_I6], %i6 + ldx [%sp + PTREGS_OFF + PT_V9_I7], %i7 + ldx [%sp + PTREGS_OFF + PT_V9_TPC], %l2 + ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %o2 + + ld [%sp + PTREGS_OFF + PT_V9_Y], %o3 + wr %o3, %g0, %y + wrpr %l4, 0x0, %pil + wrpr %g0, 0x1, %tl + andn %l1, TSTATE_SYSCALL, %l1 + wrpr %l1, %g0, %tstate + wrpr %l2, %g0, %tpc + wrpr %o2, %g0, %tnpc + + brnz,pn %l3, kern_rtt + mov PRIMARY_CONTEXT, %l7 + +661: ldxa [%l7 + %l7] ASI_DMMU, %l0 + .section .sun4v_1insn_patch, "ax" + .word 661b + ldxa [%l7 + %l7] ASI_MMU, %l0 + .previous + + sethi %hi(sparc64_kern_pri_nuc_bits), %l1 + ldx [%l1 + %lo(sparc64_kern_pri_nuc_bits)], %l1 + or %l0, %l1, %l0 + +661: stxa %l0, [%l7] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %l0, [%l7] ASI_MMU + .previous + + sethi %hi(KERNBASE), %l7 + flush %l7 + rdpr %wstate, %l1 + rdpr %otherwin, %l2 + srl %l1, 3, %l1 + +661: wrpr %l2, %g0, %canrestore + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + .word 0x89880000 ! normalw + .previous + + wrpr %l1, %g0, %wstate + brnz,pt %l2, user_rtt_restore +661: wrpr %g0, %g0, %otherwin + .section .fast_win_ctrl_1insn_patch, "ax" + .word 661b + nop + .previous + + ldx [%g6 + TI_FLAGS], %g3 + wr %g0, ASI_AIUP, %asi + rdpr %cwp, %g1 + andcc %g3, _TIF_32BIT, %g0 + sub %g1, 1, %g1 + bne,pt %xcc, user_rtt_fill_32bit + wrpr %g1, %cwp + ba,a,pt %xcc, user_rtt_fill_64bit + nop + +user_rtt_fill_fixup_dax: + ba,pt %xcc, user_rtt_fill_fixup_common + mov 1, %g3 + +user_rtt_fill_fixup_mna: + ba,pt %xcc, user_rtt_fill_fixup_common + mov 2, %g3 + +user_rtt_fill_fixup: + ba,pt %xcc, user_rtt_fill_fixup_common + clr %g3 + +user_rtt_pre_restore: + add %g1, 1, %g1 + wrpr %g1, 0x0, %cwp + +user_rtt_restore: + restore + rdpr %canrestore, %g1 + wrpr %g1, 0x0, %cleanwin + retry + nop + +kern_rtt: rdpr %canrestore, %g1 + brz,pn %g1, kern_rtt_fill + nop +kern_rtt_restore: + stw %g0, [%sp + PTREGS_OFF + PT_V9_MAGIC] + restore + retry + +to_kernel: +#ifdef CONFIG_PREEMPTION + ldsw [%g6 + TI_PRE_COUNT], %l5 + brnz %l5, kern_fpucheck + ldx [%g6 + TI_FLAGS], %l5 + andcc %l5, _TIF_NEED_RESCHED, %g0 + be,pt %xcc, kern_fpucheck + nop + cmp %l4, 0 + bne,pn %xcc, kern_fpucheck + nop + call preempt_schedule_irq + nop + ba,pt %xcc, rtrap +#endif +kern_fpucheck: ldub [%g6 + TI_FPDEPTH], %l5 + brz,pt %l5, rt_continue + srl %l5, 1, %o0 + add %g6, TI_FPSAVED, %l6 + ldub [%l6 + %o0], %l2 + sub %l5, 2, %l5 + + add %g6, TI_GSR, %o1 + andcc %l2, (FPRS_FEF|FPRS_DU), %g0 + be,pt %icc, 2f + and %l2, FPRS_DL, %l6 + andcc %l2, FPRS_FEF, %g0 + be,pn %icc, 5f + sll %o0, 3, %o5 + rd %fprs, %g1 + + wr %g1, FPRS_FEF, %fprs + ldx [%o1 + %o5], %g1 + add %g6, TI_XFSR, %o1 + sll %o0, 8, %o2 + add %g6, TI_FPREGS, %o3 + brz,pn %l6, 1f + add %g6, TI_FPREGS+0x40, %o4 + + membar #Sync + ldda [%o3 + %o2] ASI_BLK_P, %f0 + ldda [%o4 + %o2] ASI_BLK_P, %f16 + membar #Sync +1: andcc %l2, FPRS_DU, %g0 + be,pn %icc, 1f + wr %g1, 0, %gsr + add %o2, 0x80, %o2 + membar #Sync + ldda [%o3 + %o2] ASI_BLK_P, %f32 + ldda [%o4 + %o2] ASI_BLK_P, %f48 +1: membar #Sync + ldx [%o1 + %o5], %fsr +2: stb %l5, [%g6 + TI_FPDEPTH] + ba,pt %xcc, rt_continue + nop +5: wr %g0, FPRS_FEF, %fprs + sll %o0, 8, %o2 + + add %g6, TI_FPREGS+0x80, %o3 + add %g6, TI_FPREGS+0xc0, %o4 + membar #Sync + ldda [%o3 + %o2] ASI_BLK_P, %f32 + ldda [%o4 + %o2] ASI_BLK_P, %f48 + membar #Sync + wr %g0, FPRS_DU, %fprs + ba,pt %xcc, rt_continue + stb %l5, [%g6 + TI_FPDEPTH] diff --git a/arch/sparc/kernel/sbus.c b/arch/sparc/kernel/sbus.c new file mode 100644 index 000000000..bb2794ce7 --- /dev/null +++ b/arch/sparc/kernel/sbus.c @@ -0,0 +1,679 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sbus.c: UltraSparc SBUS controller support. + * + * Copyright (C) 1999 David S. Miller (davem@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iommu_common.h" + +#define MAP_BASE ((u32)0xc0000000) + +/* Offsets from iommu_regs */ +#define SYSIO_IOMMUREG_BASE 0x2400UL +#define IOMMU_CONTROL (0x2400UL - 0x2400UL) /* IOMMU control register */ +#define IOMMU_TSBBASE (0x2408UL - 0x2400UL) /* TSB base address register */ +#define IOMMU_FLUSH (0x2410UL - 0x2400UL) /* IOMMU flush register */ +#define IOMMU_VADIAG (0x4400UL - 0x2400UL) /* SBUS virtual address diagnostic */ +#define IOMMU_TAGCMP (0x4408UL - 0x2400UL) /* TLB tag compare diagnostics */ +#define IOMMU_LRUDIAG (0x4500UL - 0x2400UL) /* IOMMU LRU queue diagnostics */ +#define IOMMU_TAGDIAG (0x4580UL - 0x2400UL) /* TLB tag diagnostics */ +#define IOMMU_DRAMDIAG (0x4600UL - 0x2400UL) /* TLB data RAM diagnostics */ + +#define IOMMU_DRAM_VALID (1UL << 30UL) + +/* Offsets from strbuf_regs */ +#define SYSIO_STRBUFREG_BASE 0x2800UL +#define STRBUF_CONTROL (0x2800UL - 0x2800UL) /* Control */ +#define STRBUF_PFLUSH (0x2808UL - 0x2800UL) /* Page flush/invalidate */ +#define STRBUF_FSYNC (0x2810UL - 0x2800UL) /* Flush synchronization */ +#define STRBUF_DRAMDIAG (0x5000UL - 0x2800UL) /* data RAM diagnostic */ +#define STRBUF_ERRDIAG (0x5400UL - 0x2800UL) /* error status diagnostics */ +#define STRBUF_PTAGDIAG (0x5800UL - 0x2800UL) /* Page tag diagnostics */ +#define STRBUF_LTAGDIAG (0x5900UL - 0x2800UL) /* Line tag diagnostics */ + +#define STRBUF_TAG_VALID 0x02UL + +/* Enable 64-bit DVMA mode for the given device. */ +void sbus_set_sbus64(struct device *dev, int bursts) +{ + struct iommu *iommu = dev->archdata.iommu; + struct platform_device *op = to_platform_device(dev); + const struct linux_prom_registers *regs; + unsigned long cfg_reg; + int slot; + u64 val; + + regs = of_get_property(op->dev.of_node, "reg", NULL); + if (!regs) { + printk(KERN_ERR "sbus_set_sbus64: Cannot find regs for %pOF\n", + op->dev.of_node); + return; + } + slot = regs->which_io; + + cfg_reg = iommu->write_complete_reg; + switch (slot) { + case 0: + cfg_reg += 0x20UL; + break; + case 1: + cfg_reg += 0x28UL; + break; + case 2: + cfg_reg += 0x30UL; + break; + case 3: + cfg_reg += 0x38UL; + break; + case 13: + cfg_reg += 0x40UL; + break; + case 14: + cfg_reg += 0x48UL; + break; + case 15: + cfg_reg += 0x50UL; + break; + + default: + return; + } + + val = upa_readq(cfg_reg); + if (val & (1UL << 14UL)) { + /* Extended transfer mode already enabled. */ + return; + } + + val |= (1UL << 14UL); + + if (bursts & DMA_BURST8) + val |= (1UL << 1UL); + if (bursts & DMA_BURST16) + val |= (1UL << 2UL); + if (bursts & DMA_BURST32) + val |= (1UL << 3UL); + if (bursts & DMA_BURST64) + val |= (1UL << 4UL); + upa_writeq(val, cfg_reg); +} +EXPORT_SYMBOL(sbus_set_sbus64); + +/* INO number to IMAP register offset for SYSIO external IRQ's. + * This should conform to both Sunfire/Wildfire server and Fusion + * desktop designs. + */ +#define SYSIO_IMAP_SLOT0 0x2c00UL +#define SYSIO_IMAP_SLOT1 0x2c08UL +#define SYSIO_IMAP_SLOT2 0x2c10UL +#define SYSIO_IMAP_SLOT3 0x2c18UL +#define SYSIO_IMAP_SCSI 0x3000UL +#define SYSIO_IMAP_ETH 0x3008UL +#define SYSIO_IMAP_BPP 0x3010UL +#define SYSIO_IMAP_AUDIO 0x3018UL +#define SYSIO_IMAP_PFAIL 0x3020UL +#define SYSIO_IMAP_KMS 0x3028UL +#define SYSIO_IMAP_FLPY 0x3030UL +#define SYSIO_IMAP_SHW 0x3038UL +#define SYSIO_IMAP_KBD 0x3040UL +#define SYSIO_IMAP_MS 0x3048UL +#define SYSIO_IMAP_SER 0x3050UL +#define SYSIO_IMAP_TIM0 0x3060UL +#define SYSIO_IMAP_TIM1 0x3068UL +#define SYSIO_IMAP_UE 0x3070UL +#define SYSIO_IMAP_CE 0x3078UL +#define SYSIO_IMAP_SBERR 0x3080UL +#define SYSIO_IMAP_PMGMT 0x3088UL +#define SYSIO_IMAP_GFX 0x3090UL +#define SYSIO_IMAP_EUPA 0x3098UL + +#define bogon ((unsigned long) -1) +static unsigned long sysio_irq_offsets[] = { + /* SBUS Slot 0 --> 3, level 1 --> 7 */ + SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, + SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, SYSIO_IMAP_SLOT0, + SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, + SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, SYSIO_IMAP_SLOT1, + SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, + SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, SYSIO_IMAP_SLOT2, + SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, + SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, SYSIO_IMAP_SLOT3, + + /* Onboard devices (not relevant/used on SunFire). */ + SYSIO_IMAP_SCSI, + SYSIO_IMAP_ETH, + SYSIO_IMAP_BPP, + bogon, + SYSIO_IMAP_AUDIO, + SYSIO_IMAP_PFAIL, + bogon, + bogon, + SYSIO_IMAP_KMS, + SYSIO_IMAP_FLPY, + SYSIO_IMAP_SHW, + SYSIO_IMAP_KBD, + SYSIO_IMAP_MS, + SYSIO_IMAP_SER, + bogon, + bogon, + SYSIO_IMAP_TIM0, + SYSIO_IMAP_TIM1, + bogon, + bogon, + SYSIO_IMAP_UE, + SYSIO_IMAP_CE, + SYSIO_IMAP_SBERR, + SYSIO_IMAP_PMGMT, +}; + +#undef bogon + +#define NUM_SYSIO_OFFSETS ARRAY_SIZE(sysio_irq_offsets) + +/* Convert Interrupt Mapping register pointer to associated + * Interrupt Clear register pointer, SYSIO specific version. + */ +#define SYSIO_ICLR_UNUSED0 0x3400UL +#define SYSIO_ICLR_SLOT0 0x3408UL +#define SYSIO_ICLR_SLOT1 0x3448UL +#define SYSIO_ICLR_SLOT2 0x3488UL +#define SYSIO_ICLR_SLOT3 0x34c8UL +static unsigned long sysio_imap_to_iclr(unsigned long imap) +{ + unsigned long diff = SYSIO_ICLR_UNUSED0 - SYSIO_IMAP_SLOT0; + return imap + diff; +} + +static unsigned int sbus_build_irq(struct platform_device *op, unsigned int ino) +{ + struct iommu *iommu = op->dev.archdata.iommu; + unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; + unsigned long imap, iclr; + int sbus_level = 0; + + imap = sysio_irq_offsets[ino]; + if (imap == ((unsigned long)-1)) { + prom_printf("get_irq_translations: Bad SYSIO INO[%x]\n", + ino); + prom_halt(); + } + imap += reg_base; + + /* SYSIO inconsistency. For external SLOTS, we have to select + * the right ICLR register based upon the lower SBUS irq level + * bits. + */ + if (ino >= 0x20) { + iclr = sysio_imap_to_iclr(imap); + } else { + int sbus_slot = (ino & 0x18)>>3; + + sbus_level = ino & 0x7; + + switch(sbus_slot) { + case 0: + iclr = reg_base + SYSIO_ICLR_SLOT0; + break; + case 1: + iclr = reg_base + SYSIO_ICLR_SLOT1; + break; + case 2: + iclr = reg_base + SYSIO_ICLR_SLOT2; + break; + default: + case 3: + iclr = reg_base + SYSIO_ICLR_SLOT3; + break; + } + + iclr += ((unsigned long)sbus_level - 1UL) * 8UL; + } + return build_irq(sbus_level, iclr, imap); +} + +/* Error interrupt handling. */ +#define SYSIO_UE_AFSR 0x0030UL +#define SYSIO_UE_AFAR 0x0038UL +#define SYSIO_UEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */ +#define SYSIO_UEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */ +#define SYSIO_UEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */ +#define SYSIO_UEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO is cause */ +#define SYSIO_UEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */ +#define SYSIO_UEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/ +#define SYSIO_UEAFSR_RESV1 0x03ff000000000000UL /* Reserved */ +#define SYSIO_UEAFSR_DOFF 0x0000e00000000000UL /* Doubleword Offset */ +#define SYSIO_UEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */ +#define SYSIO_UEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */ +#define SYSIO_UEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */ +static irqreturn_t sysio_ue_handler(int irq, void *dev_id) +{ + struct platform_device *op = dev_id; + struct iommu *iommu = op->dev.archdata.iommu; + unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; + unsigned long afsr_reg, afar_reg; + unsigned long afsr, afar, error_bits; + int reported, portid; + + afsr_reg = reg_base + SYSIO_UE_AFSR; + afar_reg = reg_base + SYSIO_UE_AFAR; + + /* Latch error status. */ + afsr = upa_readq(afsr_reg); + afar = upa_readq(afar_reg); + + /* Clear primary/secondary error status bits. */ + error_bits = afsr & + (SYSIO_UEAFSR_PPIO | SYSIO_UEAFSR_PDRD | SYSIO_UEAFSR_PDWR | + SYSIO_UEAFSR_SPIO | SYSIO_UEAFSR_SDRD | SYSIO_UEAFSR_SDWR); + upa_writeq(error_bits, afsr_reg); + + portid = of_getintprop_default(op->dev.of_node, "portid", -1); + + /* Log the error. */ + printk("SYSIO[%x]: Uncorrectable ECC Error, primary error type[%s]\n", + portid, + (((error_bits & SYSIO_UEAFSR_PPIO) ? + "PIO" : + ((error_bits & SYSIO_UEAFSR_PDRD) ? + "DVMA Read" : + ((error_bits & SYSIO_UEAFSR_PDWR) ? + "DVMA Write" : "???"))))); + printk("SYSIO[%x]: DOFF[%lx] SIZE[%lx] MID[%lx]\n", + portid, + (afsr & SYSIO_UEAFSR_DOFF) >> 45UL, + (afsr & SYSIO_UEAFSR_SIZE) >> 42UL, + (afsr & SYSIO_UEAFSR_MID) >> 37UL); + printk("SYSIO[%x]: AFAR[%016lx]\n", portid, afar); + printk("SYSIO[%x]: Secondary UE errors [", portid); + reported = 0; + if (afsr & SYSIO_UEAFSR_SPIO) { + reported++; + printk("(PIO)"); + } + if (afsr & SYSIO_UEAFSR_SDRD) { + reported++; + printk("(DVMA Read)"); + } + if (afsr & SYSIO_UEAFSR_SDWR) { + reported++; + printk("(DVMA Write)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + return IRQ_HANDLED; +} + +#define SYSIO_CE_AFSR 0x0040UL +#define SYSIO_CE_AFAR 0x0048UL +#define SYSIO_CEAFSR_PPIO 0x8000000000000000UL /* Primary PIO cause */ +#define SYSIO_CEAFSR_PDRD 0x4000000000000000UL /* Primary DVMA read cause */ +#define SYSIO_CEAFSR_PDWR 0x2000000000000000UL /* Primary DVMA write cause */ +#define SYSIO_CEAFSR_SPIO 0x1000000000000000UL /* Secondary PIO cause */ +#define SYSIO_CEAFSR_SDRD 0x0800000000000000UL /* Secondary DVMA read cause */ +#define SYSIO_CEAFSR_SDWR 0x0400000000000000UL /* Secondary DVMA write cause*/ +#define SYSIO_CEAFSR_RESV1 0x0300000000000000UL /* Reserved */ +#define SYSIO_CEAFSR_ESYND 0x00ff000000000000UL /* Syndrome Bits */ +#define SYSIO_CEAFSR_DOFF 0x0000e00000000000UL /* Double Offset */ +#define SYSIO_CEAFSR_SIZE 0x00001c0000000000UL /* Bad transfer size 2^SIZE */ +#define SYSIO_CEAFSR_MID 0x000003e000000000UL /* UPA MID causing the fault */ +#define SYSIO_CEAFSR_RESV2 0x0000001fffffffffUL /* Reserved */ +static irqreturn_t sysio_ce_handler(int irq, void *dev_id) +{ + struct platform_device *op = dev_id; + struct iommu *iommu = op->dev.archdata.iommu; + unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; + unsigned long afsr_reg, afar_reg; + unsigned long afsr, afar, error_bits; + int reported, portid; + + afsr_reg = reg_base + SYSIO_CE_AFSR; + afar_reg = reg_base + SYSIO_CE_AFAR; + + /* Latch error status. */ + afsr = upa_readq(afsr_reg); + afar = upa_readq(afar_reg); + + /* Clear primary/secondary error status bits. */ + error_bits = afsr & + (SYSIO_CEAFSR_PPIO | SYSIO_CEAFSR_PDRD | SYSIO_CEAFSR_PDWR | + SYSIO_CEAFSR_SPIO | SYSIO_CEAFSR_SDRD | SYSIO_CEAFSR_SDWR); + upa_writeq(error_bits, afsr_reg); + + portid = of_getintprop_default(op->dev.of_node, "portid", -1); + + printk("SYSIO[%x]: Correctable ECC Error, primary error type[%s]\n", + portid, + (((error_bits & SYSIO_CEAFSR_PPIO) ? + "PIO" : + ((error_bits & SYSIO_CEAFSR_PDRD) ? + "DVMA Read" : + ((error_bits & SYSIO_CEAFSR_PDWR) ? + "DVMA Write" : "???"))))); + + /* XXX Use syndrome and afar to print out module string just like + * XXX UDB CE trap handler does... -DaveM + */ + printk("SYSIO[%x]: DOFF[%lx] ECC Syndrome[%lx] Size[%lx] MID[%lx]\n", + portid, + (afsr & SYSIO_CEAFSR_DOFF) >> 45UL, + (afsr & SYSIO_CEAFSR_ESYND) >> 48UL, + (afsr & SYSIO_CEAFSR_SIZE) >> 42UL, + (afsr & SYSIO_CEAFSR_MID) >> 37UL); + printk("SYSIO[%x]: AFAR[%016lx]\n", portid, afar); + + printk("SYSIO[%x]: Secondary CE errors [", portid); + reported = 0; + if (afsr & SYSIO_CEAFSR_SPIO) { + reported++; + printk("(PIO)"); + } + if (afsr & SYSIO_CEAFSR_SDRD) { + reported++; + printk("(DVMA Read)"); + } + if (afsr & SYSIO_CEAFSR_SDWR) { + reported++; + printk("(DVMA Write)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + return IRQ_HANDLED; +} + +#define SYSIO_SBUS_AFSR 0x2010UL +#define SYSIO_SBUS_AFAR 0x2018UL +#define SYSIO_SBAFSR_PLE 0x8000000000000000UL /* Primary Late PIO Error */ +#define SYSIO_SBAFSR_PTO 0x4000000000000000UL /* Primary SBUS Timeout */ +#define SYSIO_SBAFSR_PBERR 0x2000000000000000UL /* Primary SBUS Error ACK */ +#define SYSIO_SBAFSR_SLE 0x1000000000000000UL /* Secondary Late PIO Error */ +#define SYSIO_SBAFSR_STO 0x0800000000000000UL /* Secondary SBUS Timeout */ +#define SYSIO_SBAFSR_SBERR 0x0400000000000000UL /* Secondary SBUS Error ACK */ +#define SYSIO_SBAFSR_RESV1 0x03ff000000000000UL /* Reserved */ +#define SYSIO_SBAFSR_RD 0x0000800000000000UL /* Primary was late PIO read */ +#define SYSIO_SBAFSR_RESV2 0x0000600000000000UL /* Reserved */ +#define SYSIO_SBAFSR_SIZE 0x00001c0000000000UL /* Size of transfer */ +#define SYSIO_SBAFSR_MID 0x000003e000000000UL /* MID causing the error */ +#define SYSIO_SBAFSR_RESV3 0x0000001fffffffffUL /* Reserved */ +static irqreturn_t sysio_sbus_error_handler(int irq, void *dev_id) +{ + struct platform_device *op = dev_id; + struct iommu *iommu = op->dev.archdata.iommu; + unsigned long afsr_reg, afar_reg, reg_base; + unsigned long afsr, afar, error_bits; + int reported, portid; + + reg_base = iommu->write_complete_reg - 0x2000UL; + afsr_reg = reg_base + SYSIO_SBUS_AFSR; + afar_reg = reg_base + SYSIO_SBUS_AFAR; + + afsr = upa_readq(afsr_reg); + afar = upa_readq(afar_reg); + + /* Clear primary/secondary error status bits. */ + error_bits = afsr & + (SYSIO_SBAFSR_PLE | SYSIO_SBAFSR_PTO | SYSIO_SBAFSR_PBERR | + SYSIO_SBAFSR_SLE | SYSIO_SBAFSR_STO | SYSIO_SBAFSR_SBERR); + upa_writeq(error_bits, afsr_reg); + + portid = of_getintprop_default(op->dev.of_node, "portid", -1); + + /* Log the error. */ + printk("SYSIO[%x]: SBUS Error, primary error type[%s] read(%d)\n", + portid, + (((error_bits & SYSIO_SBAFSR_PLE) ? + "Late PIO Error" : + ((error_bits & SYSIO_SBAFSR_PTO) ? + "Time Out" : + ((error_bits & SYSIO_SBAFSR_PBERR) ? + "Error Ack" : "???")))), + (afsr & SYSIO_SBAFSR_RD) ? 1 : 0); + printk("SYSIO[%x]: size[%lx] MID[%lx]\n", + portid, + (afsr & SYSIO_SBAFSR_SIZE) >> 42UL, + (afsr & SYSIO_SBAFSR_MID) >> 37UL); + printk("SYSIO[%x]: AFAR[%016lx]\n", portid, afar); + printk("SYSIO[%x]: Secondary SBUS errors [", portid); + reported = 0; + if (afsr & SYSIO_SBAFSR_SLE) { + reported++; + printk("(Late PIO Error)"); + } + if (afsr & SYSIO_SBAFSR_STO) { + reported++; + printk("(Time Out)"); + } + if (afsr & SYSIO_SBAFSR_SBERR) { + reported++; + printk("(Error Ack)"); + } + if (!reported) + printk("(none)"); + printk("]\n"); + + /* XXX check iommu/strbuf for further error status XXX */ + + return IRQ_HANDLED; +} + +#define ECC_CONTROL 0x0020UL +#define SYSIO_ECNTRL_ECCEN 0x8000000000000000UL /* Enable ECC Checking */ +#define SYSIO_ECNTRL_UEEN 0x4000000000000000UL /* Enable UE Interrupts */ +#define SYSIO_ECNTRL_CEEN 0x2000000000000000UL /* Enable CE Interrupts */ + +#define SYSIO_UE_INO 0x34 +#define SYSIO_CE_INO 0x35 +#define SYSIO_SBUSERR_INO 0x36 + +static void __init sysio_register_error_handlers(struct platform_device *op) +{ + struct iommu *iommu = op->dev.archdata.iommu; + unsigned long reg_base = iommu->write_complete_reg - 0x2000UL; + unsigned int irq; + u64 control; + int portid; + + portid = of_getintprop_default(op->dev.of_node, "portid", -1); + + irq = sbus_build_irq(op, SYSIO_UE_INO); + if (request_irq(irq, sysio_ue_handler, 0, + "SYSIO_UE", op) < 0) { + prom_printf("SYSIO[%x]: Cannot register UE interrupt.\n", + portid); + prom_halt(); + } + + irq = sbus_build_irq(op, SYSIO_CE_INO); + if (request_irq(irq, sysio_ce_handler, 0, + "SYSIO_CE", op) < 0) { + prom_printf("SYSIO[%x]: Cannot register CE interrupt.\n", + portid); + prom_halt(); + } + + irq = sbus_build_irq(op, SYSIO_SBUSERR_INO); + if (request_irq(irq, sysio_sbus_error_handler, 0, + "SYSIO_SBERR", op) < 0) { + prom_printf("SYSIO[%x]: Cannot register SBUS Error interrupt.\n", + portid); + prom_halt(); + } + + /* Now turn the error interrupts on and also enable ECC checking. */ + upa_writeq((SYSIO_ECNTRL_ECCEN | + SYSIO_ECNTRL_UEEN | + SYSIO_ECNTRL_CEEN), + reg_base + ECC_CONTROL); + + control = upa_readq(iommu->write_complete_reg); + control |= 0x100UL; /* SBUS Error Interrupt Enable */ + upa_writeq(control, iommu->write_complete_reg); +} + +/* Boot time initialization. */ +static void __init sbus_iommu_init(struct platform_device *op) +{ + const struct linux_prom64_registers *pr; + struct device_node *dp = op->dev.of_node; + struct iommu *iommu; + struct strbuf *strbuf; + unsigned long regs, reg_base; + int i, portid; + u64 control; + + pr = of_get_property(dp, "reg", NULL); + if (!pr) { + prom_printf("sbus_iommu_init: Cannot map SYSIO " + "control registers.\n"); + prom_halt(); + } + regs = pr->phys_addr; + + iommu = kzalloc_obj(*iommu, GFP_ATOMIC); + strbuf = kzalloc_obj(*strbuf, GFP_ATOMIC); + if (!iommu || !strbuf) + goto fatal_memory_error; + + op->dev.archdata.iommu = iommu; + op->dev.archdata.stc = strbuf; + op->dev.archdata.numa_node = NUMA_NO_NODE; + + reg_base = regs + SYSIO_IOMMUREG_BASE; + iommu->iommu_control = reg_base + IOMMU_CONTROL; + iommu->iommu_tsbbase = reg_base + IOMMU_TSBBASE; + iommu->iommu_flush = reg_base + IOMMU_FLUSH; + iommu->iommu_tags = iommu->iommu_control + + (IOMMU_TAGDIAG - IOMMU_CONTROL); + + reg_base = regs + SYSIO_STRBUFREG_BASE; + strbuf->strbuf_control = reg_base + STRBUF_CONTROL; + strbuf->strbuf_pflush = reg_base + STRBUF_PFLUSH; + strbuf->strbuf_fsync = reg_base + STRBUF_FSYNC; + + strbuf->strbuf_enabled = 1; + + strbuf->strbuf_flushflag = (volatile unsigned long *) + ((((unsigned long)&strbuf->__flushflag_buf[0]) + + 63UL) + & ~63UL); + strbuf->strbuf_flushflag_pa = (unsigned long) + __pa(strbuf->strbuf_flushflag); + + /* The SYSIO SBUS control register is used for dummy reads + * in order to ensure write completion. + */ + iommu->write_complete_reg = regs + 0x2000UL; + + portid = of_getintprop_default(op->dev.of_node, "portid", -1); + printk(KERN_INFO "SYSIO: UPA portID %x, at %016lx\n", + portid, regs); + + /* Setup for TSB_SIZE=7, TBW_SIZE=0, MMU_DE=1, MMU_EN=1 */ + if (iommu_table_init(iommu, IO_TSB_SIZE, MAP_BASE, 0xffffffff, -1)) + goto fatal_memory_error; + + control = upa_readq(iommu->iommu_control); + control = ((7UL << 16UL) | + (0UL << 2UL) | + (1UL << 1UL) | + (1UL << 0UL)); + upa_writeq(control, iommu->iommu_control); + + /* Clean out any cruft in the IOMMU using + * diagnostic accesses. + */ + for (i = 0; i < 16; i++) { + unsigned long dram, tag; + + dram = iommu->iommu_control + (IOMMU_DRAMDIAG - IOMMU_CONTROL); + tag = iommu->iommu_control + (IOMMU_TAGDIAG - IOMMU_CONTROL); + + dram += (unsigned long)i * 8UL; + tag += (unsigned long)i * 8UL; + upa_writeq(0, dram); + upa_writeq(0, tag); + } + upa_readq(iommu->write_complete_reg); + + /* Give the TSB to SYSIO. */ + upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase); + + /* Setup streaming buffer, DE=1 SB_EN=1 */ + control = (1UL << 1UL) | (1UL << 0UL); + upa_writeq(control, strbuf->strbuf_control); + + /* Clear out the tags using diagnostics. */ + for (i = 0; i < 16; i++) { + unsigned long ptag, ltag; + + ptag = strbuf->strbuf_control + + (STRBUF_PTAGDIAG - STRBUF_CONTROL); + ltag = strbuf->strbuf_control + + (STRBUF_LTAGDIAG - STRBUF_CONTROL); + ptag += (unsigned long)i * 8UL; + ltag += (unsigned long)i * 8UL; + + upa_writeq(0UL, ptag); + upa_writeq(0UL, ltag); + } + + /* Enable DVMA arbitration for all devices/slots. */ + control = upa_readq(iommu->write_complete_reg); + control |= 0x3fUL; + upa_writeq(control, iommu->write_complete_reg); + + /* Now some Xfire specific grot... */ + if (this_is_starfire) + starfire_hookup(portid); + + sysio_register_error_handlers(op); + return; + +fatal_memory_error: + kfree(iommu); + kfree(strbuf); + prom_printf("sbus_iommu_init: Fatal memory allocation error.\n"); +} + +static int __init sbus_init(void) +{ + struct device_node *dp; + + for_each_node_by_name(dp, "sbus") { + struct platform_device *op = of_find_device_by_node(dp); + + sbus_iommu_init(op); + of_propagate_archdata(op); + } + + return 0; +} + +subsys_initcall(sbus_init); diff --git a/arch/sparc/kernel/setup.c b/arch/sparc/kernel/setup.c new file mode 100644 index 000000000..4975867d9 --- /dev/null +++ b/arch/sparc/kernel/setup.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include + +static const struct ctl_table sparc_sysctl_table[] = { + { + .procname = "reboot-cmd", + .data = reboot_command, + .maxlen = 256, + .mode = 0644, + .proc_handler = proc_dostring, + }, + { + .procname = "stop-a", + .data = &stop_a_enabled, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, + { + .procname = "scons-poweroff", + .data = &scons_pwroff, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#ifdef CONFIG_SPARC64 + { + .procname = "tsb-ratio", + .data = &sysctl_tsb_ratio, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec, + }, +#endif +}; + + +static int __init init_sparc_sysctls(void) +{ + register_sysctl_init("kernel", sparc_sysctl_table); + return 0; +} + +arch_initcall(init_sparc_sysctls); diff --git a/arch/sparc/kernel/setup_32.c b/arch/sparc/kernel/setup_32.c new file mode 100644 index 000000000..1b0db16cd --- /dev/null +++ b/arch/sparc/kernel/setup_32.c @@ -0,0 +1,408 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/sparc/kernel/setup.c + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 2000 Anton Blanchard (anton@samba.org) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" + +/* Typing sync at the prom prompt calls the function pointed to by + * romvec->pv_synchook which I set to the following function. + * This should sync all filesystems and return, for now it just + * prints out pretty messages and returns. + */ + +/* Pretty sick eh? */ +static void prom_sync_me(void) +{ + unsigned long prom_tbr, flags; + + /* XXX Badly broken. FIX! - Anton */ + local_irq_save(flags); + __asm__ __volatile__("rd %%tbr, %0\n\t" : "=r" (prom_tbr)); + __asm__ __volatile__("wr %0, 0x0, %%tbr\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" : : "r" (&trapbase[0])); + + prom_printf("PROM SYNC COMMAND...\n"); + show_mem(); + if (!is_idle_task(current)) { + local_irq_enable(); + ksys_sync(); + local_irq_disable(); + } + prom_printf("Returning to prom\n"); + + __asm__ __volatile__("wr %0, 0x0, %%tbr\n\t" + "nop\n\t" + "nop\n\t" + "nop\n\t" : : "r" (prom_tbr)); + local_irq_restore(flags); +} + +static unsigned int boot_flags __initdata = 0; +#define BOOTME_DEBUG 0x1 + +/* Exported for mm/init.c:paging_init. */ +unsigned long cmdline_memory_size __initdata = 0; + +/* which CPU booted us (0xff = not set) */ +unsigned char boot_cpu_id = 0xff; /* 0xff will make it into DATA section... */ + +static void +prom_console_write(struct console *con, const char *s, unsigned int n) +{ + prom_write(s, n); +} + +static struct console prom_early_console = { + .name = "earlyprom", + .write = prom_console_write, + .flags = CON_PRINTBUFFER | CON_BOOT, + .index = -1, +}; + +/* + * Process kernel command line switches that are specific to the + * SPARC or that require special low-level processing. + */ +static void __init process_switch(char c) +{ + switch (c) { + case 'd': + boot_flags |= BOOTME_DEBUG; + break; + case 's': + break; + case 'h': + prom_printf("boot_flags_init: Halt!\n"); + prom_halt(); + break; + case 'p': + prom_early_console.flags &= ~CON_BOOT; + break; + default: + printk("Unknown boot switch (-%c)\n", c); + break; + } +} + +static void __init boot_flags_init(char *commands) +{ + while (*commands) { + /* Move to the start of the next "argument". */ + while (*commands == ' ') + commands++; + + /* Process any command switches, otherwise skip it. */ + if (*commands == '\0') + break; + if (*commands == '-') { + commands++; + while (*commands && *commands != ' ') + process_switch(*commands++); + continue; + } + if (!strncmp(commands, "mem=", 4)) { + /* + * "mem=XXX[kKmM] overrides the PROM-reported + * memory size. + */ + cmdline_memory_size = simple_strtoul(commands + 4, + &commands, 0); + if (*commands == 'K' || *commands == 'k') { + cmdline_memory_size <<= 10; + commands++; + } else if (*commands=='M' || *commands=='m') { + cmdline_memory_size <<= 20; + commands++; + } + } + while (*commands && *commands != ' ') + commands++; + } +} + +extern unsigned short root_flags; +extern unsigned short root_dev; +extern unsigned short ram_flags; +#define RAMDISK_IMAGE_START_MASK 0x07FF +#define RAMDISK_PROMPT_FLAG 0x8000 +#define RAMDISK_LOAD_FLAG 0x4000 + +extern int root_mountflags; + +char reboot_command[COMMAND_LINE_SIZE]; + +struct cpuid_patch_entry { + unsigned int addr; + unsigned int sun4d[3]; + unsigned int leon[3]; +}; +extern struct cpuid_patch_entry __cpuid_patch, __cpuid_patch_end; + +static void __init per_cpu_patch(void) +{ + struct cpuid_patch_entry *p; + + if (sparc_cpu_model == sun4m) { + /* Nothing to do, this is what the unpatched code + * targets. + */ + return; + } + + p = &__cpuid_patch; + while (p < &__cpuid_patch_end) { + unsigned long addr = p->addr; + unsigned int *insns; + + switch (sparc_cpu_model) { + case sun4d: + insns = &p->sun4d[0]; + break; + + case sparc_leon: + insns = &p->leon[0]; + break; + default: + prom_printf("Unknown cpu type, halting.\n"); + prom_halt(); + } + *(unsigned int *) (addr + 0) = insns[0]; + flushi(addr + 0); + *(unsigned int *) (addr + 4) = insns[1]; + flushi(addr + 4); + *(unsigned int *) (addr + 8) = insns[2]; + flushi(addr + 8); + + p++; + } +} + +struct leon_1insn_patch_entry { + unsigned int addr; + unsigned int insn; +}; + +enum sparc_cpu sparc_cpu_model; +EXPORT_SYMBOL(sparc_cpu_model); + +static __init void leon_patch(void) +{ + struct leon_1insn_patch_entry *start = (void *)__leon_1insn_patch; + struct leon_1insn_patch_entry *end = (void *)__leon_1insn_patch_end; + + /* Default instruction is leon - no patching */ + if (sparc_cpu_model == sparc_leon) + return; + + while (start < end) { + unsigned long addr = start->addr; + + *(unsigned int *)(addr) = start->insn; + flushi(addr); + + start++; + } +} + +struct tt_entry *sparc_ttable; + +/* Called from head_32.S - before we have setup anything + * in the kernel. Be very careful with what you do here. + */ +void __init sparc32_start_kernel(struct linux_romvec *rp) +{ + prom_init(rp); + + /* Set sparc_cpu_model */ + sparc_cpu_model = sun_unknown; + if (!strcmp(&cputypval[0], "sun4m")) + sparc_cpu_model = sun4m; + if (!strcmp(&cputypval[0], "sun4s")) + sparc_cpu_model = sun4m; /* CP-1200 with PROM 2.30 -E */ + if (!strcmp(&cputypval[0], "sun4d")) + sparc_cpu_model = sun4d; + if (!strcmp(&cputypval[0], "sun4e")) + sparc_cpu_model = sun4e; + if (!strcmp(&cputypval[0], "sun4u")) + sparc_cpu_model = sun4u; + if (!strncmp(&cputypval[0], "leon" , 4)) + sparc_cpu_model = sparc_leon; + + leon_patch(); + start_kernel(); +} + +void __init setup_arch(char **cmdline_p) +{ + int i; + unsigned long highest_paddr; + + sparc_ttable = &trapbase[0]; + + /* Initialize PROM console and command line. */ + *cmdline_p = prom_getbootargs(); + strscpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); + parse_early_param(); + + boot_flags_init(*cmdline_p); + + register_console(&prom_early_console); + + switch(sparc_cpu_model) { + case sun4m: + pr_info("ARCH: SUN4M\n"); + break; + case sun4d: + pr_info("ARCH: SUN4D\n"); + break; + case sun4e: + pr_info("ARCH: SUN4E\n"); + break; + case sun4u: + pr_info("ARCH: SUN4U\n"); + break; + case sparc_leon: + pr_info("ARCH: LEON\n"); + break; + default: + pr_info("ARCH: UNKNOWN!\n"); + break; + } + + idprom_init(); + load_mmu(); + + phys_base = 0xffffffffUL; + highest_paddr = 0UL; + for (i = 0; sp_banks[i].num_bytes != 0; i++) { + unsigned long top; + + if (sp_banks[i].base_addr < phys_base) + phys_base = sp_banks[i].base_addr; + top = sp_banks[i].base_addr + + sp_banks[i].num_bytes; + if (highest_paddr < top) + highest_paddr = top; + } + pfn_base = phys_base >> PAGE_SHIFT; + + if (!root_flags) + root_mountflags &= ~MS_RDONLY; + ROOT_DEV = old_decode_dev(root_dev); +#ifdef CONFIG_BLK_DEV_RAM + rd_image_start = ram_flags & RAMDISK_IMAGE_START_MASK; +#endif + + prom_setsync(prom_sync_me); + + if((boot_flags & BOOTME_DEBUG) && (linux_dbvec != NULL) && + ((*(short *)linux_dbvec) != -1)) { + printk("Booted under KADB. Syncing trap table.\n"); + (*(linux_dbvec->teach_debugger))(); + } + + /* Run-time patch instructions to match the cpu model */ + per_cpu_patch(); + + paging_init(); + + smp_setup_cpu_possible_map(); +} + +extern int stop_a_enabled; + +void sun_do_break(void) +{ + if (!stop_a_enabled) + return; + + printk("\n"); + flush_user_windows(); + + prom_cmdline(); +} +EXPORT_SYMBOL(sun_do_break); + +int stop_a_enabled = 1; + +static int __init topology_init(void) +{ + int i, ncpus, err; + + /* Count the number of physically present processors in + * the machine, even on uniprocessor, so that /proc/cpuinfo + * output is consistent with 2.4.x + */ + ncpus = 0; + while (!cpu_find_by_instance(ncpus, NULL, NULL)) + ncpus++; + ncpus_probed = ncpus; + + err = 0; + for_each_online_cpu(i) { + struct cpu *p = kzalloc_obj(*p); + if (!p) + err = -ENOMEM; + else + register_cpu(p, i); + } + + return err; +} + +subsys_initcall(topology_init); + +#if defined(CONFIG_SPARC32) && !defined(CONFIG_SMP) +void __init arch_cpu_finalize_init(void) +{ + cpu_data(0).udelay_val = loops_per_jiffy; +} +#endif diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c new file mode 100644 index 000000000..63615f5c9 --- /dev/null +++ b/arch/sparc/kernel/setup_64.c @@ -0,0 +1,697 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/sparc64/kernel/setup.c + * + * Copyright (C) 1995,1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_IP_PNP +#include +#endif + +#include "entry.h" +#include "kernel.h" + +/* Used to synchronize accesses to NatSemi SUPER I/O chip configure + * operations in asm/ns87303.h + */ +DEFINE_SPINLOCK(ns87303_lock); +EXPORT_SYMBOL(ns87303_lock); + +static void +prom_console_write(struct console *con, const char *s, unsigned int n) +{ + prom_write(s, n); +} + +/* Exported for mm/init.c:paging_init. */ +unsigned long cmdline_memory_size = 0; + +static struct console prom_early_console = { + .name = "earlyprom", + .write = prom_console_write, + .flags = CON_PRINTBUFFER | CON_BOOT | CON_ANYTIME, + .index = -1, +}; + +/* + * Process kernel command line switches that are specific to the + * SPARC or that require special low-level processing. + */ +static void __init process_switch(char c) +{ + switch (c) { + case 'd': + case 's': + break; + case 'h': + prom_printf("boot_flags_init: Halt!\n"); + prom_halt(); + break; + case 'p': + prom_early_console.flags &= ~CON_BOOT; + break; + case 'P': + /* Force UltraSPARC-III P-Cache on. */ + if (tlb_type != cheetah) { + printk("BOOT: Ignoring P-Cache force option.\n"); + break; + } + cheetah_pcache_forced_on = 1; + add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); + cheetah_enable_pcache(); + break; + + default: + printk("Unknown boot switch (-%c)\n", c); + break; + } +} + +static void __init boot_flags_init(char *commands) +{ + while (*commands) { + /* Move to the start of the next "argument". */ + while (*commands == ' ') + commands++; + + /* Process any command switches, otherwise skip it. */ + if (*commands == '\0') + break; + if (*commands == '-') { + commands++; + while (*commands && *commands != ' ') + process_switch(*commands++); + continue; + } + if (!strncmp(commands, "mem=", 4)) + cmdline_memory_size = memparse(commands + 4, &commands); + + while (*commands && *commands != ' ') + commands++; + } +} + +extern unsigned short root_flags; +extern unsigned short root_dev; +extern unsigned short ram_flags; +#define RAMDISK_IMAGE_START_MASK 0x07FF +#define RAMDISK_PROMPT_FLAG 0x8000 +#define RAMDISK_LOAD_FLAG 0x4000 + +extern int root_mountflags; + +char reboot_command[COMMAND_LINE_SIZE]; + +static void __init per_cpu_patch(void) +{ + struct cpuid_patch_entry *p; + unsigned long ver; + int is_jbus; + + if (tlb_type == spitfire && !this_is_starfire) + return; + + is_jbus = 0; + if (tlb_type != hypervisor) { + __asm__ ("rdpr %%ver, %0" : "=r" (ver)); + is_jbus = ((ver >> 32UL) == __JALAPENO_ID || + (ver >> 32UL) == __SERRANO_ID); + } + + p = &__cpuid_patch; + while (p < &__cpuid_patch_end) { + unsigned long addr = p->addr; + unsigned int *insns; + + switch (tlb_type) { + case spitfire: + insns = &p->starfire[0]; + break; + case cheetah: + case cheetah_plus: + if (is_jbus) + insns = &p->cheetah_jbus[0]; + else + insns = &p->cheetah_safari[0]; + break; + case hypervisor: + insns = &p->sun4v[0]; + break; + default: + prom_printf("Unknown cpu type, halting.\n"); + prom_halt(); + } + + *(unsigned int *) (addr + 0) = insns[0]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 0)); + + *(unsigned int *) (addr + 4) = insns[1]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 4)); + + *(unsigned int *) (addr + 8) = insns[2]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 8)); + + *(unsigned int *) (addr + 12) = insns[3]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 12)); + + p++; + } +} + +void sun4v_patch_1insn_range(struct sun4v_1insn_patch_entry *start, + struct sun4v_1insn_patch_entry *end) +{ + while (start < end) { + unsigned long addr = start->addr; + + *(unsigned int *) (addr + 0) = start->insn; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 0)); + + start++; + } +} + +void sun4v_patch_2insn_range(struct sun4v_2insn_patch_entry *start, + struct sun4v_2insn_patch_entry *end) +{ + while (start < end) { + unsigned long addr = start->addr; + + *(unsigned int *) (addr + 0) = start->insns[0]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 0)); + + *(unsigned int *) (addr + 4) = start->insns[1]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 4)); + + start++; + } +} + +void sun_m7_patch_2insn_range(struct sun4v_2insn_patch_entry *start, + struct sun4v_2insn_patch_entry *end) +{ + while (start < end) { + unsigned long addr = start->addr; + + *(unsigned int *) (addr + 0) = start->insns[0]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 0)); + + *(unsigned int *) (addr + 4) = start->insns[1]; + wmb(); + __asm__ __volatile__("flush %0" : : "r" (addr + 4)); + + start++; + } +} + +static void __init sun4v_patch(void) +{ + extern void sun4v_hvapi_init(void); + + if (tlb_type != hypervisor) + return; + + sun4v_patch_1insn_range(&__sun4v_1insn_patch, + &__sun4v_1insn_patch_end); + + sun4v_patch_2insn_range(&__sun4v_2insn_patch, + &__sun4v_2insn_patch_end); + + switch (sun4v_chip_type) { + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_M8: + case SUN4V_CHIP_SPARC_SN: + sun4v_patch_1insn_range(&__sun_m7_1insn_patch, + &__sun_m7_1insn_patch_end); + sun_m7_patch_2insn_range(&__sun_m7_2insn_patch, + &__sun_m7_2insn_patch_end); + break; + default: + break; + } + + if (sun4v_chip_type != SUN4V_CHIP_NIAGARA1) { + sun4v_patch_1insn_range(&__fast_win_ctrl_1insn_patch, + &__fast_win_ctrl_1insn_patch_end); + } + + sun4v_hvapi_init(); +} + +static void __init popc_patch(void) +{ + struct popc_3insn_patch_entry *p3; + struct popc_6insn_patch_entry *p6; + + p3 = &__popc_3insn_patch; + while (p3 < &__popc_3insn_patch_end) { + unsigned long i, addr = p3->addr; + + for (i = 0; i < 3; i++) { + *(unsigned int *) (addr + (i * 4)) = p3->insns[i]; + wmb(); + __asm__ __volatile__("flush %0" + : : "r" (addr + (i * 4))); + } + + p3++; + } + + p6 = &__popc_6insn_patch; + while (p6 < &__popc_6insn_patch_end) { + unsigned long i, addr = p6->addr; + + for (i = 0; i < 6; i++) { + *(unsigned int *) (addr + (i * 4)) = p6->insns[i]; + wmb(); + __asm__ __volatile__("flush %0" + : : "r" (addr + (i * 4))); + } + + p6++; + } +} + +static void __init pause_patch(void) +{ + struct pause_patch_entry *p; + + p = &__pause_3insn_patch; + while (p < &__pause_3insn_patch_end) { + unsigned long i, addr = p->addr; + + for (i = 0; i < 3; i++) { + *(unsigned int *) (addr + (i * 4)) = p->insns[i]; + wmb(); + __asm__ __volatile__("flush %0" + : : "r" (addr + (i * 4))); + } + + p++; + } +} + +void __init start_early_boot(void) +{ + int cpu; + + check_if_starfire(); + per_cpu_patch(); + sun4v_patch(); + smp_init_cpu_poke(); + + cpu = hard_smp_processor_id(); + if (cpu >= NR_CPUS) { + prom_printf("Serious problem, boot cpu id (%d) >= NR_CPUS (%d)\n", + cpu, NR_CPUS); + prom_halt(); + } + current_thread_info()->cpu = cpu; + + time_init_early(); + prom_init_report(); + start_kernel(); +} + +/* On Ultra, we support all of the v8 capabilities. */ +unsigned long sparc64_elf_hwcap = (HWCAP_SPARC_FLUSH | HWCAP_SPARC_STBAR | + HWCAP_SPARC_SWAP | HWCAP_SPARC_MULDIV | + HWCAP_SPARC_V9); +EXPORT_SYMBOL(sparc64_elf_hwcap); + +static const char *hwcaps[] = { + "flush", "stbar", "swap", "muldiv", "v9", + "ultra3", "blkinit", "n2", + + /* These strings are as they appear in the machine description + * 'hwcap-list' property for cpu nodes. + */ + "mul32", "div32", "fsmuld", "v8plus", "popc", "vis", "vis2", + "ASIBlkInit", "fmaf", "vis3", "hpc", "random", "trans", "fjfmau", + "ima", "cspare", "pause", "cbcond", NULL /*reserved for crypto */, + "adp", +}; + +static const char *crypto_hwcaps[] = { + "aes", "des", "kasumi", "camellia", "md5", "sha1", "sha256", + "sha512", "mpmul", "montmul", "montsqr", "crc32c", +}; + +void cpucap_info(struct seq_file *m) +{ + unsigned long caps = sparc64_elf_hwcap; + int i, printed = 0; + + seq_puts(m, "cpucaps\t\t: "); + for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { + unsigned long bit = 1UL << i; + if (hwcaps[i] && (caps & bit)) { + seq_printf(m, "%s%s", + printed ? "," : "", hwcaps[i]); + printed++; + } + } + if (caps & HWCAP_SPARC_CRYPTO) { + unsigned long cfr; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + for (i = 0; i < ARRAY_SIZE(crypto_hwcaps); i++) { + unsigned long bit = 1UL << i; + if (cfr & bit) { + seq_printf(m, "%s%s", + printed ? "," : "", crypto_hwcaps[i]); + printed++; + } + } + } + seq_putc(m, '\n'); +} + +static void __init report_one_hwcap(int *printed, const char *name) +{ + if ((*printed) == 0) + printk(KERN_INFO "CPU CAPS: ["); + printk(KERN_CONT "%s%s", + (*printed) ? "," : "", name); + if (++(*printed) == 8) { + printk(KERN_CONT "]\n"); + *printed = 0; + } +} + +static void __init report_crypto_hwcaps(int *printed) +{ + unsigned long cfr; + int i; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + + for (i = 0; i < ARRAY_SIZE(crypto_hwcaps); i++) { + unsigned long bit = 1UL << i; + if (cfr & bit) + report_one_hwcap(printed, crypto_hwcaps[i]); + } +} + +static void __init report_hwcaps(unsigned long caps) +{ + int i, printed = 0; + + for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { + unsigned long bit = 1UL << i; + if (hwcaps[i] && (caps & bit)) + report_one_hwcap(&printed, hwcaps[i]); + } + if (caps & HWCAP_SPARC_CRYPTO) + report_crypto_hwcaps(&printed); + if (printed != 0) + printk(KERN_CONT "]\n"); +} + +static unsigned long __init mdesc_cpu_hwcap_list(void) +{ + struct mdesc_handle *hp; + unsigned long caps = 0; + const char *prop; + int len; + u64 pn; + + hp = mdesc_grab(); + if (!hp) + return 0; + + pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "cpu"); + if (pn == MDESC_NODE_NULL) + goto out; + + prop = mdesc_get_property(hp, pn, "hwcap-list", &len); + if (!prop) + goto out; + + while (len) { + int i, plen; + + for (i = 0; i < ARRAY_SIZE(hwcaps); i++) { + unsigned long bit = 1UL << i; + + if (hwcaps[i] && !strcmp(prop, hwcaps[i])) { + caps |= bit; + break; + } + } + for (i = 0; i < ARRAY_SIZE(crypto_hwcaps); i++) { + if (!strcmp(prop, crypto_hwcaps[i])) + caps |= HWCAP_SPARC_CRYPTO; + } + + plen = strlen(prop) + 1; + prop += plen; + len -= plen; + } + +out: + mdesc_release(hp); + return caps; +} + +/* This yields a mask that user programs can use to figure out what + * instruction set this cpu supports. + */ +static void __init init_sparc64_elf_hwcap(void) +{ + unsigned long cap = sparc64_elf_hwcap; + unsigned long mdesc_caps; + + if (tlb_type == cheetah || tlb_type == cheetah_plus) + cap |= HWCAP_SPARC_ULTRA3; + else if (tlb_type == hypervisor) { + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M6 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M7 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M8 || + sun4v_chip_type == SUN4V_CHIP_SPARC_SN || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) + cap |= HWCAP_SPARC_BLKINIT; + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M6 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M7 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M8 || + sun4v_chip_type == SUN4V_CHIP_SPARC_SN || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) + cap |= HWCAP_SPARC_N2; + } + + cap |= (AV_SPARC_MUL32 | AV_SPARC_DIV32 | AV_SPARC_V8PLUS); + + mdesc_caps = mdesc_cpu_hwcap_list(); + if (!mdesc_caps) { + if (tlb_type == spitfire) + cap |= AV_SPARC_VIS; + if (tlb_type == cheetah || tlb_type == cheetah_plus) + cap |= AV_SPARC_VIS | AV_SPARC_VIS2; + if (tlb_type == cheetah_plus) { + unsigned long impl, ver; + + __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver)); + impl = ((ver >> 32) & 0xffff); + if (impl == PANTHER_IMPL) + cap |= AV_SPARC_POPC; + } + if (tlb_type == hypervisor) { + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA1) + cap |= AV_SPARC_ASI_BLK_INIT; + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M6 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M7 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M8 || + sun4v_chip_type == SUN4V_CHIP_SPARC_SN || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) + cap |= (AV_SPARC_VIS | AV_SPARC_VIS2 | + AV_SPARC_ASI_BLK_INIT | + AV_SPARC_POPC); + if (sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || + sun4v_chip_type == SUN4V_CHIP_NIAGARA5 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M6 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M7 || + sun4v_chip_type == SUN4V_CHIP_SPARC_M8 || + sun4v_chip_type == SUN4V_CHIP_SPARC_SN || + sun4v_chip_type == SUN4V_CHIP_SPARC64X) + cap |= (AV_SPARC_VIS3 | AV_SPARC_HPC | + AV_SPARC_FMAF); + } + } + sparc64_elf_hwcap = cap | mdesc_caps; + + report_hwcaps(sparc64_elf_hwcap); + + if (sparc64_elf_hwcap & AV_SPARC_POPC) + popc_patch(); + if (sparc64_elf_hwcap & AV_SPARC_PAUSE) + pause_patch(); +} + +static void __init alloc_irqstack_bootmem(void) +{ + unsigned int i, node; + + for_each_possible_cpu(i) { + node = cpu_to_node(i); + + softirq_stack[i] = memblock_alloc_node(THREAD_SIZE, + THREAD_SIZE, node); + if (!softirq_stack[i]) + panic("%s: Failed to allocate %lu bytes align=%lx nid=%d\n", + __func__, THREAD_SIZE, THREAD_SIZE, node); + hardirq_stack[i] = memblock_alloc_node(THREAD_SIZE, + THREAD_SIZE, node); + if (!hardirq_stack[i]) + panic("%s: Failed to allocate %lu bytes align=%lx nid=%d\n", + __func__, THREAD_SIZE, THREAD_SIZE, node); + } +} + +void __init setup_arch(char **cmdline_p) +{ + /* Initialize PROM console and command line. */ + *cmdline_p = prom_getbootargs(); + strscpy(boot_command_line, *cmdline_p, COMMAND_LINE_SIZE); + parse_early_param(); + + boot_flags_init(*cmdline_p); +#ifdef CONFIG_EARLYFB + if (btext_find_display()) +#endif + register_console(&prom_early_console); + + if (tlb_type == hypervisor) + pr_info("ARCH: SUN4V\n"); + else + pr_info("ARCH: SUN4U\n"); + + idprom_init(); + + if (!root_flags) + root_mountflags &= ~MS_RDONLY; + ROOT_DEV = old_decode_dev(root_dev); +#ifdef CONFIG_BLK_DEV_RAM + rd_image_start = ram_flags & RAMDISK_IMAGE_START_MASK; +#endif + +#ifdef CONFIG_IP_PNP + if (!ic_set_manually) { + phandle chosen = prom_finddevice("/chosen"); + u32 cl, sv, gw; + + cl = prom_getintdefault (chosen, "client-ip", 0); + sv = prom_getintdefault (chosen, "server-ip", 0); + gw = prom_getintdefault (chosen, "gateway-ip", 0); + if (cl && sv) { + ic_myaddr = cl; + ic_servaddr = sv; + if (gw) + ic_gateway = gw; +#if defined(CONFIG_IP_PNP_BOOTP) || defined(CONFIG_IP_PNP_RARP) + ic_proto_enabled = 0; +#endif + } + } +#endif + + /* Get boot processor trap_block[] setup. */ + init_cur_cpu_trap(current_thread_info()); + + paging_init(); + init_sparc64_elf_hwcap(); + /* + * Once the OF device tree and MDESC have been setup and nr_cpus has + * been parsed, we know the list of possible cpus. Therefore we can + * allocate the IRQ stacks. + */ + alloc_irqstack_bootmem(); +} + +extern int stop_a_enabled; + +void sun_do_break(void) +{ + if (!stop_a_enabled) + return; + + prom_printf("\n"); + flush_user_windows(); + + prom_cmdline(); +} +EXPORT_SYMBOL(sun_do_break); + +int stop_a_enabled = 1; +EXPORT_SYMBOL(stop_a_enabled); diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c new file mode 100644 index 000000000..a23cdd745 --- /dev/null +++ b/arch/sparc/kernel/signal32.c @@ -0,0 +1,786 @@ +// SPDX-License-Identifier: GPL-2.0 +/* arch/sparc64/kernel/signal32.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) + * Copyright (C) 1997 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "sigutil.h" +#include "kernel.h" + +/* This magic should be in g_upper[0] for all upper parts + * to be valid. + */ +#define SIGINFO_EXTRA_V8PLUS_MAGIC 0x130e269 +typedef struct { + unsigned int g_upper[8]; + unsigned int o_upper[8]; + unsigned int asi; +} siginfo_extra_v8plus_t; + +struct signal_frame32 { + struct sparc_stackf32 ss; + __siginfo32_t info; + /* __siginfo_fpu_t * */ u32 fpu_save; + unsigned int insns[2]; + unsigned int extramask[_COMPAT_NSIG_WORDS - 1]; + unsigned int extra_size; /* Should be sizeof(siginfo_extra_v8plus_t) */ + /* Only valid if (info.si_regs.psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS */ + siginfo_extra_v8plus_t v8plus; + /* __siginfo_rwin_t * */u32 rwin_save; +} __attribute__((aligned(8))); + +struct rt_signal_frame32 { + struct sparc_stackf32 ss; + compat_siginfo_t info; + struct pt_regs32 regs; + compat_sigset_t mask; + /* __siginfo_fpu_t * */ u32 fpu_save; + unsigned int insns[2]; + compat_stack_t stack; + unsigned int extra_size; /* Should be sizeof(siginfo_extra_v8plus_t) */ + /* Only valid if (regs.psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS */ + siginfo_extra_v8plus_t v8plus; + /* __siginfo_rwin_t * */u32 rwin_save; +} __attribute__((aligned(8))); + +/* Checks if the fp is valid. We always build signal frames which are + * 16-byte aligned, therefore we can always enforce that the restore + * frame has that property as well. + */ +static bool invalid_frame_pointer(void __user *fp, int fplen) +{ + if ((((unsigned long) fp) & 15) || + ((unsigned long)fp) > 0x100000000ULL - fplen) + return true; + return false; +} + +void do_sigreturn32(struct pt_regs *regs) +{ + struct signal_frame32 __user *sf; + compat_uptr_t fpu_save; + compat_uptr_t rwin_save; + unsigned int psr, ufp; + unsigned int pc, npc; + sigset_t set; + compat_sigset_t seta; + int err, i; + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + synchronize_user_stack(); + + regs->u_regs[UREG_FP] &= 0x00000000ffffffffUL; + sf = (struct signal_frame32 __user *) regs->u_regs[UREG_FP]; + + /* 1. Make sure we are not getting garbage from the user */ + if (invalid_frame_pointer(sf, sizeof(*sf))) + goto segv; + + if (get_user(ufp, &sf->info.si_regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) + goto segv; + + if (__get_user(pc, &sf->info.si_regs.pc) || + __get_user(npc, &sf->info.si_regs.npc)) + goto segv; + + if ((pc | npc) & 3) + goto segv; + + if (test_thread_flag(TIF_32BIT)) { + pc &= 0xffffffff; + npc &= 0xffffffff; + } + regs->tpc = pc; + regs->tnpc = npc; + + /* 2. Restore the state */ + err = __get_user(regs->y, &sf->info.si_regs.y); + err |= __get_user(psr, &sf->info.si_regs.psr); + + for (i = UREG_G1; i <= UREG_I7; i++) + err |= __get_user(regs->u_regs[i], &sf->info.si_regs.u_regs[i]); + if ((psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS) { + err |= __get_user(i, &sf->v8plus.g_upper[0]); + if (i == SIGINFO_EXTRA_V8PLUS_MAGIC) { + unsigned long asi; + + for (i = UREG_G1; i <= UREG_I7; i++) + err |= __get_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]); + err |= __get_user(asi, &sf->v8plus.asi); + regs->tstate &= ~TSTATE_ASI; + regs->tstate |= ((asi & 0xffUL) << 24UL); + } + } + + /* User can only change condition codes in %tstate. */ + regs->tstate &= ~(TSTATE_ICC|TSTATE_XCC); + regs->tstate |= psr_to_tstate_icc(psr); + + /* Prevent syscall restart. */ + pt_regs_clear_syscall(regs); + + err |= __get_user(fpu_save, &sf->fpu_save); + if (!err && fpu_save) + err |= restore_fpu_state(regs, compat_ptr(fpu_save)); + err |= __get_user(rwin_save, &sf->rwin_save); + if (!err && rwin_save) { + if (restore_rwin_state(compat_ptr(rwin_save))) + goto segv; + } + err |= __get_user(seta.sig[0], &sf->info.si_mask); + err |= copy_from_user(&seta.sig[1], &sf->extramask, + (_COMPAT_NSIG_WORDS - 1) * sizeof(unsigned int)); + if (err) + goto segv; + + set.sig[0] = seta.sig[0] + (((long)seta.sig[1]) << 32); + set_current_blocked(&set); + return; + +segv: + force_sig(SIGSEGV); +} + +asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) +{ + struct rt_signal_frame32 __user *sf; + unsigned int psr, pc, npc, ufp; + compat_uptr_t fpu_save; + compat_uptr_t rwin_save; + sigset_t set; + int err, i; + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + synchronize_user_stack(); + regs->u_regs[UREG_FP] &= 0x00000000ffffffffUL; + sf = (struct rt_signal_frame32 __user *) regs->u_regs[UREG_FP]; + + /* 1. Make sure we are not getting garbage from the user */ + if (invalid_frame_pointer(sf, sizeof(*sf))) + goto segv; + + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) + goto segv; + + if (__get_user(pc, &sf->regs.pc) || + __get_user(npc, &sf->regs.npc)) + goto segv; + + if ((pc | npc) & 3) + goto segv; + + if (test_thread_flag(TIF_32BIT)) { + pc &= 0xffffffff; + npc &= 0xffffffff; + } + regs->tpc = pc; + regs->tnpc = npc; + + /* 2. Restore the state */ + err = __get_user(regs->y, &sf->regs.y); + err |= __get_user(psr, &sf->regs.psr); + + for (i = UREG_G1; i <= UREG_I7; i++) + err |= __get_user(regs->u_regs[i], &sf->regs.u_regs[i]); + if ((psr & (PSR_VERS|PSR_IMPL)) == PSR_V8PLUS) { + err |= __get_user(i, &sf->v8plus.g_upper[0]); + if (i == SIGINFO_EXTRA_V8PLUS_MAGIC) { + unsigned long asi; + + for (i = UREG_G1; i <= UREG_I7; i++) + err |= __get_user(((u32 *)regs->u_regs)[2*i], &sf->v8plus.g_upper[i]); + err |= __get_user(asi, &sf->v8plus.asi); + regs->tstate &= ~TSTATE_ASI; + regs->tstate |= ((asi & 0xffUL) << 24UL); + } + } + + /* User can only change condition codes in %tstate. */ + regs->tstate &= ~(TSTATE_ICC|TSTATE_XCC); + regs->tstate |= psr_to_tstate_icc(psr); + + /* Prevent syscall restart. */ + pt_regs_clear_syscall(regs); + + err |= __get_user(fpu_save, &sf->fpu_save); + if (!err && fpu_save) + err |= restore_fpu_state(regs, compat_ptr(fpu_save)); + err |= get_compat_sigset(&set, &sf->mask); + err |= compat_restore_altstack(&sf->stack); + if (err) + goto segv; + + err |= __get_user(rwin_save, &sf->rwin_save); + if (!err && rwin_save) { + if (restore_rwin_state(compat_ptr(rwin_save))) + goto segv; + } + + set_current_blocked(&set); + return; +segv: + force_sig(SIGSEGV); +} + +static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) +{ + unsigned long sp; + + regs->u_regs[UREG_FP] &= 0x00000000ffffffffUL; + sp = regs->u_regs[UREG_FP]; + + /* + * If we are on the alternate signal stack and would overflow it, don't. + * Return an always-bogus address instead so we will die with SIGSEGV. + */ + if (on_sig_stack(sp) && !likely(on_sig_stack(sp - framesize))) + return (void __user *) -1L; + + /* This is the X/Open sanctioned signal stack switching. */ + sp = sigsp(sp, ksig) - framesize; + + /* Always align the stack frame. This handles two cases. First, + * sigaltstack need not be mindful of platform specific stack + * alignment. Second, if we took this signal because the stack + * is not aligned properly, we'd like to take the signal cleanly + * and report that. + */ + sp &= ~15UL; + + return (void __user *) sp; +} + +/* The I-cache flush instruction only works in the primary ASI, which + * right now is the nucleus, aka. kernel space. + * + * Therefore we have to kick the instructions out using the kernel + * side linear mapping of the physical address backing the user + * instructions. + */ +static void flush_signal_insns(unsigned long address) +{ + unsigned long pstate, paddr; + pte_t *ptep, pte; + pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; + pmd_t *pmdp; + + /* Commit all stores of the instructions we are about to flush. */ + wmb(); + + /* Disable cross-call reception. In this way even a very wide + * munmap() on another cpu can't tear down the page table + * hierarchy from underneath us, since that can't complete + * until the IPI tlb flush returns. + */ + + __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); + __asm__ __volatile__("wrpr %0, %1, %%pstate" + : : "r" (pstate), "i" (PSTATE_IE)); + + pgdp = pgd_offset(current->mm, address); + if (pgd_none(*pgdp)) + goto out_irqs_on; + p4dp = p4d_offset(pgdp, address); + if (p4d_none(*p4dp)) + goto out_irqs_on; + pudp = pud_offset(p4dp, address); + if (pud_none(*pudp)) + goto out_irqs_on; + pmdp = pmd_offset(pudp, address); + if (pmd_none(*pmdp)) + goto out_irqs_on; + + ptep = pte_offset_map(pmdp, address); + if (!ptep) + goto out_irqs_on; + pte = *ptep; + if (!pte_present(pte)) + goto out_unmap; + + paddr = (unsigned long) page_address(pte_page(pte)); + + __asm__ __volatile__("flush %0 + %1" + : /* no outputs */ + : "r" (paddr), + "r" (address & (PAGE_SIZE - 1)) + : "memory"); + +out_unmap: + pte_unmap(ptep); +out_irqs_on: + __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate)); + +} + +static int setup_frame32(struct ksignal *ksig, struct pt_regs *regs, + sigset_t *oldset) +{ + struct signal_frame32 __user *sf; + int i, err, wsaved; + void __user *tail; + int sigframe_size; + u32 psr; + compat_sigset_t seta; + + /* 1. Make sure everything is clean */ + synchronize_user_stack(); + save_and_clear_fpu(); + + wsaved = get_thread_wsaved(); + + sigframe_size = sizeof(*sf); + if (current_thread_info()->fpsaved[0] & FPRS_FEF) + sigframe_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sigframe_size += sizeof(__siginfo_rwin_t); + + sf = (struct signal_frame32 __user *) + get_sigframe(ksig, regs, sigframe_size); + + if (invalid_frame_pointer(sf, sigframe_size)) { + if (show_unhandled_signals) + pr_info("%s[%d] bad frame in setup_frame32: %08lx TPC %08lx O7 %08lx\n", + current->comm, current->pid, (unsigned long)sf, + regs->tpc, regs->u_regs[UREG_I7]); + force_sigsegv(ksig->sig); + return -EINVAL; + } + + tail = (sf + 1); + + /* 2. Save the current process state */ + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + err = put_user(regs->tpc, &sf->info.si_regs.pc); + err |= __put_user(regs->tnpc, &sf->info.si_regs.npc); + err |= __put_user(regs->y, &sf->info.si_regs.y); + psr = tstate_to_psr(regs->tstate); + if (current_thread_info()->fpsaved[0] & FPRS_FEF) + psr |= PSR_EF; + err |= __put_user(psr, &sf->info.si_regs.psr); + for (i = 0; i < 16; i++) + err |= __put_user(regs->u_regs[i], &sf->info.si_regs.u_regs[i]); + err |= __put_user(sizeof(siginfo_extra_v8plus_t), &sf->extra_size); + err |= __put_user(SIGINFO_EXTRA_V8PLUS_MAGIC, &sf->v8plus.g_upper[0]); + for (i = 1; i < 16; i++) + err |= __put_user(((u32 *)regs->u_regs)[2*i], + &sf->v8plus.g_upper[i]); + err |= __put_user((regs->tstate & TSTATE_ASI) >> 24UL, + &sf->v8plus.asi); + + if (psr & PSR_EF) { + __siginfo_fpu_t __user *fp = tail; + tail += sizeof(*fp); + err |= save_fpu_state(regs, fp); + err |= __put_user((u64)fp, &sf->fpu_save); + } else { + err |= __put_user(0, &sf->fpu_save); + } + if (wsaved) { + __siginfo_rwin_t __user *rwp = tail; + tail += sizeof(*rwp); + err |= save_rwin_state(wsaved, rwp); + err |= __put_user((u64)rwp, &sf->rwin_save); + set_thread_wsaved(0); + } else { + err |= __put_user(0, &sf->rwin_save); + } + + /* If these change we need to know - assignments to seta relies on these sizes */ + BUILD_BUG_ON(_NSIG_WORDS != 1); + BUILD_BUG_ON(_COMPAT_NSIG_WORDS != 2); + seta.sig[1] = (oldset->sig[0] >> 32); + seta.sig[0] = oldset->sig[0]; + + err |= __put_user(seta.sig[0], &sf->info.si_mask); + err |= __copy_to_user(sf->extramask, &seta.sig[1], + (_COMPAT_NSIG_WORDS - 1) * sizeof(unsigned int)); + + if (!wsaved) { + err |= raw_copy_in_user((u32 __user *)sf, + (u32 __user *)(regs->u_regs[UREG_FP]), + sizeof(struct reg_window32)); + } else { + struct reg_window *rp; + + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + for (i = 0; i < 8; i++) + err |= __put_user(rp->locals[i], &sf->ss.locals[i]); + for (i = 0; i < 6; i++) + err |= __put_user(rp->ins[i], &sf->ss.ins[i]); + err |= __put_user(rp->ins[6], &sf->ss.fp); + err |= __put_user(rp->ins[7], &sf->ss.callers_pc); + } + if (err) + return err; + + /* 3. signal handler back-trampoline and parameters */ + regs->u_regs[UREG_FP] = (unsigned long) sf; + regs->u_regs[UREG_I0] = ksig->sig; + regs->u_regs[UREG_I1] = (unsigned long) &sf->info; + regs->u_regs[UREG_I2] = (unsigned long) &sf->info; + + /* 4. signal handler */ + regs->tpc = (unsigned long) ksig->ka.sa.sa_handler; + regs->tnpc = (regs->tpc + 4); + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + + /* 5. return to kernel instructions */ + if (ksig->ka.ka_restorer) { + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; + } else { + unsigned long address = ((unsigned long)&(sf->insns[0])); + + regs->u_regs[UREG_I7] = (unsigned long) (&(sf->insns[0]) - 2); + + err = __put_user(0x821020d8, &sf->insns[0]); /*mov __NR_sigreturn, %g1*/ + err |= __put_user(0x91d02010, &sf->insns[1]); /*t 0x10*/ + if (err) + return err; + flush_signal_insns(address); + } + return 0; +} + +static int setup_rt_frame32(struct ksignal *ksig, struct pt_regs *regs, + sigset_t *oldset) +{ + struct rt_signal_frame32 __user *sf; + int i, err, wsaved; + void __user *tail; + int sigframe_size; + u32 psr; + + /* 1. Make sure everything is clean */ + synchronize_user_stack(); + save_and_clear_fpu(); + + wsaved = get_thread_wsaved(); + + sigframe_size = sizeof(*sf); + if (current_thread_info()->fpsaved[0] & FPRS_FEF) + sigframe_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sigframe_size += sizeof(__siginfo_rwin_t); + + sf = (struct rt_signal_frame32 __user *) + get_sigframe(ksig, regs, sigframe_size); + + if (invalid_frame_pointer(sf, sigframe_size)) { + if (show_unhandled_signals) + pr_info("%s[%d] bad frame in setup_rt_frame32: %08lx TPC %08lx O7 %08lx\n", + current->comm, current->pid, (unsigned long)sf, + regs->tpc, regs->u_regs[UREG_I7]); + force_sigsegv(ksig->sig); + return -EINVAL; + } + + tail = (sf + 1); + + /* 2. Save the current process state */ + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + err = put_user(regs->tpc, &sf->regs.pc); + err |= __put_user(regs->tnpc, &sf->regs.npc); + err |= __put_user(regs->y, &sf->regs.y); + psr = tstate_to_psr(regs->tstate); + if (current_thread_info()->fpsaved[0] & FPRS_FEF) + psr |= PSR_EF; + err |= __put_user(psr, &sf->regs.psr); + for (i = 0; i < 16; i++) + err |= __put_user(regs->u_regs[i], &sf->regs.u_regs[i]); + err |= __put_user(sizeof(siginfo_extra_v8plus_t), &sf->extra_size); + err |= __put_user(SIGINFO_EXTRA_V8PLUS_MAGIC, &sf->v8plus.g_upper[0]); + for (i = 1; i < 16; i++) + err |= __put_user(((u32 *)regs->u_regs)[2*i], + &sf->v8plus.g_upper[i]); + err |= __put_user((regs->tstate & TSTATE_ASI) >> 24UL, + &sf->v8plus.asi); + + if (psr & PSR_EF) { + __siginfo_fpu_t __user *fp = tail; + tail += sizeof(*fp); + err |= save_fpu_state(regs, fp); + err |= __put_user((u64)fp, &sf->fpu_save); + } else { + err |= __put_user(0, &sf->fpu_save); + } + if (wsaved) { + __siginfo_rwin_t __user *rwp = tail; + tail += sizeof(*rwp); + err |= save_rwin_state(wsaved, rwp); + err |= __put_user((u64)rwp, &sf->rwin_save); + set_thread_wsaved(0); + } else { + err |= __put_user(0, &sf->rwin_save); + } + + /* Update the siginfo structure. */ + err |= copy_siginfo_to_user32(&sf->info, &ksig->info); + + /* Setup sigaltstack */ + err |= __compat_save_altstack(&sf->stack, regs->u_regs[UREG_FP]); + + err |= put_compat_sigset(&sf->mask, oldset, sizeof(compat_sigset_t)); + + if (!wsaved) { + err |= raw_copy_in_user((u32 __user *)sf, + (u32 __user *)(regs->u_regs[UREG_FP]), + sizeof(struct reg_window32)); + } else { + struct reg_window *rp; + + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + for (i = 0; i < 8; i++) + err |= __put_user(rp->locals[i], &sf->ss.locals[i]); + for (i = 0; i < 6; i++) + err |= __put_user(rp->ins[i], &sf->ss.ins[i]); + err |= __put_user(rp->ins[6], &sf->ss.fp); + err |= __put_user(rp->ins[7], &sf->ss.callers_pc); + } + if (err) + return err; + + /* 3. signal handler back-trampoline and parameters */ + regs->u_regs[UREG_FP] = (unsigned long) sf; + regs->u_regs[UREG_I0] = ksig->sig; + regs->u_regs[UREG_I1] = (unsigned long) &sf->info; + regs->u_regs[UREG_I2] = (unsigned long) &sf->regs; + + /* 4. signal handler */ + regs->tpc = (unsigned long) ksig->ka.sa.sa_handler; + regs->tnpc = (regs->tpc + 4); + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + + /* 5. return to kernel instructions */ + if (ksig->ka.ka_restorer) + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; + else { + unsigned long address = ((unsigned long)&(sf->insns[0])); + + regs->u_regs[UREG_I7] = (unsigned long) (&(sf->insns[0]) - 2); + + /* mov __NR_rt_sigreturn, %g1 */ + err |= __put_user(0x82102065, &sf->insns[0]); + + /* t 0x10 */ + err |= __put_user(0x91d02010, &sf->insns[1]); + if (err) + return err; + + flush_signal_insns(address); + } + return 0; +} + +static inline void handle_signal32(struct ksignal *ksig, + struct pt_regs *regs) +{ + sigset_t *oldset = sigmask_to_save(); + int err; + + if (ksig->ka.sa.sa_flags & SA_SIGINFO) + err = setup_rt_frame32(ksig, regs, oldset); + else + err = setup_frame32(ksig, regs, oldset); + + signal_setup_done(err, ksig, 0); +} + +static inline void syscall_restart32(unsigned long orig_i0, struct pt_regs *regs, + struct sigaction *sa) +{ + switch (regs->u_regs[UREG_I0]) { + case ERESTART_RESTARTBLOCK: + case ERESTARTNOHAND: + no_system_call_restart: + regs->u_regs[UREG_I0] = EINTR; + regs->tstate |= TSTATE_ICARRY; + break; + case ERESTARTSYS: + if (!(sa->sa_flags & SA_RESTART)) + goto no_system_call_restart; + fallthrough; + case ERESTARTNOINTR: + regs->u_regs[UREG_I0] = orig_i0; + regs->tpc -= 4; + regs->tnpc -= 4; + } +} + +/* Note that 'init' is a special process: it doesn't get signals it doesn't + * want to handle. Thus you cannot kill init even with a SIGKILL even by + * mistake. + */ +void do_signal32(struct pt_regs * regs) +{ + struct ksignal ksig; + unsigned long orig_i0 = 0; + int restart_syscall = 0; + bool has_handler = get_signal(&ksig); + + if (pt_regs_is_syscall(regs) && + (regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY))) { + restart_syscall = 1; + orig_i0 = regs->u_regs[UREG_G6]; + } + + if (has_handler) { + if (restart_syscall) + syscall_restart32(orig_i0, regs, &ksig.ka.sa); + handle_signal32(&ksig, regs); + } else { + if (restart_syscall) { + switch (regs->u_regs[UREG_I0]) { + case ERESTARTNOHAND: + case ERESTARTSYS: + case ERESTARTNOINTR: + /* replay the system call when we are done */ + regs->u_regs[UREG_I0] = orig_i0; + regs->tpc -= 4; + regs->tnpc -= 4; + pt_regs_clear_syscall(regs); + fallthrough; + case ERESTART_RESTARTBLOCK: + regs->u_regs[UREG_G1] = __NR_restart_syscall; + regs->tpc -= 4; + regs->tnpc -= 4; + pt_regs_clear_syscall(regs); + } + } + restore_saved_sigmask(); + } +} + +struct sigstack32 { + u32 the_stack; + int cur_status; +}; + +asmlinkage int do_sys32_sigstack(u32 u_ssptr, u32 u_ossptr, unsigned long sp) +{ + struct sigstack32 __user *ssptr = + (struct sigstack32 __user *)((unsigned long)(u_ssptr)); + struct sigstack32 __user *ossptr = + (struct sigstack32 __user *)((unsigned long)(u_ossptr)); + int ret = -EFAULT; + + /* First see if old state is wanted. */ + if (ossptr) { + if (put_user(current->sas_ss_sp + current->sas_ss_size, + &ossptr->the_stack) || + __put_user(on_sig_stack(sp), &ossptr->cur_status)) + goto out; + } + + /* Now see if we want to update the new state. */ + if (ssptr) { + u32 ss_sp; + + if (get_user(ss_sp, &ssptr->the_stack)) + goto out; + + /* If the current stack was set with sigaltstack, don't + * swap stacks while we are on it. + */ + ret = -EPERM; + if (current->sas_ss_sp && on_sig_stack(sp)) + goto out; + + /* Since we don't know the extent of the stack, and we don't + * track onstack-ness, but rather calculate it, we must + * presume a size. Ho hum this interface is lossy. + */ + current->sas_ss_sp = (unsigned long)ss_sp - SIGSTKSZ; + current->sas_ss_size = SIGSTKSZ; + } + + ret = 0; +out: + return ret; +} + +/* + * Compile-time assertions for siginfo_t offsets. Check NSIG* as well, as + * changes likely come with new fields that should be added below. + */ +static_assert(NSIGILL == 11); +static_assert(NSIGFPE == 15); +static_assert(NSIGSEGV == 10); +static_assert(NSIGBUS == 5); +static_assert(NSIGTRAP == 6); +static_assert(NSIGCHLD == 6); +static_assert(NSIGSYS == 2); +static_assert(sizeof(compat_siginfo_t) == 128); +static_assert(__alignof__(compat_siginfo_t) == 4); +static_assert(offsetof(compat_siginfo_t, si_signo) == 0x00); +static_assert(offsetof(compat_siginfo_t, si_errno) == 0x04); +static_assert(offsetof(compat_siginfo_t, si_code) == 0x08); +static_assert(offsetof(compat_siginfo_t, si_pid) == 0x0c); +static_assert(offsetof(compat_siginfo_t, si_uid) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_tid) == 0x0c); +static_assert(offsetof(compat_siginfo_t, si_overrun) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_status) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_utime) == 0x18); +static_assert(offsetof(compat_siginfo_t, si_stime) == 0x1c); +static_assert(offsetof(compat_siginfo_t, si_value) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_int) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_ptr) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_addr) == 0x0c); +static_assert(offsetof(compat_siginfo_t, si_trapno) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_addr_lsb) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_lower) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_upper) == 0x18); +static_assert(offsetof(compat_siginfo_t, si_pkey) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_perf_data) == 0x10); +static_assert(offsetof(compat_siginfo_t, si_perf_type) == 0x14); +static_assert(offsetof(compat_siginfo_t, si_perf_flags) == 0x18); +static_assert(offsetof(compat_siginfo_t, si_band) == 0x0c); +static_assert(offsetof(compat_siginfo_t, si_fd) == 0x10); diff --git a/arch/sparc/kernel/signal_32.c b/arch/sparc/kernel/signal_32.c new file mode 100644 index 000000000..478014d2e --- /dev/null +++ b/arch/sparc/kernel/signal_32.c @@ -0,0 +1,565 @@ +// SPDX-License-Identifier: GPL-2.0 +/* linux/arch/sparc/kernel/signal.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) + * Copyright (C) 1997 Eddie C. Dost (ecd@skynet.be) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* do_coredum */ +#include +#include + +#include +#include +#include /* flush_sig_insns */ +#include + +#include "sigutil.h" +#include "kernel.h" + +extern void fpsave(unsigned long *fpregs, unsigned long *fsr, + void *fpqueue, unsigned long *fpqdepth); +extern void fpload(unsigned long *fpregs, unsigned long *fsr); + +struct signal_frame { + struct sparc_stackf ss; + __siginfo32_t info; + __siginfo_fpu_t __user *fpu_save; + unsigned long insns[2] __attribute__ ((aligned (8))); + unsigned int extramask[_NSIG_WORDS - 1]; + unsigned int extra_size; /* Should be 0 */ + __siginfo_rwin_t __user *rwin_save; +} __attribute__((aligned(8))); + +struct rt_signal_frame { + struct sparc_stackf ss; + siginfo_t info; + struct pt_regs regs; + sigset_t mask; + __siginfo_fpu_t __user *fpu_save; + unsigned int insns[2]; + stack_t stack; + unsigned int extra_size; /* Should be 0 */ + __siginfo_rwin_t __user *rwin_save; +} __attribute__((aligned(8))); + +/* Align macros */ +#define SF_ALIGNEDSZ (((sizeof(struct signal_frame) + 7) & (~7))) +#define RT_ALIGNEDSZ (((sizeof(struct rt_signal_frame) + 7) & (~7))) + +/* Checks if the fp is valid. We always build signal frames which are + * 16-byte aligned, therefore we can always enforce that the restore + * frame has that property as well. + */ +static inline bool invalid_frame_pointer(void __user *fp, int fplen) +{ + if ((((unsigned long) fp) & 15) || !access_ok(fp, fplen)) + return true; + + return false; +} + +asmlinkage void do_sigreturn(struct pt_regs *regs) +{ + unsigned long up_psr, pc, npc, ufp; + struct signal_frame __user *sf; + sigset_t set; + __siginfo_fpu_t __user *fpu_save; + __siginfo_rwin_t __user *rwin_save; + int err; + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + synchronize_user_stack(); + + sf = (struct signal_frame __user *) regs->u_regs[UREG_FP]; + + /* 1. Make sure we are not getting garbage from the user */ + if (invalid_frame_pointer(sf, sizeof(*sf))) + goto segv_and_exit; + + if (get_user(ufp, &sf->info.si_regs.u_regs[UREG_FP])) + goto segv_and_exit; + + if (ufp & 0x7) + goto segv_and_exit; + + err = __get_user(pc, &sf->info.si_regs.pc); + err |= __get_user(npc, &sf->info.si_regs.npc); + + if ((pc | npc) & 3) + goto segv_and_exit; + + /* 2. Restore the state */ + up_psr = regs->psr; + err |= __copy_from_user(regs, &sf->info.si_regs, sizeof(struct pt_regs)); + + /* User can only change condition codes and FPU enabling in %psr. */ + regs->psr = (up_psr & ~(PSR_ICC | PSR_EF)) + | (regs->psr & (PSR_ICC | PSR_EF)); + + /* Prevent syscall restart. */ + pt_regs_clear_syscall(regs); + + err |= __get_user(fpu_save, &sf->fpu_save); + if (fpu_save) + err |= restore_fpu_state(regs, fpu_save); + err |= __get_user(rwin_save, &sf->rwin_save); + if (rwin_save) + err |= restore_rwin_state(rwin_save); + + /* This is pretty much atomic, no amount locking would prevent + * the races which exist anyways. + */ + err |= __get_user(set.sig[0], &sf->info.si_mask); + err |= __copy_from_user(&set.sig[1], &sf->extramask, + (_NSIG_WORDS-1) * sizeof(unsigned int)); + + if (err) + goto segv_and_exit; + + set_current_blocked(&set); + return; + +segv_and_exit: + force_sig(SIGSEGV); +} + +asmlinkage void do_rt_sigreturn(struct pt_regs *regs) +{ + struct rt_signal_frame __user *sf; + unsigned int psr, pc, npc, ufp; + __siginfo_fpu_t __user *fpu_save; + __siginfo_rwin_t __user *rwin_save; + sigset_t set; + int err; + + synchronize_user_stack(); + sf = (struct rt_signal_frame __user *) regs->u_regs[UREG_FP]; + if (invalid_frame_pointer(sf, sizeof(*sf))) + goto segv; + + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) + goto segv; + + err = __get_user(pc, &sf->regs.pc); + err |= __get_user(npc, &sf->regs.npc); + err |= ((pc | npc) & 0x03); + + err |= __get_user(regs->y, &sf->regs.y); + err |= __get_user(psr, &sf->regs.psr); + + err |= __copy_from_user(®s->u_regs[UREG_G1], + &sf->regs.u_regs[UREG_G1], 15 * sizeof(u32)); + + regs->psr = (regs->psr & ~PSR_ICC) | (psr & PSR_ICC); + + /* Prevent syscall restart. */ + pt_regs_clear_syscall(regs); + + err |= __get_user(fpu_save, &sf->fpu_save); + if (!err && fpu_save) + err |= restore_fpu_state(regs, fpu_save); + err |= __copy_from_user(&set, &sf->mask, sizeof(sigset_t)); + err |= restore_altstack(&sf->stack); + + if (err) + goto segv; + + regs->pc = pc; + regs->npc = npc; + + err |= __get_user(rwin_save, &sf->rwin_save); + if (!err && rwin_save) { + if (restore_rwin_state(rwin_save)) + goto segv; + } + + set_current_blocked(&set); + return; +segv: + force_sig(SIGSEGV); +} + +static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) +{ + unsigned long sp = regs->u_regs[UREG_FP]; + + /* + * If we are on the alternate signal stack and would overflow it, don't. + * Return an always-bogus address instead so we will die with SIGSEGV. + */ + if (on_sig_stack(sp) && !likely(on_sig_stack(sp - framesize))) + return (void __user *) -1L; + + /* This is the X/Open sanctioned signal stack switching. */ + sp = sigsp(sp, ksig) - framesize; + + /* Always align the stack frame. This handles two cases. First, + * sigaltstack need not be mindful of platform specific stack + * alignment. Second, if we took this signal because the stack + * is not aligned properly, we'd like to take the signal cleanly + * and report that. + */ + sp &= ~15UL; + + return (void __user *) sp; +} + +static int setup_frame(struct ksignal *ksig, struct pt_regs *regs, + sigset_t *oldset) +{ + struct signal_frame __user *sf; + int sigframe_size, err, wsaved; + void __user *tail; + + /* 1. Make sure everything is clean */ + synchronize_user_stack(); + + wsaved = current_thread_info()->w_saved; + + sigframe_size = sizeof(*sf); + if (used_math()) + sigframe_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sigframe_size += sizeof(__siginfo_rwin_t); + + sf = (struct signal_frame __user *) + get_sigframe(ksig, regs, sigframe_size); + + if (invalid_frame_pointer(sf, sigframe_size)) { + force_exit_sig(SIGILL); + return -EINVAL; + } + + tail = sf + 1; + + /* 2. Save the current process state */ + err = __copy_to_user(&sf->info.si_regs, regs, sizeof(struct pt_regs)); + + err |= __put_user(0, &sf->extra_size); + + if (used_math()) { + __siginfo_fpu_t __user *fp = tail; + tail += sizeof(*fp); + err |= save_fpu_state(regs, fp); + err |= __put_user(fp, &sf->fpu_save); + } else { + err |= __put_user(0, &sf->fpu_save); + } + if (wsaved) { + __siginfo_rwin_t __user *rwp = tail; + tail += sizeof(*rwp); + err |= save_rwin_state(wsaved, rwp); + err |= __put_user(rwp, &sf->rwin_save); + } else { + err |= __put_user(0, &sf->rwin_save); + } + + err |= __put_user(oldset->sig[0], &sf->info.si_mask); + err |= __copy_to_user(sf->extramask, &oldset->sig[1], + (_NSIG_WORDS - 1) * sizeof(unsigned int)); + if (!wsaved) { + err |= __copy_to_user(sf, (char *) regs->u_regs[UREG_FP], + sizeof(struct reg_window32)); + } else { + struct reg_window32 *rp; + + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + err |= __copy_to_user(sf, rp, sizeof(struct reg_window32)); + } + if (err) + return err; + + /* 3. signal handler back-trampoline and parameters */ + regs->u_regs[UREG_FP] = (unsigned long) sf; + regs->u_regs[UREG_I0] = ksig->sig; + regs->u_regs[UREG_I1] = (unsigned long) &sf->info; + regs->u_regs[UREG_I2] = (unsigned long) &sf->info; + + /* 4. signal handler */ + regs->pc = (unsigned long) ksig->ka.sa.sa_handler; + regs->npc = (regs->pc + 4); + + /* 5. return to kernel instructions */ + if (ksig->ka.ka_restorer) + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; + else { + regs->u_regs[UREG_I7] = (unsigned long)(&(sf->insns[0]) - 2); + + /* mov __NR_sigreturn, %g1 */ + err |= __put_user(0x821020d8, &sf->insns[0]); + + /* t 0x10 */ + err |= __put_user(0x91d02010, &sf->insns[1]); + if (err) + return err; + + /* Flush instruction space. */ + flush_sig_insns(current->mm, (unsigned long) &(sf->insns[0])); + } + return 0; +} + +static int setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs, + sigset_t *oldset) +{ + struct rt_signal_frame __user *sf; + int sigframe_size, wsaved; + void __user *tail; + unsigned int psr; + int err; + + synchronize_user_stack(); + wsaved = current_thread_info()->w_saved; + sigframe_size = sizeof(*sf); + if (used_math()) + sigframe_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sigframe_size += sizeof(__siginfo_rwin_t); + sf = (struct rt_signal_frame __user *) + get_sigframe(ksig, regs, sigframe_size); + if (invalid_frame_pointer(sf, sigframe_size)) { + force_exit_sig(SIGILL); + return -EINVAL; + } + + tail = sf + 1; + err = __put_user(regs->pc, &sf->regs.pc); + err |= __put_user(regs->npc, &sf->regs.npc); + err |= __put_user(regs->y, &sf->regs.y); + psr = regs->psr; + if (used_math()) + psr |= PSR_EF; + err |= __put_user(psr, &sf->regs.psr); + err |= __copy_to_user(&sf->regs.u_regs, regs->u_regs, sizeof(regs->u_regs)); + err |= __put_user(0, &sf->extra_size); + + if (psr & PSR_EF) { + __siginfo_fpu_t __user *fp = tail; + tail += sizeof(*fp); + err |= save_fpu_state(regs, fp); + err |= __put_user(fp, &sf->fpu_save); + } else { + err |= __put_user(0, &sf->fpu_save); + } + if (wsaved) { + __siginfo_rwin_t __user *rwp = tail; + tail += sizeof(*rwp); + err |= save_rwin_state(wsaved, rwp); + err |= __put_user(rwp, &sf->rwin_save); + } else { + err |= __put_user(0, &sf->rwin_save); + } + err |= __copy_to_user(&sf->mask, &oldset->sig[0], sizeof(sigset_t)); + + /* Setup sigaltstack */ + err |= __save_altstack(&sf->stack, regs->u_regs[UREG_FP]); + + if (!wsaved) { + err |= __copy_to_user(sf, (char *) regs->u_regs[UREG_FP], + sizeof(struct reg_window32)); + } else { + struct reg_window32 *rp; + + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + err |= __copy_to_user(sf, rp, sizeof(struct reg_window32)); + } + + err |= copy_siginfo_to_user(&sf->info, &ksig->info); + + if (err) + return err; + + regs->u_regs[UREG_FP] = (unsigned long) sf; + regs->u_regs[UREG_I0] = ksig->sig; + regs->u_regs[UREG_I1] = (unsigned long) &sf->info; + regs->u_regs[UREG_I2] = (unsigned long) &sf->regs; + + regs->pc = (unsigned long) ksig->ka.sa.sa_handler; + regs->npc = (regs->pc + 4); + + if (ksig->ka.ka_restorer) + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; + else { + regs->u_regs[UREG_I7] = (unsigned long)(&(sf->insns[0]) - 2); + + /* mov __NR_rt_sigreturn, %g1 */ + err |= __put_user(0x82102065, &sf->insns[0]); + + /* t 0x10 */ + err |= __put_user(0x91d02010, &sf->insns[1]); + if (err) + return err; + + /* Flush instruction space. */ + flush_sig_insns(current->mm, (unsigned long) &(sf->insns[0])); + } + return 0; +} + +static inline void +handle_signal(struct ksignal *ksig, struct pt_regs *regs) +{ + sigset_t *oldset = sigmask_to_save(); + int err; + + if (ksig->ka.sa.sa_flags & SA_SIGINFO) + err = setup_rt_frame(ksig, regs, oldset); + else + err = setup_frame(ksig, regs, oldset); + signal_setup_done(err, ksig, 0); +} + +static inline void syscall_restart(unsigned long orig_i0, struct pt_regs *regs, + struct sigaction *sa) +{ + switch(regs->u_regs[UREG_I0]) { + case ERESTART_RESTARTBLOCK: + case ERESTARTNOHAND: + no_system_call_restart: + regs->u_regs[UREG_I0] = EINTR; + regs->psr |= PSR_C; + break; + case ERESTARTSYS: + if (!(sa->sa_flags & SA_RESTART)) + goto no_system_call_restart; + fallthrough; + case ERESTARTNOINTR: + regs->u_regs[UREG_I0] = orig_i0; + regs->pc -= 4; + regs->npc -= 4; + } +} + +/* Note that 'init' is a special process: it doesn't get signals it doesn't + * want to handle. Thus you cannot kill init even with a SIGKILL even by + * mistake. + */ +static void do_signal(struct pt_regs *regs, unsigned long orig_i0) +{ + struct ksignal ksig; + int restart_syscall; + bool has_handler; + + /* It's a lot of work and synchronization to add a new ptrace + * register for GDB to save and restore in order to get + * orig_i0 correct for syscall restarts when debugging. + * + * Although it should be the case that most of the global + * registers are volatile across a system call, glibc already + * depends upon that fact that we preserve them. So we can't + * just use any global register to save away the orig_i0 value. + * + * In particular %g2, %g3, %g4, and %g5 are all assumed to be + * preserved across a system call trap by various pieces of + * code in glibc. + * + * %g7 is used as the "thread register". %g6 is not used in + * any fixed manner. %g6 is used as a scratch register and + * a compiler temporary, but its value is never used across + * a system call. Therefore %g6 is usable for orig_i0 storage. + */ + if (pt_regs_is_syscall(regs) && (regs->psr & PSR_C)) + regs->u_regs[UREG_G6] = orig_i0; + + has_handler = get_signal(&ksig); + + /* If the debugger messes with the program counter, it clears + * the software "in syscall" bit, directing us to not perform + * a syscall restart. + */ + restart_syscall = 0; + if (pt_regs_is_syscall(regs) && (regs->psr & PSR_C)) { + restart_syscall = 1; + orig_i0 = regs->u_regs[UREG_G6]; + } + + if (has_handler) { + if (restart_syscall) + syscall_restart(orig_i0, regs, &ksig.ka.sa); + handle_signal(&ksig, regs); + } else { + if (restart_syscall) { + switch (regs->u_regs[UREG_I0]) { + case ERESTARTNOHAND: + case ERESTARTSYS: + case ERESTARTNOINTR: + /* replay the system call when we are done */ + regs->u_regs[UREG_I0] = orig_i0; + regs->pc -= 4; + regs->npc -= 4; + pt_regs_clear_syscall(regs); + fallthrough; + case ERESTART_RESTARTBLOCK: + regs->u_regs[UREG_G1] = __NR_restart_syscall; + regs->pc -= 4; + regs->npc -= 4; + pt_regs_clear_syscall(regs); + } + } + restore_saved_sigmask(); + } +} + +void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, + unsigned long thread_info_flags) +{ + if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs, orig_i0); + if (thread_info_flags & _TIF_NOTIFY_RESUME) + resume_user_mode_work(regs); +} + +asmlinkage int do_sys_sigstack(struct sigstack __user *ssptr, + struct sigstack __user *ossptr, + unsigned long sp) +{ + int ret = -EFAULT; + + /* First see if old state is wanted. */ + if (ossptr) { + if (put_user(current->sas_ss_sp + current->sas_ss_size, + &ossptr->the_stack) || + __put_user(on_sig_stack(sp), &ossptr->cur_status)) + goto out; + } + + /* Now see if we want to update the new state. */ + if (ssptr) { + char *ss_sp; + + if (get_user(ss_sp, &ssptr->the_stack)) + goto out; + /* If the current stack was set with sigaltstack, don't + swap stacks while we are on it. */ + ret = -EPERM; + if (current->sas_ss_sp && on_sig_stack(sp)) + goto out; + + /* Since we don't know the extent of the stack, and we don't + track onstack-ness, but rather calculate it, we must + presume a size. Ho hum this interface is lossy. */ + current->sas_ss_sp = (unsigned long)ss_sp - SIGSTKSZ; + current->sas_ss_size = SIGSTKSZ; + } + ret = 0; +out: + return ret; +} diff --git a/arch/sparc/kernel/signal_64.c b/arch/sparc/kernel/signal_64.c new file mode 100644 index 000000000..2d64566a1 --- /dev/null +++ b/arch/sparc/kernel/signal_64.c @@ -0,0 +1,595 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/sparc64/kernel/signal.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + * Copyright (C) 1995, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996 Miguel de Icaza (miguel@nuclecu.unam.mx) + * Copyright (C) 1997 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sigutil.h" +#include "systbls.h" +#include "kernel.h" +#include "entry.h" + +/* {set, get}context() needed for 64-bit SparcLinux userland. */ +asmlinkage void sparc64_set_context(struct pt_regs *regs) +{ + struct ucontext __user *ucp = (struct ucontext __user *) + regs->u_regs[UREG_I0]; + enum ctx_state prev_state = exception_enter(); + mc_gregset_t __user *grp; + unsigned long pc, npc, tstate; + unsigned long fp, i7; + unsigned char fenab; + int err; + + synchronize_user_stack(); + if (get_thread_wsaved() || + (((unsigned long)ucp) & (sizeof(unsigned long)-1)) || + (!__access_ok(ucp, sizeof(*ucp)))) + goto do_sigsegv; + grp = &ucp->uc_mcontext.mc_gregs; + err = __get_user(pc, &((*grp)[MC_PC])); + err |= __get_user(npc, &((*grp)[MC_NPC])); + if (err || ((pc | npc) & 3)) + goto do_sigsegv; + if (regs->u_regs[UREG_I1]) { + sigset_t set; + + if (_NSIG_WORDS == 1) { + if (__get_user(set.sig[0], &ucp->uc_sigmask.sig[0])) + goto do_sigsegv; + } else { + if (__copy_from_user(&set, &ucp->uc_sigmask, sizeof(sigset_t))) + goto do_sigsegv; + } + set_current_blocked(&set); + } + if (test_thread_flag(TIF_32BIT)) { + pc &= 0xffffffff; + npc &= 0xffffffff; + } + regs->tpc = pc; + regs->tnpc = npc; + err |= __get_user(regs->y, &((*grp)[MC_Y])); + err |= __get_user(tstate, &((*grp)[MC_TSTATE])); + regs->tstate &= ~(TSTATE_ASI | TSTATE_ICC | TSTATE_XCC); + regs->tstate |= (tstate & (TSTATE_ASI | TSTATE_ICC | TSTATE_XCC)); + err |= __get_user(regs->u_regs[UREG_G1], (&(*grp)[MC_G1])); + err |= __get_user(regs->u_regs[UREG_G2], (&(*grp)[MC_G2])); + err |= __get_user(regs->u_regs[UREG_G3], (&(*grp)[MC_G3])); + err |= __get_user(regs->u_regs[UREG_G4], (&(*grp)[MC_G4])); + err |= __get_user(regs->u_regs[UREG_G5], (&(*grp)[MC_G5])); + err |= __get_user(regs->u_regs[UREG_G6], (&(*grp)[MC_G6])); + + /* Skip %g7 as that's the thread register in userspace. */ + + err |= __get_user(regs->u_regs[UREG_I0], (&(*grp)[MC_O0])); + err |= __get_user(regs->u_regs[UREG_I1], (&(*grp)[MC_O1])); + err |= __get_user(regs->u_regs[UREG_I2], (&(*grp)[MC_O2])); + err |= __get_user(regs->u_regs[UREG_I3], (&(*grp)[MC_O3])); + err |= __get_user(regs->u_regs[UREG_I4], (&(*grp)[MC_O4])); + err |= __get_user(regs->u_regs[UREG_I5], (&(*grp)[MC_O5])); + err |= __get_user(regs->u_regs[UREG_I6], (&(*grp)[MC_O6])); + err |= __get_user(regs->u_regs[UREG_I7], (&(*grp)[MC_O7])); + + err |= __get_user(fp, &(ucp->uc_mcontext.mc_fp)); + err |= __get_user(i7, &(ucp->uc_mcontext.mc_i7)); + err |= __put_user(fp, + (&(((struct reg_window __user *)(STACK_BIAS+regs->u_regs[UREG_I6]))->ins[6]))); + err |= __put_user(i7, + (&(((struct reg_window __user *)(STACK_BIAS+regs->u_regs[UREG_I6]))->ins[7]))); + + err |= __get_user(fenab, &(ucp->uc_mcontext.mc_fpregs.mcfpu_enab)); + if (fenab) { + unsigned long *fpregs = current_thread_info()->fpregs; + unsigned long fprs; + + fprs_write(0); + err |= __get_user(fprs, &(ucp->uc_mcontext.mc_fpregs.mcfpu_fprs)); + if (fprs & FPRS_DL) + err |= copy_from_user(fpregs, + &(ucp->uc_mcontext.mc_fpregs.mcfpu_fregs), + (sizeof(unsigned int) * 32)); + if (fprs & FPRS_DU) + err |= copy_from_user(fpregs+16, + ((unsigned long __user *)&(ucp->uc_mcontext.mc_fpregs.mcfpu_fregs))+16, + (sizeof(unsigned int) * 32)); + err |= __get_user(current_thread_info()->xfsr[0], + &(ucp->uc_mcontext.mc_fpregs.mcfpu_fsr)); + err |= __get_user(current_thread_info()->gsr[0], + &(ucp->uc_mcontext.mc_fpregs.mcfpu_gsr)); + regs->tstate &= ~TSTATE_PEF; + } + if (err) + goto do_sigsegv; +out: + exception_exit(prev_state); + return; +do_sigsegv: + force_sig(SIGSEGV); + goto out; +} + +asmlinkage void sparc64_get_context(struct pt_regs *regs) +{ + struct ucontext __user *ucp = (struct ucontext __user *) + regs->u_regs[UREG_I0]; + enum ctx_state prev_state = exception_enter(); + mc_gregset_t __user *grp; + mcontext_t __user *mcp; + unsigned long fp, i7; + unsigned char fenab; + int err; + + synchronize_user_stack(); + if (get_thread_wsaved() || clear_user(ucp, sizeof(*ucp))) + goto do_sigsegv; + +#if 1 + fenab = 0; /* IMO get_context is like any other system call, thus modifies FPU state -jj */ +#else + fenab = (current_thread_info()->fpsaved[0] & FPRS_FEF); +#endif + + mcp = &ucp->uc_mcontext; + grp = &mcp->mc_gregs; + + /* Skip over the trap instruction, first. */ + if (test_thread_flag(TIF_32BIT)) { + regs->tpc = (regs->tnpc & 0xffffffff); + regs->tnpc = (regs->tnpc + 4) & 0xffffffff; + } else { + regs->tpc = regs->tnpc; + regs->tnpc += 4; + } + err = 0; + if (_NSIG_WORDS == 1) + err |= __put_user(current->blocked.sig[0], + (unsigned long __user *)&ucp->uc_sigmask); + else + err |= __copy_to_user(&ucp->uc_sigmask, ¤t->blocked, + sizeof(sigset_t)); + + err |= __put_user(regs->tstate, &((*grp)[MC_TSTATE])); + err |= __put_user(regs->tpc, &((*grp)[MC_PC])); + err |= __put_user(regs->tnpc, &((*grp)[MC_NPC])); + err |= __put_user(regs->y, &((*grp)[MC_Y])); + err |= __put_user(regs->u_regs[UREG_G1], &((*grp)[MC_G1])); + err |= __put_user(regs->u_regs[UREG_G2], &((*grp)[MC_G2])); + err |= __put_user(regs->u_regs[UREG_G3], &((*grp)[MC_G3])); + err |= __put_user(regs->u_regs[UREG_G4], &((*grp)[MC_G4])); + err |= __put_user(regs->u_regs[UREG_G5], &((*grp)[MC_G5])); + err |= __put_user(regs->u_regs[UREG_G6], &((*grp)[MC_G6])); + err |= __put_user(regs->u_regs[UREG_G7], &((*grp)[MC_G7])); + err |= __put_user(regs->u_regs[UREG_I0], &((*grp)[MC_O0])); + err |= __put_user(regs->u_regs[UREG_I1], &((*grp)[MC_O1])); + err |= __put_user(regs->u_regs[UREG_I2], &((*grp)[MC_O2])); + err |= __put_user(regs->u_regs[UREG_I3], &((*grp)[MC_O3])); + err |= __put_user(regs->u_regs[UREG_I4], &((*grp)[MC_O4])); + err |= __put_user(regs->u_regs[UREG_I5], &((*grp)[MC_O5])); + err |= __put_user(regs->u_regs[UREG_I6], &((*grp)[MC_O6])); + err |= __put_user(regs->u_regs[UREG_I7], &((*grp)[MC_O7])); + + err |= __get_user(fp, + (&(((struct reg_window __user *)(STACK_BIAS+regs->u_regs[UREG_I6]))->ins[6]))); + err |= __get_user(i7, + (&(((struct reg_window __user *)(STACK_BIAS+regs->u_regs[UREG_I6]))->ins[7]))); + err |= __put_user(fp, &(mcp->mc_fp)); + err |= __put_user(i7, &(mcp->mc_i7)); + + err |= __put_user(fenab, &(mcp->mc_fpregs.mcfpu_enab)); + if (fenab) { + unsigned long *fpregs = current_thread_info()->fpregs; + unsigned long fprs; + + fprs = current_thread_info()->fpsaved[0]; + if (fprs & FPRS_DL) + err |= copy_to_user(&(mcp->mc_fpregs.mcfpu_fregs), fpregs, + (sizeof(unsigned int) * 32)); + if (fprs & FPRS_DU) + err |= copy_to_user( + ((unsigned long __user *)&(mcp->mc_fpregs.mcfpu_fregs))+16, fpregs+16, + (sizeof(unsigned int) * 32)); + err |= __put_user(current_thread_info()->xfsr[0], &(mcp->mc_fpregs.mcfpu_fsr)); + err |= __put_user(current_thread_info()->gsr[0], &(mcp->mc_fpregs.mcfpu_gsr)); + err |= __put_user(fprs, &(mcp->mc_fpregs.mcfpu_fprs)); + } + if (err) + goto do_sigsegv; +out: + exception_exit(prev_state); + return; +do_sigsegv: + force_sig(SIGSEGV); + goto out; +} + +/* Checks if the fp is valid. We always build rt signal frames which + * are 16-byte aligned, therefore we can always enforce that the + * restore frame has that property as well. + */ +static bool invalid_frame_pointer(void __user *fp) +{ + if (((unsigned long) fp) & 15) + return true; + return false; +} + +struct rt_signal_frame { + struct sparc_stackf ss; + siginfo_t info; + struct pt_regs regs; + __siginfo_fpu_t __user *fpu_save; + stack_t stack; + sigset_t mask; + __siginfo_rwin_t *rwin_save; +}; + +void do_rt_sigreturn(struct pt_regs *regs) +{ + unsigned long tpc, tnpc, tstate, ufp; + struct rt_signal_frame __user *sf; + __siginfo_fpu_t __user *fpu_save; + __siginfo_rwin_t __user *rwin_save; + sigset_t set; + int err; + + /* Always make any pending restarted system calls return -EINTR */ + current->restart_block.fn = do_no_restart_syscall; + + synchronize_user_stack (); + sf = (struct rt_signal_frame __user *) + (regs->u_regs [UREG_FP] + STACK_BIAS); + + /* 1. Make sure we are not getting garbage from the user */ + if (invalid_frame_pointer(sf)) + goto segv; + + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) + goto segv; + + if ((ufp + STACK_BIAS) & 0x7) + goto segv; + + err = __get_user(tpc, &sf->regs.tpc); + err |= __get_user(tnpc, &sf->regs.tnpc); + if (test_thread_flag(TIF_32BIT)) { + tpc &= 0xffffffff; + tnpc &= 0xffffffff; + } + err |= ((tpc | tnpc) & 3); + + /* 2. Restore the state */ + err |= __get_user(regs->y, &sf->regs.y); + err |= __get_user(tstate, &sf->regs.tstate); + err |= copy_from_user(regs->u_regs, sf->regs.u_regs, sizeof(regs->u_regs)); + + /* User can only change condition codes and %asi in %tstate. */ + regs->tstate &= ~(TSTATE_ASI | TSTATE_ICC | TSTATE_XCC); + regs->tstate |= (tstate & (TSTATE_ASI | TSTATE_ICC | TSTATE_XCC)); + + err |= __get_user(fpu_save, &sf->fpu_save); + if (!err && fpu_save) + err |= restore_fpu_state(regs, fpu_save); + + err |= __copy_from_user(&set, &sf->mask, sizeof(sigset_t)); + err |= restore_altstack(&sf->stack); + if (err) + goto segv; + + err |= __get_user(rwin_save, &sf->rwin_save); + if (!err && rwin_save) { + if (restore_rwin_state(rwin_save)) + goto segv; + } + + regs->tpc = tpc; + regs->tnpc = tnpc; + + /* Prevent syscall restart. */ + pt_regs_clear_syscall(regs); + + set_current_blocked(&set); + return; +segv: + force_sig(SIGSEGV); +} + +static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) +{ + unsigned long sp = regs->u_regs[UREG_FP] + STACK_BIAS; + + /* + * If we are on the alternate signal stack and would overflow it, don't. + * Return an always-bogus address instead so we will die with SIGSEGV. + */ + if (on_sig_stack(sp) && !likely(on_sig_stack(sp - framesize))) + return (void __user *) -1L; + + /* This is the X/Open sanctioned signal stack switching. */ + sp = sigsp(sp, ksig) - framesize; + + /* Always align the stack frame. This handles two cases. First, + * sigaltstack need not be mindful of platform specific stack + * alignment. Second, if we took this signal because the stack + * is not aligned properly, we'd like to take the signal cleanly + * and report that. + */ + sp &= ~15UL; + + return (void __user *) sp; +} + +static inline int +setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs) +{ + struct rt_signal_frame __user *sf; + int wsaved, err, sf_size; + void __user *tail; + + /* 1. Make sure everything is clean */ + synchronize_user_stack(); + save_and_clear_fpu(); + + wsaved = get_thread_wsaved(); + + sf_size = sizeof(struct rt_signal_frame); + if (current_thread_info()->fpsaved[0] & FPRS_FEF) + sf_size += sizeof(__siginfo_fpu_t); + if (wsaved) + sf_size += sizeof(__siginfo_rwin_t); + sf = (struct rt_signal_frame __user *) + get_sigframe(ksig, regs, sf_size); + + if (invalid_frame_pointer (sf)) { + if (show_unhandled_signals) + pr_info("%s[%d] bad frame in setup_rt_frame: %016lx TPC %016lx O7 %016lx\n", + current->comm, current->pid, (unsigned long)sf, + regs->tpc, regs->u_regs[UREG_I7]); + force_sigsegv(ksig->sig); + return -EINVAL; + } + + tail = (sf + 1); + + /* 2. Save the current process state */ + err = copy_to_user(&sf->regs, regs, sizeof (*regs)); + + if (current_thread_info()->fpsaved[0] & FPRS_FEF) { + __siginfo_fpu_t __user *fpu_save = tail; + tail += sizeof(__siginfo_fpu_t); + err |= save_fpu_state(regs, fpu_save); + err |= __put_user((u64)fpu_save, &sf->fpu_save); + } else { + err |= __put_user(0, &sf->fpu_save); + } + if (wsaved) { + __siginfo_rwin_t __user *rwin_save = tail; + tail += sizeof(__siginfo_rwin_t); + err |= save_rwin_state(wsaved, rwin_save); + err |= __put_user((u64)rwin_save, &sf->rwin_save); + set_thread_wsaved(0); + } else { + err |= __put_user(0, &sf->rwin_save); + } + + /* Setup sigaltstack */ + err |= __save_altstack(&sf->stack, regs->u_regs[UREG_FP]); + + err |= copy_to_user(&sf->mask, sigmask_to_save(), sizeof(sigset_t)); + + if (!wsaved) { + err |= raw_copy_in_user((u64 __user *)sf, + (u64 __user *)(regs->u_regs[UREG_FP] + + STACK_BIAS), + sizeof(struct reg_window)); + } else { + struct reg_window *rp; + + rp = ¤t_thread_info()->reg_window[wsaved - 1]; + err |= copy_to_user(sf, rp, sizeof(struct reg_window)); + } + if (ksig->ka.sa.sa_flags & SA_SIGINFO) + err |= copy_siginfo_to_user(&sf->info, &ksig->info); + else { + err |= __put_user(ksig->sig, &sf->info.si_signo); + err |= __put_user(SI_NOINFO, &sf->info.si_code); + } + if (err) + return err; + + /* 3. signal handler back-trampoline and parameters */ + regs->u_regs[UREG_FP] = ((unsigned long) sf) - STACK_BIAS; + regs->u_regs[UREG_I0] = ksig->sig; + regs->u_regs[UREG_I1] = (unsigned long) &sf->info; + + /* The sigcontext is passed in this way because of how it + * is defined in GLIBC's /usr/include/bits/sigcontext.h + * for sparc64. It includes the 128 bytes of siginfo_t. + */ + regs->u_regs[UREG_I2] = (unsigned long) &sf->info; + + /* 5. signal handler */ + regs->tpc = (unsigned long) ksig->ka.sa.sa_handler; + regs->tnpc = (regs->tpc + 4); + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + /* 4. return to kernel instructions */ + regs->u_regs[UREG_I7] = (unsigned long)ksig->ka.ka_restorer; + return 0; +} + +static inline void syscall_restart(unsigned long orig_i0, struct pt_regs *regs, + struct sigaction *sa) +{ + switch (regs->u_regs[UREG_I0]) { + case ERESTART_RESTARTBLOCK: + case ERESTARTNOHAND: + no_system_call_restart: + regs->u_regs[UREG_I0] = EINTR; + regs->tstate |= (TSTATE_ICARRY|TSTATE_XCARRY); + break; + case ERESTARTSYS: + if (!(sa->sa_flags & SA_RESTART)) + goto no_system_call_restart; + fallthrough; + case ERESTARTNOINTR: + regs->u_regs[UREG_I0] = orig_i0; + regs->tpc -= 4; + regs->tnpc -= 4; + } +} + +/* Note that 'init' is a special process: it doesn't get signals it doesn't + * want to handle. Thus you cannot kill init even with a SIGKILL even by + * mistake. + */ +static void do_signal(struct pt_regs *regs, unsigned long orig_i0) +{ + struct ksignal ksig; + int restart_syscall; + bool has_handler; + + /* It's a lot of work and synchronization to add a new ptrace + * register for GDB to save and restore in order to get + * orig_i0 correct for syscall restarts when debugging. + * + * Although it should be the case that most of the global + * registers are volatile across a system call, glibc already + * depends upon that fact that we preserve them. So we can't + * just use any global register to save away the orig_i0 value. + * + * In particular %g2, %g3, %g4, and %g5 are all assumed to be + * preserved across a system call trap by various pieces of + * code in glibc. + * + * %g7 is used as the "thread register". %g6 is not used in + * any fixed manner. %g6 is used as a scratch register and + * a compiler temporary, but its value is never used across + * a system call. Therefore %g6 is usable for orig_i0 storage. + */ + if (pt_regs_is_syscall(regs) && + (regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY))) + regs->u_regs[UREG_G6] = orig_i0; + +#ifdef CONFIG_COMPAT + if (test_thread_flag(TIF_32BIT)) { + do_signal32(regs); + return; + } +#endif + + has_handler = get_signal(&ksig); + + restart_syscall = 0; + if (pt_regs_is_syscall(regs) && + (regs->tstate & (TSTATE_XCARRY | TSTATE_ICARRY))) { + restart_syscall = 1; + orig_i0 = regs->u_regs[UREG_G6]; + } + + if (has_handler) { + if (restart_syscall) + syscall_restart(orig_i0, regs, &ksig.ka.sa); + signal_setup_done(setup_rt_frame(&ksig, regs), &ksig, 0); + } else { + if (restart_syscall) { + switch (regs->u_regs[UREG_I0]) { + case ERESTARTNOHAND: + case ERESTARTSYS: + case ERESTARTNOINTR: + /* replay the system call when we are done */ + regs->u_regs[UREG_I0] = orig_i0; + regs->tpc -= 4; + regs->tnpc -= 4; + pt_regs_clear_syscall(regs); + fallthrough; + case ERESTART_RESTARTBLOCK: + regs->u_regs[UREG_G1] = __NR_restart_syscall; + regs->tpc -= 4; + regs->tnpc -= 4; + pt_regs_clear_syscall(regs); + } + } + restore_saved_sigmask(); + } +} + +void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long thread_info_flags) +{ + user_exit(); + if (thread_info_flags & _TIF_UPROBE) + uprobe_notify_resume(regs); + if (thread_info_flags & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL)) + do_signal(regs, orig_i0); + if (thread_info_flags & _TIF_NOTIFY_RESUME) + resume_user_mode_work(regs); + user_enter(); +} + +/* + * Compile-time assertions for siginfo_t offsets. Check NSIG* as well, as + * changes likely come with new fields that should be added below. + */ +static_assert(NSIGILL == 11); +static_assert(NSIGFPE == 15); +static_assert(NSIGSEGV == 10); +static_assert(NSIGBUS == 5); +static_assert(NSIGTRAP == 6); +static_assert(NSIGCHLD == 6); +static_assert(NSIGSYS == 2); +static_assert(sizeof(siginfo_t) == 128); +static_assert(__alignof__(siginfo_t) == 8); +static_assert(offsetof(siginfo_t, si_signo) == 0x00); +static_assert(offsetof(siginfo_t, si_errno) == 0x04); +static_assert(offsetof(siginfo_t, si_code) == 0x08); +static_assert(offsetof(siginfo_t, si_pid) == 0x10); +static_assert(offsetof(siginfo_t, si_uid) == 0x14); +static_assert(offsetof(siginfo_t, si_tid) == 0x10); +static_assert(offsetof(siginfo_t, si_overrun) == 0x14); +static_assert(offsetof(siginfo_t, si_status) == 0x18); +static_assert(offsetof(siginfo_t, si_utime) == 0x20); +static_assert(offsetof(siginfo_t, si_stime) == 0x28); +static_assert(offsetof(siginfo_t, si_value) == 0x18); +static_assert(offsetof(siginfo_t, si_int) == 0x18); +static_assert(offsetof(siginfo_t, si_ptr) == 0x18); +static_assert(offsetof(siginfo_t, si_addr) == 0x10); +static_assert(offsetof(siginfo_t, si_trapno) == 0x18); +static_assert(offsetof(siginfo_t, si_addr_lsb) == 0x18); +static_assert(offsetof(siginfo_t, si_lower) == 0x20); +static_assert(offsetof(siginfo_t, si_upper) == 0x28); +static_assert(offsetof(siginfo_t, si_pkey) == 0x20); +static_assert(offsetof(siginfo_t, si_perf_data) == 0x18); +static_assert(offsetof(siginfo_t, si_perf_type) == 0x20); +static_assert(offsetof(siginfo_t, si_perf_flags) == 0x24); +static_assert(offsetof(siginfo_t, si_band) == 0x10); +static_assert(offsetof(siginfo_t, si_fd) == 0x14); diff --git a/arch/sparc/kernel/sigutil.h b/arch/sparc/kernel/sigutil.h new file mode 100644 index 000000000..21d332d8b --- /dev/null +++ b/arch/sparc/kernel/sigutil.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SIGUTIL_H +#define _SIGUTIL_H + +int save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu); +int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu); +int save_rwin_state(int wsaved, __siginfo_rwin_t __user *rwin); +int restore_rwin_state(__siginfo_rwin_t __user *rp); + +#endif /* _SIGUTIL_H */ diff --git a/arch/sparc/kernel/sigutil_32.c b/arch/sparc/kernel/sigutil_32.c new file mode 100644 index 000000000..f25c6daa9 --- /dev/null +++ b/arch/sparc/kernel/sigutil_32.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "sigutil.h" + +int save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) +{ + int err = 0; +#ifdef CONFIG_SMP + if (test_tsk_thread_flag(current, TIF_USEDFPU)) { + put_psr(get_psr() | PSR_EF); + fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, + ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); + regs->psr &= ~(PSR_EF); + clear_tsk_thread_flag(current, TIF_USEDFPU); + } +#else + if (current == last_task_used_math) { + put_psr(get_psr() | PSR_EF); + fpsave(¤t->thread.float_regs[0], ¤t->thread.fsr, + ¤t->thread.fpqueue[0], ¤t->thread.fpqdepth); + last_task_used_math = NULL; + regs->psr &= ~(PSR_EF); + } +#endif + err |= __copy_to_user(&fpu->si_float_regs[0], + ¤t->thread.float_regs[0], + (sizeof(unsigned long) * 32)); + err |= __put_user(current->thread.fsr, &fpu->si_fsr); + err |= __put_user(current->thread.fpqdepth, &fpu->si_fpqdepth); + if (current->thread.fpqdepth != 0) + err |= __copy_to_user(&fpu->si_fpqueue[0], + ¤t->thread.fpqueue[0], + ((sizeof(unsigned long) + + (sizeof(unsigned long *)))*16)); + clear_used_math(); + return err; +} + +int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) +{ + int err; + + if (((unsigned long) fpu) & 3) + return -EFAULT; + +#ifdef CONFIG_SMP + if (test_tsk_thread_flag(current, TIF_USEDFPU)) + regs->psr &= ~PSR_EF; +#else + if (current == last_task_used_math) { + last_task_used_math = NULL; + regs->psr &= ~PSR_EF; + } +#endif + set_used_math(); + clear_tsk_thread_flag(current, TIF_USEDFPU); + + if (!access_ok(fpu, sizeof(*fpu))) + return -EFAULT; + + err = __copy_from_user(¤t->thread.float_regs[0], &fpu->si_float_regs[0], + (sizeof(unsigned long) * 32)); + err |= __get_user(current->thread.fsr, &fpu->si_fsr); + err |= __get_user(current->thread.fpqdepth, &fpu->si_fpqdepth); + if (current->thread.fpqdepth != 0) + err |= __copy_from_user(¤t->thread.fpqueue[0], + &fpu->si_fpqueue[0], + ((sizeof(unsigned long) + + (sizeof(unsigned long *)))*16)); + return err; +} + +int save_rwin_state(int wsaved, __siginfo_rwin_t __user *rwin) +{ + int i, err = __put_user(wsaved, &rwin->wsaved); + + for (i = 0; i < wsaved; i++) { + struct reg_window32 *rp; + unsigned long fp; + + rp = ¤t_thread_info()->reg_window[i]; + fp = current_thread_info()->rwbuf_stkptrs[i]; + err |= copy_to_user(&rwin->reg_window[i], rp, + sizeof(struct reg_window32)); + err |= __put_user(fp, &rwin->rwbuf_stkptrs[i]); + } + return err; +} + +int restore_rwin_state(__siginfo_rwin_t __user *rp) +{ + struct thread_info *t = current_thread_info(); + int i, wsaved, err; + + if (((unsigned long) rp) & 3) + return -EFAULT; + + get_user(wsaved, &rp->wsaved); + if (wsaved > NSWINS) + return -EFAULT; + + err = 0; + for (i = 0; i < wsaved; i++) { + err |= copy_from_user(&t->reg_window[i], + &rp->reg_window[i], + sizeof(struct reg_window32)); + err |= __get_user(t->rwbuf_stkptrs[i], + &rp->rwbuf_stkptrs[i]); + } + if (err) + return err; + + t->w_saved = wsaved; + synchronize_user_stack(); + if (t->w_saved) + return -EFAULT; + return 0; + +} diff --git a/arch/sparc/kernel/sigutil_64.c b/arch/sparc/kernel/sigutil_64.c new file mode 100644 index 000000000..512e4639e --- /dev/null +++ b/arch/sparc/kernel/sigutil_64.c @@ -0,0 +1,102 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "sigutil.h" + +int save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) +{ + unsigned long *fpregs = current_thread_info()->fpregs; + unsigned long fprs; + int err = 0; + + fprs = current_thread_info()->fpsaved[0]; + if (fprs & FPRS_DL) + err |= copy_to_user(&fpu->si_float_regs[0], fpregs, + (sizeof(unsigned int) * 32)); + if (fprs & FPRS_DU) + err |= copy_to_user(&fpu->si_float_regs[32], fpregs+16, + (sizeof(unsigned int) * 32)); + err |= __put_user(current_thread_info()->xfsr[0], &fpu->si_fsr); + err |= __put_user(current_thread_info()->gsr[0], &fpu->si_gsr); + err |= __put_user(fprs, &fpu->si_fprs); + + return err; +} + +int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) +{ + unsigned long *fpregs = current_thread_info()->fpregs; + unsigned long fprs; + int err; + + if (((unsigned long) fpu) & 7) + return -EFAULT; + + err = get_user(fprs, &fpu->si_fprs); + fprs_write(0); + regs->tstate &= ~TSTATE_PEF; + if (fprs & FPRS_DL) + err |= copy_from_user(fpregs, &fpu->si_float_regs[0], + (sizeof(unsigned int) * 32)); + if (fprs & FPRS_DU) + err |= copy_from_user(fpregs+16, &fpu->si_float_regs[32], + (sizeof(unsigned int) * 32)); + err |= __get_user(current_thread_info()->xfsr[0], &fpu->si_fsr); + err |= __get_user(current_thread_info()->gsr[0], &fpu->si_gsr); + current_thread_info()->fpsaved[0] |= fprs; + return err; +} + +int save_rwin_state(int wsaved, __siginfo_rwin_t __user *rwin) +{ + int i, err = __put_user(wsaved, &rwin->wsaved); + + for (i = 0; i < wsaved; i++) { + struct reg_window *rp = ¤t_thread_info()->reg_window[i]; + unsigned long fp = current_thread_info()->rwbuf_stkptrs[i]; + + err |= copy_to_user(&rwin->reg_window[i], rp, + sizeof(struct reg_window)); + err |= __put_user(fp, &rwin->rwbuf_stkptrs[i]); + } + return err; +} + +int restore_rwin_state(__siginfo_rwin_t __user *rp) +{ + struct thread_info *t = current_thread_info(); + int i, wsaved, err; + + if (((unsigned long) rp) & 7) + return -EFAULT; + + get_user(wsaved, &rp->wsaved); + if (wsaved > NSWINS) + return -EFAULT; + + err = 0; + for (i = 0; i < wsaved; i++) { + err |= copy_from_user(&t->reg_window[i], + &rp->reg_window[i], + sizeof(struct reg_window)); + err |= __get_user(t->rwbuf_stkptrs[i], + &rp->rwbuf_stkptrs[i]); + } + if (err) + return err; + + set_thread_wsaved(wsaved); + synchronize_user_stack(); + if (get_thread_wsaved()) + return -EFAULT; + return 0; +} diff --git a/arch/sparc/kernel/smp_32.c b/arch/sparc/kernel/smp_32.c new file mode 100644 index 000000000..87eaa7719 --- /dev/null +++ b/arch/sparc/kernel/smp_32.c @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0 +/* smp.c: Sparc SMP support. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 2004 Keith M Wesolowski (wesolows@foobazco.org) + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" +#include "irq.h" + +volatile unsigned long cpu_callin_map[NR_CPUS] = {0,}; + +cpumask_t smp_commenced_mask = CPU_MASK_NONE; + +const struct sparc32_ipi_ops *sparc32_ipi_ops; + +/* The only guaranteed locking primitive available on all Sparc + * processors is 'ldstub [%reg + immediate], %dest_reg' which atomically + * places the current byte at the effective address into dest_reg and + * places 0xff there afterwards. Pretty lame locking primitive + * compared to the Alpha and the Intel no? Most Sparcs have 'swap' + * instruction which is much better... + */ + +void smp_store_cpu_info(int id) +{ + int cpu_node; + int mid; + + cpu_data(id).udelay_val = loops_per_jiffy; + + cpu_find_by_mid(id, &cpu_node); + cpu_data(id).clock_tick = prom_getintdefault(cpu_node, + "clock-frequency", 0); + cpu_data(id).prom_node = cpu_node; + mid = cpu_get_hwmid(cpu_node); + + if (mid < 0) { + printk(KERN_NOTICE "No MID found for CPU%d at node 0x%08x", id, cpu_node); + mid = 0; + } + cpu_data(id).mid = mid; +} + +void __init smp_cpus_done(unsigned int max_cpus) +{ + unsigned long bogosum = 0; + int cpu, num = 0; + + for_each_online_cpu(cpu) { + num++; + bogosum += cpu_data(cpu).udelay_val; + } + + printk("Total of %d processors activated (%lu.%02lu BogoMIPS).\n", + num, bogosum/(500000/HZ), + (bogosum/(5000/HZ))%100); + + switch(sparc_cpu_model) { + case sun4m: + smp4m_smp_done(); + break; + case sun4d: + smp4d_smp_done(); + break; + case sparc_leon: + leon_smp_done(); + break; + case sun4e: + printk("SUN4E\n"); + BUG(); + break; + case sun4u: + printk("SUN4U\n"); + BUG(); + break; + default: + printk("UNKNOWN!\n"); + BUG(); + break; + } +} + +void cpu_panic(void) +{ + printk("CPU[%d]: Returns from cpu_idle!\n", smp_processor_id()); + panic("SMP bolixed\n"); +} + +struct linux_prom_registers smp_penguin_ctable = { 0 }; + +void arch_smp_send_reschedule(int cpu) +{ + /* + * CPU model dependent way of implementing IPI generation targeting + * a single CPU. The trap handler needs only to do trap entry/return + * to call schedule. + */ + sparc32_ipi_ops->resched(cpu); +} + +void smp_send_stop(void) +{ +} + +void arch_send_call_function_single_ipi(int cpu) +{ + /* trigger one IPI single call on one CPU */ + sparc32_ipi_ops->single(cpu); +} + +void arch_send_call_function_ipi_mask(const struct cpumask *mask) +{ + int cpu; + + /* trigger IPI mask call on each CPU */ + for_each_cpu(cpu, mask) + sparc32_ipi_ops->mask_one(cpu); +} + +void smp_resched_interrupt(void) +{ + irq_enter(); + scheduler_ipi(); + local_cpu_data().irq_resched_count++; + irq_exit(); + /* re-schedule routine called by interrupt return code. */ +} + +void smp_call_function_single_interrupt(void) +{ + irq_enter(); + generic_smp_call_function_single_interrupt(); + local_cpu_data().irq_call_count++; + irq_exit(); +} + +void smp_call_function_interrupt(void) +{ + irq_enter(); + generic_smp_call_function_interrupt(); + local_cpu_data().irq_call_count++; + irq_exit(); +} + +void __init smp_prepare_cpus(unsigned int max_cpus) +{ + int i, cpuid, extra; + + printk("Entering SMP Mode...\n"); + + extra = 0; + for (i = 0; !cpu_find_by_instance(i, NULL, &cpuid); i++) { + if (cpuid >= NR_CPUS) + extra++; + } + /* i = number of cpus */ + if (extra && max_cpus > i - extra) + printk("Warning: NR_CPUS is too low to start all cpus\n"); + + smp_store_cpu_info(boot_cpu_id); + + switch(sparc_cpu_model) { + case sun4m: + smp4m_boot_cpus(); + break; + case sun4d: + smp4d_boot_cpus(); + break; + case sparc_leon: + leon_boot_cpus(); + break; + case sun4e: + printk("SUN4E\n"); + BUG(); + break; + case sun4u: + printk("SUN4U\n"); + BUG(); + break; + default: + printk("UNKNOWN!\n"); + BUG(); + break; + } +} + +/* Set this up early so that things like the scheduler can init + * properly. We use the same cpu mask for both the present and + * possible cpu map. + */ +void __init smp_setup_cpu_possible_map(void) +{ + int instance, mid; + + instance = 0; + while (!cpu_find_by_instance(instance, NULL, &mid)) { + if (mid < NR_CPUS) { + set_cpu_possible(mid, true); + set_cpu_present(mid, true); + } + instance++; + } +} + +void __init smp_prepare_boot_cpu(void) +{ + int cpuid = hard_smp_processor_id(); + + if (cpuid >= NR_CPUS) { + prom_printf("Serious problem, boot cpu id >= NR_CPUS\n"); + prom_halt(); + } + if (cpuid != 0) + printk("boot cpu id != 0, this could work but is untested\n"); + + current_thread_info()->cpu = cpuid; + set_cpu_online(cpuid, true); + set_cpu_possible(cpuid, true); +} + +int __cpu_up(unsigned int cpu, struct task_struct *tidle) +{ + int ret=0; + + switch(sparc_cpu_model) { + case sun4m: + ret = smp4m_boot_one_cpu(cpu, tidle); + break; + case sun4d: + ret = smp4d_boot_one_cpu(cpu, tidle); + break; + case sparc_leon: + ret = leon_boot_one_cpu(cpu, tidle); + break; + case sun4e: + printk("SUN4E\n"); + BUG(); + break; + case sun4u: + printk("SUN4U\n"); + BUG(); + break; + default: + printk("UNKNOWN!\n"); + BUG(); + break; + } + + if (!ret) { + cpumask_set_cpu(cpu, &smp_commenced_mask); + while (!cpu_online(cpu)) + mb(); + } + return ret; +} + +static void arch_cpu_pre_starting(void *arg) +{ + local_ops->cache_all(); + local_ops->tlb_all(); + + switch(sparc_cpu_model) { + case sun4m: + sun4m_cpu_pre_starting(arg); + break; + case sun4d: + sun4d_cpu_pre_starting(arg); + break; + case sparc_leon: + leon_cpu_pre_starting(arg); + break; + default: + BUG(); + } +} + +static void arch_cpu_pre_online(void *arg) +{ + unsigned int cpuid = hard_smp_processor_id(); + + register_percpu_ce(cpuid); + + calibrate_delay(); + smp_store_cpu_info(cpuid); + + local_ops->cache_all(); + local_ops->tlb_all(); + + switch(sparc_cpu_model) { + case sun4m: + sun4m_cpu_pre_online(arg); + break; + case sun4d: + sun4d_cpu_pre_online(arg); + break; + case sparc_leon: + leon_cpu_pre_online(arg); + break; + default: + BUG(); + } +} + +static void sparc_start_secondary(void *arg) +{ + unsigned int cpu; + + /* + * SMP booting is extremely fragile in some architectures. So run + * the cpu initialization code first before anything else. + */ + arch_cpu_pre_starting(arg); + + cpu = smp_processor_id(); + + notify_cpu_starting(cpu); + arch_cpu_pre_online(arg); + + /* Set the CPU in the cpu_online_mask */ + set_cpu_online(cpu, true); + + /* Enable local interrupts now */ + local_irq_enable(); + + wmb(); + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); + + /* We should never reach here! */ + BUG(); +} + +void smp_callin(void) +{ + sparc_start_secondary(NULL); +} + +void smp_bogo(struct seq_file *m) +{ + int i; + + for_each_online_cpu(i) { + seq_printf(m, + "Cpu%dBogo\t: %lu.%02lu\n", + i, + cpu_data(i).udelay_val/(500000/HZ), + (cpu_data(i).udelay_val/(5000/HZ))%100); + } +} + +void smp_info(struct seq_file *m) +{ + int i; + + seq_printf(m, "State:\n"); + for_each_online_cpu(i) + seq_printf(m, "CPU%d\t\t: online\n", i); +} diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c new file mode 100644 index 000000000..371460e34 --- /dev/null +++ b/arch/sparc/kernel/smp_64.c @@ -0,0 +1,1563 @@ +// SPDX-License-Identifier: GPL-2.0 +/* smp.c: Sparc64 SMP support. + * + * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cpumap.h" +#include "kernel.h" + +DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE; +cpumask_t cpu_core_map[NR_CPUS] __read_mostly = + { [0 ... NR_CPUS-1] = CPU_MASK_NONE }; + +cpumask_t cpu_core_sib_map[NR_CPUS] __read_mostly = { + [0 ... NR_CPUS-1] = CPU_MASK_NONE }; + +cpumask_t cpu_core_sib_cache_map[NR_CPUS] __read_mostly = { + [0 ... NR_CPUS - 1] = CPU_MASK_NONE }; + +EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); +EXPORT_SYMBOL(cpu_core_map); +EXPORT_SYMBOL(cpu_core_sib_map); +EXPORT_SYMBOL(cpu_core_sib_cache_map); + +static cpumask_t smp_commenced_mask; + +static DEFINE_PER_CPU(bool, poke); +static bool cpu_poke; + +void smp_info(struct seq_file *m) +{ + int i; + + seq_printf(m, "State:\n"); + for_each_online_cpu(i) + seq_printf(m, "CPU%d:\t\tonline\n", i); +} + +void smp_bogo(struct seq_file *m) +{ + int i; + + for_each_online_cpu(i) + seq_printf(m, + "Cpu%dClkTck\t: %016lx\n", + i, cpu_data(i).clock_tick); +} + +extern void setup_sparc64_timer(void); + +static volatile unsigned long callin_flag = 0; + +void smp_callin(void) +{ + int cpuid = hard_smp_processor_id(); + + __local_per_cpu_offset = __per_cpu_offset(cpuid); + + if (tlb_type == hypervisor) + sun4v_ktsb_register(); + + __flush_tlb_all(); + + setup_sparc64_timer(); + + if (cheetah_pcache_forced_on) + cheetah_enable_pcache(); + + callin_flag = 1; + __asm__ __volatile__("membar #Sync\n\t" + "flush %%g6" : : : "memory"); + + /* Clear this or we will die instantly when we + * schedule back to this idler... + */ + current_thread_info()->new_child = 0; + + /* Attach to the address space of init_task. */ + mmgrab(&init_mm); + current->active_mm = &init_mm; + + /* inform the notifiers about the new cpu */ + notify_cpu_starting(cpuid); + + while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) + rmb(); + + set_cpu_online(cpuid, true); + + local_irq_enable(); + + cpu_startup_entry(CPUHP_AP_ONLINE_IDLE); +} + +void cpu_panic(void) +{ + printk("CPU[%d]: Returns from cpu_idle!\n", smp_processor_id()); + panic("SMP bolixed\n"); +} + +/* This tick register synchronization scheme is taken entirely from + * the ia64 port, see arch/ia64/kernel/smpboot.c for details and credit. + * + * The only change I've made is to rework it so that the master + * initiates the synchonization instead of the slave. -DaveM + */ + +#define MASTER 0 +#define SLAVE (SMP_CACHE_BYTES/sizeof(unsigned long)) + +#define NUM_ROUNDS 64 /* magic value */ +#define NUM_ITERS 5 /* likewise */ + +static DEFINE_RAW_SPINLOCK(itc_sync_lock); +static unsigned long go[SLAVE + 1]; + +#define DEBUG_TICK_SYNC 0 + +static inline long get_delta (long *rt, long *master) +{ + unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0; + unsigned long tcenter, t0, t1, tm; + unsigned long i; + + for (i = 0; i < NUM_ITERS; i++) { + t0 = tick_ops->get_tick(); + go[MASTER] = 1; + membar_safe("#StoreLoad"); + while (!(tm = go[SLAVE])) + rmb(); + go[SLAVE] = 0; + wmb(); + t1 = tick_ops->get_tick(); + + if (t1 - t0 < best_t1 - best_t0) + best_t0 = t0, best_t1 = t1, best_tm = tm; + } + + *rt = best_t1 - best_t0; + *master = best_tm - best_t0; + + /* average best_t0 and best_t1 without overflow: */ + tcenter = (best_t0/2 + best_t1/2); + if (best_t0 % 2 + best_t1 % 2 == 2) + tcenter++; + return tcenter - best_tm; +} + +void smp_synchronize_tick_client(void) +{ + long i, delta, adj, adjust_latency = 0, done = 0; + unsigned long flags, rt, master_time_stamp; +#if DEBUG_TICK_SYNC + struct { + long rt; /* roundtrip time */ + long master; /* master's timestamp */ + long diff; /* difference between midpoint and master's timestamp */ + long lat; /* estimate of itc adjustment latency */ + } t[NUM_ROUNDS]; +#endif + + go[MASTER] = 1; + + while (go[MASTER]) + rmb(); + + local_irq_save(flags); + { + for (i = 0; i < NUM_ROUNDS; i++) { + delta = get_delta(&rt, &master_time_stamp); + if (delta == 0) + done = 1; /* let's lock on to this... */ + + if (!done) { + if (i > 0) { + adjust_latency += -delta; + adj = -delta + adjust_latency/4; + } else + adj = -delta; + + tick_ops->add_tick(adj); + } +#if DEBUG_TICK_SYNC + t[i].rt = rt; + t[i].master = master_time_stamp; + t[i].diff = delta; + t[i].lat = adjust_latency/4; +#endif + } + } + local_irq_restore(flags); + +#if DEBUG_TICK_SYNC + for (i = 0; i < NUM_ROUNDS; i++) + printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n", + t[i].rt, t[i].master, t[i].diff, t[i].lat); +#endif + + printk(KERN_INFO "CPU %d: synchronized TICK with master CPU " + "(last diff %ld cycles, maxerr %lu cycles)\n", + smp_processor_id(), delta, rt); +} + +static void smp_start_sync_tick_client(int cpu); + +static void smp_synchronize_one_tick(int cpu) +{ + unsigned long flags, i; + + go[MASTER] = 0; + + smp_start_sync_tick_client(cpu); + + /* wait for client to be ready */ + while (!go[MASTER]) + rmb(); + + /* now let the client proceed into his loop */ + go[MASTER] = 0; + membar_safe("#StoreLoad"); + + raw_spin_lock_irqsave(&itc_sync_lock, flags); + { + for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) { + while (!go[MASTER]) + rmb(); + go[MASTER] = 0; + wmb(); + go[SLAVE] = tick_ops->get_tick(); + membar_safe("#StoreLoad"); + } + } + raw_spin_unlock_irqrestore(&itc_sync_lock, flags); +} + +#if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU) +static void ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg, + void **descrp) +{ + extern unsigned long sparc64_ttable_tl0; + extern unsigned long kern_locked_tte_data; + struct hvtramp_descr *hdesc; + unsigned long trampoline_ra; + struct trap_per_cpu *tb; + u64 tte_vaddr, tte_data; + unsigned long hv_err; + int i; + + hdesc = kzalloc_flex(*hdesc, maps, num_kernel_image_mappings); + if (!hdesc) { + printk(KERN_ERR "ldom_startcpu_cpuid: Cannot allocate " + "hvtramp_descr.\n"); + return; + } + *descrp = hdesc; + + hdesc->cpu = cpu; + hdesc->num_mappings = num_kernel_image_mappings; + + tb = &trap_block[cpu]; + + hdesc->fault_info_va = (unsigned long) &tb->fault_info; + hdesc->fault_info_pa = kimage_addr_to_ra(&tb->fault_info); + + hdesc->thread_reg = thread_reg; + + tte_vaddr = (unsigned long) KERNBASE; + tte_data = kern_locked_tte_data; + + for (i = 0; i < hdesc->num_mappings; i++) { + hdesc->maps[i].vaddr = tte_vaddr; + hdesc->maps[i].tte = tte_data; + tte_vaddr += 0x400000; + tte_data += 0x400000; + } + + trampoline_ra = kimage_addr_to_ra(hv_cpu_startup); + + hv_err = sun4v_cpu_start(cpu, trampoline_ra, + kimage_addr_to_ra(&sparc64_ttable_tl0), + __pa(hdesc)); + if (hv_err) + printk(KERN_ERR "ldom_startcpu_cpuid: sun4v_cpu_start() " + "gives error %lu\n", hv_err); +} +#endif + +extern unsigned long sparc64_cpu_startup; + +/* The OBP cpu startup callback truncates the 3rd arg cookie to + * 32-bits (I think) so to be safe we have it read the pointer + * contained here so we work on >4GB machines. -DaveM + */ +static struct thread_info *cpu_new_thread = NULL; + +static int smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle) +{ + unsigned long entry = + (unsigned long)(&sparc64_cpu_startup); + unsigned long cookie = + (unsigned long)(&cpu_new_thread); + void *descr = NULL; + int timeout, ret; + + callin_flag = 0; + cpu_new_thread = task_thread_info(idle); + + if (tlb_type == hypervisor) { +#if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU) + if (ldom_domaining_enabled) + ldom_startcpu_cpuid(cpu, + (unsigned long) cpu_new_thread, + &descr); + else +#endif + prom_startcpu_cpuid(cpu, entry, cookie); + } else { + struct device_node *dp = of_find_node_by_cpuid(cpu); + + prom_startcpu(dp->phandle, entry, cookie); + } + + for (timeout = 0; timeout < 50000; timeout++) { + if (callin_flag) + break; + udelay(100); + } + + if (callin_flag) { + ret = 0; + } else { + printk("Processor %d is stuck.\n", cpu); + ret = -ENODEV; + } + cpu_new_thread = NULL; + + kfree(descr); + + return ret; +} + +static void spitfire_xcall_helper(u64 data0, u64 data1, u64 data2, u64 pstate, unsigned long cpu) +{ + u64 result, target; + int stuck, tmp; + + if (this_is_starfire) { + /* map to real upaid */ + cpu = (((cpu & 0x3c) << 1) | + ((cpu & 0x40) >> 4) | + (cpu & 0x3)); + } + + target = (cpu << 14) | 0x70; +again: + /* Ok, this is the real Spitfire Errata #54. + * One must read back from a UDB internal register + * after writes to the UDB interrupt dispatch, but + * before the membar Sync for that write. + * So we use the high UDB control register (ASI 0x7f, + * ADDR 0x20) for the dummy read. -DaveM + */ + tmp = 0x40; + __asm__ __volatile__( + "wrpr %1, %2, %%pstate\n\t" + "stxa %4, [%0] %3\n\t" + "stxa %5, [%0+%8] %3\n\t" + "add %0, %8, %0\n\t" + "stxa %6, [%0+%8] %3\n\t" + "membar #Sync\n\t" + "stxa %%g0, [%7] %3\n\t" + "membar #Sync\n\t" + "mov 0x20, %%g1\n\t" + "ldxa [%%g1] 0x7f, %%g0\n\t" + "membar #Sync" + : "=r" (tmp) + : "r" (pstate), "i" (PSTATE_IE), "i" (ASI_INTR_W), + "r" (data0), "r" (data1), "r" (data2), "r" (target), + "r" (0x10), "0" (tmp) + : "g1"); + + /* NOTE: PSTATE_IE is still clear. */ + stuck = 100000; + do { + __asm__ __volatile__("ldxa [%%g0] %1, %0" + : "=r" (result) + : "i" (ASI_INTR_DISPATCH_STAT)); + if (result == 0) { + __asm__ __volatile__("wrpr %0, 0x0, %%pstate" + : : "r" (pstate)); + return; + } + stuck -= 1; + if (stuck == 0) + break; + } while (result & 0x1); + __asm__ __volatile__("wrpr %0, 0x0, %%pstate" + : : "r" (pstate)); + if (stuck == 0) { + printk("CPU[%d]: mondo stuckage result[%016llx]\n", + smp_processor_id(), result); + } else { + udelay(2); + goto again; + } +} + +static void spitfire_xcall_deliver(struct trap_per_cpu *tb, int cnt) +{ + u64 *mondo, data0, data1, data2; + u16 *cpu_list; + u64 pstate; + int i; + + __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); + cpu_list = __va(tb->cpu_list_pa); + mondo = __va(tb->cpu_mondo_block_pa); + data0 = mondo[0]; + data1 = mondo[1]; + data2 = mondo[2]; + for (i = 0; i < cnt; i++) + spitfire_xcall_helper(data0, data1, data2, pstate, cpu_list[i]); +} + +/* Cheetah now allows to send the whole 64-bytes of data in the interrupt + * packet, but we have no use for that. However we do take advantage of + * the new pipelining feature (ie. dispatch to multiple cpus simultaneously). + */ +static void cheetah_xcall_deliver(struct trap_per_cpu *tb, int cnt) +{ + int nack_busy_id, is_jbus, need_more; + u64 *mondo, pstate, ver, busy_mask; + u16 *cpu_list; + + cpu_list = __va(tb->cpu_list_pa); + mondo = __va(tb->cpu_mondo_block_pa); + + /* Unfortunately, someone at Sun had the brilliant idea to make the + * busy/nack fields hard-coded by ITID number for this Ultra-III + * derivative processor. + */ + __asm__ ("rdpr %%ver, %0" : "=r" (ver)); + is_jbus = ((ver >> 32) == __JALAPENO_ID || + (ver >> 32) == __SERRANO_ID); + + __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); + +retry: + need_more = 0; + __asm__ __volatile__("wrpr %0, %1, %%pstate\n\t" + : : "r" (pstate), "i" (PSTATE_IE)); + + /* Setup the dispatch data registers. */ + __asm__ __volatile__("stxa %0, [%3] %6\n\t" + "stxa %1, [%4] %6\n\t" + "stxa %2, [%5] %6\n\t" + "membar #Sync\n\t" + : /* no outputs */ + : "r" (mondo[0]), "r" (mondo[1]), "r" (mondo[2]), + "r" (0x40), "r" (0x50), "r" (0x60), + "i" (ASI_INTR_W)); + + nack_busy_id = 0; + busy_mask = 0; + { + int i; + + for (i = 0; i < cnt; i++) { + u64 target, nr; + + nr = cpu_list[i]; + if (nr == 0xffff) + continue; + + target = (nr << 14) | 0x70; + if (is_jbus) { + busy_mask |= (0x1UL << (nr * 2)); + } else { + target |= (nack_busy_id << 24); + busy_mask |= (0x1UL << + (nack_busy_id * 2)); + } + __asm__ __volatile__( + "stxa %%g0, [%0] %1\n\t" + "membar #Sync\n\t" + : /* no outputs */ + : "r" (target), "i" (ASI_INTR_W)); + nack_busy_id++; + if (nack_busy_id == 32) { + need_more = 1; + break; + } + } + } + + /* Now, poll for completion. */ + { + u64 dispatch_stat, nack_mask; + long stuck; + + stuck = 100000 * nack_busy_id; + nack_mask = busy_mask << 1; + do { + __asm__ __volatile__("ldxa [%%g0] %1, %0" + : "=r" (dispatch_stat) + : "i" (ASI_INTR_DISPATCH_STAT)); + if (!(dispatch_stat & (busy_mask | nack_mask))) { + __asm__ __volatile__("wrpr %0, 0x0, %%pstate" + : : "r" (pstate)); + if (unlikely(need_more)) { + int i, this_cnt = 0; + for (i = 0; i < cnt; i++) { + if (cpu_list[i] == 0xffff) + continue; + cpu_list[i] = 0xffff; + this_cnt++; + if (this_cnt == 32) + break; + } + goto retry; + } + return; + } + if (!--stuck) + break; + } while (dispatch_stat & busy_mask); + + __asm__ __volatile__("wrpr %0, 0x0, %%pstate" + : : "r" (pstate)); + + if (dispatch_stat & busy_mask) { + /* Busy bits will not clear, continue instead + * of freezing up on this cpu. + */ + printk("CPU[%d]: mondo stuckage result[%016llx]\n", + smp_processor_id(), dispatch_stat); + } else { + int i, this_busy_nack = 0; + + /* Delay some random time with interrupts enabled + * to prevent deadlock. + */ + udelay(2 * nack_busy_id); + + /* Clear out the mask bits for cpus which did not + * NACK us. + */ + for (i = 0; i < cnt; i++) { + u64 check_mask, nr; + + nr = cpu_list[i]; + if (nr == 0xffff) + continue; + + if (is_jbus) + check_mask = (0x2UL << (2*nr)); + else + check_mask = (0x2UL << + this_busy_nack); + if ((dispatch_stat & check_mask) == 0) + cpu_list[i] = 0xffff; + this_busy_nack += 2; + if (this_busy_nack == 64) + break; + } + + goto retry; + } + } +} + +#define CPU_MONDO_COUNTER(cpuid) (cpu_mondo_counter[cpuid]) +#define MONDO_USEC_WAIT_MIN 2 +#define MONDO_USEC_WAIT_MAX 100 +#define MONDO_RETRY_LIMIT 500000 + +/* Multi-cpu list version. + * + * Deliver xcalls to 'cnt' number of cpus in 'cpu_list'. + * Sometimes not all cpus receive the mondo, requiring us to re-send + * the mondo until all cpus have received, or cpus are truly stuck + * unable to receive mondo, and we timeout. + * Occasionally a target cpu strand is borrowed briefly by hypervisor to + * perform guest service, such as PCIe error handling. Consider the + * service time, 1 second overall wait is reasonable for 1 cpu. + * Here two in-between mondo check wait time are defined: 2 usec for + * single cpu quick turn around and up to 100usec for large cpu count. + * Deliver mondo to large number of cpus could take longer, we adjusts + * the retry count as long as target cpus are making forward progress. + */ +static void hypervisor_xcall_deliver(struct trap_per_cpu *tb, int cnt) +{ + int this_cpu, tot_cpus, prev_sent, i, rem; + int usec_wait, retries, tot_retries; + u16 first_cpu = 0xffff; + unsigned long xc_rcvd = 0; + unsigned long status; + int ecpuerror_id = 0; + int enocpu_id = 0; + u16 *cpu_list; + u16 cpu; + + this_cpu = smp_processor_id(); + cpu_list = __va(tb->cpu_list_pa); + usec_wait = cnt * MONDO_USEC_WAIT_MIN; + if (usec_wait > MONDO_USEC_WAIT_MAX) + usec_wait = MONDO_USEC_WAIT_MAX; + retries = tot_retries = 0; + tot_cpus = cnt; + prev_sent = 0; + + do { + int n_sent, mondo_delivered, target_cpu_busy; + + status = sun4v_cpu_mondo_send(cnt, + tb->cpu_list_pa, + tb->cpu_mondo_block_pa); + + /* HV_EOK means all cpus received the xcall, we're done. */ + if (likely(status == HV_EOK)) + goto xcall_done; + + /* If not these non-fatal errors, panic */ + if (unlikely((status != HV_EWOULDBLOCK) && + (status != HV_ECPUERROR) && + (status != HV_ENOCPU))) + goto fatal_errors; + + /* First, see if we made any forward progress. + * + * Go through the cpu_list, count the target cpus that have + * received our mondo (n_sent), and those that did not (rem). + * Re-pack cpu_list with the cpus remain to be retried in the + * front - this simplifies tracking the truly stalled cpus. + * + * The hypervisor indicates successful sends by setting + * cpu list entries to the value 0xffff. + * + * EWOULDBLOCK means some target cpus did not receive the + * mondo and retry usually helps. + * + * ECPUERROR means at least one target cpu is in error state, + * it's usually safe to skip the faulty cpu and retry. + * + * ENOCPU means one of the target cpu doesn't belong to the + * domain, perhaps offlined which is unexpected, but not + * fatal and it's okay to skip the offlined cpu. + */ + rem = 0; + n_sent = 0; + for (i = 0; i < cnt; i++) { + cpu = cpu_list[i]; + if (likely(cpu == 0xffff)) { + n_sent++; + } else if ((status == HV_ECPUERROR) && + (sun4v_cpu_state(cpu) == HV_CPU_STATE_ERROR)) { + ecpuerror_id = cpu + 1; + } else if (status == HV_ENOCPU && !cpu_online(cpu)) { + enocpu_id = cpu + 1; + } else { + cpu_list[rem++] = cpu; + } + } + + /* No cpu remained, we're done. */ + if (rem == 0) + break; + + /* Otherwise, update the cpu count for retry. */ + cnt = rem; + + /* Record the overall number of mondos received by the + * first of the remaining cpus. + */ + if (first_cpu != cpu_list[0]) { + first_cpu = cpu_list[0]; + xc_rcvd = CPU_MONDO_COUNTER(first_cpu); + } + + /* Was any mondo delivered successfully? */ + mondo_delivered = (n_sent > prev_sent); + prev_sent = n_sent; + + /* or, was any target cpu busy processing other mondos? */ + target_cpu_busy = (xc_rcvd < CPU_MONDO_COUNTER(first_cpu)); + xc_rcvd = CPU_MONDO_COUNTER(first_cpu); + + /* Retry count is for no progress. If we're making progress, + * reset the retry count. + */ + if (likely(mondo_delivered || target_cpu_busy)) { + tot_retries += retries; + retries = 0; + } else if (unlikely(retries > MONDO_RETRY_LIMIT)) { + goto fatal_mondo_timeout; + } + + /* Delay a little bit to let other cpus catch up on + * their cpu mondo queue work. + */ + if (!mondo_delivered) + udelay(usec_wait); + + retries++; + } while (1); + +xcall_done: + if (unlikely(ecpuerror_id > 0)) { + pr_crit("CPU[%d]: SUN4V mondo cpu error, target cpu(%d) was in error state\n", + this_cpu, ecpuerror_id - 1); + } else if (unlikely(enocpu_id > 0)) { + pr_crit("CPU[%d]: SUN4V mondo cpu error, target cpu(%d) does not belong to the domain\n", + this_cpu, enocpu_id - 1); + } + return; + +fatal_errors: + /* fatal errors include bad alignment, etc */ + pr_crit("CPU[%d]: Args were cnt(%d) cpulist_pa(%lx) mondo_block_pa(%lx)\n", + this_cpu, tot_cpus, tb->cpu_list_pa, tb->cpu_mondo_block_pa); + panic("Unexpected SUN4V mondo error %lu\n", status); + +fatal_mondo_timeout: + /* some cpus being non-responsive to the cpu mondo */ + pr_crit("CPU[%d]: SUN4V mondo timeout, cpu(%d) made no forward progress after %d retries. Total target cpus(%d).\n", + this_cpu, first_cpu, (tot_retries + retries), tot_cpus); + panic("SUN4V mondo timeout panic\n"); +} + +static void (*xcall_deliver_impl)(struct trap_per_cpu *, int); + +static void xcall_deliver(u64 data0, u64 data1, u64 data2, const cpumask_t *mask) +{ + struct trap_per_cpu *tb; + int this_cpu, i, cnt; + unsigned long flags; + u16 *cpu_list; + u64 *mondo; + + /* We have to do this whole thing with interrupts fully disabled. + * Otherwise if we send an xcall from interrupt context it will + * corrupt both our mondo block and cpu list state. + * + * One consequence of this is that we cannot use timeout mechanisms + * that depend upon interrupts being delivered locally. So, for + * example, we cannot sample jiffies and expect it to advance. + * + * Fortunately, udelay() uses %stick/%tick so we can use that. + */ + local_irq_save(flags); + + this_cpu = smp_processor_id(); + tb = &trap_block[this_cpu]; + + mondo = __va(tb->cpu_mondo_block_pa); + mondo[0] = data0; + mondo[1] = data1; + mondo[2] = data2; + wmb(); + + cpu_list = __va(tb->cpu_list_pa); + + /* Setup the initial cpu list. */ + cnt = 0; + for_each_cpu(i, mask) { + if (i == this_cpu || !cpu_online(i)) + continue; + cpu_list[cnt++] = i; + } + + if (cnt) + xcall_deliver_impl(tb, cnt); + + local_irq_restore(flags); +} + +/* Send cross call to all processors mentioned in MASK_P + * except self. Really, there are only two cases currently, + * "cpu_online_mask" and "mm_cpumask(mm)". + */ +static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 data2, const cpumask_t *mask) +{ + u64 data0 = (((u64)ctx)<<32 | (((u64)func) & 0xffffffff)); + + xcall_deliver(data0, data1, data2, mask); +} + +/* Send cross call to all processors except self. */ +static void smp_cross_call(unsigned long *func, u32 ctx, u64 data1, u64 data2) +{ + smp_cross_call_masked(func, ctx, data1, data2, cpu_online_mask); +} + +extern unsigned long xcall_sync_tick; + +static void smp_start_sync_tick_client(int cpu) +{ + xcall_deliver((u64) &xcall_sync_tick, 0, 0, + cpumask_of(cpu)); +} + +extern unsigned long xcall_call_function; + +void arch_send_call_function_ipi_mask(const struct cpumask *mask) +{ + xcall_deliver((u64) &xcall_call_function, 0, 0, mask); +} + +extern unsigned long xcall_call_function_single; + +void arch_send_call_function_single_ipi(int cpu) +{ + xcall_deliver((u64) &xcall_call_function_single, 0, 0, + cpumask_of(cpu)); +} + +void __irq_entry smp_call_function_client(int irq, struct pt_regs *regs) +{ + clear_softint(1 << irq); + irq_enter(); + generic_smp_call_function_interrupt(); + irq_exit(); +} + +void __irq_entry smp_call_function_single_client(int irq, struct pt_regs *regs) +{ + clear_softint(1 << irq); + irq_enter(); + generic_smp_call_function_single_interrupt(); + irq_exit(); +} + +static void tsb_sync(void *info) +{ + struct trap_per_cpu *tp = &trap_block[raw_smp_processor_id()]; + struct mm_struct *mm = info; + + /* It is not valid to test "current->active_mm == mm" here. + * + * The value of "current" is not changed atomically with + * switch_mm(). But that's OK, we just need to check the + * current cpu's trap block PGD physical address. + */ + if (tp->pgd_paddr == __pa(mm->pgd)) + tsb_context_switch(mm); +} + +void smp_tsb_sync(struct mm_struct *mm) +{ + smp_call_function_many(mm_cpumask(mm), tsb_sync, mm, 1); +} + +extern unsigned long xcall_flush_tlb_mm; +extern unsigned long xcall_flush_tlb_page; +extern unsigned long xcall_flush_tlb_kernel_range; +extern unsigned long xcall_fetch_glob_regs; +extern unsigned long xcall_fetch_glob_pmu; +extern unsigned long xcall_fetch_glob_pmu_n4; +extern unsigned long xcall_receive_signal; +extern unsigned long xcall_new_mmu_context_version; +#ifdef CONFIG_KGDB +extern unsigned long xcall_kgdb_capture; +#endif + +#ifdef DCACHE_ALIASING_POSSIBLE +extern unsigned long xcall_flush_dcache_page_cheetah; +#endif +extern unsigned long xcall_flush_dcache_page_spitfire; + +static inline void __local_flush_dcache_folio(struct folio *folio) +{ + unsigned int i, nr = folio_nr_pages(folio); + +#ifdef DCACHE_ALIASING_POSSIBLE + for (i = 0; i < nr; i++) + __flush_dcache_page(folio_address(folio) + i * PAGE_SIZE, + ((tlb_type == spitfire) && + folio_flush_mapping(folio) != NULL)); +#else + if (folio_flush_mapping(folio) != NULL && + tlb_type == spitfire) { + unsigned long pfn = folio_pfn(folio) + for (i = 0; i < nr; i++) + __flush_icache_page((pfn + i) * PAGE_SIZE); + } +#endif +} + +void smp_flush_dcache_folio_impl(struct folio *folio, int cpu) +{ + int this_cpu; + + if (tlb_type == hypervisor) + return; + +#ifdef CONFIG_DEBUG_DCFLUSH + atomic_inc(&dcpage_flushes); +#endif + + this_cpu = get_cpu(); + + if (cpu == this_cpu) { + __local_flush_dcache_folio(folio); + } else if (cpu_online(cpu)) { + void *pg_addr = folio_address(folio); + u64 data0 = 0; + + if (tlb_type == spitfire) { + data0 = ((u64)&xcall_flush_dcache_page_spitfire); + if (folio_flush_mapping(folio) != NULL) + data0 |= ((u64)1 << 32); + } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { +#ifdef DCACHE_ALIASING_POSSIBLE + data0 = ((u64)&xcall_flush_dcache_page_cheetah); +#endif + } + if (data0) { + unsigned int i, nr = folio_nr_pages(folio); + + for (i = 0; i < nr; i++) { + xcall_deliver(data0, __pa(pg_addr), + (u64) pg_addr, cpumask_of(cpu)); +#ifdef CONFIG_DEBUG_DCFLUSH + atomic_inc(&dcpage_flushes_xcall); +#endif + pg_addr += PAGE_SIZE; + } + } + } + + put_cpu(); +} + +void flush_dcache_folio_all(struct mm_struct *mm, struct folio *folio) +{ + void *pg_addr; + u64 data0; + + if (tlb_type == hypervisor) + return; + + preempt_disable(); + +#ifdef CONFIG_DEBUG_DCFLUSH + atomic_inc(&dcpage_flushes); +#endif + data0 = 0; + pg_addr = folio_address(folio); + if (tlb_type == spitfire) { + data0 = ((u64)&xcall_flush_dcache_page_spitfire); + if (folio_flush_mapping(folio) != NULL) + data0 |= ((u64)1 << 32); + } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { +#ifdef DCACHE_ALIASING_POSSIBLE + data0 = ((u64)&xcall_flush_dcache_page_cheetah); +#endif + } + if (data0) { + unsigned int i, nr = folio_nr_pages(folio); + + for (i = 0; i < nr; i++) { + xcall_deliver(data0, __pa(pg_addr), + (u64) pg_addr, cpu_online_mask); +#ifdef CONFIG_DEBUG_DCFLUSH + atomic_inc(&dcpage_flushes_xcall); +#endif + pg_addr += PAGE_SIZE; + } + } + __local_flush_dcache_folio(folio); + + preempt_enable(); +} + +#ifdef CONFIG_KGDB +void kgdb_roundup_cpus(void) +{ + smp_cross_call(&xcall_kgdb_capture, 0, 0, 0); +} +#endif + +void smp_fetch_global_regs(void) +{ + smp_cross_call(&xcall_fetch_glob_regs, 0, 0, 0); +} + +void smp_fetch_global_pmu(void) +{ + if (tlb_type == hypervisor && + sun4v_chip_type >= SUN4V_CHIP_NIAGARA4) + smp_cross_call(&xcall_fetch_glob_pmu_n4, 0, 0, 0); + else + smp_cross_call(&xcall_fetch_glob_pmu, 0, 0, 0); +} + +/* We know that the window frames of the user have been flushed + * to the stack before we get here because all callers of us + * are flush_tlb_*() routines, and these run after flush_cache_*() + * which performs the flushw. + * + * mm->cpu_vm_mask is a bit mask of which cpus an address + * space has (potentially) executed on, this is the heuristic + * we use to limit cross calls. + */ + +/* This currently is only used by the hugetlb arch pre-fault + * hook on UltraSPARC-III+ and later when changing the pagesize + * bits of the context register for an address space. + */ +void smp_flush_tlb_mm(struct mm_struct *mm) +{ + u32 ctx = CTX_HWBITS(mm->context); + + get_cpu(); + + smp_cross_call_masked(&xcall_flush_tlb_mm, + ctx, 0, 0, + mm_cpumask(mm)); + + __flush_tlb_mm(ctx, SECONDARY_CONTEXT); + + put_cpu(); +} + +struct tlb_pending_info { + unsigned long ctx; + unsigned long nr; + unsigned long *vaddrs; +}; + +static void tlb_pending_func(void *info) +{ + struct tlb_pending_info *t = info; + + __flush_tlb_pending(t->ctx, t->nr, t->vaddrs); +} + +void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long *vaddrs) +{ + u32 ctx = CTX_HWBITS(mm->context); + struct tlb_pending_info info; + + get_cpu(); + + info.ctx = ctx; + info.nr = nr; + info.vaddrs = vaddrs; + + smp_call_function_many(mm_cpumask(mm), tlb_pending_func, + &info, 1); + + __flush_tlb_pending(ctx, nr, vaddrs); + + put_cpu(); +} + +void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr) +{ + unsigned long context = CTX_HWBITS(mm->context); + + get_cpu(); + + smp_cross_call_masked(&xcall_flush_tlb_page, + context, vaddr, 0, + mm_cpumask(mm)); + + __flush_tlb_page(context, vaddr); + + put_cpu(); +} + +void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + start &= PAGE_MASK; + end = PAGE_ALIGN(end); + if (start != end) { + smp_cross_call(&xcall_flush_tlb_kernel_range, + 0, start, end); + + __flush_tlb_kernel_range(start, end); + } +} + +/* CPU capture. */ +/* #define CAPTURE_DEBUG */ +extern unsigned long xcall_capture; + +static atomic_t smp_capture_depth = ATOMIC_INIT(0); +static atomic_t smp_capture_registry = ATOMIC_INIT(0); +static unsigned long penguins_are_doing_time; + +void smp_capture(void) +{ + int result = atomic_add_return(1, &smp_capture_depth); + + if (result == 1) { + int ncpus = num_online_cpus(); + +#ifdef CAPTURE_DEBUG + printk("CPU[%d]: Sending penguins to jail...", + smp_processor_id()); +#endif + penguins_are_doing_time = 1; + atomic_inc(&smp_capture_registry); + smp_cross_call(&xcall_capture, 0, 0, 0); + while (atomic_read(&smp_capture_registry) != ncpus) + rmb(); +#ifdef CAPTURE_DEBUG + printk("done\n"); +#endif + } +} + +void smp_release(void) +{ + if (atomic_dec_and_test(&smp_capture_depth)) { +#ifdef CAPTURE_DEBUG + printk("CPU[%d]: Giving pardon to " + "imprisoned penguins\n", + smp_processor_id()); +#endif + penguins_are_doing_time = 0; + membar_safe("#StoreLoad"); + atomic_dec(&smp_capture_registry); + } +} + +/* Imprisoned penguins run with %pil == PIL_NORMAL_MAX, but PSTATE_IE + * set, so they can service tlb flush xcalls... + */ +extern void prom_world(int); + +void __irq_entry smp_penguin_jailcell(int irq, struct pt_regs *regs) +{ + clear_softint(1 << irq); + + preempt_disable(); + + __asm__ __volatile__("flushw"); + prom_world(1); + atomic_inc(&smp_capture_registry); + membar_safe("#StoreLoad"); + while (penguins_are_doing_time) + rmb(); + atomic_dec(&smp_capture_registry); + prom_world(0); + + preempt_enable(); +} + +void __init smp_prepare_cpus(unsigned int max_cpus) +{ +} + +void __init smp_setup_processor_id(void) +{ + if (tlb_type == spitfire) + xcall_deliver_impl = spitfire_xcall_deliver; + else if (tlb_type == cheetah || tlb_type == cheetah_plus) + xcall_deliver_impl = cheetah_xcall_deliver; + else + xcall_deliver_impl = hypervisor_xcall_deliver; +} + +void smp_fill_in_sib_core_maps(void) +{ + unsigned int i; + + for_each_present_cpu(i) { + unsigned int j; + + cpumask_clear(&cpu_core_map[i]); + if (cpu_data(i).core_id == 0) { + cpumask_set_cpu(i, &cpu_core_map[i]); + continue; + } + + for_each_present_cpu(j) { + if (cpu_data(i).core_id == + cpu_data(j).core_id) + cpumask_set_cpu(j, &cpu_core_map[i]); + } + } + + for_each_present_cpu(i) { + unsigned int j; + + for_each_present_cpu(j) { + if (cpu_data(i).max_cache_id == + cpu_data(j).max_cache_id) + cpumask_set_cpu(j, &cpu_core_sib_cache_map[i]); + + if (cpu_data(i).sock_id == cpu_data(j).sock_id) + cpumask_set_cpu(j, &cpu_core_sib_map[i]); + } + } + + for_each_present_cpu(i) { + unsigned int j; + + cpumask_clear(&per_cpu(cpu_sibling_map, i)); + if (cpu_data(i).proc_id == -1) { + cpumask_set_cpu(i, &per_cpu(cpu_sibling_map, i)); + continue; + } + + for_each_present_cpu(j) { + if (cpu_data(i).proc_id == + cpu_data(j).proc_id) + cpumask_set_cpu(j, &per_cpu(cpu_sibling_map, i)); + } + } +} + +int __cpu_up(unsigned int cpu, struct task_struct *tidle) +{ + int ret = smp_boot_one_cpu(cpu, tidle); + + if (!ret) { + cpumask_set_cpu(cpu, &smp_commenced_mask); + while (!cpu_online(cpu)) + mb(); + if (!cpu_online(cpu)) { + ret = -ENODEV; + } else { + /* On SUN4V, writes to %tick and %stick are + * not allowed. + */ + if (tlb_type != hypervisor) + smp_synchronize_one_tick(cpu); + } + } + return ret; +} + +#ifdef CONFIG_HOTPLUG_CPU +void cpu_play_dead(void) +{ + int cpu = smp_processor_id(); + unsigned long pstate; + + idle_task_exit(); + + if (tlb_type == hypervisor) { + struct trap_per_cpu *tb = &trap_block[cpu]; + + sun4v_cpu_qconf(HV_CPU_QUEUE_CPU_MONDO, + tb->cpu_mondo_pa, 0); + sun4v_cpu_qconf(HV_CPU_QUEUE_DEVICE_MONDO, + tb->dev_mondo_pa, 0); + sun4v_cpu_qconf(HV_CPU_QUEUE_RES_ERROR, + tb->resum_mondo_pa, 0); + sun4v_cpu_qconf(HV_CPU_QUEUE_NONRES_ERROR, + tb->nonresum_mondo_pa, 0); + } + + cpumask_clear_cpu(cpu, &smp_commenced_mask); + membar_safe("#Sync"); + + local_irq_disable(); + + __asm__ __volatile__( + "rdpr %%pstate, %0\n\t" + "wrpr %0, %1, %%pstate" + : "=r" (pstate) + : "i" (PSTATE_IE)); + + while (1) + barrier(); +} + +int __cpu_disable(void) +{ + int cpu = smp_processor_id(); + cpuinfo_sparc *c; + int i; + + for_each_cpu(i, &cpu_core_map[cpu]) + cpumask_clear_cpu(cpu, &cpu_core_map[i]); + cpumask_clear(&cpu_core_map[cpu]); + + for_each_cpu(i, &per_cpu(cpu_sibling_map, cpu)) + cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i)); + cpumask_clear(&per_cpu(cpu_sibling_map, cpu)); + + c = &cpu_data(cpu); + + c->core_id = 0; + c->proc_id = -1; + + smp_wmb(); + + /* Make sure no interrupts point to this cpu. */ + fixup_irqs(); + + local_irq_enable(); + mdelay(1); + local_irq_disable(); + + set_cpu_online(cpu, false); + + cpu_map_rebuild(); + + return 0; +} + +void __cpu_die(unsigned int cpu) +{ + int i; + + for (i = 0; i < 100; i++) { + smp_rmb(); + if (!cpumask_test_cpu(cpu, &smp_commenced_mask)) + break; + msleep(100); + } + if (cpumask_test_cpu(cpu, &smp_commenced_mask)) { + printk(KERN_ERR "CPU %u didn't die...\n", cpu); + } else { +#if defined(CONFIG_SUN_LDOMS) + unsigned long hv_err; + int limit = 100; + + do { + hv_err = sun4v_cpu_stop(cpu); + if (hv_err == HV_EOK) { + set_cpu_present(cpu, false); + break; + } + } while (--limit > 0); + if (limit <= 0) { + printk(KERN_ERR "sun4v_cpu_stop() fails err=%lu\n", + hv_err); + } +#endif + } +} +#endif + +void __init smp_cpus_done(unsigned int max_cpus) +{ +} + +static void send_cpu_ipi(int cpu) +{ + xcall_deliver((u64) &xcall_receive_signal, + 0, 0, cpumask_of(cpu)); +} + +void scheduler_poke(void) +{ + if (!cpu_poke) + return; + + if (!__this_cpu_read(poke)) + return; + + __this_cpu_write(poke, false); + set_softint(1 << PIL_SMP_RECEIVE_SIGNAL); +} + +static unsigned long send_cpu_poke(int cpu) +{ + unsigned long hv_err; + + per_cpu(poke, cpu) = true; + hv_err = sun4v_cpu_poke(cpu); + if (hv_err != HV_EOK) { + per_cpu(poke, cpu) = false; + pr_err_ratelimited("%s: sun4v_cpu_poke() fails err=%lu\n", + __func__, hv_err); + } + + return hv_err; +} + +void arch_smp_send_reschedule(int cpu) +{ + if (cpu == smp_processor_id()) { + WARN_ON_ONCE(preemptible()); + set_softint(1 << PIL_SMP_RECEIVE_SIGNAL); + return; + } + + /* Use cpu poke to resume idle cpu if supported. */ + if (cpu_poke && idle_cpu(cpu)) { + unsigned long ret; + + ret = send_cpu_poke(cpu); + if (ret == HV_EOK) + return; + } + + /* Use IPI in following cases: + * - cpu poke not supported + * - cpu not idle + * - send_cpu_poke() returns with error + */ + send_cpu_ipi(cpu); +} + +void smp_init_cpu_poke(void) +{ + unsigned long major; + unsigned long minor; + int ret; + + if (tlb_type != hypervisor) + return; + + ret = sun4v_hvapi_get(HV_GRP_CORE, &major, &minor); + if (ret) { + pr_debug("HV_GRP_CORE is not registered\n"); + return; + } + + if (major == 1 && minor >= 6) { + /* CPU POKE is registered. */ + cpu_poke = true; + return; + } + + pr_debug("CPU_POKE not supported\n"); +} + +void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs) +{ + clear_softint(1 << irq); + scheduler_ipi(); +} + +static void stop_this_cpu(void *dummy) +{ + set_cpu_online(smp_processor_id(), false); + prom_stopself(); +} + +void smp_send_stop(void) +{ + int cpu; + + if (tlb_type == hypervisor) { + int this_cpu = smp_processor_id(); +#ifdef CONFIG_SERIAL_SUNHV + sunhv_migrate_hvcons_irq(this_cpu); +#endif + for_each_online_cpu(cpu) { + if (cpu == this_cpu) + continue; + + set_cpu_online(cpu, false); +#ifdef CONFIG_SUN_LDOMS + if (ldom_domaining_enabled) { + unsigned long hv_err; + hv_err = sun4v_cpu_stop(cpu); + if (hv_err) + printk(KERN_ERR "sun4v_cpu_stop() " + "failed err=%lu\n", hv_err); + } else +#endif + prom_stopcpu_cpuid(cpu); + } + } else + smp_call_function(stop_this_cpu, NULL, 0); +} + +static int __init pcpu_cpu_distance(unsigned int from, unsigned int to) +{ + if (cpu_to_node(from) == cpu_to_node(to)) + return LOCAL_DISTANCE; + else + return REMOTE_DISTANCE; +} + +static int __init pcpu_cpu_to_node(int cpu) +{ + return cpu_to_node(cpu); +} + +void __init setup_per_cpu_areas(void) +{ + unsigned long delta; + unsigned int cpu; + int rc = -EINVAL; + + if (pcpu_chosen_fc != PCPU_FC_PAGE) { + rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE, + PERCPU_DYNAMIC_RESERVE, 4 << 20, + pcpu_cpu_distance, + pcpu_cpu_to_node); + if (rc) + pr_warn("PERCPU: %s allocator failed (%d), " + "falling back to page size\n", + pcpu_fc_names[pcpu_chosen_fc], rc); + } + if (rc < 0) + rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE, + pcpu_cpu_to_node); + if (rc < 0) + panic("cannot initialize percpu area (err=%d)", rc); + + delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start; + for_each_possible_cpu(cpu) + __per_cpu_offset(cpu) = delta + pcpu_unit_offsets[cpu]; + + /* Setup %g5 for the boot cpu. */ + __local_per_cpu_offset = __per_cpu_offset(smp_processor_id()); + + of_fill_in_cpu_data(); + if (tlb_type == hypervisor) + mdesc_fill_in_cpu_data(cpu_all_mask); +} diff --git a/arch/sparc/kernel/sparc_ksyms.c b/arch/sparc/kernel/sparc_ksyms.c new file mode 100644 index 000000000..09aa69e42 --- /dev/null +++ b/arch/sparc/kernel/sparc_ksyms.c @@ -0,0 +1,12 @@ +/* + * arch/sparc/kernel/ksyms.c: Sparc specific ksyms support. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + */ + +#include +#include + +/* This is needed only for drivers/sbus/char/openprom.c */ +EXPORT_SYMBOL(saved_command_line); diff --git a/arch/sparc/kernel/spiterrs.S b/arch/sparc/kernel/spiterrs.S new file mode 100644 index 000000000..5427af440 --- /dev/null +++ b/arch/sparc/kernel/spiterrs.S @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + /* We need to carefully read the error status, ACK the errors, + * prevent recursive traps, and pass the information on to C + * code for logging. + * + * We pass the AFAR in as-is, and we encode the status + * information as described in asm-sparc64/sfafsr.h + */ + .type __spitfire_access_error,#function +__spitfire_access_error: + /* Disable ESTATE error reporting so that we do not take + * recursive traps and RED state the processor. + */ + stxa %g0, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + mov UDBE_UE, %g1 + ldxa [%g0] ASI_AFSR, %g4 ! Get AFSR + + /* __spitfire_cee_trap branches here with AFSR in %g4 and + * UDBE_CE in %g1. It only clears ESTATE_ERR_CE in the ESTATE + * Error Enable register. + */ +__spitfire_cee_trap_continue: + ldxa [%g0] ASI_AFAR, %g5 ! Get AFAR + + rdpr %tt, %g3 + and %g3, 0x1ff, %g3 ! Paranoia + sllx %g3, SFSTAT_TRAP_TYPE_SHIFT, %g3 + or %g4, %g3, %g4 + rdpr %tl, %g3 + cmp %g3, 1 + mov 1, %g3 + bleu %xcc, 1f + sllx %g3, SFSTAT_TL_GT_ONE_SHIFT, %g3 + + or %g4, %g3, %g4 + + /* Read in the UDB error register state, clearing the sticky + * error bits as-needed. We only clear them if the UE bit is + * set. Likewise, __spitfire_cee_trap below will only do so + * if the CE bit is set. + * + * NOTE: UltraSparc-I/II have high and low UDB error + * registers, corresponding to the two UDB units + * present on those chips. UltraSparc-IIi only + * has a single UDB, called "SDB" in the manual. + * For IIi the upper UDB register always reads + * as zero so for our purposes things will just + * work with the checks below. + */ +1: ldxa [%g0] ASI_UDBH_ERROR_R, %g3 + and %g3, 0x3ff, %g7 ! Paranoia + sllx %g7, SFSTAT_UDBH_SHIFT, %g7 + or %g4, %g7, %g4 + andcc %g3, %g1, %g3 ! UDBE_UE or UDBE_CE + be,pn %xcc, 1f + nop + stxa %g3, [%g0] ASI_UDB_ERROR_W + membar #Sync + +1: mov 0x18, %g3 + ldxa [%g3] ASI_UDBL_ERROR_R, %g3 + and %g3, 0x3ff, %g7 ! Paranoia + sllx %g7, SFSTAT_UDBL_SHIFT, %g7 + or %g4, %g7, %g4 + andcc %g3, %g1, %g3 ! UDBE_UE or UDBE_CE + be,pn %xcc, 1f + nop + mov 0x18, %g7 + stxa %g3, [%g7] ASI_UDB_ERROR_W + membar #Sync + +1: /* Ok, now that we've latched the error state, clear the + * sticky bits in the AFSR. + */ + stxa %g4, [%g0] ASI_AFSR + membar #Sync + + rdpr %tl, %g2 + cmp %g2, 1 + rdpr %pil, %g2 + bleu,pt %xcc, 1f + wrpr %g0, PIL_NORMAL_MAX, %pil + + ba,pt %xcc, etraptl1 + rd %pc, %g7 + + ba,a,pt %xcc, 2f + nop + +1: ba,pt %xcc, etrap_irq + rd %pc, %g7 + +2: +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + mov %l4, %o1 + mov %l5, %o2 + call spitfire_access_error + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + .size __spitfire_access_error,.-__spitfire_access_error + + /* This is the trap handler entry point for ECC correctable + * errors. They are corrected, but we listen for the trap so + * that the event can be logged. + * + * Disrupting errors are either: + * 1) single-bit ECC errors during UDB reads to system + * memory + * 2) data parity errors during write-back events + * + * As far as I can make out from the manual, the CEE trap is + * only for correctable errors during memory read accesses by + * the front-end of the processor. + * + * The code below is only for trap level 1 CEE events, as it + * is the only situation where we can safely record and log. + * For trap level >1 we just clear the CE bit in the AFSR and + * return. + * + * This is just like __spiftire_access_error above, but it + * specifically handles correctable errors. If an + * uncorrectable error is indicated in the AFSR we will branch + * directly above to __spitfire_access_error to handle it + * instead. Uncorrectable therefore takes priority over + * correctable, and the error logging C code will notice this + * case by inspecting the trap type. + */ + .type __spitfire_cee_trap,#function +__spitfire_cee_trap: + ldxa [%g0] ASI_AFSR, %g4 ! Get AFSR + mov 1, %g3 + sllx %g3, SFAFSR_UE_SHIFT, %g3 + andcc %g4, %g3, %g0 ! Check for UE + bne,pn %xcc, __spitfire_access_error + nop + + /* Ok, in this case we only have a correctable error. + * Indicate we only wish to capture that state in register + * %g1, and we only disable CE error reporting unlike UE + * handling which disables all errors. + */ + ldxa [%g0] ASI_ESTATE_ERROR_EN, %g3 + andn %g3, ESTATE_ERR_CE, %g3 + stxa %g3, [%g0] ASI_ESTATE_ERROR_EN + membar #Sync + + /* Preserve AFSR in %g4, indicate UDB state to capture in %g1 */ + ba,pt %xcc, __spitfire_cee_trap_continue + mov UDBE_CE, %g1 + .size __spitfire_cee_trap,.-__spitfire_cee_trap + + .type __spitfire_data_access_exception_tl1,#function +__spitfire_data_access_exception_tl1: + rdpr %pstate, %g4 + wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate + mov TLB_SFSR, %g3 + mov DMMU_SFAR, %g5 + ldxa [%g3] ASI_DMMU, %g4 ! Get SFSR + ldxa [%g5] ASI_DMMU, %g5 ! Get SFAR + stxa %g0, [%g3] ASI_DMMU ! Clear SFSR.FaultValid bit + membar #Sync + rdpr %tt, %g3 + cmp %g3, 0x80 ! first win spill/fill trap + blu,pn %xcc, 1f + cmp %g3, 0xff ! last win spill/fill trap + bgu,pn %xcc, 1f + nop + ba,pt %xcc, winfix_dax + rdpr %tpc, %g3 +1: sethi %hi(109f), %g7 + ba,pt %xcc, etraptl1 +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call spitfire_data_access_exception_tl1 + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + .size __spitfire_data_access_exception_tl1,.-__spitfire_data_access_exception_tl1 + + .type __spitfire_data_access_exception,#function +__spitfire_data_access_exception: + rdpr %pstate, %g4 + wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate + mov TLB_SFSR, %g3 + mov DMMU_SFAR, %g5 + ldxa [%g3] ASI_DMMU, %g4 ! Get SFSR + ldxa [%g5] ASI_DMMU, %g5 ! Get SFAR + stxa %g0, [%g3] ASI_DMMU ! Clear SFSR.FaultValid bit + membar #Sync + sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call spitfire_data_access_exception + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + .size __spitfire_data_access_exception,.-__spitfire_data_access_exception + + .type __spitfire_insn_access_exception_tl1,#function +__spitfire_insn_access_exception_tl1: + rdpr %pstate, %g4 + wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate + mov TLB_SFSR, %g3 + ldxa [%g3] ASI_IMMU, %g4 ! Get SFSR + rdpr %tpc, %g5 ! IMMU has no SFAR, use TPC + stxa %g0, [%g3] ASI_IMMU ! Clear FaultValid bit + membar #Sync + sethi %hi(109f), %g7 + ba,pt %xcc, etraptl1 +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call spitfire_insn_access_exception_tl1 + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + .size __spitfire_insn_access_exception_tl1,.-__spitfire_insn_access_exception_tl1 + + .type __spitfire_insn_access_exception,#function +__spitfire_insn_access_exception: + rdpr %pstate, %g4 + wrpr %g4, PSTATE_MG|PSTATE_AG, %pstate + mov TLB_SFSR, %g3 + ldxa [%g3] ASI_IMMU, %g4 ! Get SFSR + rdpr %tpc, %g5 ! IMMU has no SFAR, use TPC + stxa %g0, [%g3] ASI_IMMU ! Clear FaultValid bit + membar #Sync + sethi %hi(109f), %g7 + ba,pt %xcc, etrap +109: or %g7, %lo(109b), %g7 + mov %l4, %o1 + mov %l5, %o2 + call spitfire_insn_access_exception + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + .size __spitfire_insn_access_exception,.-__spitfire_insn_access_exception diff --git a/arch/sparc/kernel/sstate.c b/arch/sparc/kernel/sstate.c new file mode 100644 index 000000000..3bcc4ddc6 --- /dev/null +++ b/arch/sparc/kernel/sstate.c @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* sstate.c: System soft state support. + * + * Copyright (C) 2007, 2008 David S. Miller + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "kernel.h" + +static int hv_supports_soft_state; + +static void do_set_sstate(unsigned long state, const char *msg) +{ + unsigned long err; + + if (!hv_supports_soft_state) + return; + + err = sun4v_mach_set_soft_state(state, kimage_addr_to_ra(msg)); + if (err) { + printk(KERN_WARNING "SSTATE: Failed to set soft-state to " + "state[%lx] msg[%s], err=%lu\n", + state, msg, err); + } +} + +static const char booting_msg[32] __attribute__((aligned(32))) = + "Linux booting"; +static const char running_msg[32] __attribute__((aligned(32))) = + "Linux running"; +static const char halting_msg[32] __attribute__((aligned(32))) = + "Linux halting"; +static const char poweroff_msg[32] __attribute__((aligned(32))) = + "Linux powering off"; +static const char rebooting_msg[32] __attribute__((aligned(32))) = + "Linux rebooting"; +static const char panicking_msg[32] __attribute__((aligned(32))) = + "Linux panicking"; + +static int sstate_reboot_call(struct notifier_block *np, unsigned long type, void *_unused) +{ + const char *msg; + + switch (type) { + case SYS_DOWN: + default: + msg = rebooting_msg; + break; + + case SYS_HALT: + msg = halting_msg; + break; + + case SYS_POWER_OFF: + msg = poweroff_msg; + break; + } + + do_set_sstate(HV_SOFT_STATE_TRANSITION, msg); + + return NOTIFY_OK; +} + +static struct notifier_block sstate_reboot_notifier = { + .notifier_call = sstate_reboot_call, +}; + +static int sstate_panic_event(struct notifier_block *n, unsigned long event, void *ptr) +{ + do_set_sstate(HV_SOFT_STATE_TRANSITION, panicking_msg); + + return NOTIFY_DONE; +} + +static struct notifier_block sstate_panic_block = { + .notifier_call = sstate_panic_event, + .priority = INT_MAX, +}; + +static int __init sstate_init(void) +{ + unsigned long major, minor; + + if (tlb_type != hypervisor) + return 0; + + major = 1; + minor = 0; + if (sun4v_hvapi_register(HV_GRP_SOFT_STATE, major, &minor)) + return 0; + + hv_supports_soft_state = 1; + + prom_sun4v_guest_soft_state(); + + do_set_sstate(HV_SOFT_STATE_TRANSITION, booting_msg); + + atomic_notifier_chain_register(&panic_notifier_list, + &sstate_panic_block); + register_reboot_notifier(&sstate_reboot_notifier); + + return 0; +} + +core_initcall(sstate_init); + +static int __init sstate_running(void) +{ + do_set_sstate(HV_SOFT_STATE_NORMAL, running_msg); + return 0; +} + +late_initcall(sstate_running); diff --git a/arch/sparc/kernel/stacktrace.c b/arch/sparc/kernel/stacktrace.c new file mode 100644 index 000000000..d8eb1d149 --- /dev/null +++ b/arch/sparc/kernel/stacktrace.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kstack.h" + +static void __save_stack_trace(struct thread_info *tp, + struct stack_trace *trace, + bool skip_sched) +{ + unsigned long ksp, fp; +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + struct task_struct *t; + int graph = 0; +#endif + + if (tp == current_thread_info()) { + stack_trace_flush(); + __asm__ __volatile__("mov %%fp, %0" : "=r" (ksp)); + } else { + ksp = tp->ksp; + } + + fp = ksp + STACK_BIAS; +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + t = tp->task; +#endif + do { + struct sparc_stackf *sf; + struct pt_regs *regs; + unsigned long pc; + + if (!kstack_valid(tp, fp)) + break; + + sf = (struct sparc_stackf *) fp; + regs = (struct pt_regs *) (sf + 1); + + if (kstack_is_trap_frame(tp, regs)) { + if (!(regs->tstate & TSTATE_PRIV)) + break; + pc = regs->tpc; + fp = regs->u_regs[UREG_I6] + STACK_BIAS; + } else { + pc = sf->callers_pc; + fp = (unsigned long)sf->fp + STACK_BIAS; + } + + if (trace->skip > 0) + trace->skip--; + else if (!skip_sched || !in_sched_functions(pc)) { + trace->entries[trace->nr_entries++] = pc; +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + if ((pc + 8UL) == (unsigned long) &return_to_handler) { + struct ftrace_ret_stack *ret_stack; + ret_stack = ftrace_graph_get_ret_stack(t, + graph); + if (ret_stack) { + pc = ret_stack->ret; + if (trace->nr_entries < + trace->max_entries) + trace->entries[trace->nr_entries++] = pc; + graph++; + } + } +#endif + } + } while (trace->nr_entries < trace->max_entries); +} + +void save_stack_trace(struct stack_trace *trace) +{ + __save_stack_trace(current_thread_info(), trace, false); +} +EXPORT_SYMBOL_GPL(save_stack_trace); + +void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace) +{ + struct thread_info *tp = task_thread_info(tsk); + + __save_stack_trace(tp, trace, true); +} +EXPORT_SYMBOL_GPL(save_stack_trace_tsk); diff --git a/arch/sparc/kernel/starfire.c b/arch/sparc/kernel/starfire.c new file mode 100644 index 000000000..d8ee9a69a --- /dev/null +++ b/arch/sparc/kernel/starfire.c @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * starfire.c: Starfire/E10000 support. + * + * Copyright (C) 1998 David S. Miller (davem@redhat.com) + * Copyright (C) 2000 Anton Blanchard (anton@samba.org) + */ + +#include +#include + +#include +#include +#include +#include +#include + +/* + * A few places around the kernel check this to see if + * they need to call us to do things in a Starfire specific + * way. + */ +int this_is_starfire = 0; + +void check_if_starfire(void) +{ + phandle ssnode = prom_finddevice("/ssp-serial"); + if (ssnode != 0 && (s32)ssnode != -1) + this_is_starfire = 1; +} + +/* + * Each Starfire board has 32 registers which perform translation + * and delivery of traditional interrupt packets into the extended + * Starfire hardware format. Essentially UPAID's now have 2 more + * bits than in all previous Sun5 systems. + */ +struct starfire_irqinfo { + unsigned long imap_slots[32]; + unsigned long tregs[32]; + struct starfire_irqinfo *next; + int upaid, hwmid; +}; + +static struct starfire_irqinfo *sflist = NULL; + +/* Beam me up Scott(McNeil)y... */ +void starfire_hookup(int upaid) +{ + struct starfire_irqinfo *p; + unsigned long treg_base, hwmid, i; + + p = kmalloc_obj(*p); + if (!p) { + prom_printf("starfire_hookup: No memory, this is insane.\n"); + prom_halt(); + } + treg_base = 0x100fc000000UL; + hwmid = ((upaid & 0x3c) << 1) | + ((upaid & 0x40) >> 4) | + (upaid & 0x3); + p->hwmid = hwmid; + treg_base += (hwmid << 33UL); + treg_base += 0x200UL; + for (i = 0; i < 32; i++) { + p->imap_slots[i] = 0UL; + p->tregs[i] = treg_base + (i * 0x10UL); + /* Lets play it safe and not overwrite existing mappings */ + if (upa_readl(p->tregs[i]) != 0) + p->imap_slots[i] = 0xdeadbeaf; + } + p->upaid = upaid; + p->next = sflist; + sflist = p; +} + +unsigned int starfire_translate(unsigned long imap, + unsigned int upaid) +{ + struct starfire_irqinfo *p; + unsigned int bus_hwmid; + unsigned int i; + + bus_hwmid = (((unsigned long)imap) >> 33) & 0x7f; + for (p = sflist; p != NULL; p = p->next) + if (p->hwmid == bus_hwmid) + break; + if (p == NULL) { + prom_printf("XFIRE: Cannot find irqinfo for imap %016lx\n", + ((unsigned long)imap)); + prom_halt(); + } + for (i = 0; i < 32; i++) { + if (p->imap_slots[i] == imap || + p->imap_slots[i] == 0UL) + break; + } + if (i == 32) { + printk("starfire_translate: Are you kidding me?\n"); + panic("Lucy in the sky...."); + } + p->imap_slots[i] = imap; + + /* map to real upaid */ + upaid = (((upaid & 0x3c) << 1) | + ((upaid & 0x40) >> 4) | + (upaid & 0x3)); + + upa_writel(upaid, p->tregs[i]); + + return i; +} diff --git a/arch/sparc/kernel/sun4d_irq.c b/arch/sparc/kernel/sun4d_irq.c new file mode 100644 index 000000000..95683ecdc --- /dev/null +++ b/arch/sparc/kernel/sun4d_irq.c @@ -0,0 +1,519 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SS1000/SC2000 interrupt handling. + * + * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Heavily based on arch/sparc/kernel/irq.c. + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" +#include "irq.h" + +/* Sun4d interrupts fall roughly into two categories. SBUS and + * cpu local. CPU local interrupts cover the timer interrupts + * and whatnot, and we encode those as normal PILs between + * 0 and 15. + * SBUS interrupts are encodes as a combination of board, level and slot. + */ + +struct sun4d_handler_data { + unsigned int cpuid; /* target cpu */ + unsigned int real_irq; /* interrupt level */ +}; + + +static unsigned int sun4d_encode_irq(int board, int lvl, int slot) +{ + return (board + 1) << 5 | (lvl << 2) | slot; +} + +struct sun4d_timer_regs { + u32 l10_timer_limit; + u32 l10_cur_countx; + u32 l10_limit_noclear; + u32 ctrl; + u32 l10_cur_count; +}; + +static struct sun4d_timer_regs __iomem *sun4d_timers; + +#define SUN4D_TIMER_IRQ 10 + +/* Specify which cpu handle interrupts from which board. + * Index is board - value is cpu. + */ +static unsigned char board_to_cpu[32]; + +static int pil_to_sbus[] = { + 0, + 0, + 1, + 2, + 0, + 3, + 0, + 4, + 0, + 5, + 0, + 6, + 0, + 7, + 0, + 0, +}; + +/* Exported for sun4d_smp.c */ +DEFINE_SPINLOCK(sun4d_imsk_lock); + +/* SBUS interrupts are encoded integers including the board number + * (plus one), the SBUS level, and the SBUS slot number. Sun4D + * IRQ dispatch is done by: + * + * 1) Reading the BW local interrupt table in order to get the bus + * interrupt mask. + * + * This table is indexed by SBUS interrupt level which can be + * derived from the PIL we got interrupted on. + * + * 2) For each bus showing interrupt pending from #1, read the + * SBI interrupt state register. This will indicate which slots + * have interrupts pending for that SBUS interrupt level. + * + * 3) Call the genreric IRQ support. + */ +static void sun4d_sbus_handler_irq(int sbusl) +{ + unsigned int bus_mask; + unsigned int sbino, slot; + unsigned int sbil; + + bus_mask = bw_get_intr_mask(sbusl) & 0x3ffff; + bw_clear_intr_mask(sbusl, bus_mask); + + sbil = (sbusl << 2); + /* Loop for each pending SBI */ + for (sbino = 0; bus_mask; sbino++, bus_mask >>= 1) { + unsigned int idx, mask; + + if (!(bus_mask & 1)) + continue; + /* XXX This seems to ACK the irq twice. acquire_sbi() + * XXX uses swap, therefore this writes 0xf << sbil, + * XXX then later release_sbi() will write the individual + * XXX bits which were set again. + */ + mask = acquire_sbi(SBI2DEVID(sbino), 0xf << sbil); + mask &= (0xf << sbil); + + /* Loop for each pending SBI slot */ + slot = (1 << sbil); + for (idx = 0; mask != 0; idx++, slot <<= 1) { + unsigned int pil; + struct irq_bucket *p; + + if (!(mask & slot)) + continue; + + mask &= ~slot; + pil = sun4d_encode_irq(sbino, sbusl, idx); + + p = irq_map[pil]; + while (p) { + struct irq_bucket *next; + + next = p->next; + generic_handle_irq(p->irq); + p = next; + } + release_sbi(SBI2DEVID(sbino), slot); + } + } +} + +void sun4d_handler_irq(unsigned int pil, struct pt_regs *regs) +{ + struct pt_regs *old_regs; + /* SBUS IRQ level (1 - 7) */ + int sbusl = pil_to_sbus[pil]; + + /* FIXME: Is this necessary?? */ + cc_get_ipen(); + + cc_set_iclr(1 << pil); + +#ifdef CONFIG_SMP + /* + * Check IPI data structures after IRQ has been cleared. Hard and Soft + * IRQ can happen at the same time, so both cases are always handled. + */ + if (pil == SUN4D_IPI_IRQ) + sun4d_ipi_interrupt(); +#endif + + old_regs = set_irq_regs(regs); + irq_enter(); + if (sbusl == 0) { + /* cpu interrupt */ + struct irq_bucket *p; + + p = irq_map[pil]; + while (p) { + struct irq_bucket *next; + + next = p->next; + generic_handle_irq(p->irq); + p = next; + } + } else { + /* SBUS interrupt */ + sun4d_sbus_handler_irq(sbusl); + } + irq_exit(); + set_irq_regs(old_regs); +} + + +static void sun4d_mask_irq(struct irq_data *data) +{ + struct sun4d_handler_data *handler_data = irq_data_get_irq_handler_data(data); + unsigned int real_irq; +#ifdef CONFIG_SMP + int cpuid = handler_data->cpuid; + unsigned long flags; +#endif + real_irq = handler_data->real_irq; +#ifdef CONFIG_SMP + spin_lock_irqsave(&sun4d_imsk_lock, flags); + cc_set_imsk_other(cpuid, cc_get_imsk_other(cpuid) | (1 << real_irq)); + spin_unlock_irqrestore(&sun4d_imsk_lock, flags); +#else + cc_set_imsk(cc_get_imsk() | (1 << real_irq)); +#endif +} + +static void sun4d_unmask_irq(struct irq_data *data) +{ + struct sun4d_handler_data *handler_data = irq_data_get_irq_handler_data(data); + unsigned int real_irq; +#ifdef CONFIG_SMP + int cpuid = handler_data->cpuid; + unsigned long flags; +#endif + real_irq = handler_data->real_irq; + +#ifdef CONFIG_SMP + spin_lock_irqsave(&sun4d_imsk_lock, flags); + cc_set_imsk_other(cpuid, cc_get_imsk_other(cpuid) & ~(1 << real_irq)); + spin_unlock_irqrestore(&sun4d_imsk_lock, flags); +#else + cc_set_imsk(cc_get_imsk() & ~(1 << real_irq)); +#endif +} + +static unsigned int sun4d_startup_irq(struct irq_data *data) +{ + irq_link(data->irq); + sun4d_unmask_irq(data); + return 0; +} + +static void sun4d_shutdown_irq(struct irq_data *data) +{ + sun4d_mask_irq(data); + irq_unlink(data->irq); +} + +static struct irq_chip sun4d_irq = { + .name = "sun4d", + .irq_startup = sun4d_startup_irq, + .irq_shutdown = sun4d_shutdown_irq, + .irq_unmask = sun4d_unmask_irq, + .irq_mask = sun4d_mask_irq, +}; + +#ifdef CONFIG_SMP +/* Setup IRQ distribution scheme. */ +void __init sun4d_distribute_irqs(void) +{ + struct device_node *dp; + + int cpuid = cpu_logical_map(1); + + if (cpuid == -1) + cpuid = cpu_logical_map(0); + for_each_node_by_name(dp, "sbi") { + int devid = of_getintprop_default(dp, "device-id", 0); + int board = of_getintprop_default(dp, "board#", 0); + board_to_cpu[board] = cpuid; + set_sbi_tid(devid, cpuid << 3); + } + printk(KERN_ERR "All sbus IRQs directed to CPU%d\n", cpuid); +} +#endif + +static void sun4d_clear_clock_irq(void) +{ + sbus_readl(&sun4d_timers->l10_timer_limit); +} + +static void sun4d_load_profile_irq(int cpu, unsigned int limit) +{ + unsigned int value = limit ? timer_value(limit) : 0; + bw_set_prof_limit(cpu, value); +} + +static void __init sun4d_load_profile_irqs(void) +{ + int cpu = 0, mid; + + while (!cpu_find_by_instance(cpu, NULL, &mid)) { + sun4d_load_profile_irq(mid >> 3, 0); + cpu++; + } +} + +static unsigned int _sun4d_build_device_irq(unsigned int real_irq, + unsigned int pil, + unsigned int board) +{ + struct sun4d_handler_data *handler_data; + unsigned int irq; + + irq = irq_alloc(real_irq, pil); + if (irq == 0) { + prom_printf("IRQ: allocate for %d %d %d failed\n", + real_irq, pil, board); + goto err_out; + } + + handler_data = irq_get_handler_data(irq); + if (unlikely(handler_data)) + goto err_out; + + handler_data = kzalloc_obj(struct sun4d_handler_data, GFP_ATOMIC); + if (unlikely(!handler_data)) { + prom_printf("IRQ: kzalloc(sun4d_handler_data) failed.\n"); + prom_halt(); + } + handler_data->cpuid = board_to_cpu[board]; + handler_data->real_irq = real_irq; + irq_set_chip_and_handler_name(irq, &sun4d_irq, + handle_level_irq, "level"); + irq_set_handler_data(irq, handler_data); + +err_out: + return irq; +} + + + +static unsigned int sun4d_build_device_irq(struct platform_device *op, + unsigned int real_irq) +{ + struct device_node *dp = op->dev.of_node; + struct device_node *board_parent, *bus = dp->parent; + char *bus_connection; + const struct linux_prom_registers *regs; + unsigned int pil; + unsigned int irq; + int board, slot; + int sbusl; + + irq = real_irq; + while (bus) { + if (of_node_name_eq(bus, "sbi")) { + bus_connection = "io-unit"; + break; + } + + if (of_node_name_eq(bus, "bootbus")) { + bus_connection = "cpu-unit"; + break; + } + + bus = bus->parent; + } + if (!bus) + goto err_out; + + regs = of_get_property(dp, "reg", NULL); + if (!regs) + goto err_out; + + slot = regs->which_io; + + /* + * If Bus nodes parent is not io-unit/cpu-unit or the io-unit/cpu-unit + * lacks a "board#" property, something is very wrong. + */ + if (!of_node_name_eq(bus->parent, bus_connection)) { + printk(KERN_ERR "%pOF: Error, parent is not %s.\n", + bus, bus_connection); + goto err_out; + } + board_parent = bus->parent; + board = of_getintprop_default(board_parent, "board#", -1); + if (board == -1) { + printk(KERN_ERR "%pOF: Error, lacks board# property.\n", + board_parent); + goto err_out; + } + + sbusl = pil_to_sbus[real_irq]; + if (sbusl) + pil = sun4d_encode_irq(board, sbusl, slot); + else + pil = real_irq; + + irq = _sun4d_build_device_irq(real_irq, pil, board); +err_out: + return irq; +} + +static unsigned int sun4d_build_timer_irq(unsigned int board, + unsigned int real_irq) +{ + return _sun4d_build_device_irq(real_irq, real_irq, board); +} + + +static void __init sun4d_fixup_trap_table(void) +{ +#ifdef CONFIG_SMP + unsigned long flags; + struct tt_entry *trap_table = &sparc_ttable[SP_TRAP_IRQ1 + (14 - 1)]; + + /* Adjust so that we jump directly to smp4d_ticker */ + lvl14_save[2] += smp4d_ticker - real_irq_entry; + + /* For SMP we use the level 14 ticker, however the bootup code + * has copied the firmware's level 14 vector into the boot cpu's + * trap table, we must fix this now or we get squashed. + */ + local_irq_save(flags); + patchme_maybe_smp_msg[0] = 0x01000000; /* NOP out the branch */ + trap_table->inst_one = lvl14_save[0]; + trap_table->inst_two = lvl14_save[1]; + trap_table->inst_three = lvl14_save[2]; + trap_table->inst_four = lvl14_save[3]; + local_ops->cache_all(); + local_irq_restore(flags); +#endif +} + +static void __init sun4d_init_timers(void) +{ + struct device_node *dp; + struct resource res; + unsigned int irq; + const u32 *reg; + int err; + int board; + + dp = of_find_node_by_name(NULL, "cpu-unit"); + if (!dp) { + prom_printf("sun4d_init_timers: Unable to find cpu-unit\n"); + prom_halt(); + } + + /* Which cpu-unit we use is arbitrary, we can view the bootbus timer + * registers via any cpu's mapping. The first 'reg' property is the + * bootbus. + */ + reg = of_get_property(dp, "reg", NULL); + if (!reg) { + prom_printf("sun4d_init_timers: No reg property\n"); + prom_halt(); + } + + board = of_getintprop_default(dp, "board#", -1); + if (board == -1) { + prom_printf("sun4d_init_timers: No board# property on cpu-unit\n"); + prom_halt(); + } + + of_node_put(dp); + + res.start = reg[1]; + res.end = reg[2] - 1; + res.flags = reg[0] & 0xff; + sun4d_timers = of_ioremap(&res, BW_TIMER_LIMIT, + sizeof(struct sun4d_timer_regs), "user timer"); + if (!sun4d_timers) { + prom_printf("sun4d_init_timers: Can't map timer regs\n"); + prom_halt(); + } + +#ifdef CONFIG_SMP + sparc_config.cs_period = SBUS_CLOCK_RATE * 2; /* 2 seconds */ +#else + sparc_config.cs_period = SBUS_CLOCK_RATE / HZ; /* 1/HZ sec */ + sparc_config.features |= FEAT_L10_CLOCKEVENT; +#endif + sparc_config.features |= FEAT_L10_CLOCKSOURCE; + sbus_writel(timer_value(sparc_config.cs_period), + &sun4d_timers->l10_timer_limit); + + master_l10_counter = &sun4d_timers->l10_cur_count; + + irq = sun4d_build_timer_irq(board, SUN4D_TIMER_IRQ); + err = request_irq(irq, timer_interrupt, IRQF_TIMER, "timer", NULL); + if (err) { + prom_printf("sun4d_init_timers: request_irq() failed with %d\n", + err); + prom_halt(); + } + sun4d_load_profile_irqs(); + sun4d_fixup_trap_table(); +} + +void __init sun4d_init_sbi_irq(void) +{ + struct device_node *dp; + int target_cpu; + + target_cpu = boot_cpu_id; + for_each_node_by_name(dp, "sbi") { + int devid = of_getintprop_default(dp, "device-id", 0); + int board = of_getintprop_default(dp, "board#", 0); + unsigned int mask; + + set_sbi_tid(devid, target_cpu << 3); + board_to_cpu[board] = target_cpu; + + /* Get rid of pending irqs from PROM */ + mask = acquire_sbi(devid, 0xffffffff); + if (mask) { + printk(KERN_ERR "Clearing pending IRQs %08x on SBI %d\n", + mask, board); + release_sbi(devid, mask); + } + } +} + +void __init sun4d_init_IRQ(void) +{ + local_irq_disable(); + + sparc_config.init_timers = sun4d_init_timers; + sparc_config.build_device_irq = sun4d_build_device_irq; + sparc_config.clock_rate = SBUS_CLOCK_RATE; + sparc_config.clear_clock_irq = sun4d_clear_clock_irq; + sparc_config.load_profile_irq = sun4d_load_profile_irq; + + /* Cannot enable interrupts until OBP ticker is disabled. */ +} diff --git a/arch/sparc/kernel/sun4d_smp.c b/arch/sparc/kernel/sun4d_smp.c new file mode 100644 index 000000000..9a62a5cf3 --- /dev/null +++ b/arch/sparc/kernel/sun4d_smp.c @@ -0,0 +1,415 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Sparc SS1000/SC2000 SMP support. + * + * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * + * Based on sun4m's smp.c, which is: + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" +#include "irq.h" + +#define IRQ_CROSS_CALL 15 + +static volatile int smp_processors_ready; +static int smp_highest_cpu; + +static inline unsigned long sun4d_swap(volatile unsigned long *ptr, unsigned long val) +{ + __asm__ __volatile__("swap [%1], %0\n\t" : + "=&r" (val), "=&r" (ptr) : + "0" (val), "1" (ptr)); + return val; +} + +static void smp4d_ipi_init(void); + +static unsigned char cpu_leds[32]; + +static inline void show_leds(int cpuid) +{ + cpuid &= 0x1e; + __asm__ __volatile__ ("stba %0, [%1] %2" : : + "r" ((cpu_leds[cpuid] << 4) | cpu_leds[cpuid+1]), + "r" (ECSR_BASE(cpuid) | BB_LEDS), + "i" (ASI_M_CTL)); +} + +void sun4d_cpu_pre_starting(void *arg) +{ + int cpuid = hard_smp_processor_id(); + + /* Show we are alive */ + cpu_leds[cpuid] = 0x6; + show_leds(cpuid); + + /* Enable level15 interrupt, disable level14 interrupt for now */ + cc_set_imsk((cc_get_imsk() & ~0x8000) | 0x4000); +} + +void sun4d_cpu_pre_online(void *arg) +{ + unsigned long flags; + int cpuid; + + cpuid = hard_smp_processor_id(); + + /* Unblock the master CPU _only_ when the scheduler state + * of all secondary CPUs will be up-to-date, so after + * the SMP initialization the master will be just allowed + * to call the scheduler code. + */ + sun4d_swap((unsigned long *)&cpu_callin_map[cpuid], 1); + local_ops->cache_all(); + local_ops->tlb_all(); + + while ((unsigned long)current_set[cpuid] < PAGE_OFFSET) + barrier(); + + while (current_set[cpuid]->cpu != cpuid) + barrier(); + + /* Fix idle thread fields. */ + __asm__ __volatile__("ld [%0], %%g6\n\t" + : : "r" (¤t_set[cpuid]) + : "memory" /* paranoid */); + + cpu_leds[cpuid] = 0x9; + show_leds(cpuid); + + /* Attach to the address space of init_task. */ + mmgrab(&init_mm); + current->active_mm = &init_mm; + + local_ops->cache_all(); + local_ops->tlb_all(); + + while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) + barrier(); + + spin_lock_irqsave(&sun4d_imsk_lock, flags); + cc_set_imsk(cc_get_imsk() & ~0x4000); /* Allow PIL 14 as well */ + spin_unlock_irqrestore(&sun4d_imsk_lock, flags); +} + +/* + * Cycle through the processors asking the PROM to start each one. + */ +void __init smp4d_boot_cpus(void) +{ + smp4d_ipi_init(); + if (boot_cpu_id) + current_set[0] = NULL; + local_ops->cache_all(); +} + +int smp4d_boot_one_cpu(int i, struct task_struct *idle) +{ + unsigned long *entry = &sun4d_cpu_startup; + int timeout; + int cpu_node; + + cpu_find_by_instance(i, &cpu_node, NULL); + current_set[i] = task_thread_info(idle); + /* + * Initialize the contexts table + * Since the call to prom_startcpu() trashes the structure, + * we need to re-initialize it for each cpu + */ + smp_penguin_ctable.which_io = 0; + smp_penguin_ctable.phys_addr = (unsigned int) srmmu_ctx_table_phys; + smp_penguin_ctable.reg_size = 0; + + /* whirrr, whirrr, whirrrrrrrrr... */ + printk(KERN_INFO "Starting CPU %d at %p\n", i, entry); + local_ops->cache_all(); + prom_startcpu(cpu_node, + &smp_penguin_ctable, 0, (char *)entry); + + printk(KERN_INFO "prom_startcpu returned :)\n"); + + /* wheee... it's going... */ + for (timeout = 0; timeout < 10000; timeout++) { + if (cpu_callin_map[i]) + break; + udelay(200); + } + + if (!(cpu_callin_map[i])) { + printk(KERN_ERR "Processor %d is stuck.\n", i); + return -ENODEV; + + } + local_ops->cache_all(); + return 0; +} + +void __init smp4d_smp_done(void) +{ + int i, first; + int *prev; + + /* setup cpu list for irq rotation */ + first = 0; + prev = &first; + for_each_online_cpu(i) { + *prev = i; + prev = &cpu_data(i).next; + } + *prev = first; + local_ops->cache_all(); + + /* Ok, they are spinning and ready to go. */ + smp_processors_ready = 1; + sun4d_distribute_irqs(); +} + +/* Memory structure giving interrupt handler information about IPI generated */ +struct sun4d_ipi_work { + int single; + int msk; + int resched; +}; + +static DEFINE_PER_CPU_SHARED_ALIGNED(struct sun4d_ipi_work, sun4d_ipi_work); + +/* Initialize IPIs on the SUN4D SMP machine */ +static void __init smp4d_ipi_init(void) +{ + int cpu; + struct sun4d_ipi_work *work; + + printk(KERN_INFO "smp4d: setup IPI at IRQ %d\n", SUN4D_IPI_IRQ); + + for_each_possible_cpu(cpu) { + work = &per_cpu(sun4d_ipi_work, cpu); + work->single = work->msk = work->resched = 0; + } +} + +void sun4d_ipi_interrupt(void) +{ + struct sun4d_ipi_work *work = this_cpu_ptr(&sun4d_ipi_work); + + if (work->single) { + work->single = 0; + smp_call_function_single_interrupt(); + } + if (work->msk) { + work->msk = 0; + smp_call_function_interrupt(); + } + if (work->resched) { + work->resched = 0; + smp_resched_interrupt(); + } +} + +/* +-------+-------------+-----------+------------------------------------+ + * | bcast | devid | sid | levels mask | + * +-------+-------------+-----------+------------------------------------+ + * 31 30 23 22 15 14 0 + */ +#define IGEN_MESSAGE(bcast, devid, sid, levels) \ + (((bcast) << 31) | ((devid) << 23) | ((sid) << 15) | (levels)) + +static void sun4d_send_ipi(int cpu, int level) +{ + cc_set_igen(IGEN_MESSAGE(0, cpu << 3, 6 + ((level >> 1) & 7), 1 << (level - 1))); +} + +static void sun4d_ipi_single(int cpu) +{ + struct sun4d_ipi_work *work = &per_cpu(sun4d_ipi_work, cpu); + + /* Mark work */ + work->single = 1; + + /* Generate IRQ on the CPU */ + sun4d_send_ipi(cpu, SUN4D_IPI_IRQ); +} + +static void sun4d_ipi_mask_one(int cpu) +{ + struct sun4d_ipi_work *work = &per_cpu(sun4d_ipi_work, cpu); + + /* Mark work */ + work->msk = 1; + + /* Generate IRQ on the CPU */ + sun4d_send_ipi(cpu, SUN4D_IPI_IRQ); +} + +static void sun4d_ipi_resched(int cpu) +{ + struct sun4d_ipi_work *work = &per_cpu(sun4d_ipi_work, cpu); + + /* Mark work */ + work->resched = 1; + + /* Generate IRQ on the CPU (any IRQ will cause resched) */ + sun4d_send_ipi(cpu, SUN4D_IPI_IRQ); +} + +static struct smp_funcall { + void *func; + unsigned long arg1; + unsigned long arg2; + unsigned long arg3; + unsigned long arg4; + unsigned long arg5; + unsigned char processors_in[NR_CPUS]; /* Set when ipi entered. */ + unsigned char processors_out[NR_CPUS]; /* Set when ipi exited. */ +} ccall_info __attribute__((aligned(8))); + +static DEFINE_SPINLOCK(cross_call_lock); + +/* Cross calls must be serialized, at least currently. */ +static void sun4d_cross_call(void *func, cpumask_t mask, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4) +{ + if (smp_processors_ready) { + register int high = smp_highest_cpu; + unsigned long flags; + + spin_lock_irqsave(&cross_call_lock, flags); + + { + /* + * If you make changes here, make sure + * gcc generates proper code... + */ + register void *f asm("i0") = func; + register unsigned long a1 asm("i1") = arg1; + register unsigned long a2 asm("i2") = arg2; + register unsigned long a3 asm("i3") = arg3; + register unsigned long a4 asm("i4") = arg4; + register unsigned long a5 asm("i5") = 0; + + __asm__ __volatile__( + "std %0, [%6]\n\t" + "std %2, [%6 + 8]\n\t" + "std %4, [%6 + 16]\n\t" : : + "r"(f), "r"(a1), "r"(a2), "r"(a3), "r"(a4), "r"(a5), + "r" (&ccall_info.func)); + } + + /* Init receive/complete mapping, plus fire the IPI's off. */ + { + register int i; + + cpumask_clear_cpu(smp_processor_id(), &mask); + cpumask_and(&mask, cpu_online_mask, &mask); + for (i = 0; i <= high; i++) { + if (cpumask_test_cpu(i, &mask)) { + ccall_info.processors_in[i] = 0; + ccall_info.processors_out[i] = 0; + sun4d_send_ipi(i, IRQ_CROSS_CALL); + } + } + } + + { + register int i; + + i = 0; + do { + if (!cpumask_test_cpu(i, &mask)) + continue; + while (!ccall_info.processors_in[i]) + barrier(); + } while (++i <= high); + + i = 0; + do { + if (!cpumask_test_cpu(i, &mask)) + continue; + while (!ccall_info.processors_out[i]) + barrier(); + } while (++i <= high); + } + + spin_unlock_irqrestore(&cross_call_lock, flags); + } +} + +/* Running cross calls. */ +void smp4d_cross_call_irq(void) +{ + void (*func)(unsigned long, unsigned long, unsigned long, unsigned long, + unsigned long) = ccall_info.func; + int i = hard_smp_processor_id(); + + ccall_info.processors_in[i] = 1; + func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4, + ccall_info.arg5); + ccall_info.processors_out[i] = 1; +} + +void smp4d_percpu_timer_interrupt(struct pt_regs *regs) +{ + struct pt_regs *old_regs; + int cpu = hard_smp_processor_id(); + struct clock_event_device *ce; + static int cpu_tick[NR_CPUS]; + static char led_mask[] = { 0xe, 0xd, 0xb, 0x7, 0xb, 0xd }; + + old_regs = set_irq_regs(regs); + bw_get_prof_limit(cpu); + bw_clear_intr_mask(0, 1); /* INTR_TABLE[0] & 1 is Profile IRQ */ + + cpu_tick[cpu]++; + if (!(cpu_tick[cpu] & 15)) { + if (cpu_tick[cpu] == 0x60) + cpu_tick[cpu] = 0; + cpu_leds[cpu] = led_mask[cpu_tick[cpu] >> 4]; + show_leds(cpu); + } + + ce = &per_cpu(sparc32_clockevent, cpu); + + irq_enter(); + ce->event_handler(ce); + irq_exit(); + + set_irq_regs(old_regs); +} + +static const struct sparc32_ipi_ops sun4d_ipi_ops = { + .cross_call = sun4d_cross_call, + .resched = sun4d_ipi_resched, + .single = sun4d_ipi_single, + .mask_one = sun4d_ipi_mask_one, +}; + +void __init sun4d_init_smp(void) +{ + int i; + + /* Patch ipi15 trap table */ + t_nmi[1] = t_nmi[1] + (linux_trap_ipi15_sun4d - linux_trap_ipi15_sun4m); + + sparc32_ipi_ops = &sun4d_ipi_ops; + + for (i = 0; i < NR_CPUS; i++) { + ccall_info.processors_in[i] = 1; + ccall_info.processors_out[i] = 1; + } +} diff --git a/arch/sparc/kernel/sun4m_irq.c b/arch/sparc/kernel/sun4m_irq.c new file mode 100644 index 000000000..fe8cfb1cd --- /dev/null +++ b/arch/sparc/kernel/sun4m_irq.c @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun4m irq support + * + * djhr: Hacked out of irq.c into a CPU dependent version. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1995 Miguel de Icaza (miguel@nuclecu.unam.mx) + * Copyright (C) 1995 Pete A. Zaitcev (zaitcev@yahoo.com) + * Copyright (C) 1996 Dave Redman (djhr@tadpole.co.uk) + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "irq.h" +#include "kernel.h" + +/* Sample sun4m IRQ layout: + * + * 0x22 - Power + * 0x24 - ESP SCSI + * 0x26 - Lance ethernet + * 0x2b - Floppy + * 0x2c - Zilog uart + * 0x32 - SBUS level 0 + * 0x33 - Parallel port, SBUS level 1 + * 0x35 - SBUS level 2 + * 0x37 - SBUS level 3 + * 0x39 - Audio, Graphics card, SBUS level 4 + * 0x3b - SBUS level 5 + * 0x3d - SBUS level 6 + * + * Each interrupt source has a mask bit in the interrupt registers. + * When the mask bit is set, this blocks interrupt deliver. So you + * clear the bit to enable the interrupt. + * + * Interrupts numbered less than 0x10 are software triggered interrupts + * and unused by Linux. + * + * Interrupt level assignment on sun4m: + * + * level source + * ------------------------------------------------------------ + * 1 softint-1 + * 2 softint-2, VME/SBUS level 1 + * 3 softint-3, VME/SBUS level 2 + * 4 softint-4, onboard SCSI + * 5 softint-5, VME/SBUS level 3 + * 6 softint-6, onboard ETHERNET + * 7 softint-7, VME/SBUS level 4 + * 8 softint-8, onboard VIDEO + * 9 softint-9, VME/SBUS level 5, Module Interrupt + * 10 softint-10, system counter/timer + * 11 softint-11, VME/SBUS level 6, Floppy + * 12 softint-12, Keyboard/Mouse, Serial + * 13 softint-13, VME/SBUS level 7, ISDN Audio + * 14 softint-14, per-processor counter/timer + * 15 softint-15, Asynchronous Errors (broadcast) + * + * Each interrupt source is masked distinctly in the sun4m interrupt + * registers. The PIL level alone is therefore ambiguous, since multiple + * interrupt sources map to a single PIL. + * + * This ambiguity is resolved in the 'intr' property for device nodes + * in the OF device tree. Each 'intr' property entry is composed of + * two 32-bit words. The first word is the IRQ priority value, which + * is what we're intersted in. The second word is the IRQ vector, which + * is unused. + * + * The low 4 bits of the IRQ priority indicate the PIL, and the upper + * 4 bits indicate onboard vs. SBUS leveled vs. VME leveled. 0x20 + * means onboard, 0x30 means SBUS leveled, and 0x40 means VME leveled. + * + * For example, an 'intr' IRQ priority value of 0x24 is onboard SCSI + * whereas a value of 0x33 is SBUS level 2. Here are some sample + * 'intr' property IRQ priority values from ss4, ss5, ss10, ss20, and + * Tadpole S3 GX systems. + * + * esp: 0x24 onboard ESP SCSI + * le: 0x26 onboard Lance ETHERNET + * p9100: 0x32 SBUS level 1 P9100 video + * bpp: 0x33 SBUS level 2 BPP parallel port device + * DBRI: 0x39 SBUS level 5 DBRI ISDN audio + * SUNW,leo: 0x39 SBUS level 5 LEO video + * pcmcia: 0x3b SBUS level 6 PCMCIA controller + * uctrl: 0x3b SBUS level 6 UCTRL device + * modem: 0x3d SBUS level 7 MODEM + * zs: 0x2c onboard keyboard/mouse/serial + * floppy: 0x2b onboard Floppy + * power: 0x22 onboard power device (XXX unknown mask bit XXX) + */ + + +/* Code in entry.S needs to get at these register mappings. */ +struct sun4m_irq_percpu __iomem *sun4m_irq_percpu[SUN4M_NCPUS]; +struct sun4m_irq_global __iomem *sun4m_irq_global; + +struct sun4m_handler_data { + bool percpu; + long mask; +}; + +/* Dave Redman (djhr@tadpole.co.uk) + * The sun4m interrupt registers. + */ +#define SUN4M_INT_ENABLE 0x80000000 +#define SUN4M_INT_E14 0x00000080 +#define SUN4M_INT_E10 0x00080000 + +#define SUN4M_INT_MASKALL 0x80000000 /* mask all interrupts */ +#define SUN4M_INT_MODULE_ERR 0x40000000 /* module error */ +#define SUN4M_INT_M2S_WRITE_ERR 0x20000000 /* write buffer error */ +#define SUN4M_INT_ECC_ERR 0x10000000 /* ecc memory error */ +#define SUN4M_INT_VME_ERR 0x08000000 /* vme async error */ +#define SUN4M_INT_FLOPPY 0x00400000 /* floppy disk */ +#define SUN4M_INT_MODULE 0x00200000 /* module interrupt */ +#define SUN4M_INT_VIDEO 0x00100000 /* onboard video */ +#define SUN4M_INT_REALTIME 0x00080000 /* system timer */ +#define SUN4M_INT_SCSI 0x00040000 /* onboard scsi */ +#define SUN4M_INT_AUDIO 0x00020000 /* audio/isdn */ +#define SUN4M_INT_ETHERNET 0x00010000 /* onboard ethernet */ +#define SUN4M_INT_SERIAL 0x00008000 /* serial ports */ +#define SUN4M_INT_KBDMS 0x00004000 /* keyboard/mouse */ +#define SUN4M_INT_SBUSBITS 0x00003F80 /* sbus int bits */ +#define SUN4M_INT_VMEBITS 0x0000007F /* vme int bits */ + +#define SUN4M_INT_ERROR (SUN4M_INT_MODULE_ERR | \ + SUN4M_INT_M2S_WRITE_ERR | \ + SUN4M_INT_ECC_ERR | \ + SUN4M_INT_VME_ERR) + +#define SUN4M_INT_SBUS(x) (1 << (x+7)) +#define SUN4M_INT_VME(x) (1 << (x)) + +/* Interrupt levels used by OBP */ +#define OBP_INT_LEVEL_SOFT 0x10 +#define OBP_INT_LEVEL_ONBOARD 0x20 +#define OBP_INT_LEVEL_SBUS 0x30 +#define OBP_INT_LEVEL_VME 0x40 + +#define SUN4M_TIMER_IRQ (OBP_INT_LEVEL_ONBOARD | 10) +#define SUN4M_PROFILE_IRQ (OBP_INT_LEVEL_ONBOARD | 14) + +static unsigned long sun4m_imask[0x50] = { + /* 0x00 - SMP */ + 0, SUN4M_SOFT_INT(1), + SUN4M_SOFT_INT(2), SUN4M_SOFT_INT(3), + SUN4M_SOFT_INT(4), SUN4M_SOFT_INT(5), + SUN4M_SOFT_INT(6), SUN4M_SOFT_INT(7), + SUN4M_SOFT_INT(8), SUN4M_SOFT_INT(9), + SUN4M_SOFT_INT(10), SUN4M_SOFT_INT(11), + SUN4M_SOFT_INT(12), SUN4M_SOFT_INT(13), + SUN4M_SOFT_INT(14), SUN4M_SOFT_INT(15), + /* 0x10 - soft */ + 0, SUN4M_SOFT_INT(1), + SUN4M_SOFT_INT(2), SUN4M_SOFT_INT(3), + SUN4M_SOFT_INT(4), SUN4M_SOFT_INT(5), + SUN4M_SOFT_INT(6), SUN4M_SOFT_INT(7), + SUN4M_SOFT_INT(8), SUN4M_SOFT_INT(9), + SUN4M_SOFT_INT(10), SUN4M_SOFT_INT(11), + SUN4M_SOFT_INT(12), SUN4M_SOFT_INT(13), + SUN4M_SOFT_INT(14), SUN4M_SOFT_INT(15), + /* 0x20 - onboard */ + 0, 0, 0, 0, + SUN4M_INT_SCSI, 0, SUN4M_INT_ETHERNET, 0, + SUN4M_INT_VIDEO, SUN4M_INT_MODULE, + SUN4M_INT_REALTIME, SUN4M_INT_FLOPPY, + (SUN4M_INT_SERIAL | SUN4M_INT_KBDMS), + SUN4M_INT_AUDIO, SUN4M_INT_E14, SUN4M_INT_MODULE_ERR, + /* 0x30 - sbus */ + 0, 0, SUN4M_INT_SBUS(0), SUN4M_INT_SBUS(1), + 0, SUN4M_INT_SBUS(2), 0, SUN4M_INT_SBUS(3), + 0, SUN4M_INT_SBUS(4), 0, SUN4M_INT_SBUS(5), + 0, SUN4M_INT_SBUS(6), 0, 0, + /* 0x40 - vme */ + 0, 0, SUN4M_INT_VME(0), SUN4M_INT_VME(1), + 0, SUN4M_INT_VME(2), 0, SUN4M_INT_VME(3), + 0, SUN4M_INT_VME(4), 0, SUN4M_INT_VME(5), + 0, SUN4M_INT_VME(6), 0, 0 +}; + +static void sun4m_mask_irq(struct irq_data *data) +{ + struct sun4m_handler_data *handler_data; + int cpu = smp_processor_id(); + + handler_data = irq_data_get_irq_handler_data(data); + if (handler_data->mask) { + unsigned long flags; + + local_irq_save(flags); + if (handler_data->percpu) { + sbus_writel(handler_data->mask, &sun4m_irq_percpu[cpu]->set); + } else { + sbus_writel(handler_data->mask, &sun4m_irq_global->mask_set); + } + local_irq_restore(flags); + } +} + +static void sun4m_unmask_irq(struct irq_data *data) +{ + struct sun4m_handler_data *handler_data; + int cpu = smp_processor_id(); + + handler_data = irq_data_get_irq_handler_data(data); + if (handler_data->mask) { + unsigned long flags; + + local_irq_save(flags); + if (handler_data->percpu) { + sbus_writel(handler_data->mask, &sun4m_irq_percpu[cpu]->clear); + } else { + sbus_writel(handler_data->mask, &sun4m_irq_global->mask_clear); + } + local_irq_restore(flags); + } +} + +static unsigned int sun4m_startup_irq(struct irq_data *data) +{ + irq_link(data->irq); + sun4m_unmask_irq(data); + return 0; +} + +static void sun4m_shutdown_irq(struct irq_data *data) +{ + sun4m_mask_irq(data); + irq_unlink(data->irq); +} + +static struct irq_chip sun4m_irq = { + .name = "sun4m", + .irq_startup = sun4m_startup_irq, + .irq_shutdown = sun4m_shutdown_irq, + .irq_mask = sun4m_mask_irq, + .irq_unmask = sun4m_unmask_irq, +}; + + +static unsigned int sun4m_build_device_irq(struct platform_device *op, + unsigned int real_irq) +{ + struct sun4m_handler_data *handler_data; + unsigned int irq; + unsigned int pil; + + if (real_irq >= OBP_INT_LEVEL_VME) { + prom_printf("Bogus sun4m IRQ %u\n", real_irq); + prom_halt(); + } + pil = (real_irq & 0xf); + irq = irq_alloc(real_irq, pil); + + if (irq == 0) + goto out; + + handler_data = irq_get_handler_data(irq); + if (unlikely(handler_data)) + goto out; + + handler_data = kzalloc_obj(struct sun4m_handler_data, GFP_ATOMIC); + if (unlikely(!handler_data)) { + prom_printf("IRQ: kzalloc(sun4m_handler_data) failed.\n"); + prom_halt(); + } + + handler_data->mask = sun4m_imask[real_irq]; + handler_data->percpu = real_irq < OBP_INT_LEVEL_ONBOARD; + irq_set_chip_and_handler_name(irq, &sun4m_irq, + handle_level_irq, "level"); + irq_set_handler_data(irq, handler_data); + +out: + return irq; +} + +struct sun4m_timer_percpu { + u32 l14_limit; + u32 l14_count; + u32 l14_limit_noclear; + u32 user_timer_start_stop; +}; + +static struct sun4m_timer_percpu __iomem *timers_percpu[SUN4M_NCPUS]; + +struct sun4m_timer_global { + u32 l10_limit; + u32 l10_count; + u32 l10_limit_noclear; + u32 reserved; + u32 timer_config; +}; + +static struct sun4m_timer_global __iomem *timers_global; + +static void sun4m_clear_clock_irq(void) +{ + sbus_readl(&timers_global->l10_limit); +} + +void sun4m_nmi(struct pt_regs *regs) +{ + unsigned long afsr, afar, si; + + printk(KERN_ERR "Aieee: sun4m NMI received!\n"); + /* XXX HyperSparc hack XXX */ + __asm__ __volatile__("mov 0x500, %%g1\n\t" + "lda [%%g1] 0x4, %0\n\t" + "mov 0x600, %%g1\n\t" + "lda [%%g1] 0x4, %1\n\t" : + "=r" (afsr), "=r" (afar)); + printk(KERN_ERR "afsr=%08lx afar=%08lx\n", afsr, afar); + si = sbus_readl(&sun4m_irq_global->pending); + printk(KERN_ERR "si=%08lx\n", si); + if (si & SUN4M_INT_MODULE_ERR) + printk(KERN_ERR "Module async error\n"); + if (si & SUN4M_INT_M2S_WRITE_ERR) + printk(KERN_ERR "MBus/SBus async error\n"); + if (si & SUN4M_INT_ECC_ERR) + printk(KERN_ERR "ECC memory error\n"); + if (si & SUN4M_INT_VME_ERR) + printk(KERN_ERR "VME async error\n"); + printk(KERN_ERR "you lose buddy boy...\n"); + show_regs(regs); + prom_halt(); +} + +void sun4m_unmask_profile_irq(void) +{ + unsigned long flags; + + local_irq_save(flags); + sbus_writel(sun4m_imask[SUN4M_PROFILE_IRQ], &sun4m_irq_global->mask_clear); + local_irq_restore(flags); +} + +void sun4m_clear_profile_irq(int cpu) +{ + sbus_readl(&timers_percpu[cpu]->l14_limit); +} + +static void sun4m_load_profile_irq(int cpu, unsigned int limit) +{ + unsigned int value = limit ? timer_value(limit) : 0; + sbus_writel(value, &timers_percpu[cpu]->l14_limit); +} + +static void __init sun4m_init_timers(void) +{ + struct device_node *dp = of_find_node_by_name(NULL, "counter"); + int i, err, len, num_cpu_timers; + unsigned int irq; + const u32 *addr; + + if (!dp) { + printk(KERN_ERR "sun4m_init_timers: No 'counter' node.\n"); + return; + } + + addr = of_get_property(dp, "address", &len); + of_node_put(dp); + if (!addr) { + printk(KERN_ERR "sun4m_init_timers: No 'address' prop.\n"); + return; + } + + num_cpu_timers = (len / sizeof(u32)) - 1; + for (i = 0; i < num_cpu_timers; i++) { + timers_percpu[i] = (void __iomem *) + (unsigned long) addr[i]; + } + timers_global = (void __iomem *) + (unsigned long) addr[num_cpu_timers]; + + /* Every per-cpu timer works in timer mode */ + sbus_writel(0x00000000, &timers_global->timer_config); + +#ifdef CONFIG_SMP + sparc_config.cs_period = SBUS_CLOCK_RATE * 2; /* 2 seconds */ + sparc_config.features |= FEAT_L14_ONESHOT; +#else + sparc_config.cs_period = SBUS_CLOCK_RATE / HZ; /* 1/HZ sec */ + sparc_config.features |= FEAT_L10_CLOCKEVENT; +#endif + sparc_config.features |= FEAT_L10_CLOCKSOURCE; + sbus_writel(timer_value(sparc_config.cs_period), + &timers_global->l10_limit); + + master_l10_counter = &timers_global->l10_count; + + irq = sun4m_build_device_irq(NULL, SUN4M_TIMER_IRQ); + + err = request_irq(irq, timer_interrupt, IRQF_TIMER, "timer", NULL); + if (err) { + printk(KERN_ERR "sun4m_init_timers: Register IRQ error %d.\n", + err); + return; + } + + for (i = 0; i < num_cpu_timers; i++) + sbus_writel(0, &timers_percpu[i]->l14_limit); + if (num_cpu_timers == 4) + sbus_writel(SUN4M_INT_E14, &sun4m_irq_global->mask_set); + +#ifdef CONFIG_SMP + { + unsigned long flags; + struct tt_entry *trap_table = &sparc_ttable[SP_TRAP_IRQ1 + (14 - 1)]; + + /* For SMP we use the level 14 ticker, however the bootup code + * has copied the firmware's level 14 vector into the boot cpu's + * trap table, we must fix this now or we get squashed. + */ + local_irq_save(flags); + trap_table->inst_one = lvl14_save[0]; + trap_table->inst_two = lvl14_save[1]; + trap_table->inst_three = lvl14_save[2]; + trap_table->inst_four = lvl14_save[3]; + local_ops->cache_all(); + local_irq_restore(flags); + } +#endif +} + +void __init sun4m_init_IRQ(void) +{ + struct device_node *dp = of_find_node_by_name(NULL, "interrupt"); + int len, i, mid, num_cpu_iregs; + const u32 *addr; + + if (!dp) { + printk(KERN_ERR "sun4m_init_IRQ: No 'interrupt' node.\n"); + return; + } + + addr = of_get_property(dp, "address", &len); + of_node_put(dp); + if (!addr) { + printk(KERN_ERR "sun4m_init_IRQ: No 'address' prop.\n"); + return; + } + + num_cpu_iregs = (len / sizeof(u32)) - 1; + for (i = 0; i < num_cpu_iregs; i++) { + sun4m_irq_percpu[i] = (void __iomem *) + (unsigned long) addr[i]; + } + sun4m_irq_global = (void __iomem *) + (unsigned long) addr[num_cpu_iregs]; + + local_irq_disable(); + + sbus_writel(~SUN4M_INT_MASKALL, &sun4m_irq_global->mask_set); + for (i = 0; !cpu_find_by_instance(i, NULL, &mid); i++) + sbus_writel(~0x17fff, &sun4m_irq_percpu[mid]->clear); + + if (num_cpu_iregs == 4) + sbus_writel(0, &sun4m_irq_global->interrupt_target); + + sparc_config.init_timers = sun4m_init_timers; + sparc_config.build_device_irq = sun4m_build_device_irq; + sparc_config.clock_rate = SBUS_CLOCK_RATE; + sparc_config.clear_clock_irq = sun4m_clear_clock_irq; + sparc_config.load_profile_irq = sun4m_load_profile_irq; + + + /* Cannot enable interrupts until OBP ticker is disabled. */ +} diff --git a/arch/sparc/kernel/sun4m_smp.c b/arch/sparc/kernel/sun4m_smp.c new file mode 100644 index 000000000..056df034e --- /dev/null +++ b/arch/sparc/kernel/sun4m_smp.c @@ -0,0 +1,275 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sun4m SMP support. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "irq.h" +#include "kernel.h" + +#define IRQ_IPI_SINGLE 12 +#define IRQ_IPI_MASK 13 +#define IRQ_IPI_RESCHED 14 +#define IRQ_CROSS_CALL 15 + +static inline unsigned long +swap_ulong(volatile unsigned long *ptr, unsigned long val) +{ + __asm__ __volatile__("swap [%1], %0\n\t" : + "=&r" (val), "=&r" (ptr) : + "0" (val), "1" (ptr)); + return val; +} + +void sun4m_cpu_pre_starting(void *arg) +{ +} + +void sun4m_cpu_pre_online(void *arg) +{ + int cpuid = hard_smp_processor_id(); + + /* Allow master to continue. The master will then give us the + * go-ahead by setting the smp_commenced_mask and will wait without + * timeouts until our setup is completed fully (signified by + * our bit being set in the cpu_online_mask). + */ + swap_ulong(&cpu_callin_map[cpuid], 1); + + /* XXX: What's up with all the flushes? */ + local_ops->cache_all(); + local_ops->tlb_all(); + + /* Fix idle thread fields. */ + __asm__ __volatile__("ld [%0], %%g6\n\t" + : : "r" (¤t_set[cpuid]) + : "memory" /* paranoid */); + + /* Attach to the address space of init_task. */ + mmgrab(&init_mm); + current->active_mm = &init_mm; + + while (!cpumask_test_cpu(cpuid, &smp_commenced_mask)) + mb(); +} + +/* + * Cycle through the processors asking the PROM to start each one. + */ +void __init smp4m_boot_cpus(void) +{ + sun4m_unmask_profile_irq(); + local_ops->cache_all(); +} + +int smp4m_boot_one_cpu(int i, struct task_struct *idle) +{ + unsigned long *entry = &sun4m_cpu_startup; + int timeout; + int cpu_node; + + cpu_find_by_mid(i, &cpu_node); + current_set[i] = task_thread_info(idle); + + /* See trampoline.S for details... */ + entry += ((i - 1) * 3); + + /* + * Initialize the contexts table + * Since the call to prom_startcpu() trashes the structure, + * we need to re-initialize it for each cpu + */ + smp_penguin_ctable.which_io = 0; + smp_penguin_ctable.phys_addr = (unsigned int) srmmu_ctx_table_phys; + smp_penguin_ctable.reg_size = 0; + + /* whirrr, whirrr, whirrrrrrrrr... */ + printk(KERN_INFO "Starting CPU %d at %p\n", i, entry); + local_ops->cache_all(); + prom_startcpu(cpu_node, &smp_penguin_ctable, 0, (char *)entry); + + /* wheee... it's going... */ + for (timeout = 0; timeout < 10000; timeout++) { + if (cpu_callin_map[i]) + break; + udelay(200); + } + + if (!(cpu_callin_map[i])) { + printk(KERN_ERR "Processor %d is stuck.\n", i); + return -ENODEV; + } + + local_ops->cache_all(); + return 0; +} + +void __init smp4m_smp_done(void) +{ + int i, first; + int *prev; + + /* setup cpu list for irq rotation */ + first = 0; + prev = &first; + for_each_online_cpu(i) { + *prev = i; + prev = &cpu_data(i).next; + } + *prev = first; + local_ops->cache_all(); + + /* Ok, they are spinning and ready to go. */ +} + +static void sun4m_send_ipi(int cpu, int level) +{ + sbus_writel(SUN4M_SOFT_INT(level), &sun4m_irq_percpu[cpu]->set); +} + +static void sun4m_ipi_resched(int cpu) +{ + sun4m_send_ipi(cpu, IRQ_IPI_RESCHED); +} + +static void sun4m_ipi_single(int cpu) +{ + sun4m_send_ipi(cpu, IRQ_IPI_SINGLE); +} + +static void sun4m_ipi_mask_one(int cpu) +{ + sun4m_send_ipi(cpu, IRQ_IPI_MASK); +} + +static struct smp_funcall { + void *func; + unsigned long arg1; + unsigned long arg2; + unsigned long arg3; + unsigned long arg4; + unsigned long arg5; + unsigned long processors_in[SUN4M_NCPUS]; /* Set when ipi entered. */ + unsigned long processors_out[SUN4M_NCPUS]; /* Set when ipi exited. */ +} ccall_info; + +static DEFINE_SPINLOCK(cross_call_lock); + +/* Cross calls must be serialized, at least currently. */ +static void sun4m_cross_call(void *func, cpumask_t mask, unsigned long arg1, + unsigned long arg2, unsigned long arg3, + unsigned long arg4) +{ + register int ncpus = SUN4M_NCPUS; + unsigned long flags; + + spin_lock_irqsave(&cross_call_lock, flags); + + /* Init function glue. */ + ccall_info.func = func; + ccall_info.arg1 = arg1; + ccall_info.arg2 = arg2; + ccall_info.arg3 = arg3; + ccall_info.arg4 = arg4; + ccall_info.arg5 = 0; + + /* Init receive/complete mapping, plus fire the IPI's off. */ + { + register int i; + + cpumask_clear_cpu(smp_processor_id(), &mask); + cpumask_and(&mask, cpu_online_mask, &mask); + for (i = 0; i < ncpus; i++) { + if (cpumask_test_cpu(i, &mask)) { + ccall_info.processors_in[i] = 0; + ccall_info.processors_out[i] = 0; + sun4m_send_ipi(i, IRQ_CROSS_CALL); + } else { + ccall_info.processors_in[i] = 1; + ccall_info.processors_out[i] = 1; + } + } + } + + { + register int i; + + i = 0; + do { + if (!cpumask_test_cpu(i, &mask)) + continue; + while (!ccall_info.processors_in[i]) + barrier(); + } while (++i < ncpus); + + i = 0; + do { + if (!cpumask_test_cpu(i, &mask)) + continue; + while (!ccall_info.processors_out[i]) + barrier(); + } while (++i < ncpus); + } + spin_unlock_irqrestore(&cross_call_lock, flags); +} + +/* Running cross calls. */ +void smp4m_cross_call_irq(void) +{ + void (*func)(unsigned long, unsigned long, unsigned long, unsigned long, + unsigned long) = ccall_info.func; + int i = smp_processor_id(); + + ccall_info.processors_in[i] = 1; + func(ccall_info.arg1, ccall_info.arg2, ccall_info.arg3, ccall_info.arg4, + ccall_info.arg5); + ccall_info.processors_out[i] = 1; +} + +void smp4m_percpu_timer_interrupt(struct pt_regs *regs) +{ + struct pt_regs *old_regs; + struct clock_event_device *ce; + int cpu = smp_processor_id(); + + old_regs = set_irq_regs(regs); + + ce = &per_cpu(sparc32_clockevent, cpu); + + if (clockevent_state_periodic(ce)) + sun4m_clear_profile_irq(cpu); + else + sparc_config.load_profile_irq(cpu, 0); /* Is this needless? */ + + irq_enter(); + ce->event_handler(ce); + irq_exit(); + + set_irq_regs(old_regs); +} + +static const struct sparc32_ipi_ops sun4m_ipi_ops = { + .cross_call = sun4m_cross_call, + .resched = sun4m_ipi_resched, + .single = sun4m_ipi_single, + .mask_one = sun4m_ipi_mask_one, +}; + +void __init sun4m_init_smp(void) +{ + sparc32_ipi_ops = &sun4m_ipi_ops; +} diff --git a/arch/sparc/kernel/sun4v_ivec.S b/arch/sparc/kernel/sun4v_ivec.S new file mode 100644 index 000000000..6478ef4f6 --- /dev/null +++ b/arch/sparc/kernel/sun4v_ivec.S @@ -0,0 +1,357 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* sun4v_ivec.S: Sun4v interrupt vector handling. + * + * Copyright (C) 2006 + */ + +#include +#include +#include + + .text + .align 32 + +sun4v_cpu_mondo: + /* Head offset in %g2, tail offset in %g4. + * If they are the same, no work. + */ + mov INTRQ_CPU_MONDO_HEAD, %g2 + ldxa [%g2] ASI_QUEUE, %g2 + mov INTRQ_CPU_MONDO_TAIL, %g4 + ldxa [%g4] ASI_QUEUE, %g4 + cmp %g2, %g4 + be,pn %xcc, sun4v_cpu_mondo_queue_empty + nop + + /* Get &trap_block[smp_processor_id()] into %g4. */ + ldxa [%g0] ASI_SCRATCHPAD, %g4 + sub %g4, TRAP_PER_CPU_FAULT_INFO, %g4 + + /* Get smp_processor_id() into %g3 */ + sethi %hi(trap_block), %g5 + or %g5, %lo(trap_block), %g5 + sub %g4, %g5, %g3 + srlx %g3, TRAP_BLOCK_SZ_SHIFT, %g3 + + /* Increment cpu_mondo_counter[smp_processor_id()] */ + sethi %hi(cpu_mondo_counter), %g5 + or %g5, %lo(cpu_mondo_counter), %g5 + sllx %g3, 3, %g3 + add %g5, %g3, %g5 + ldx [%g5], %g3 + add %g3, 1, %g3 + stx %g3, [%g5] + + /* Get CPU mondo queue base phys address into %g7. */ + ldx [%g4 + TRAP_PER_CPU_CPU_MONDO_PA], %g7 + + /* Now get the cross-call arguments and handler PC, same + * layout as sun4u: + * + * 1st 64-bit word: low half is 32-bit PC, put into %g3 and jmpl to it + * high half is context arg to MMU flushes, into %g5 + * 2nd 64-bit word: 64-bit arg, load into %g1 + * 3rd 64-bit word: 64-bit arg, load into %g7 + */ + ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g3 + add %g2, 0x8, %g2 + srlx %g3, 32, %g5 + ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g1 + add %g2, 0x8, %g2 + srl %g3, 0, %g3 + ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g7 + add %g2, 0x40 - 0x8 - 0x8, %g2 + + /* Update queue head pointer. */ + lduw [%g4 + TRAP_PER_CPU_CPU_MONDO_QMASK], %g4 + and %g2, %g4, %g2 + + mov INTRQ_CPU_MONDO_HEAD, %g4 + stxa %g2, [%g4] ASI_QUEUE + membar #Sync + + jmpl %g3, %g0 + nop + +sun4v_cpu_mondo_queue_empty: + retry + +sun4v_dev_mondo: + /* Head offset in %g2, tail offset in %g4. */ + mov INTRQ_DEVICE_MONDO_HEAD, %g2 + ldxa [%g2] ASI_QUEUE, %g2 + mov INTRQ_DEVICE_MONDO_TAIL, %g4 + ldxa [%g4] ASI_QUEUE, %g4 + cmp %g2, %g4 + be,pn %xcc, sun4v_dev_mondo_queue_empty + nop + + /* Get &trap_block[smp_processor_id()] into %g4. */ + ldxa [%g0] ASI_SCRATCHPAD, %g4 + sub %g4, TRAP_PER_CPU_FAULT_INFO, %g4 + + /* Get DEV mondo queue base phys address into %g5. */ + ldx [%g4 + TRAP_PER_CPU_DEV_MONDO_PA], %g5 + + /* Load IVEC into %g3. */ + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + add %g2, 0x40, %g2 + + /* XXX There can be a full 64-byte block of data here. + * XXX This is how we can get at MSI vector data. + * XXX Current we do not capture this, but when we do we'll + * XXX need to add a 64-byte storage area in the struct ino_bucket + * XXX or the struct irq_desc. + */ + + /* Update queue head pointer, this frees up some registers. */ + lduw [%g4 + TRAP_PER_CPU_DEV_MONDO_QMASK], %g4 + and %g2, %g4, %g2 + + mov INTRQ_DEVICE_MONDO_HEAD, %g4 + stxa %g2, [%g4] ASI_QUEUE + membar #Sync + + TRAP_LOAD_IRQ_WORK_PA(%g1, %g4) + + /* For VIRQs, cookie is encoded as ~bucket_phys_addr */ + brlz,pt %g3, 1f + xnor %g3, %g0, %g4 + + /* Get __pa(&ivector_table[IVEC]) into %g4. */ + sethi %hi(ivector_table_pa), %g4 + ldx [%g4 + %lo(ivector_table_pa)], %g4 + sllx %g3, 4, %g3 + add %g4, %g3, %g4 + +1: ldx [%g1], %g2 + stxa %g2, [%g4] ASI_PHYS_USE_EC + stx %g4, [%g1] + + /* Signal the interrupt by setting (1 << pil) in %softint. */ + wr %g0, 1 << PIL_DEVICE_IRQ, %set_softint + +sun4v_dev_mondo_queue_empty: + retry + +sun4v_res_mondo: + /* Head offset in %g2, tail offset in %g4. */ + mov INTRQ_RESUM_MONDO_HEAD, %g2 + ldxa [%g2] ASI_QUEUE, %g2 + mov INTRQ_RESUM_MONDO_TAIL, %g4 + ldxa [%g4] ASI_QUEUE, %g4 + cmp %g2, %g4 + be,pn %xcc, sun4v_res_mondo_queue_empty + nop + + /* Get &trap_block[smp_processor_id()] into %g3. */ + ldxa [%g0] ASI_SCRATCHPAD, %g3 + sub %g3, TRAP_PER_CPU_FAULT_INFO, %g3 + + /* Get RES mondo queue base phys address into %g5. */ + ldx [%g3 + TRAP_PER_CPU_RESUM_MONDO_PA], %g5 + + /* Get RES kernel buffer base phys address into %g7. */ + ldx [%g3 + TRAP_PER_CPU_RESUM_KBUF_PA], %g7 + + /* If the first word is non-zero, queue is full. */ + ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g1 + brnz,pn %g1, sun4v_res_mondo_queue_full + nop + + lduw [%g3 + TRAP_PER_CPU_RESUM_QMASK], %g4 + + /* Remember this entry's offset in %g1. */ + mov %g2, %g1 + + /* Copy 64-byte queue entry into kernel buffer. */ + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + + /* Update queue head pointer. */ + and %g2, %g4, %g2 + + mov INTRQ_RESUM_MONDO_HEAD, %g4 + stxa %g2, [%g4] ASI_QUEUE + membar #Sync + + /* Disable interrupts and save register state so we can call + * C code. The etrap handling will leave %g4 in %l4 for us + * when it's done. + */ + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + mov %g1, %g4 + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + /* Log the event. */ + add %sp, PTREGS_OFF, %o0 + call sun4v_resum_error + mov %l4, %o1 + + /* Return from trap. */ + ba,pt %xcc, rtrap_irq + nop + +sun4v_res_mondo_queue_empty: + retry + +sun4v_res_mondo_queue_full: + /* The queue is full, consolidate our damage by setting + * the head equal to the tail. We'll just trap again otherwise. + * Call C code to log the event. + */ + mov INTRQ_RESUM_MONDO_HEAD, %g2 + stxa %g4, [%g2] ASI_QUEUE + membar #Sync + + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + call sun4v_resum_overflow + add %sp, PTREGS_OFF, %o0 + + ba,pt %xcc, rtrap_irq + nop + +sun4v_nonres_mondo: + /* Head offset in %g2, tail offset in %g4. */ + mov INTRQ_NONRESUM_MONDO_HEAD, %g2 + ldxa [%g2] ASI_QUEUE, %g2 + mov INTRQ_NONRESUM_MONDO_TAIL, %g4 + ldxa [%g4] ASI_QUEUE, %g4 + cmp %g2, %g4 + be,pn %xcc, sun4v_nonres_mondo_queue_empty + nop + + /* Get &trap_block[smp_processor_id()] into %g3. */ + ldxa [%g0] ASI_SCRATCHPAD, %g3 + sub %g3, TRAP_PER_CPU_FAULT_INFO, %g3 + + /* Get RES mondo queue base phys address into %g5. */ + ldx [%g3 + TRAP_PER_CPU_NONRESUM_MONDO_PA], %g5 + + /* Get RES kernel buffer base phys address into %g7. */ + ldx [%g3 + TRAP_PER_CPU_NONRESUM_KBUF_PA], %g7 + + /* If the first word is non-zero, queue is full. */ + ldxa [%g7 + %g2] ASI_PHYS_USE_EC, %g1 + brnz,pn %g1, sun4v_nonres_mondo_queue_full + nop + + lduw [%g3 + TRAP_PER_CPU_NONRESUM_QMASK], %g4 + + /* Remember this entry's offset in %g1. */ + mov %g2, %g1 + + /* Copy 64-byte queue entry into kernel buffer. */ + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + ldxa [%g5 + %g2] ASI_PHYS_USE_EC, %g3 + stxa %g3, [%g7 + %g2] ASI_PHYS_USE_EC + add %g2, 0x08, %g2 + + /* Update queue head pointer. */ + and %g2, %g4, %g2 + + mov INTRQ_NONRESUM_MONDO_HEAD, %g4 + stxa %g2, [%g4] ASI_QUEUE + membar #Sync + + /* Disable interrupts and save register state so we can call + * C code. The etrap handling will leave %g4 in %l4 for us + * when it's done. + */ + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + mov %g1, %g4 + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + /* Log the event. */ + add %sp, PTREGS_OFF, %o0 + call sun4v_nonresum_error + mov %l4, %o1 + + /* Return from trap. */ + ba,pt %xcc, rtrap_irq + nop + +sun4v_nonres_mondo_queue_empty: + retry + +sun4v_nonres_mondo_queue_full: + /* The queue is full, consolidate our damage by setting + * the head equal to the tail. We'll just trap again otherwise. + * Call C code to log the event. + */ + mov INTRQ_NONRESUM_MONDO_HEAD, %g2 + stxa %g4, [%g2] ASI_QUEUE + membar #Sync + + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + ba,pt %xcc, etrap_irq + rd %pc, %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + call sun4v_nonresum_overflow + add %sp, PTREGS_OFF, %o0 + + ba,pt %xcc, rtrap_irq + nop diff --git a/arch/sparc/kernel/sun4v_mcd.S b/arch/sparc/kernel/sun4v_mcd.S new file mode 100644 index 000000000..a419b7318 --- /dev/null +++ b/arch/sparc/kernel/sun4v_mcd.S @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* sun4v_mcd.S: Sun4v memory corruption detected precise exception handler + * + * Copyright (c) 2015 Oracle and/or its affiliates. All rights reserved. + * Authors: Bob Picco , + * Khalid Aziz + */ + .text + .align 32 + +sun4v_mcd_detect_precise: + mov %l4, %o1 + mov %l5, %o2 + call sun4v_mem_corrupt_detect_precise + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + nop diff --git a/arch/sparc/kernel/sun4v_tlb_miss.S b/arch/sparc/kernel/sun4v_tlb_miss.S new file mode 100644 index 000000000..7ac9f3367 --- /dev/null +++ b/arch/sparc/kernel/sun4v_tlb_miss.S @@ -0,0 +1,437 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* sun4v_tlb_miss.S: Sun4v TLB miss handlers. + * + * Copyright (C) 2006 + */ + + .text + .align 32 + + /* Load ITLB fault information into VADDR and CTX, using BASE. */ +#define LOAD_ITLB_INFO(BASE, VADDR, CTX) \ + ldx [BASE + HV_FAULT_I_ADDR_OFFSET], VADDR; \ + ldx [BASE + HV_FAULT_I_CTX_OFFSET], CTX; + + /* Load DTLB fault information into VADDR and CTX, using BASE. */ +#define LOAD_DTLB_INFO(BASE, VADDR, CTX) \ + ldx [BASE + HV_FAULT_D_ADDR_OFFSET], VADDR; \ + ldx [BASE + HV_FAULT_D_CTX_OFFSET], CTX; + + /* DEST = (VADDR >> 22) + * + * Branch to ZERO_CTX_LABEL if context is zero. + */ +#define COMPUTE_TAG_TARGET(DEST, VADDR, CTX, ZERO_CTX_LABEL) \ + srlx VADDR, 22, DEST; \ + brz,pn CTX, ZERO_CTX_LABEL; \ + nop; + + /* Create TSB pointer. This is something like: + * + * index_mask = (512 << (tsb_reg & 0x7UL)) - 1UL; + * tsb_base = tsb_reg & ~0x7UL; + * tsb_index = ((vaddr >> HASH_SHIFT) & tsb_mask); + * tsb_ptr = tsb_base + (tsb_index * 16); + */ +#define COMPUTE_TSB_PTR(TSB_PTR, VADDR, HASH_SHIFT, TMP1, TMP2) \ + and TSB_PTR, 0x7, TMP1; \ + mov 512, TMP2; \ + andn TSB_PTR, 0x7, TSB_PTR; \ + sllx TMP2, TMP1, TMP2; \ + srlx VADDR, HASH_SHIFT, TMP1; \ + sub TMP2, 1, TMP2; \ + and TMP1, TMP2, TMP1; \ + sllx TMP1, 4, TMP1; \ + add TSB_PTR, TMP1, TSB_PTR; + +sun4v_itlb_miss: + /* Load MMU Miss base into %g2. */ + ldxa [%g0] ASI_SCRATCHPAD, %g2 + + /* Load UTSB reg into %g1. */ + mov SCRATCHPAD_UTSBREG1, %g1 + ldxa [%g1] ASI_SCRATCHPAD, %g1 + + LOAD_ITLB_INFO(%g2, %g4, %g5) + COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_itlb_4v) + COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) + + /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ + ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 + cmp %g2, %g6 + bne,a,pn %xcc, tsb_miss_page_table_walk + mov FAULT_CODE_ITLB, %g3 + andcc %g3, _PAGE_EXEC_4V, %g0 + be,a,pn %xcc, tsb_do_fault + mov FAULT_CODE_ITLB, %g3 + + /* We have a valid entry, make hypervisor call to load + * I-TLB and return from trap. + * + * %g3: PTE + * %g4: vaddr + */ +sun4v_itlb_load: + ldxa [%g0] ASI_SCRATCHPAD, %g6 + mov %o0, %g1 ! save %o0 + mov %o1, %g2 ! save %o1 + mov %o2, %g5 ! save %o2 + mov %o3, %g7 ! save %o3 + mov %g4, %o0 ! vaddr + ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 ! ctx + mov %g3, %o2 ! PTE + mov HV_MMU_IMMU, %o3 ! flags + ta HV_MMU_MAP_ADDR_TRAP + brnz,pn %o0, sun4v_itlb_error + mov %g2, %o1 ! restore %o1 + mov %g1, %o0 ! restore %o0 + mov %g5, %o2 ! restore %o2 + mov %g7, %o3 ! restore %o3 + + retry + +sun4v_dtlb_miss: + /* Load MMU Miss base into %g2. */ + ldxa [%g0] ASI_SCRATCHPAD, %g2 + + /* Load UTSB reg into %g1. */ + mov SCRATCHPAD_UTSBREG1, %g1 + ldxa [%g1] ASI_SCRATCHPAD, %g1 + + LOAD_DTLB_INFO(%g2, %g4, %g5) + COMPUTE_TAG_TARGET(%g6, %g4, %g5, kvmap_dtlb_4v) + COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g3, %g7) + + /* Load TSB tag/pte into %g2/%g3 and compare the tag. */ + ldda [%g1] ASI_QUAD_LDD_PHYS_4V, %g2 + cmp %g2, %g6 + bne,a,pn %xcc, tsb_miss_page_table_walk + mov FAULT_CODE_DTLB, %g3 + + /* We have a valid entry, make hypervisor call to load + * D-TLB and return from trap. + * + * %g3: PTE + * %g4: vaddr + */ +sun4v_dtlb_load: + ldxa [%g0] ASI_SCRATCHPAD, %g6 + mov %o0, %g1 ! save %o0 + mov %o1, %g2 ! save %o1 + mov %o2, %g5 ! save %o2 + mov %o3, %g7 ! save %o3 + mov %g4, %o0 ! vaddr + ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 ! ctx + mov %g3, %o2 ! PTE + mov HV_MMU_DMMU, %o3 ! flags + ta HV_MMU_MAP_ADDR_TRAP + brnz,pn %o0, sun4v_dtlb_error + mov %g2, %o1 ! restore %o1 + mov %g1, %o0 ! restore %o0 + mov %g5, %o2 ! restore %o2 + mov %g7, %o3 ! restore %o3 + + retry + +sun4v_dtlb_prot: + SET_GL(1) + + /* Load MMU Miss base into %g5. */ + ldxa [%g0] ASI_SCRATCHPAD, %g5 + + ldx [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5 + rdpr %tl, %g1 + cmp %g1, 1 + bgu,pn %xcc, winfix_trampoline + mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 + ba,pt %xcc, sparc64_realfault_common + nop + + /* Called from trap table: + * %g4: vaddr + * %g5: context + * %g6: TAG TARGET + */ +sun4v_itsb_miss: + mov SCRATCHPAD_UTSBREG1, %g1 + ldxa [%g1] ASI_SCRATCHPAD, %g1 + brz,pn %g5, kvmap_itlb_4v + mov FAULT_CODE_ITLB, %g3 + ba,a,pt %xcc, sun4v_tsb_miss_common + + /* Called from trap table: + * %g4: vaddr + * %g5: context + * %g6: TAG TARGET + */ +sun4v_dtsb_miss: + mov SCRATCHPAD_UTSBREG1, %g1 + ldxa [%g1] ASI_SCRATCHPAD, %g1 + brz,pn %g5, kvmap_dtlb_4v + mov FAULT_CODE_DTLB, %g3 + + /* fallthrough */ + +sun4v_tsb_miss_common: + COMPUTE_TSB_PTR(%g1, %g4, PAGE_SHIFT, %g5, %g7) + + sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + mov SCRATCHPAD_UTSBREG2, %g5 + ldxa [%g5] ASI_SCRATCHPAD, %g5 + cmp %g5, -1 + be,pt %xcc, 80f + nop + COMPUTE_TSB_PTR(%g5, %g4, REAL_HPAGE_SHIFT, %g2, %g7) + + /* That clobbered %g2, reload it. */ + ldxa [%g0] ASI_SCRATCHPAD, %g2 + sub %g2, TRAP_PER_CPU_FAULT_INFO, %g2 + +80: stx %g5, [%g2 + TRAP_PER_CPU_TSB_HUGE_TEMP] +#endif + + ba,pt %xcc, tsb_miss_page_table_walk_sun4v_fastpath + ldx [%g2 + TRAP_PER_CPU_PGD_PADDR], %g7 + +sun4v_itlb_error: + rdpr %tl, %g1 + cmp %g1, 1 + ble,pt %icc, sun4v_bad_ra + or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_ITLB, %g1 + + sethi %hi(sun4v_err_itlb_vaddr), %g1 + stx %g4, [%g1 + %lo(sun4v_err_itlb_vaddr)] + sethi %hi(sun4v_err_itlb_ctx), %g1 + ldxa [%g0] ASI_SCRATCHPAD, %g6 + ldx [%g6 + HV_FAULT_I_CTX_OFFSET], %o1 + stx %o1, [%g1 + %lo(sun4v_err_itlb_ctx)] + sethi %hi(sun4v_err_itlb_pte), %g1 + stx %g3, [%g1 + %lo(sun4v_err_itlb_pte)] + sethi %hi(sun4v_err_itlb_error), %g1 + stx %o0, [%g1 + %lo(sun4v_err_itlb_error)] + + sethi %hi(1f), %g7 + rdpr %tl, %g4 + ba,pt %xcc, etraptl1 +1: or %g7, %lo(1f), %g7 + mov %l4, %o1 + call sun4v_itlb_error_report + add %sp, PTREGS_OFF, %o0 + + /* NOTREACHED */ + +sun4v_dtlb_error: + rdpr %tl, %g1 + cmp %g1, 1 + ble,pt %icc, sun4v_bad_ra + or %g0, FAULT_CODE_BAD_RA | FAULT_CODE_DTLB, %g1 + + sethi %hi(sun4v_err_dtlb_vaddr), %g1 + stx %g4, [%g1 + %lo(sun4v_err_dtlb_vaddr)] + sethi %hi(sun4v_err_dtlb_ctx), %g1 + ldxa [%g0] ASI_SCRATCHPAD, %g6 + ldx [%g6 + HV_FAULT_D_CTX_OFFSET], %o1 + stx %o1, [%g1 + %lo(sun4v_err_dtlb_ctx)] + sethi %hi(sun4v_err_dtlb_pte), %g1 + stx %g3, [%g1 + %lo(sun4v_err_dtlb_pte)] + sethi %hi(sun4v_err_dtlb_error), %g1 + stx %o0, [%g1 + %lo(sun4v_err_dtlb_error)] + + sethi %hi(1f), %g7 + rdpr %tl, %g4 + ba,pt %xcc, etraptl1 +1: or %g7, %lo(1f), %g7 + mov %l4, %o1 + call sun4v_dtlb_error_report + add %sp, PTREGS_OFF, %o0 + + /* NOTREACHED */ + +sun4v_bad_ra: + or %g0, %g4, %g5 + ba,pt %xcc, sparc64_realfault_common + or %g1, %g0, %g4 + + /* NOTREACHED */ + + /* Instruction Access Exception, tl0. */ +sun4v_iacc: + ldxa [%g0] ASI_SCRATCHPAD, %g2 + ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 + ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 + ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 + sllx %g3, 16, %g3 + or %g5, %g3, %g5 + ba,pt %xcc, etrap + rd %pc, %g7 + mov %l4, %o1 + mov %l5, %o2 + call sun4v_insn_access_exception + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + + /* Instruction Access Exception, tl1. */ +sun4v_iacc_tl1: + ldxa [%g0] ASI_SCRATCHPAD, %g2 + ldx [%g2 + HV_FAULT_I_TYPE_OFFSET], %g3 + ldx [%g2 + HV_FAULT_I_ADDR_OFFSET], %g4 + ldx [%g2 + HV_FAULT_I_CTX_OFFSET], %g5 + sllx %g3, 16, %g3 + or %g5, %g3, %g5 + ba,pt %xcc, etraptl1 + rd %pc, %g7 + mov %l4, %o1 + mov %l5, %o2 + call sun4v_insn_access_exception_tl1 + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + + /* Data Access Exception, tl0. */ +sun4v_dacc: + ldxa [%g0] ASI_SCRATCHPAD, %g2 + ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 + ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 + ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 + sllx %g3, 16, %g3 + or %g5, %g3, %g5 + ba,pt %xcc, etrap + rd %pc, %g7 + mov %l4, %o1 + mov %l5, %o2 + call sun4v_data_access_exception + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + + /* Data Access Exception, tl1. */ +sun4v_dacc_tl1: + ldxa [%g0] ASI_SCRATCHPAD, %g2 + ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 + ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 + ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 + sllx %g3, 16, %g3 + or %g5, %g3, %g5 + ba,pt %xcc, etraptl1 + rd %pc, %g7 + mov %l4, %o1 + mov %l5, %o2 + call sun4v_data_access_exception_tl1 + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + + /* Memory Address Unaligned. */ +sun4v_mna: + /* Window fixup? */ + rdpr %tl, %g2 + cmp %g2, 1 + ble,pt %icc, 1f + nop + + SET_GL(1) + ldxa [%g0] ASI_SCRATCHPAD, %g2 + ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g5 + mov HV_FAULT_TYPE_UNALIGNED, %g3 + ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g4 + sllx %g3, 16, %g3 + or %g4, %g3, %g4 + ba,pt %xcc, winfix_mna + rdpr %tpc, %g3 + /* not reached */ + +1: ldxa [%g0] ASI_SCRATCHPAD, %g2 + mov HV_FAULT_TYPE_UNALIGNED, %g3 + ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 + ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 + sllx %g3, 16, %g3 + or %g5, %g3, %g5 + + ba,pt %xcc, etrap + rd %pc, %g7 + mov %l4, %o1 + mov %l5, %o2 + call sun4v_do_mna + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + nop + + /* Privileged Action. */ +sun4v_privact: + ba,pt %xcc, etrap + rd %pc, %g7 + call do_privact + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + + /* Unaligned ldd float, tl0. */ +sun4v_lddfmna: + ldxa [%g0] ASI_SCRATCHPAD, %g2 + ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 + ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 + ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 + sllx %g3, 16, %g3 + or %g5, %g3, %g5 + ba,pt %xcc, etrap + rd %pc, %g7 + mov %l4, %o1 + mov %l5, %o2 + call handle_lddfmna + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + + /* Unaligned std float, tl0. */ +sun4v_stdfmna: + ldxa [%g0] ASI_SCRATCHPAD, %g2 + ldx [%g2 + HV_FAULT_D_TYPE_OFFSET], %g3 + ldx [%g2 + HV_FAULT_D_ADDR_OFFSET], %g4 + ldx [%g2 + HV_FAULT_D_CTX_OFFSET], %g5 + sllx %g3, 16, %g3 + or %g5, %g3, %g5 + ba,pt %xcc, etrap + rd %pc, %g7 + mov %l4, %o1 + mov %l5, %o2 + call handle_stdfmna + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define SUN4V_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl sun4v_patch_tlb_handlers + .type sun4v_patch_tlb_handlers,#function +sun4v_patch_tlb_handlers: + SUN4V_DO_PATCH(tl0_iamiss, sun4v_itlb_miss) + SUN4V_DO_PATCH(tl1_iamiss, sun4v_itlb_miss) + SUN4V_DO_PATCH(tl0_damiss, sun4v_dtlb_miss) + SUN4V_DO_PATCH(tl1_damiss, sun4v_dtlb_miss) + SUN4V_DO_PATCH(tl0_daprot, sun4v_dtlb_prot) + SUN4V_DO_PATCH(tl1_daprot, sun4v_dtlb_prot) + SUN4V_DO_PATCH(tl0_iax, sun4v_iacc) + SUN4V_DO_PATCH(tl1_iax, sun4v_iacc_tl1) + SUN4V_DO_PATCH(tl0_dax, sun4v_dacc) + SUN4V_DO_PATCH(tl1_dax, sun4v_dacc_tl1) + SUN4V_DO_PATCH(tl0_mna, sun4v_mna) + SUN4V_DO_PATCH(tl1_mna, sun4v_mna) + SUN4V_DO_PATCH(tl0_lddfmna, sun4v_lddfmna) + SUN4V_DO_PATCH(tl0_stdfmna, sun4v_stdfmna) + SUN4V_DO_PATCH(tl0_privact, sun4v_privact) + retl + nop + .size sun4v_patch_tlb_handlers,.-sun4v_patch_tlb_handlers diff --git a/arch/sparc/kernel/sys32.S b/arch/sparc/kernel/sys32.S new file mode 100644 index 000000000..a3d308f20 --- /dev/null +++ b/arch/sparc/kernel/sys32.S @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sys32.S: I-cache tricks for 32-bit compatibility layer simple + * conversions. + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#include + +/* NOTE: call as jump breaks return stack, we have to avoid that */ + + .text + + .globl sys32_mmap2 +sys32_mmap2: + sethi %hi(sys_mmap), %g1 + jmpl %g1 + %lo(sys_mmap), %g0 + sllx %o5, 12, %o5 diff --git a/arch/sparc/kernel/sys_sparc32.c b/arch/sparc/kernel/sys_sparc32.c new file mode 100644 index 000000000..04432b82b --- /dev/null +++ b/arch/sparc/kernel/sys_sparc32.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +/* sys_sparc32.c: Conversion between 32bit and 64bit native syscalls. + * + * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 1997, 2007 David S. Miller (davem@davemloft.net) + * + * These routines maintain argument size conversion between 32bit and 64bit + * environment. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "systbls.h" + +COMPAT_SYSCALL_DEFINE3(truncate64, const char __user *, path, u32, high, u32, low) +{ + return ksys_truncate(path, ((u64)high << 32) | low); +} + +COMPAT_SYSCALL_DEFINE3(ftruncate64, unsigned int, fd, u32, high, u32, low) +{ + return ksys_ftruncate(fd, ((u64)high << 32) | low, FTRUNCATE_LFS); +} + +static int cp_compat_stat64(struct kstat *stat, + struct compat_stat64 __user *statbuf) +{ + int err; + + err = put_user(huge_encode_dev(stat->dev), &statbuf->st_dev); + err |= put_user(stat->ino, &statbuf->st_ino); + err |= put_user(stat->mode, &statbuf->st_mode); + err |= put_user(stat->nlink, &statbuf->st_nlink); + err |= put_user(from_kuid_munged(current_user_ns(), stat->uid), &statbuf->st_uid); + err |= put_user(from_kgid_munged(current_user_ns(), stat->gid), &statbuf->st_gid); + err |= put_user(huge_encode_dev(stat->rdev), &statbuf->st_rdev); + err |= put_user(0, (unsigned long __user *) &statbuf->__pad3[0]); + err |= put_user(stat->size, &statbuf->st_size); + err |= put_user(stat->blksize, &statbuf->st_blksize); + err |= put_user(0, (unsigned int __user *) &statbuf->__pad4[0]); + err |= put_user(0, (unsigned int __user *) &statbuf->__pad4[4]); + err |= put_user(stat->blocks, &statbuf->st_blocks); + err |= put_user(stat->atime.tv_sec, &statbuf->st_atime); + err |= put_user(stat->atime.tv_nsec, &statbuf->st_atime_nsec); + err |= put_user(stat->mtime.tv_sec, &statbuf->st_mtime); + err |= put_user(stat->mtime.tv_nsec, &statbuf->st_mtime_nsec); + err |= put_user(stat->ctime.tv_sec, &statbuf->st_ctime); + err |= put_user(stat->ctime.tv_nsec, &statbuf->st_ctime_nsec); + err |= put_user(0, &statbuf->__unused4); + err |= put_user(0, &statbuf->__unused5); + + return err; +} + +COMPAT_SYSCALL_DEFINE2(stat64, const char __user *, filename, + struct compat_stat64 __user *, statbuf) +{ + struct kstat stat; + int error = vfs_stat(filename, &stat); + + if (!error) + error = cp_compat_stat64(&stat, statbuf); + return error; +} + +COMPAT_SYSCALL_DEFINE2(lstat64, const char __user *, filename, + struct compat_stat64 __user *, statbuf) +{ + struct kstat stat; + int error = vfs_lstat(filename, &stat); + + if (!error) + error = cp_compat_stat64(&stat, statbuf); + return error; +} + +COMPAT_SYSCALL_DEFINE2(fstat64, unsigned int, fd, + struct compat_stat64 __user *, statbuf) +{ + struct kstat stat; + int error = vfs_fstat(fd, &stat); + + if (!error) + error = cp_compat_stat64(&stat, statbuf); + return error; +} + +COMPAT_SYSCALL_DEFINE4(fstatat64, unsigned int, dfd, + const char __user *, filename, + struct compat_stat64 __user *, statbuf, int, flag) +{ + struct kstat stat; + int error; + + error = vfs_fstatat(dfd, filename, &stat, flag); + if (error) + return error; + return cp_compat_stat64(&stat, statbuf); +} + +COMPAT_SYSCALL_DEFINE3(sparc_sigaction, int, sig, + struct compat_old_sigaction __user *,act, + struct compat_old_sigaction __user *,oact) +{ + WARN_ON_ONCE(sig >= 0); + return compat_sys_sigaction(-sig, act, oact); +} + +COMPAT_SYSCALL_DEFINE5(rt_sigaction, int, sig, + struct compat_sigaction __user *,act, + struct compat_sigaction __user *,oact, + void __user *,restorer, + compat_size_t,sigsetsize) +{ + struct k_sigaction new_ka, old_ka; + int ret; + + /* XXX: Don't preclude handling different sized sigset_t's. */ + if (sigsetsize != sizeof(compat_sigset_t)) + return -EINVAL; + + if (act) { + u32 u_handler, u_restorer; + + new_ka.ka_restorer = restorer; + ret = get_user(u_handler, &act->sa_handler); + new_ka.sa.sa_handler = compat_ptr(u_handler); + ret |= get_compat_sigset(&new_ka.sa.sa_mask, &act->sa_mask); + ret |= get_user(new_ka.sa.sa_flags, &act->sa_flags); + ret |= get_user(u_restorer, &act->sa_restorer); + new_ka.sa.sa_restorer = compat_ptr(u_restorer); + if (ret) + return -EFAULT; + } + + ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); + + if (!ret && oact) { + ret = put_user(ptr_to_compat(old_ka.sa.sa_handler), &oact->sa_handler); + ret |= put_compat_sigset(&oact->sa_mask, &old_ka.sa.sa_mask, + sizeof(oact->sa_mask)); + ret |= put_user(old_ka.sa.sa_flags, &oact->sa_flags); + ret |= put_user(ptr_to_compat(old_ka.sa.sa_restorer), &oact->sa_restorer); + if (ret) + ret = -EFAULT; + } + + return ret; +} + +COMPAT_SYSCALL_DEFINE5(pread64, unsigned int, fd, char __user *, ubuf, + compat_size_t, count, u32, poshi, u32, poslo) +{ + return ksys_pread64(fd, ubuf, count, ((u64)poshi << 32) | poslo); +} + +COMPAT_SYSCALL_DEFINE5(pwrite64, unsigned int, fd, char __user *, ubuf, + compat_size_t, count, u32, poshi, u32, poslo) +{ + return ksys_pwrite64(fd, ubuf, count, ((u64)poshi << 32) | poslo); +} + +COMPAT_SYSCALL_DEFINE4(readahead, int, fd, u32, offhi, u32, offlo, + compat_size_t, count) +{ + return ksys_readahead(fd, ((u64)offhi << 32) | offlo, count); +} + +COMPAT_SYSCALL_DEFINE5(fadvise64, int, fd, u32, offhi, u32, offlo, + compat_size_t, len, int, advice) +{ + return ksys_fadvise64_64(fd, ((u64)offhi << 32) | offlo, len, advice); +} + +COMPAT_SYSCALL_DEFINE6(fadvise64_64, int, fd, u32, offhi, u32, offlo, + u32, lenhi, u32, lenlo, int, advice) +{ + return ksys_fadvise64_64(fd, + ((u64)offhi << 32) | offlo, + ((u64)lenhi << 32) | lenlo, + advice); +} + +COMPAT_SYSCALL_DEFINE6(sync_file_range, unsigned int, fd, u32, off_high, u32, off_low, + u32, nb_high, u32, nb_low, unsigned int, flags) +{ + return ksys_sync_file_range(fd, + ((u64)off_high << 32) | off_low, + ((u64)nb_high << 32) | nb_low, + flags); +} + +COMPAT_SYSCALL_DEFINE6(fallocate, int, fd, int, mode, u32, offhi, u32, offlo, + u32, lenhi, u32, lenlo) +{ + return ksys_fallocate(fd, mode, ((loff_t)offhi << 32) | offlo, + ((loff_t)lenhi << 32) | lenlo); +} diff --git a/arch/sparc/kernel/sys_sparc_32.c b/arch/sparc/kernel/sys_sparc_32.c new file mode 100644 index 000000000..fb31bc0c5 --- /dev/null +++ b/arch/sparc/kernel/sys_sparc_32.c @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: GPL-2.0 +/* linux/arch/sparc/kernel/sys_sparc.c + * + * This file contains various random system calls that + * have a non-standard calling sequence on the Linux/sparc + * platform. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "systbls.h" + +/* #define DEBUG_UNIMP_SYSCALL */ + +/* XXX Make this per-binary type, this way we can detect the type of + * XXX a binary. Every Sparc executable calls this very early on. + */ +SYSCALL_DEFINE0(getpagesize) +{ + return PAGE_SIZE; /* Possibly older binaries want 8192 on sun4's? */ +} + +unsigned long arch_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags) +{ + struct vm_unmapped_area_info info = {}; + bool file_hugepage = false; + + if (filp && is_file_hugepages(filp)) + file_hugepage = true; + + if (flags & MAP_FIXED) { + /* We do not accept a shared mapping if it would violate + * cache aliasing constraints. + */ + if (!file_hugepage && (flags & MAP_SHARED) && + ((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1))) + return -EINVAL; + return addr; + } + + /* See asm-sparc/uaccess.h */ + if (len > TASK_SIZE - PAGE_SIZE) + return -ENOMEM; + if (!addr) + addr = TASK_UNMAPPED_BASE; + + info.length = len; + info.low_limit = addr; + info.high_limit = TASK_SIZE; + if (!file_hugepage) { + info.align_mask = (flags & MAP_SHARED) ? + (PAGE_MASK & (SHMLBA - 1)) : 0; + info.align_offset = pgoff << PAGE_SHIFT; + } else { + info.align_mask = huge_page_mask_align(filp); + } + return vm_unmapped_area(&info); +} + +/* + * sys_pipe() is the normal C calling standard for creating + * a pipe. It's not the way unix traditionally does this, though. + */ +SYSCALL_DEFINE0(sparc_pipe) +{ + int fd[2]; + int error; + + error = do_pipe_flags(fd, 0); + if (error) + goto out; + current_pt_regs()->u_regs[UREG_I1] = fd[1]; + error = fd[0]; +out: + return error; +} + +int sparc_mmap_check(unsigned long addr, unsigned long len) +{ + /* See asm-sparc/uaccess.h */ + if (len > TASK_SIZE - PAGE_SIZE || addr + len > TASK_SIZE - PAGE_SIZE) + return -EINVAL; + + return 0; +} + +/* Linux version of mmap */ + +SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, unsigned long, fd, + unsigned long, pgoff) +{ + /* Make sure the shift for mmap2 is constant (12), no matter what PAGE_SIZE + we have. */ + return ksys_mmap_pgoff(addr, len, prot, flags, fd, + pgoff >> (PAGE_SHIFT - 12)); +} + +SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, unsigned long, fd, + unsigned long, off) +{ + /* no alignment check? */ + return ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); +} + +SYSCALL_DEFINE5(sparc_remap_file_pages, unsigned long, start, unsigned long, size, + unsigned long, prot, unsigned long, pgoff, + unsigned long, flags) +{ + /* This works on an existing mmap so we don't need to validate + * the range as that was done at the original mmap call. + */ + return sys_remap_file_pages(start, size, prot, + (pgoff >> (PAGE_SHIFT - 12)), flags); +} + +SYSCALL_DEFINE0(nis_syscall) +{ + static int count = 0; + struct pt_regs *regs = current_pt_regs(); + + if (count++ > 5) + return -ENOSYS; + printk ("%s[%d]: Unimplemented SPARC system call %d\n", + current->comm, task_pid_nr(current), (int)regs->u_regs[1]); +#ifdef DEBUG_UNIMP_SYSCALL + show_regs (regs); +#endif + return -ENOSYS; +} + +/* #define DEBUG_SPARC_BREAKPOINT */ + +asmlinkage void +sparc_breakpoint (struct pt_regs *regs) +{ + +#ifdef DEBUG_SPARC_BREAKPOINT + printk ("TRAP: Entering kernel PC=%x, nPC=%x\n", regs->pc, regs->npc); +#endif + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->pc); + +#ifdef DEBUG_SPARC_BREAKPOINT + printk ("TRAP: Returning to space: PC=%x nPC=%x\n", regs->pc, regs->npc); +#endif +} + +SYSCALL_DEFINE3(sparc_sigaction, int, sig, + struct old_sigaction __user *,act, + struct old_sigaction __user *,oact) +{ + WARN_ON_ONCE(sig >= 0); + return sys_sigaction(-sig, act, oact); +} + +SYSCALL_DEFINE5(rt_sigaction, int, sig, + const struct sigaction __user *, act, + struct sigaction __user *, oact, + void __user *, restorer, + size_t, sigsetsize) +{ + struct k_sigaction new_ka, old_ka; + int ret; + + /* XXX: Don't preclude handling different sized sigset_t's. */ + if (sigsetsize != sizeof(sigset_t)) + return -EINVAL; + + if (act) { + new_ka.ka_restorer = restorer; + if (copy_from_user(&new_ka.sa, act, sizeof(*act))) + return -EFAULT; + } + + ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); + + if (!ret && oact) { + if (copy_to_user(oact, &old_ka.sa, sizeof(*oact))) + return -EFAULT; + } + + return ret; +} + +SYSCALL_DEFINE2(getdomainname, char __user *, name, int, len) +{ + int nlen, err; + char tmp[__NEW_UTS_LEN + 1]; + + if (len < 0) + return -EINVAL; + + down_read(&uts_sem); + + nlen = strlen(utsname()->domainname) + 1; + err = -EINVAL; + if (nlen > len) + goto out_unlock; + memcpy(tmp, utsname()->domainname, nlen); + + up_read(&uts_sem); + + if (copy_to_user(name, tmp, nlen)) + return -EFAULT; + return 0; + +out_unlock: + up_read(&uts_sem); + return err; +} diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c new file mode 100644 index 000000000..ecefcffcf --- /dev/null +++ b/arch/sparc/kernel/sys_sparc_64.c @@ -0,0 +1,723 @@ +// SPDX-License-Identifier: GPL-2.0 +/* linux/arch/sparc64/kernel/sys_sparc.c + * + * This file contains various random system calls that + * have a non-standard calling sequence on the Linux/sparc + * platform. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "entry.h" +#include "kernel.h" +#include "systbls.h" + +/* #define DEBUG_UNIMP_SYSCALL */ + +SYSCALL_DEFINE0(getpagesize) +{ + return PAGE_SIZE; +} + +/* Does addr --> addr+len fall within 4GB of the VA-space hole or + * overflow past the end of the 64-bit address space? + */ +static inline int invalid_64bit_range(unsigned long addr, unsigned long len) +{ + unsigned long va_exclude_start, va_exclude_end; + + va_exclude_start = VA_EXCLUDE_START; + va_exclude_end = VA_EXCLUDE_END; + + if (unlikely(len >= va_exclude_start)) + return 1; + + if (unlikely((addr + len) < addr)) + return 1; + + if (unlikely((addr >= va_exclude_start && addr < va_exclude_end) || + ((addr + len) >= va_exclude_start && + (addr + len) < va_exclude_end))) + return 1; + + return 0; +} + +/* These functions differ from the default implementations in + * mm/mmap.c in two ways: + * + * 1) For file backed MAP_SHARED mmap()'s we D-cache color align, + * for fixed such mappings we just validate what the user gave us. + * 2) For 64-bit tasks we avoid mapping anything within 4GB of + * the spitfire/niagara VA-hole. + */ + +static inline unsigned long COLOR_ALIGN(unsigned long addr, + unsigned long pgoff) +{ + unsigned long base = (addr+SHMLBA-1)&~(SHMLBA-1); + unsigned long off = (pgoff<mm; + struct vm_area_struct * vma; + unsigned long task_size = TASK_SIZE; + int do_color_align; + struct vm_unmapped_area_info info = {}; + bool file_hugepage = false; + + if (filp && is_file_hugepages(filp)) + file_hugepage = true; + + if (flags & MAP_FIXED) { + /* We do not accept a shared mapping if it would violate + * cache aliasing constraints. + */ + if (!file_hugepage && (flags & MAP_SHARED) && + ((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1))) + return -EINVAL; + return addr; + } + + if (test_thread_flag(TIF_32BIT)) + task_size = STACK_TOP32; + if (unlikely(len > task_size || len >= VA_EXCLUDE_START)) + return -ENOMEM; + + do_color_align = 0; + if ((filp || (flags & MAP_SHARED)) && !file_hugepage) + do_color_align = 1; + + if (addr) { + if (do_color_align) + addr = COLOR_ALIGN(addr, pgoff); + else + addr = PAGE_ALIGN(addr); + + vma = find_vma(mm, addr); + if (task_size - len >= addr && + (!vma || addr + len <= vm_start_gap(vma))) + return addr; + } + + info.length = len; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = min(task_size, VA_EXCLUDE_START); + info.align_mask = get_align_mask(filp, flags); + if (!file_hugepage) + info.align_offset = pgoff << PAGE_SHIFT; + addr = vm_unmapped_area(&info); + + if ((addr & ~PAGE_MASK) && task_size > VA_EXCLUDE_END) { + VM_BUG_ON(addr != -ENOMEM); + info.low_limit = VA_EXCLUDE_END; + info.high_limit = task_size; + addr = vm_unmapped_area(&info); + } + + return addr; +} + +unsigned long +arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, + const unsigned long len, const unsigned long pgoff, + const unsigned long flags, vm_flags_t vm_flags) +{ + struct vm_area_struct *vma; + struct mm_struct *mm = current->mm; + unsigned long task_size = STACK_TOP32; + unsigned long addr = addr0; + int do_color_align; + struct vm_unmapped_area_info info = {}; + bool file_hugepage = false; + + /* This should only ever run for 32-bit processes. */ + BUG_ON(!test_thread_flag(TIF_32BIT)); + + if (filp && is_file_hugepages(filp)) + file_hugepage = true; + + if (flags & MAP_FIXED) { + /* We do not accept a shared mapping if it would violate + * cache aliasing constraints. + */ + if (!file_hugepage && (flags & MAP_SHARED) && + ((addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1))) + return -EINVAL; + return addr; + } + + if (unlikely(len > task_size)) + return -ENOMEM; + + do_color_align = 0; + if ((filp || (flags & MAP_SHARED)) && !file_hugepage) + do_color_align = 1; + + /* requesting a specific address */ + if (addr) { + if (do_color_align) + addr = COLOR_ALIGN(addr, pgoff); + else + addr = PAGE_ALIGN(addr); + + vma = find_vma(mm, addr); + if (task_size - len >= addr && + (!vma || addr + len <= vm_start_gap(vma))) + return addr; + } + + info.flags = VM_UNMAPPED_AREA_TOPDOWN; + info.length = len; + info.low_limit = PAGE_SIZE; + info.high_limit = mm->mmap_base; + info.align_mask = get_align_mask(filp, flags); + if (!file_hugepage) + info.align_offset = pgoff << PAGE_SHIFT; + addr = vm_unmapped_area(&info); + + /* + * A failed mmap() very likely causes application failure, + * so fall back to the bottom-up function here. This scenario + * can happen with large stack limits and large mmap() + * allocations. + */ + if (addr & ~PAGE_MASK) { + VM_BUG_ON(addr != -ENOMEM); + info.flags = 0; + info.low_limit = TASK_UNMAPPED_BASE; + info.high_limit = STACK_TOP32; + addr = vm_unmapped_area(&info); + } + + return addr; +} + +/* Try to align mapping such that we align it as much as possible. */ +unsigned long get_fb_unmapped_area(struct file *filp, unsigned long orig_addr, unsigned long len, unsigned long pgoff, unsigned long flags) +{ + unsigned long align_goal, addr = -ENOMEM; + + if (flags & MAP_FIXED) { + /* Ok, don't mess with it. */ + return mm_get_unmapped_area(NULL, orig_addr, len, pgoff, flags); + } + flags &= ~MAP_SHARED; + + align_goal = PAGE_SIZE; + if (len >= (4UL * 1024 * 1024)) + align_goal = (4UL * 1024 * 1024); + else if (len >= (512UL * 1024)) + align_goal = (512UL * 1024); + else if (len >= (64UL * 1024)) + align_goal = (64UL * 1024); + + do { + addr = mm_get_unmapped_area(NULL, orig_addr, + len + (align_goal - PAGE_SIZE), pgoff, flags); + if (!(addr & ~PAGE_MASK)) { + addr = (addr + (align_goal - 1UL)) & ~(align_goal - 1UL); + break; + } + + if (align_goal == (4UL * 1024 * 1024)) + align_goal = (512UL * 1024); + else if (align_goal == (512UL * 1024)) + align_goal = (64UL * 1024); + else + align_goal = PAGE_SIZE; + } while ((addr & ~PAGE_MASK) && align_goal > PAGE_SIZE); + + /* Mapping is smaller than 64K or larger areas could not + * be obtained. + */ + if (addr & ~PAGE_MASK) + addr = mm_get_unmapped_area(NULL, orig_addr, len, pgoff, flags); + + return addr; +} +EXPORT_SYMBOL(get_fb_unmapped_area); + +/* Essentially the same as PowerPC. */ +static unsigned long mmap_rnd(void) +{ + unsigned long rnd = 0UL; + + if (current->flags & PF_RANDOMIZE) { + unsigned long val = get_random_long(); + if (test_thread_flag(TIF_32BIT)) + rnd = (val % (1UL << (23UL-PAGE_SHIFT))); + else + rnd = (val % (1UL << (30UL-PAGE_SHIFT))); + } + return rnd << PAGE_SHIFT; +} + +void arch_pick_mmap_layout(struct mm_struct *mm, const struct rlimit *rlim_stack) +{ + unsigned long random_factor = mmap_rnd(); + unsigned long gap; + + /* + * Fall back to the standard layout if the personality + * bit is set, or if the expected stack growth is unlimited: + */ + gap = rlim_stack->rlim_cur; + if (!test_thread_flag(TIF_32BIT) || + (current->personality & ADDR_COMPAT_LAYOUT) || + gap == RLIM_INFINITY || + sysctl_legacy_va_layout) { + mm->mmap_base = TASK_UNMAPPED_BASE + random_factor; + mm_flags_clear(MMF_TOPDOWN, mm); + } else { + /* We know it's 32-bit */ + unsigned long task_size = STACK_TOP32; + + if (gap < 128 * 1024 * 1024) + gap = 128 * 1024 * 1024; + if (gap > (task_size / 6 * 5)) + gap = (task_size / 6 * 5); + + mm->mmap_base = PAGE_ALIGN(task_size - gap - random_factor); + mm_flags_set(MMF_TOPDOWN, mm); + } +} + +/* + * sys_pipe() is the normal C calling standard for creating + * a pipe. It's not the way unix traditionally does this, though. + */ +SYSCALL_DEFINE0(sparc_pipe) +{ + int fd[2]; + int error; + + error = do_pipe_flags(fd, 0); + if (error) + goto out; + current_pt_regs()->u_regs[UREG_I1] = fd[1]; + error = fd[0]; +out: + return error; +} + +/* + * sys_ipc() is the de-multiplexer for the SysV IPC calls.. + * + * This is really horribly ugly. + */ + +SYSCALL_DEFINE6(sparc_ipc, unsigned int, call, int, first, unsigned long, second, + unsigned long, third, void __user *, ptr, long, fifth) +{ + long err; + + if (!IS_ENABLED(CONFIG_SYSVIPC)) + return -ENOSYS; + + /* No need for backward compatibility. We can start fresh... */ + if (call <= SEMTIMEDOP) { + switch (call) { + case SEMOP: + err = ksys_semtimedop(first, ptr, + (unsigned int)second, NULL); + goto out; + case SEMTIMEDOP: + err = ksys_semtimedop(first, ptr, (unsigned int)second, + (const struct __kernel_timespec __user *) + (unsigned long) fifth); + goto out; + case SEMGET: + err = ksys_semget(first, (int)second, (int)third); + goto out; + case SEMCTL: { + err = ksys_old_semctl(first, second, + (int)third | IPC_64, + (unsigned long) ptr); + goto out; + } + default: + err = -ENOSYS; + goto out; + } + } + if (call <= MSGCTL) { + switch (call) { + case MSGSND: + err = ksys_msgsnd(first, ptr, (size_t)second, + (int)third); + goto out; + case MSGRCV: + err = ksys_msgrcv(first, ptr, (size_t)second, fifth, + (int)third); + goto out; + case MSGGET: + err = ksys_msgget((key_t)first, (int)second); + goto out; + case MSGCTL: + err = ksys_old_msgctl(first, (int)second | IPC_64, ptr); + goto out; + default: + err = -ENOSYS; + goto out; + } + } + if (call <= SHMCTL) { + switch (call) { + case SHMAT: { + ulong raddr; + err = do_shmat(first, ptr, (int)second, &raddr, SHMLBA); + if (!err) { + if (put_user(raddr, + (ulong __user *) third)) + err = -EFAULT; + } + goto out; + } + case SHMDT: + err = ksys_shmdt(ptr); + goto out; + case SHMGET: + err = ksys_shmget(first, (size_t)second, (int)third); + goto out; + case SHMCTL: + err = ksys_old_shmctl(first, (int)second | IPC_64, ptr); + goto out; + default: + err = -ENOSYS; + goto out; + } + } else { + err = -ENOSYS; + } +out: + return err; +} + +SYSCALL_DEFINE1(sparc64_personality, unsigned long, personality) +{ + long ret; + + if (personality(current->personality) == PER_LINUX32 && + personality(personality) == PER_LINUX) + personality |= PER_LINUX32; + ret = sys_personality(personality); + if (personality(ret) == PER_LINUX32) + ret &= ~PER_LINUX32; + + return ret; +} + +int sparc_mmap_check(unsigned long addr, unsigned long len) +{ + if (test_thread_flag(TIF_32BIT)) { + if (len >= STACK_TOP32) + return -EINVAL; + + if (addr > STACK_TOP32 - len) + return -EINVAL; + } else { + if (len >= VA_EXCLUDE_START) + return -EINVAL; + + if (invalid_64bit_range(addr, len)) + return -EINVAL; + } + + return 0; +} + +/* Linux version of mmap */ +SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len, + unsigned long, prot, unsigned long, flags, unsigned long, fd, + unsigned long, off) +{ + unsigned long retval = -EINVAL; + + if ((off + PAGE_ALIGN(len)) < off) + goto out; + if (off & ~PAGE_MASK) + goto out; + retval = ksys_mmap_pgoff(addr, len, prot, flags, fd, off >> PAGE_SHIFT); +out: + return retval; +} + +SYSCALL_DEFINE2(64_munmap, unsigned long, addr, size_t, len) +{ + if (invalid_64bit_range(addr, len)) + return -EINVAL; + + return vm_munmap(addr, len); +} + +SYSCALL_DEFINE5(64_mremap, unsigned long, addr, unsigned long, old_len, + unsigned long, new_len, unsigned long, flags, + unsigned long, new_addr) +{ + if (test_thread_flag(TIF_32BIT)) + return -EINVAL; + return sys_mremap(addr, old_len, new_len, flags, new_addr); +} + +SYSCALL_DEFINE0(nis_syscall) +{ + static int count; + struct pt_regs *regs = current_pt_regs(); + + /* Don't make the system unusable, if someone goes stuck */ + if (count++ > 5) + return -ENOSYS; + + printk ("Unimplemented SPARC system call %ld\n",regs->u_regs[1]); +#ifdef DEBUG_UNIMP_SYSCALL + show_regs (regs); +#endif + + return -ENOSYS; +} + +/* #define DEBUG_SPARC_BREAKPOINT */ + +asmlinkage void sparc_breakpoint(struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } +#ifdef DEBUG_SPARC_BREAKPOINT + printk ("TRAP: Entering kernel PC=%lx, nPC=%lx\n", regs->tpc, regs->tnpc); +#endif + force_sig_fault(SIGTRAP, TRAP_BRKPT, (void __user *)regs->tpc); +#ifdef DEBUG_SPARC_BREAKPOINT + printk ("TRAP: Returning to space: PC=%lx nPC=%lx\n", regs->tpc, regs->tnpc); +#endif + exception_exit(prev_state); +} + +SYSCALL_DEFINE2(getdomainname, char __user *, name, int, len) +{ + int nlen, err; + char tmp[__NEW_UTS_LEN + 1]; + + if (len < 0) + return -EINVAL; + + down_read(&uts_sem); + + nlen = strlen(utsname()->domainname) + 1; + err = -EINVAL; + if (nlen > len) + goto out_unlock; + memcpy(tmp, utsname()->domainname, nlen); + + up_read(&uts_sem); + + if (copy_to_user(name, tmp, nlen)) + return -EFAULT; + return 0; + +out_unlock: + up_read(&uts_sem); + return err; +} + +SYSCALL_DEFINE1(sparc_adjtimex, struct __kernel_timex __user *, txc_p) +{ + struct __kernel_timex txc; + struct __kernel_old_timeval *tv = (void *)&txc.time; + int ret; + + /* Copy the user data space into the kernel copy + * structure. But bear in mind that the structures + * may change + */ + if (copy_from_user(&txc, txc_p, sizeof(txc))) + return -EFAULT; + + /* + * override for sparc64 specific timeval type: tv_usec + * is 32 bit wide instead of 64-bit in __kernel_timex + */ + txc.time.tv_usec = tv->tv_usec; + ret = do_adjtimex(&txc); + tv->tv_usec = txc.time.tv_usec; + + return copy_to_user(txc_p, &txc, sizeof(txc)) ? -EFAULT : ret; +} + +SYSCALL_DEFINE2(sparc_clock_adjtime, const clockid_t, which_clock, + struct __kernel_timex __user *, txc_p) +{ + struct __kernel_timex txc; + struct __kernel_old_timeval *tv = (void *)&txc.time; + int ret; + + if (!IS_ENABLED(CONFIG_POSIX_TIMERS)) { + pr_err_once("process %d (%s) attempted a POSIX timer syscall " + "while CONFIG_POSIX_TIMERS is not set\n", + current->pid, current->comm); + + return -ENOSYS; + } + + /* Copy the user data space into the kernel copy + * structure. But bear in mind that the structures + * may change + */ + if (copy_from_user(&txc, txc_p, sizeof(txc))) + return -EFAULT; + + /* + * override for sparc64 specific timeval type: tv_usec + * is 32 bit wide instead of 64-bit in __kernel_timex + */ + txc.time.tv_usec = tv->tv_usec; + ret = do_clock_adjtime(which_clock, &txc); + tv->tv_usec = txc.time.tv_usec; + + return copy_to_user(txc_p, &txc, sizeof(txc)) ? -EFAULT : ret; +} + +SYSCALL_DEFINE5(utrap_install, utrap_entry_t, type, + utrap_handler_t, new_p, utrap_handler_t, new_d, + utrap_handler_t __user *, old_p, + utrap_handler_t __user *, old_d) +{ + if (type < UT_INSTRUCTION_EXCEPTION || type > UT_TRAP_INSTRUCTION_31) + return -EINVAL; + if (new_p == (utrap_handler_t)(long)UTH_NOCHANGE) { + if (old_p) { + if (!current_thread_info()->utraps) { + if (put_user(NULL, old_p)) + return -EFAULT; + } else { + if (put_user((utrap_handler_t)(current_thread_info()->utraps[type]), old_p)) + return -EFAULT; + } + } + if (old_d) { + if (put_user(NULL, old_d)) + return -EFAULT; + } + return 0; + } + if (!current_thread_info()->utraps) { + current_thread_info()->utraps = + kzalloc_objs(long, UT_TRAP_INSTRUCTION_31 + 1); + if (!current_thread_info()->utraps) + return -ENOMEM; + current_thread_info()->utraps[0] = 1; + } else { + if ((utrap_handler_t)current_thread_info()->utraps[type] != new_p && + current_thread_info()->utraps[0] > 1) { + unsigned long *p = current_thread_info()->utraps; + + current_thread_info()->utraps = + kmalloc_objs(long, UT_TRAP_INSTRUCTION_31 + 1); + if (!current_thread_info()->utraps) { + current_thread_info()->utraps = p; + return -ENOMEM; + } + p[0]--; + current_thread_info()->utraps[0] = 1; + memcpy(current_thread_info()->utraps+1, p+1, + UT_TRAP_INSTRUCTION_31*sizeof(long)); + } + } + if (old_p) { + if (put_user((utrap_handler_t)(current_thread_info()->utraps[type]), old_p)) + return -EFAULT; + } + if (old_d) { + if (put_user(NULL, old_d)) + return -EFAULT; + } + current_thread_info()->utraps[type] = (long)new_p; + + return 0; +} + +SYSCALL_DEFINE1(memory_ordering, unsigned long, model) +{ + struct pt_regs *regs = current_pt_regs(); + if (model >= 3) + return -EINVAL; + regs->tstate = (regs->tstate & ~TSTATE_MM) | (model << 14); + return 0; +} + +SYSCALL_DEFINE5(rt_sigaction, int, sig, const struct sigaction __user *, act, + struct sigaction __user *, oact, void __user *, restorer, + size_t, sigsetsize) +{ + struct k_sigaction new_ka, old_ka; + int ret; + + /* XXX: Don't preclude handling different sized sigset_t's. */ + if (sigsetsize != sizeof(sigset_t)) + return -EINVAL; + + if (act) { + new_ka.ka_restorer = restorer; + if (copy_from_user(&new_ka.sa, act, sizeof(*act))) + return -EFAULT; + } + + ret = do_sigaction(sig, act ? &new_ka : NULL, oact ? &old_ka : NULL); + + if (!ret && oact) { + if (copy_to_user(oact, &old_ka.sa, sizeof(*oact))) + return -EFAULT; + } + + return ret; +} + +SYSCALL_DEFINE0(kern_features) +{ + return KERN_FEATURE_MIXED_MODE_STACK; +} diff --git a/arch/sparc/kernel/syscalls.S b/arch/sparc/kernel/syscalls.S new file mode 100644 index 000000000..96fe8763d --- /dev/null +++ b/arch/sparc/kernel/syscalls.S @@ -0,0 +1,309 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + /* SunOS's execv() call only specifies the argv argument, the + * environment settings are the same as the calling processes. + */ +sys64_execve: + set sys_execve, %g1 + jmpl %g1, %g0 + flushw + +sys64_execveat: + set sys_execveat, %g1 + jmpl %g1, %g0 + flushw + +#ifdef CONFIG_COMPAT +sunos_execv: + mov %g0, %o2 +sys32_execve: + set compat_sys_execve, %g1 + jmpl %g1, %g0 + flushw + +sys32_execveat: + set compat_sys_execveat, %g1 + jmpl %g1, %g0 + flushw +#endif + + .align 32 +#ifdef CONFIG_COMPAT +sys32_sigstack: + ba,pt %xcc, do_sys32_sigstack + mov %i6, %o2 +#endif + .align 32 +#ifdef CONFIG_COMPAT +sys32_sigreturn: + add %sp, PTREGS_OFF, %o0 + call do_sigreturn32 + add %o7, 1f-.-4, %o7 + nop +#endif +sys_rt_sigreturn: + add %sp, PTREGS_OFF, %o0 + call do_rt_sigreturn + add %o7, 1f-.-4, %o7 + nop +#ifdef CONFIG_COMPAT +sys32_rt_sigreturn: + add %sp, PTREGS_OFF, %o0 + call do_rt_sigreturn32 + add %o7, 1f-.-4, %o7 + nop +#endif + .align 32 +1: ldx [%g6 + TI_FLAGS], %l5 + andcc %l5, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 + be,pt %icc, rtrap + nop + call syscall_trace_leave + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + + /* This is how fork() was meant to be done, 8 instruction entry. + * + * I questioned the following code briefly, let me clear things + * up so you must not reason on it like I did. + * + * Know the fork_kpsr etc. we use in the sparc32 port? We don't + * need it here because the only piece of window state we copy to + * the child is the CWP register. Even if the parent sleeps, + * we are safe because we stuck it into pt_regs of the parent + * so it will not change. + * + * XXX This raises the question, whether we can do the same on + * XXX sparc32 to get rid of fork_kpsr _and_ fork_kwim. The + * XXX answer is yes. We stick fork_kpsr in UREG_G0 and + * XXX fork_kwim in UREG_G1 (global registers are considered + * XXX volatile across a system call in the sparc ABI I think + * XXX if it isn't we can use regs->y instead, anyone who depends + * XXX upon the Y register being preserved across a fork deserves + * XXX to lose). + * + * In fact we should take advantage of that fact for other things + * during system calls... + */ + .align 32 +sys_vfork: + flushw + ba,pt %xcc, sparc_vfork + add %sp, PTREGS_OFF, %o0 + + .align 32 +sys_fork: + flushw + ba,pt %xcc, sparc_fork + add %sp, PTREGS_OFF, %o0 + + .align 32 +sys_clone: + flushw + ba,pt %xcc, sparc_clone + add %sp, PTREGS_OFF, %o0 + + .align 32 +__sys_clone3: + flushw + ba,pt %xcc, sparc_clone3 + add %sp, PTREGS_OFF, %o0 + + .globl ret_from_fork +ret_from_fork: + /* Clear current_thread_info()->new_child. */ + stb %g0, [%g6 + TI_NEW_CHILD] + call schedule_tail + mov %g7, %o0 + ldx [%sp + PTREGS_OFF + PT_V9_I0], %o0 + brnz,pt %o0, ret_sys_call + ldx [%g6 + TI_FLAGS], %l0 + ldx [%sp + PTREGS_OFF + PT_V9_G1], %l1 + brz,pt %l1, ret_sys_call + nop + call %l1 + ldx [%sp + PTREGS_OFF + PT_V9_G2], %o0 + ba,pt %xcc, ret_sys_call + mov 0, %o0 + + .globl sparc_exit_group + .type sparc_exit_group,#function +sparc_exit_group: + sethi %hi(sys_exit_group), %g7 + ba,pt %xcc, 1f + or %g7, %lo(sys_exit_group), %g7 + .size sparc_exit_group,.-sparc_exit_group + + .globl sparc_exit + .type sparc_exit,#function +sparc_exit: + sethi %hi(sys_exit), %g7 + or %g7, %lo(sys_exit), %g7 +1: rdpr %pstate, %g2 + wrpr %g2, PSTATE_IE, %pstate + rdpr %otherwin, %g1 + rdpr %cansave, %g3 + add %g3, %g1, %g3 + wrpr %g3, 0x0, %cansave + wrpr %g0, 0x0, %otherwin + wrpr %g2, 0x0, %pstate + jmpl %g7, %g0 + stb %g0, [%g6 + TI_WSAVED] + .size sparc_exit,.-sparc_exit + +linux_sparc_ni_syscall: + sethi %hi(sys_ni_syscall), %l7 + ba,pt %xcc, 4f + or %l7, %lo(sys_ni_syscall), %l7 + +linux_syscall_trace32: + call syscall_trace_enter + add %sp, PTREGS_OFF, %o0 + brnz,pn %o0, 3f + mov -ENOSYS, %o0 + + /* Syscall tracing can modify the registers. */ + ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 + sethi %hi(sys_call_table32), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 + or %l7, %lo(sys_call_table32), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I1], %i1 + ldx [%sp + PTREGS_OFF + PT_V9_I2], %i2 + ldx [%sp + PTREGS_OFF + PT_V9_I3], %i3 + ldx [%sp + PTREGS_OFF + PT_V9_I4], %i4 + ldx [%sp + PTREGS_OFF + PT_V9_I5], %i5 + + cmp %g1, NR_syscalls + bgeu,pn %xcc, 3f + mov -ENOSYS, %o0 + + sll %g1, 2, %l4 + srl %i0, 0, %o0 + lduw [%l7 + %l4], %l7 + srl %i4, 0, %o4 + srl %i1, 0, %o1 + srl %i2, 0, %o2 + ba,pt %xcc, 5f + srl %i3, 0, %o3 + +linux_syscall_trace: + call syscall_trace_enter + add %sp, PTREGS_OFF, %o0 + brnz,pn %o0, 3f + mov -ENOSYS, %o0 + + /* Syscall tracing can modify the registers. */ + ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 + sethi %hi(sys_call_table64), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I0], %i0 + or %l7, %lo(sys_call_table64), %l7 + ldx [%sp + PTREGS_OFF + PT_V9_I1], %i1 + ldx [%sp + PTREGS_OFF + PT_V9_I2], %i2 + ldx [%sp + PTREGS_OFF + PT_V9_I3], %i3 + ldx [%sp + PTREGS_OFF + PT_V9_I4], %i4 + ldx [%sp + PTREGS_OFF + PT_V9_I5], %i5 + + cmp %g1, NR_syscalls + bgeu,pn %xcc, 3f + mov -ENOSYS, %o0 + + sll %g1, 2, %l4 + mov %i0, %o0 + lduw [%l7 + %l4], %l7 + mov %i1, %o1 + mov %i2, %o2 + mov %i3, %o3 + b,pt %xcc, 2f + mov %i4, %o4 + + + /* Linux 32-bit system calls enter here... */ + .align 32 + .globl linux_sparc_syscall32 +linux_sparc_syscall32: + /* Direct access to user regs, much faster. */ + cmp %g1, NR_syscalls ! IEU1 Group + bgeu,pn %xcc, linux_sparc_ni_syscall ! CTI + srl %i0, 0, %o0 ! IEU0 + sll %g1, 2, %l4 ! IEU0 Group + srl %i4, 0, %o4 ! IEU1 + lduw [%l7 + %l4], %l7 ! Load + srl %i1, 0, %o1 ! IEU0 Group + ldx [%g6 + TI_FLAGS], %l0 ! Load + + srl %i3, 0, %o3 ! IEU0 + srl %i2, 0, %o2 ! IEU0 Group + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 + bne,pn %icc, linux_syscall_trace32 ! CTI + mov %i0, %l5 ! IEU1 +5: call %l7 ! CTI Group brk forced + srl %i5, 0, %o5 ! IEU1 + ba,pt %xcc, 3f + sra %o0, 0, %o0 + + /* Linux native system calls enter here... */ + .align 32 + .globl linux_sparc_syscall +linux_sparc_syscall: + /* Direct access to user regs, much faster. */ + cmp %g1, NR_syscalls ! IEU1 Group + bgeu,pn %xcc, linux_sparc_ni_syscall ! CTI + mov %i0, %o0 ! IEU0 + sll %g1, 2, %l4 ! IEU0 Group + mov %i1, %o1 ! IEU1 + lduw [%l7 + %l4], %l7 ! Load +4: mov %i2, %o2 ! IEU0 Group + ldx [%g6 + TI_FLAGS], %l0 ! Load + + mov %i3, %o3 ! IEU1 + mov %i4, %o4 ! IEU0 Group + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 + bne,pn %icc, linux_syscall_trace ! CTI Group + mov %i0, %l5 ! IEU0 +2: call %l7 ! CTI Group brk forced + mov %i5, %o5 ! IEU0 + nop + +3: stx %o0, [%sp + PTREGS_OFF + PT_V9_I0] +ret_sys_call: + ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %g3 + mov %ulo(TSTATE_XCARRY | TSTATE_ICARRY), %g2 + sllx %g2, 32, %g2 + + cmp %o0, -ERESTART_RESTARTBLOCK + bgeu,pn %xcc, 1f + andcc %l0, (_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT|_TIF_SYSCALL_TRACEPOINT|_TIF_NOHZ), %g0 + ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc + +2: + /* System call success, clear Carry condition code. */ + andn %g3, %g2, %g3 +3: + stx %g3, [%sp + PTREGS_OFF + PT_V9_TSTATE] + bne,pn %icc, linux_syscall_trace2 + add %l1, 0x4, %l2 ! npc = npc+4 + stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC] + ba,pt %xcc, rtrap + stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] + +1: + /* Check if force_successful_syscall_return() + * was invoked. + */ + ldub [%g6 + TI_SYS_NOERROR], %l2 + brnz,pn %l2, 2b + ldx [%sp + PTREGS_OFF + PT_V9_TNPC], %l1 ! pc = npc + /* System call failure, set Carry condition code. + * Also, get abs(errno) to return to the process. + */ + sub %g0, %o0, %o0 + stx %o0, [%sp + PTREGS_OFF + PT_V9_I0] + ba,pt %xcc, 3b + or %g3, %g2, %g3 + +linux_syscall_trace2: + call syscall_trace_leave + add %sp, PTREGS_OFF, %o0 + stx %l1, [%sp + PTREGS_OFF + PT_V9_TPC] + ba,pt %xcc, rtrap + stx %l2, [%sp + PTREGS_OFF + PT_V9_TNPC] diff --git a/arch/sparc/kernel/syscalls/Makefile b/arch/sparc/kernel/syscalls/Makefile new file mode 100644 index 000000000..8440c16df --- /dev/null +++ b/arch/sparc/kernel/syscalls/Makefile @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: GPL-2.0 +kapi := arch/$(SRCARCH)/include/generated/asm +uapi := arch/$(SRCARCH)/include/generated/uapi/asm + +$(shell mkdir -p $(uapi) $(kapi)) + +syscall := $(src)/syscall.tbl +syshdr := $(srctree)/scripts/syscallhdr.sh +systbl := $(srctree)/scripts/syscalltbl.sh + +quiet_cmd_syshdr = SYSHDR $@ + cmd_syshdr = $(CONFIG_SHELL) $(syshdr) --emit-nr --abis common,$* $< $@ + +quiet_cmd_systbl = SYSTBL $@ + cmd_systbl = $(CONFIG_SHELL) $(systbl) --abis common,$* $< $@ + +$(uapi)/unistd_%.h: $(syscall) $(syshdr) FORCE + $(call if_changed,syshdr) + +$(kapi)/syscall_table_%.h: $(syscall) $(systbl) FORCE + $(call if_changed,systbl) + +uapisyshdr-y += unistd_32.h unistd_64.h +kapisyshdr-y += syscall_table_32.h \ + syscall_table_64.h + +uapisyshdr-y := $(addprefix $(uapi)/, $(uapisyshdr-y)) +kapisyshdr-y := $(addprefix $(kapi)/, $(kapisyshdr-y)) +targets += $(addprefix ../../../../, $(uapisyshdr-y) $(kapisyshdr-y)) + +PHONY += all +all: $(uapisyshdr-y) $(kapisyshdr-y) + @: diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl new file mode 100644 index 000000000..5b9fe0e81 --- /dev/null +++ b/arch/sparc/kernel/syscalls/syscall.tbl @@ -0,0 +1,520 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# system call numbers and entry vectors for sparc +# +# The format is: +# +# +# The can be common, 64, or 32 for this file. +# +0 common restart_syscall sys_restart_syscall +1 32 exit sys_exit sparc_exit +1 64 exit sparc_exit +2 common fork sys_fork +3 common read sys_read +4 common write sys_write +5 common open sys_open compat_sys_open +6 common close sys_close +7 common wait4 sys_wait4 compat_sys_wait4 +8 common creat sys_creat +9 common link sys_link +10 common unlink sys_unlink +11 32 execv sunos_execv +11 64 execv sys_nis_syscall +12 common chdir sys_chdir +13 32 chown sys_chown16 +13 64 chown sys_chown +14 common mknod sys_mknod +15 common chmod sys_chmod +16 32 lchown sys_lchown16 +16 64 lchown sys_lchown +17 common brk sys_brk +18 common perfctr sys_nis_syscall +19 common lseek sys_lseek compat_sys_lseek +20 common getpid sys_getpid +21 common capget sys_capget +22 common capset sys_capset +23 32 setuid sys_setuid16 +23 64 setuid sys_setuid +24 32 getuid sys_getuid16 +24 64 getuid sys_getuid +25 common vmsplice sys_vmsplice +26 common ptrace sys_ptrace compat_sys_ptrace +27 common alarm sys_alarm +28 common sigaltstack sys_sigaltstack compat_sys_sigaltstack +29 32 pause sys_pause +29 64 pause sys_nis_syscall +30 32 utime sys_utime32 +30 64 utime sys_utime +31 32 lchown32 sys_lchown +32 32 fchown32 sys_fchown +33 common access sys_access +34 common nice sys_nice +35 32 chown32 sys_chown +36 common sync sys_sync +37 common kill sys_kill +38 common stat sys_newstat compat_sys_newstat +39 32 sendfile sys_sendfile compat_sys_sendfile +39 64 sendfile sys_sendfile64 +40 common lstat sys_newlstat compat_sys_newlstat +41 common dup sys_dup +42 common pipe sys_sparc_pipe +43 common times sys_times compat_sys_times +44 32 getuid32 sys_getuid +45 common umount2 sys_umount +46 32 setgid sys_setgid16 +46 64 setgid sys_setgid +47 32 getgid sys_getgid16 +47 64 getgid sys_getgid +48 common signal sys_signal +49 32 geteuid sys_geteuid16 +49 64 geteuid sys_geteuid +50 32 getegid sys_getegid16 +50 64 getegid sys_getegid +51 common acct sys_acct +52 64 memory_ordering sys_memory_ordering +53 32 getgid32 sys_getgid +54 common ioctl sys_ioctl compat_sys_ioctl +55 common reboot sys_reboot +56 32 mmap2 sys_mmap2 sys32_mmap2 +57 common symlink sys_symlink +58 common readlink sys_readlink +59 32 execve sys_execve sys32_execve +59 64 execve sys64_execve +60 common umask sys_umask +61 common chroot sys_chroot +62 common fstat sys_newfstat compat_sys_newfstat +63 common fstat64 sys_fstat64 compat_sys_fstat64 +64 common getpagesize sys_getpagesize +65 common msync sys_msync +66 common vfork sys_vfork +67 common pread64 sys_pread64 compat_sys_pread64 +68 common pwrite64 sys_pwrite64 compat_sys_pwrite64 +69 32 geteuid32 sys_geteuid +70 32 getegid32 sys_getegid +71 common mmap sys_mmap +72 32 setreuid32 sys_setreuid +73 32 munmap sys_munmap +73 64 munmap sys_64_munmap +74 common mprotect sys_mprotect +75 common madvise sys_madvise +76 common vhangup sys_vhangup +77 32 truncate64 sys_truncate64 compat_sys_truncate64 +78 common mincore sys_mincore +79 32 getgroups sys_getgroups16 +79 64 getgroups sys_getgroups +80 32 setgroups sys_setgroups16 +80 64 setgroups sys_setgroups +81 common getpgrp sys_getpgrp +82 32 setgroups32 sys_setgroups +83 common setitimer sys_setitimer compat_sys_setitimer +84 32 ftruncate64 sys_ftruncate64 compat_sys_ftruncate64 +85 common swapon sys_swapon +86 common getitimer sys_getitimer compat_sys_getitimer +87 32 setuid32 sys_setuid +88 common sethostname sys_sethostname +89 32 setgid32 sys_setgid +90 common dup2 sys_dup2 +91 32 setfsuid32 sys_setfsuid +92 common fcntl sys_fcntl compat_sys_fcntl +93 common select sys_select compat_sys_select +94 32 setfsgid32 sys_setfsgid +95 common fsync sys_fsync +96 common setpriority sys_setpriority +97 common socket sys_socket +98 common connect sys_connect +99 common accept sys_accept +100 common getpriority sys_getpriority +101 common rt_sigreturn sys_rt_sigreturn sys32_rt_sigreturn +102 common rt_sigaction sys_rt_sigaction compat_sys_rt_sigaction +103 common rt_sigprocmask sys_rt_sigprocmask compat_sys_rt_sigprocmask +104 common rt_sigpending sys_rt_sigpending compat_sys_rt_sigpending +105 32 rt_sigtimedwait sys_rt_sigtimedwait_time32 compat_sys_rt_sigtimedwait_time32 +105 64 rt_sigtimedwait sys_rt_sigtimedwait +106 common rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo +107 common rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend +108 32 setresuid32 sys_setresuid +108 64 setresuid sys_setresuid +109 32 getresuid32 sys_getresuid +109 64 getresuid sys_getresuid +110 32 setresgid32 sys_setresgid +110 64 setresgid sys_setresgid +111 32 getresgid32 sys_getresgid +111 64 getresgid sys_getresgid +112 32 setregid32 sys_setregid +113 common recvmsg sys_recvmsg compat_sys_recvmsg +114 common sendmsg sys_sendmsg compat_sys_sendmsg +115 32 getgroups32 sys_getgroups +116 common gettimeofday sys_gettimeofday compat_sys_gettimeofday +117 common getrusage sys_getrusage compat_sys_getrusage +118 common getsockopt sys_getsockopt sys_getsockopt +119 common getcwd sys_getcwd +120 common readv sys_readv +121 common writev sys_writev +122 common settimeofday sys_settimeofday compat_sys_settimeofday +123 32 fchown sys_fchown16 +123 64 fchown sys_fchown +124 common fchmod sys_fchmod +125 common recvfrom sys_recvfrom compat_sys_recvfrom +126 32 setreuid sys_setreuid16 +126 64 setreuid sys_setreuid +127 32 setregid sys_setregid16 +127 64 setregid sys_setregid +128 common rename sys_rename +129 common truncate sys_truncate compat_sys_truncate +130 common ftruncate sys_ftruncate compat_sys_ftruncate +131 common flock sys_flock +132 common lstat64 sys_lstat64 compat_sys_lstat64 +133 common sendto sys_sendto +134 common shutdown sys_shutdown +135 common socketpair sys_socketpair +136 common mkdir sys_mkdir +137 common rmdir sys_rmdir +138 32 utimes sys_utimes_time32 +138 64 utimes sys_utimes +139 common stat64 sys_stat64 compat_sys_stat64 +140 common sendfile64 sys_sendfile64 +141 common getpeername sys_getpeername +142 32 futex sys_futex_time32 +142 64 futex sys_futex +143 common gettid sys_gettid +144 common getrlimit sys_getrlimit compat_sys_getrlimit +145 common setrlimit sys_setrlimit compat_sys_setrlimit +146 common pivot_root sys_pivot_root +147 common prctl sys_prctl +148 common pciconfig_read sys_pciconfig_read +149 common pciconfig_write sys_pciconfig_write +150 common getsockname sys_getsockname +151 common inotify_init sys_inotify_init +152 common inotify_add_watch sys_inotify_add_watch +153 common poll sys_poll +154 common getdents64 sys_getdents64 +155 32 fcntl64 sys_fcntl64 compat_sys_fcntl64 +156 common inotify_rm_watch sys_inotify_rm_watch +157 common statfs sys_statfs compat_sys_statfs +158 common fstatfs sys_fstatfs compat_sys_fstatfs +159 common umount sys_oldumount +160 common sched_set_affinity sys_sched_setaffinity compat_sys_sched_setaffinity +161 common sched_get_affinity sys_sched_getaffinity compat_sys_sched_getaffinity +162 common getdomainname sys_getdomainname +163 common setdomainname sys_setdomainname +164 64 utrap_install sys_utrap_install +165 common quotactl sys_quotactl +166 common set_tid_address sys_set_tid_address +167 common mount sys_mount +168 common ustat sys_ustat compat_sys_ustat +169 common setxattr sys_setxattr +170 common lsetxattr sys_lsetxattr +171 common fsetxattr sys_fsetxattr +172 common getxattr sys_getxattr +173 common lgetxattr sys_lgetxattr +174 common getdents sys_getdents compat_sys_getdents +175 common setsid sys_setsid +176 common fchdir sys_fchdir +177 common fgetxattr sys_fgetxattr +178 common listxattr sys_listxattr +179 common llistxattr sys_llistxattr +180 common flistxattr sys_flistxattr +181 common removexattr sys_removexattr +182 common lremovexattr sys_lremovexattr +183 32 sigpending sys_sigpending compat_sys_sigpending +183 64 sigpending sys_nis_syscall +184 common query_module sys_ni_syscall +185 common setpgid sys_setpgid +186 common fremovexattr sys_fremovexattr +187 common tkill sys_tkill +188 32 exit_group sys_exit_group sparc_exit_group +188 64 exit_group sparc_exit_group +189 common uname sys_newuname +190 common init_module sys_init_module +191 32 personality sys_personality sys_sparc64_personality +191 64 personality sys_sparc64_personality +192 32 remap_file_pages sys_sparc_remap_file_pages sys_remap_file_pages +192 64 remap_file_pages sys_remap_file_pages +193 common epoll_create sys_epoll_create +194 common epoll_ctl sys_epoll_ctl +195 common epoll_wait sys_epoll_wait +196 common ioprio_set sys_ioprio_set +197 common getppid sys_getppid +198 32 sigaction sys_sparc_sigaction compat_sys_sparc_sigaction +198 64 sigaction sys_nis_syscall +199 common sgetmask sys_sgetmask +200 common ssetmask sys_ssetmask +201 32 sigsuspend sys_sigsuspend +201 64 sigsuspend sys_nis_syscall +202 common oldlstat sys_newlstat compat_sys_newlstat +203 common uselib sys_uselib +204 32 readdir sys_old_readdir compat_sys_old_readdir +204 64 readdir sys_nis_syscall +205 common readahead sys_readahead compat_sys_readahead +206 common socketcall sys_socketcall compat_sys_socketcall +207 common syslog sys_syslog +208 common lookup_dcookie sys_ni_syscall +209 common fadvise64 sys_fadvise64 compat_sys_fadvise64 +210 common fadvise64_64 sys_fadvise64_64 compat_sys_fadvise64_64 +211 common tgkill sys_tgkill +212 common waitpid sys_waitpid +213 common swapoff sys_swapoff +214 common sysinfo sys_sysinfo compat_sys_sysinfo +215 32 ipc sys_ipc compat_sys_ipc +215 64 ipc sys_sparc_ipc +216 32 sigreturn sys_sigreturn sys32_sigreturn +216 64 sigreturn sys_nis_syscall +217 common clone sys_clone +218 common ioprio_get sys_ioprio_get +219 32 adjtimex sys_adjtimex_time32 +219 64 adjtimex sys_sparc_adjtimex +220 32 sigprocmask sys_sigprocmask compat_sys_sigprocmask +220 64 sigprocmask sys_nis_syscall +221 common create_module sys_ni_syscall +222 common delete_module sys_delete_module +223 common get_kernel_syms sys_ni_syscall +224 common getpgid sys_getpgid +225 common bdflush sys_ni_syscall +226 common sysfs sys_sysfs +227 common afs_syscall sys_nis_syscall +228 common setfsuid sys_setfsuid16 +229 common setfsgid sys_setfsgid16 +230 common _newselect sys_select compat_sys_select +231 32 time sys_time32 +232 common splice sys_splice +233 32 stime sys_stime32 +233 64 stime sys_stime +234 common statfs64 sys_statfs64 compat_sys_statfs64 +235 common fstatfs64 sys_fstatfs64 compat_sys_fstatfs64 +236 common _llseek sys_llseek +237 common mlock sys_mlock +238 common munlock sys_munlock +239 common mlockall sys_mlockall +240 common munlockall sys_munlockall +241 common sched_setparam sys_sched_setparam +242 common sched_getparam sys_sched_getparam +243 common sched_setscheduler sys_sched_setscheduler +244 common sched_getscheduler sys_sched_getscheduler +245 common sched_yield sys_sched_yield +246 common sched_get_priority_max sys_sched_get_priority_max +247 common sched_get_priority_min sys_sched_get_priority_min +248 32 sched_rr_get_interval sys_sched_rr_get_interval_time32 +248 64 sched_rr_get_interval sys_sched_rr_get_interval +249 32 nanosleep sys_nanosleep_time32 +249 64 nanosleep sys_nanosleep +250 32 mremap sys_mremap +250 64 mremap sys_64_mremap +251 common _sysctl sys_ni_syscall +252 common getsid sys_getsid +253 common fdatasync sys_fdatasync +254 32 nfsservctl sys_ni_syscall sys_nis_syscall +254 64 nfsservctl sys_nis_syscall +255 common sync_file_range sys_sync_file_range compat_sys_sync_file_range +256 32 clock_settime sys_clock_settime32 +256 64 clock_settime sys_clock_settime +257 32 clock_gettime sys_clock_gettime32 +257 64 clock_gettime sys_clock_gettime +258 32 clock_getres sys_clock_getres_time32 +258 64 clock_getres sys_clock_getres +259 32 clock_nanosleep sys_clock_nanosleep_time32 +259 64 clock_nanosleep sys_clock_nanosleep +260 common sched_getaffinity sys_sched_getaffinity compat_sys_sched_getaffinity +261 common sched_setaffinity sys_sched_setaffinity compat_sys_sched_setaffinity +262 32 timer_settime sys_timer_settime32 +262 64 timer_settime sys_timer_settime +263 32 timer_gettime sys_timer_gettime32 +263 64 timer_gettime sys_timer_gettime +264 common timer_getoverrun sys_timer_getoverrun +265 common timer_delete sys_timer_delete +266 common timer_create sys_timer_create compat_sys_timer_create +# 267 was vserver +267 common vserver sys_nis_syscall +268 common io_setup sys_io_setup compat_sys_io_setup +269 common io_destroy sys_io_destroy +270 common io_submit sys_io_submit compat_sys_io_submit +271 common io_cancel sys_io_cancel +272 32 io_getevents sys_io_getevents_time32 +272 64 io_getevents sys_io_getevents +273 common mq_open sys_mq_open compat_sys_mq_open +274 common mq_unlink sys_mq_unlink +275 32 mq_timedsend sys_mq_timedsend_time32 +275 64 mq_timedsend sys_mq_timedsend +276 32 mq_timedreceive sys_mq_timedreceive_time32 +276 64 mq_timedreceive sys_mq_timedreceive +277 common mq_notify sys_mq_notify compat_sys_mq_notify +278 common mq_getsetattr sys_mq_getsetattr compat_sys_mq_getsetattr +279 common waitid sys_waitid compat_sys_waitid +280 common tee sys_tee +281 common add_key sys_add_key +282 common request_key sys_request_key +283 common keyctl sys_keyctl compat_sys_keyctl +284 common openat sys_openat compat_sys_openat +285 common mkdirat sys_mkdirat +286 common mknodat sys_mknodat +287 common fchownat sys_fchownat +288 32 futimesat sys_futimesat_time32 +288 64 futimesat sys_futimesat +289 common fstatat64 sys_fstatat64 compat_sys_fstatat64 +290 common unlinkat sys_unlinkat +291 common renameat sys_renameat +292 common linkat sys_linkat +293 common symlinkat sys_symlinkat +294 common readlinkat sys_readlinkat +295 common fchmodat sys_fchmodat +296 common faccessat sys_faccessat +297 32 pselect6 sys_pselect6_time32 compat_sys_pselect6_time32 +297 64 pselect6 sys_pselect6 +298 32 ppoll sys_ppoll_time32 compat_sys_ppoll_time32 +298 64 ppoll sys_ppoll +299 common unshare sys_unshare +300 common set_robust_list sys_set_robust_list compat_sys_set_robust_list +301 common get_robust_list sys_get_robust_list compat_sys_get_robust_list +302 common migrate_pages sys_migrate_pages +303 common mbind sys_mbind +304 common get_mempolicy sys_get_mempolicy +305 common set_mempolicy sys_set_mempolicy +306 common kexec_load sys_kexec_load compat_sys_kexec_load +307 common move_pages sys_move_pages +308 common getcpu sys_getcpu +309 common epoll_pwait sys_epoll_pwait compat_sys_epoll_pwait +310 32 utimensat sys_utimensat_time32 +310 64 utimensat sys_utimensat +311 common signalfd sys_signalfd compat_sys_signalfd +312 common timerfd_create sys_timerfd_create +313 common eventfd sys_eventfd +314 common fallocate sys_fallocate compat_sys_fallocate +315 32 timerfd_settime sys_timerfd_settime32 +315 64 timerfd_settime sys_timerfd_settime +316 32 timerfd_gettime sys_timerfd_gettime32 +316 64 timerfd_gettime sys_timerfd_gettime +317 common signalfd4 sys_signalfd4 compat_sys_signalfd4 +318 common eventfd2 sys_eventfd2 +319 common epoll_create1 sys_epoll_create1 +320 common dup3 sys_dup3 +321 common pipe2 sys_pipe2 +322 common inotify_init1 sys_inotify_init1 +323 common accept4 sys_accept4 +324 common preadv sys_preadv compat_sys_preadv +325 common pwritev sys_pwritev compat_sys_pwritev +326 common rt_tgsigqueueinfo sys_rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo +327 common perf_event_open sys_perf_event_open +328 32 recvmmsg sys_recvmmsg_time32 compat_sys_recvmmsg_time32 +328 64 recvmmsg sys_recvmmsg +329 common fanotify_init sys_fanotify_init +330 common fanotify_mark sys_fanotify_mark compat_sys_fanotify_mark +331 common prlimit64 sys_prlimit64 +332 common name_to_handle_at sys_name_to_handle_at +333 common open_by_handle_at sys_open_by_handle_at compat_sys_open_by_handle_at +334 32 clock_adjtime sys_clock_adjtime32 +334 64 clock_adjtime sys_sparc_clock_adjtime +335 common syncfs sys_syncfs +336 common sendmmsg sys_sendmmsg compat_sys_sendmmsg +337 common setns sys_setns +338 common process_vm_readv sys_process_vm_readv +339 common process_vm_writev sys_process_vm_writev +340 32 kern_features sys_ni_syscall sys_kern_features +340 64 kern_features sys_kern_features +341 common kcmp sys_kcmp +342 common finit_module sys_finit_module +343 common sched_setattr sys_sched_setattr +344 common sched_getattr sys_sched_getattr +345 common renameat2 sys_renameat2 +346 common seccomp sys_seccomp +347 common getrandom sys_getrandom +348 common memfd_create sys_memfd_create +349 common bpf sys_bpf +350 32 execveat sys_execveat sys32_execveat +350 64 execveat sys64_execveat +351 common membarrier sys_membarrier +352 common userfaultfd sys_userfaultfd +353 common bind sys_bind +354 common listen sys_listen +355 common setsockopt sys_setsockopt sys_setsockopt +356 common mlock2 sys_mlock2 +357 common copy_file_range sys_copy_file_range +358 common preadv2 sys_preadv2 compat_sys_preadv2 +359 common pwritev2 sys_pwritev2 compat_sys_pwritev2 +360 common statx sys_statx +361 32 io_pgetevents sys_io_pgetevents_time32 compat_sys_io_pgetevents +361 64 io_pgetevents sys_io_pgetevents +362 common pkey_mprotect sys_pkey_mprotect +363 common pkey_alloc sys_pkey_alloc +364 common pkey_free sys_pkey_free +365 common rseq sys_rseq +# room for arch specific syscalls +392 64 semtimedop sys_semtimedop +393 common semget sys_semget +394 common semctl sys_semctl compat_sys_semctl +395 common shmget sys_shmget +396 common shmctl sys_shmctl compat_sys_shmctl +397 common shmat sys_shmat compat_sys_shmat +398 common shmdt sys_shmdt +399 common msgget sys_msgget +400 common msgsnd sys_msgsnd compat_sys_msgsnd +401 common msgrcv sys_msgrcv compat_sys_msgrcv +402 common msgctl sys_msgctl compat_sys_msgctl +403 32 clock_gettime64 sys_clock_gettime sys_clock_gettime +404 32 clock_settime64 sys_clock_settime sys_clock_settime +405 32 clock_adjtime64 sys_clock_adjtime sys_clock_adjtime +406 32 clock_getres_time64 sys_clock_getres sys_clock_getres +407 32 clock_nanosleep_time64 sys_clock_nanosleep sys_clock_nanosleep +408 32 timer_gettime64 sys_timer_gettime sys_timer_gettime +409 32 timer_settime64 sys_timer_settime sys_timer_settime +410 32 timerfd_gettime64 sys_timerfd_gettime sys_timerfd_gettime +411 32 timerfd_settime64 sys_timerfd_settime sys_timerfd_settime +412 32 utimensat_time64 sys_utimensat sys_utimensat +413 32 pselect6_time64 sys_pselect6 compat_sys_pselect6_time64 +414 32 ppoll_time64 sys_ppoll compat_sys_ppoll_time64 +416 32 io_pgetevents_time64 sys_io_pgetevents compat_sys_io_pgetevents_time64 +417 32 recvmmsg_time64 sys_recvmmsg compat_sys_recvmmsg_time64 +418 32 mq_timedsend_time64 sys_mq_timedsend sys_mq_timedsend +419 32 mq_timedreceive_time64 sys_mq_timedreceive sys_mq_timedreceive +420 32 semtimedop_time64 sys_semtimedop sys_semtimedop +421 32 rt_sigtimedwait_time64 sys_rt_sigtimedwait compat_sys_rt_sigtimedwait_time64 +422 32 futex_time64 sys_futex sys_futex +423 32 sched_rr_get_interval_time64 sys_sched_rr_get_interval sys_sched_rr_get_interval +424 common pidfd_send_signal sys_pidfd_send_signal +425 common io_uring_setup sys_io_uring_setup +426 common io_uring_enter sys_io_uring_enter +427 common io_uring_register sys_io_uring_register +428 common open_tree sys_open_tree +429 common move_mount sys_move_mount +430 common fsopen sys_fsopen +431 common fsconfig sys_fsconfig +432 common fsmount sys_fsmount +433 common fspick sys_fspick +434 common pidfd_open sys_pidfd_open +435 common clone3 __sys_clone3 +436 common close_range sys_close_range +437 common openat2 sys_openat2 +438 common pidfd_getfd sys_pidfd_getfd +439 common faccessat2 sys_faccessat2 +440 common process_madvise sys_process_madvise +441 common epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr +443 common quotactl_fd sys_quotactl_fd +444 common landlock_create_ruleset sys_landlock_create_ruleset +445 common landlock_add_rule sys_landlock_add_rule +446 common landlock_restrict_self sys_landlock_restrict_self +# 447 reserved for memfd_secret +448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common cachestat sys_cachestat +452 common fchmodat2 sys_fchmodat2 +453 common map_shadow_stack sys_map_shadow_stack +454 common futex_wake sys_futex_wake +455 common futex_wait sys_futex_wait +456 common futex_requeue sys_futex_requeue +457 common statmount sys_statmount +458 common listmount sys_listmount +459 common lsm_get_self_attr sys_lsm_get_self_attr +460 common lsm_set_self_attr sys_lsm_set_self_attr +461 common lsm_list_modules sys_lsm_list_modules +462 common mseal sys_mseal +463 common setxattrat sys_setxattrat +464 common getxattrat sys_getxattrat +465 common listxattrat sys_listxattrat +466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr +468 common file_getattr sys_file_getattr +469 common file_setattr sys_file_setattr +470 common listns sys_listns +471 common rseq_slice_yield sys_rseq_slice_yield +472 common fchroot sys_fchroot diff --git a/arch/sparc/kernel/sysfs.c b/arch/sparc/kernel/sysfs.c new file mode 100644 index 000000000..f19487e4c --- /dev/null +++ b/arch/sparc/kernel/sysfs.c @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: GPL-2.0 +/* sysfs.c: Topology sysfs support code for sparc64. + * + * Copyright (C) 2007 David S. Miller + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static DEFINE_PER_CPU(struct hv_mmu_statistics, mmu_stats) __attribute__((aligned(64))); + +#define SHOW_MMUSTAT_ULONG(NAME) \ +static ssize_t show_##NAME(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + struct hv_mmu_statistics *p = &per_cpu(mmu_stats, dev->id); \ + return sprintf(buf, "%lu\n", p->NAME); \ +} \ +static DEVICE_ATTR(NAME, 0444, show_##NAME, NULL) + +SHOW_MMUSTAT_ULONG(immu_tsb_hits_ctx0_8k_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_ticks_ctx0_8k_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_hits_ctx0_64k_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_ticks_ctx0_64k_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_hits_ctx0_4mb_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_ticks_ctx0_4mb_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_hits_ctx0_256mb_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_ticks_ctx0_256mb_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_hits_ctxnon0_8k_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_ticks_ctxnon0_8k_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_hits_ctxnon0_64k_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_ticks_ctxnon0_64k_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_hits_ctxnon0_4mb_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_ticks_ctxnon0_4mb_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_hits_ctxnon0_256mb_tte); +SHOW_MMUSTAT_ULONG(immu_tsb_ticks_ctxnon0_256mb_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_hits_ctx0_8k_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_ticks_ctx0_8k_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_hits_ctx0_64k_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_ticks_ctx0_64k_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_hits_ctx0_4mb_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_ticks_ctx0_4mb_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_hits_ctx0_256mb_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_ticks_ctx0_256mb_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_hits_ctxnon0_8k_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_ticks_ctxnon0_8k_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_hits_ctxnon0_64k_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_ticks_ctxnon0_64k_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_hits_ctxnon0_4mb_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_ticks_ctxnon0_4mb_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_hits_ctxnon0_256mb_tte); +SHOW_MMUSTAT_ULONG(dmmu_tsb_ticks_ctxnon0_256mb_tte); + +static struct attribute *mmu_stat_attrs[] = { + &dev_attr_immu_tsb_hits_ctx0_8k_tte.attr, + &dev_attr_immu_tsb_ticks_ctx0_8k_tte.attr, + &dev_attr_immu_tsb_hits_ctx0_64k_tte.attr, + &dev_attr_immu_tsb_ticks_ctx0_64k_tte.attr, + &dev_attr_immu_tsb_hits_ctx0_4mb_tte.attr, + &dev_attr_immu_tsb_ticks_ctx0_4mb_tte.attr, + &dev_attr_immu_tsb_hits_ctx0_256mb_tte.attr, + &dev_attr_immu_tsb_ticks_ctx0_256mb_tte.attr, + &dev_attr_immu_tsb_hits_ctxnon0_8k_tte.attr, + &dev_attr_immu_tsb_ticks_ctxnon0_8k_tte.attr, + &dev_attr_immu_tsb_hits_ctxnon0_64k_tte.attr, + &dev_attr_immu_tsb_ticks_ctxnon0_64k_tte.attr, + &dev_attr_immu_tsb_hits_ctxnon0_4mb_tte.attr, + &dev_attr_immu_tsb_ticks_ctxnon0_4mb_tte.attr, + &dev_attr_immu_tsb_hits_ctxnon0_256mb_tte.attr, + &dev_attr_immu_tsb_ticks_ctxnon0_256mb_tte.attr, + &dev_attr_dmmu_tsb_hits_ctx0_8k_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctx0_8k_tte.attr, + &dev_attr_dmmu_tsb_hits_ctx0_64k_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctx0_64k_tte.attr, + &dev_attr_dmmu_tsb_hits_ctx0_4mb_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctx0_4mb_tte.attr, + &dev_attr_dmmu_tsb_hits_ctx0_256mb_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctx0_256mb_tte.attr, + &dev_attr_dmmu_tsb_hits_ctxnon0_8k_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctxnon0_8k_tte.attr, + &dev_attr_dmmu_tsb_hits_ctxnon0_64k_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctxnon0_64k_tte.attr, + &dev_attr_dmmu_tsb_hits_ctxnon0_4mb_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctxnon0_4mb_tte.attr, + &dev_attr_dmmu_tsb_hits_ctxnon0_256mb_tte.attr, + &dev_attr_dmmu_tsb_ticks_ctxnon0_256mb_tte.attr, + NULL, +}; + +static struct attribute_group mmu_stat_group = { + .attrs = mmu_stat_attrs, + .name = "mmu_stats", +}; + +static long read_mmustat_enable(void *data __maybe_unused) +{ + unsigned long ra = 0; + + sun4v_mmustat_info(&ra); + + return ra != 0; +} + +static long write_mmustat_enable(void *data) +{ + unsigned long ra, orig_ra, *val = data; + + if (*val) + ra = __pa(&per_cpu(mmu_stats, smp_processor_id())); + else + ra = 0UL; + + return sun4v_mmustat_conf(ra, &orig_ra); +} + +static ssize_t show_mmustat_enable(struct device *s, + struct device_attribute *attr, char *buf) +{ + long val = work_on_cpu(s->id, read_mmustat_enable, NULL); + + return sprintf(buf, "%lx\n", val); +} + +static ssize_t store_mmustat_enable(struct device *s, + struct device_attribute *attr, const char *buf, + size_t count) +{ + unsigned long val; + long err; + int ret; + + ret = sscanf(buf, "%lu", &val); + if (ret != 1) + return -EINVAL; + + err = work_on_cpu(s->id, write_mmustat_enable, &val); + if (err) + return -EIO; + + return count; +} + +static DEVICE_ATTR(mmustat_enable, 0644, show_mmustat_enable, store_mmustat_enable); + +static int mmu_stats_supported; + +static int register_mmu_stats(struct device *s) +{ + if (!mmu_stats_supported) + return 0; + device_create_file(s, &dev_attr_mmustat_enable); + return sysfs_create_group(&s->kobj, &mmu_stat_group); +} + +#ifdef CONFIG_HOTPLUG_CPU +static void unregister_mmu_stats(struct device *s) +{ + if (!mmu_stats_supported) + return; + sysfs_remove_group(&s->kobj, &mmu_stat_group); + device_remove_file(s, &dev_attr_mmustat_enable); +} +#endif + +#define SHOW_CPUDATA_ULONG_NAME(NAME, MEMBER) \ +static ssize_t show_##NAME(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + cpuinfo_sparc *c = &cpu_data(dev->id); \ + return sprintf(buf, "%lu\n", c->MEMBER); \ +} + +#define SHOW_CPUDATA_UINT_NAME(NAME, MEMBER) \ +static ssize_t show_##NAME(struct device *dev, \ + struct device_attribute *attr, char *buf) \ +{ \ + cpuinfo_sparc *c = &cpu_data(dev->id); \ + return sprintf(buf, "%u\n", c->MEMBER); \ +} + +SHOW_CPUDATA_ULONG_NAME(clock_tick, clock_tick); +SHOW_CPUDATA_UINT_NAME(l1_dcache_size, dcache_size); +SHOW_CPUDATA_UINT_NAME(l1_dcache_line_size, dcache_line_size); +SHOW_CPUDATA_UINT_NAME(l1_icache_size, icache_size); +SHOW_CPUDATA_UINT_NAME(l1_icache_line_size, icache_line_size); +SHOW_CPUDATA_UINT_NAME(l2_cache_size, ecache_size); +SHOW_CPUDATA_UINT_NAME(l2_cache_line_size, ecache_line_size); + +static struct device_attribute cpu_core_attrs[] = { + __ATTR(clock_tick, 0444, show_clock_tick, NULL), + __ATTR(l1_dcache_size, 0444, show_l1_dcache_size, NULL), + __ATTR(l1_dcache_line_size, 0444, show_l1_dcache_line_size, NULL), + __ATTR(l1_icache_size, 0444, show_l1_icache_size, NULL), + __ATTR(l1_icache_line_size, 0444, show_l1_icache_line_size, NULL), + __ATTR(l2_cache_size, 0444, show_l2_cache_size, NULL), + __ATTR(l2_cache_line_size, 0444, show_l2_cache_line_size, NULL), +}; + +static DEFINE_PER_CPU(struct cpu, cpu_devices); + +static int register_cpu_online(unsigned int cpu) +{ + struct cpu *c = &per_cpu(cpu_devices, cpu); + struct device *s = &c->dev; + int i; + + for (i = 0; i < ARRAY_SIZE(cpu_core_attrs); i++) + device_create_file(s, &cpu_core_attrs[i]); + + register_mmu_stats(s); + return 0; +} + +static int unregister_cpu_online(unsigned int cpu) +{ +#ifdef CONFIG_HOTPLUG_CPU + struct cpu *c = &per_cpu(cpu_devices, cpu); + struct device *s = &c->dev; + int i; + + unregister_mmu_stats(s); + for (i = 0; i < ARRAY_SIZE(cpu_core_attrs); i++) + device_remove_file(s, &cpu_core_attrs[i]); +#endif + return 0; +} + +static void __init check_mmu_stats(void) +{ + unsigned long dummy1, err; + + if (tlb_type != hypervisor) + return; + + err = sun4v_mmustat_info(&dummy1); + if (!err) + mmu_stats_supported = 1; +} + +static int __init topology_init(void) +{ + int cpu, ret; + + check_mmu_stats(); + + for_each_possible_cpu(cpu) { + struct cpu *c = &per_cpu(cpu_devices, cpu); + + register_cpu(c, cpu); + } + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "sparc/topology:online", + register_cpu_online, unregister_cpu_online); + WARN_ON(ret < 0); + return 0; +} + +subsys_initcall(topology_init); diff --git a/arch/sparc/kernel/systbls.h b/arch/sparc/kernel/systbls.h new file mode 100644 index 000000000..bf014267d --- /dev/null +++ b/arch/sparc/kernel/systbls.h @@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SYSTBLS_H +#define _SYSTBLS_H + +#include +#include +#include +#include + +#include + +asmlinkage long sys_getpagesize(void); +asmlinkage long sys_sparc_pipe(void); +asmlinkage long sys_nis_syscall(void); +asmlinkage long sys_getdomainname(char __user *name, int len); +void do_rt_sigreturn(struct pt_regs *regs); +asmlinkage long sys_mmap(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long off); +asmlinkage void sparc_breakpoint(struct pt_regs *regs); + +#ifdef CONFIG_SPARC32 +asmlinkage long sys_mmap2(unsigned long addr, unsigned long len, + unsigned long prot, unsigned long flags, + unsigned long fd, unsigned long pgoff); +long sys_sparc_remap_file_pages(unsigned long start, unsigned long size, + unsigned long prot, unsigned long pgoff, + unsigned long flags); + +#endif /* CONFIG_SPARC32 */ + +#ifdef CONFIG_SPARC64 +asmlinkage long sys_sparc_ipc(unsigned int call, int first, + unsigned long second, + unsigned long third, + void __user *ptr, long fifth); +asmlinkage long sparc64_personality(unsigned long personality); +asmlinkage long sys64_munmap(unsigned long addr, size_t len); +asmlinkage unsigned long sys64_mremap(unsigned long addr, + unsigned long old_len, + unsigned long new_len, + unsigned long flags, + unsigned long new_addr); +asmlinkage long sys_utrap_install(utrap_entry_t type, + utrap_handler_t new_p, + utrap_handler_t new_d, + utrap_handler_t __user *old_p, + utrap_handler_t __user *old_d); +asmlinkage long sys_memory_ordering(unsigned long model); +asmlinkage void sparc64_set_context(struct pt_regs *regs); +asmlinkage void sparc64_get_context(struct pt_regs *regs); +asmlinkage long compat_sys_truncate64(const char __user * path, + u32 high, + u32 low); +asmlinkage long compat_sys_ftruncate64(unsigned int fd, + u32 high, + u32 low); +struct compat_stat64; +asmlinkage long compat_sys_stat64(const char __user * filename, + struct compat_stat64 __user *statbuf); +asmlinkage long compat_sys_lstat64(const char __user * filename, + struct compat_stat64 __user *statbuf); +asmlinkage long compat_sys_fstat64(unsigned int fd, + struct compat_stat64 __user * statbuf); +asmlinkage long compat_sys_fstatat64(unsigned int dfd, + const char __user *filename, + struct compat_stat64 __user * statbuf, int flag); +asmlinkage long compat_sys_pread64(unsigned int fd, + char __user *ubuf, + compat_size_t count, + u32 poshi, + u32 poslo); +asmlinkage long compat_sys_pwrite64(unsigned int fd, + char __user *ubuf, + compat_size_t count, + u32 poshi, + u32 poslo); +asmlinkage long compat_sys_readahead(int fd, + unsigned offhi, + unsigned offlo, + compat_size_t count); +long compat_sys_fadvise64(int fd, + unsigned offhi, + unsigned offlo, + compat_size_t len, int advice); +long compat_sys_fadvise64_64(int fd, + unsigned offhi, unsigned offlo, + unsigned lenhi, unsigned lenlo, + int advice); +long compat_sys_sync_file_range(unsigned int fd, + unsigned off_high, unsigned off_low, + unsigned nb_high, unsigned nb_low, + unsigned int flags); +asmlinkage long compat_sys_fallocate(int fd, int mode, u32 offhi, u32 offlo, + u32 lenhi, u32 lenlo); +asmlinkage long compat_sys_fstat64(unsigned int fd, + struct compat_stat64 __user * statbuf); +asmlinkage long compat_sys_fstatat64(unsigned int dfd, + const char __user *filename, + struct compat_stat64 __user * statbuf, + int flag); +#endif /* CONFIG_SPARC64 */ +#endif /* _SYSTBLS_H */ diff --git a/arch/sparc/kernel/systbls_32.S b/arch/sparc/kernel/systbls_32.S new file mode 100644 index 000000000..3aaffa017 --- /dev/null +++ b/arch/sparc/kernel/systbls_32.S @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* systbls.S: System call entry point tables for OS compatibility. + * The native Linux system call table lives here also. + * + * Copyright (C) 1995, 2007 David S. Miller (davem@davemloft.net) + * + * Based upon preliminary work which is: + * + * Copyright (C) 1995 Adrian M. Rodriguez (adrian@remus.rutgers.edu) + */ + +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) +#define __SYSCALL(nr, entry) .long entry + .data + .align 4 + .globl sys_call_table +sys_call_table: +#include /* 32-bit native syscalls */ diff --git a/arch/sparc/kernel/systbls_64.S b/arch/sparc/kernel/systbls_64.S new file mode 100644 index 000000000..398fe449d --- /dev/null +++ b/arch/sparc/kernel/systbls_64.S @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* systbls.S: System call entry point tables for OS compatibility. + * The native Linux system call table lives here also. + * + * Copyright (C) 1995, 1996, 2007 David S. Miller (davem@davemloft.net) + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * + * Based upon preliminary work which is: + * + * Copyright (C) 1995 Adrian M. Rodriguez (adrian@remus.rutgers.edu) + */ + +#define __SYSCALL(nr, entry) .word entry + .text + .align 4 +#ifdef CONFIG_COMPAT + .globl sys_call_table32 +sys_call_table32: +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) +#include /* Compat syscalls */ +#undef __SYSCALL_WITH_COMPAT +#endif /* CONFIG_COMPAT */ + + .align 4 + .globl sys_call_table64, sys_call_table +sys_call_table64: +sys_call_table: +#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) +#include /* 64-bit native syscalls */ diff --git a/arch/sparc/kernel/termios.c b/arch/sparc/kernel/termios.c new file mode 100644 index 000000000..ee64965c2 --- /dev/null +++ b/arch/sparc/kernel/termios.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +/* + * c_cc characters in the termio structure. Oh, how I love being + * backwardly compatible. Notice that character 4 and 5 are + * interpreted differently depending on whether ICANON is set in + * c_lflag. If it's set, they are used as _VEOF and _VEOL, otherwise + * as _VMIN and V_TIME. This is for compatibility with OSF/1 (which + * is compatible with sysV)... + */ +#define _VMIN 4 +#define _VTIME 5 + +int kernel_termios_to_user_termio(struct termio __user *termio, + struct ktermios *termios) +{ + struct termio v; + memset(&v, 0, sizeof(struct termio)); + v.c_iflag = termios->c_iflag; + v.c_oflag = termios->c_oflag; + v.c_cflag = termios->c_cflag; + v.c_lflag = termios->c_lflag; + v.c_line = termios->c_line; + memcpy(v.c_cc, termios->c_cc, NCC); + if (!(v.c_lflag & ICANON)) { + v.c_cc[_VMIN] = termios->c_cc[VMIN]; + v.c_cc[_VTIME] = termios->c_cc[VTIME]; + } + return copy_to_user(termio, &v, sizeof(struct termio)); +} + +int user_termios_to_kernel_termios(struct ktermios *k, + struct termios2 __user *u) +{ + int err; + err = get_user(k->c_iflag, &u->c_iflag); + err |= get_user(k->c_oflag, &u->c_oflag); + err |= get_user(k->c_cflag, &u->c_cflag); + err |= get_user(k->c_lflag, &u->c_lflag); + err |= get_user(k->c_line, &u->c_line); + err |= copy_from_user(k->c_cc, u->c_cc, NCCS); + if (k->c_lflag & ICANON) { + err |= get_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= get_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } else { + err |= get_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= get_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } + err |= get_user(k->c_ispeed, &u->c_ispeed); + err |= get_user(k->c_ospeed, &u->c_ospeed); + return err; +} + +int kernel_termios_to_user_termios(struct termios2 __user *u, + struct ktermios *k) +{ + int err; + err = put_user(k->c_iflag, &u->c_iflag); + err |= put_user(k->c_oflag, &u->c_oflag); + err |= put_user(k->c_cflag, &u->c_cflag); + err |= put_user(k->c_lflag, &u->c_lflag); + err |= put_user(k->c_line, &u->c_line); + err |= copy_to_user(u->c_cc, k->c_cc, NCCS); + if (!(k->c_lflag & ICANON)) { + err |= put_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= put_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } else { + err |= put_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= put_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } + err |= put_user(k->c_ispeed, &u->c_ispeed); + err |= put_user(k->c_ospeed, &u->c_ospeed); + return err; +} + +int user_termios_to_kernel_termios_1(struct ktermios *k, + struct termios __user *u) +{ + int err; + err = get_user(k->c_iflag, &u->c_iflag); + err |= get_user(k->c_oflag, &u->c_oflag); + err |= get_user(k->c_cflag, &u->c_cflag); + err |= get_user(k->c_lflag, &u->c_lflag); + err |= get_user(k->c_line, &u->c_line); + err |= copy_from_user(k->c_cc, u->c_cc, NCCS); + if (k->c_lflag & ICANON) { + err |= get_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= get_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } else { + err |= get_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= get_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } + return err; +} + +int kernel_termios_to_user_termios_1(struct termios __user *u, + struct ktermios *k) +{ + int err; + err = put_user(k->c_iflag, &u->c_iflag); + err |= put_user(k->c_oflag, &u->c_oflag); + err |= put_user(k->c_cflag, &u->c_cflag); + err |= put_user(k->c_lflag, &u->c_lflag); + err |= put_user(k->c_line, &u->c_line); + err |= copy_to_user(u->c_cc, k->c_cc, NCCS); + if (!(k->c_lflag & ICANON)) { + err |= put_user(k->c_cc[VMIN], &u->c_cc[_VMIN]); + err |= put_user(k->c_cc[VTIME], &u->c_cc[_VTIME]); + } else { + err |= put_user(k->c_cc[VEOF], &u->c_cc[VEOF]); + err |= put_user(k->c_cc[VEOL], &u->c_cc[VEOL]); + } + return err; +} diff --git a/arch/sparc/kernel/time_32.c b/arch/sparc/kernel/time_32.c new file mode 100644 index 000000000..578fd0d49 --- /dev/null +++ b/arch/sparc/kernel/time_32.c @@ -0,0 +1,356 @@ +// SPDX-License-Identifier: GPL-2.0 +/* linux/arch/sparc/kernel/time.c + * + * Copyright (C) 1995 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996 Thomas K. Dyas (tdyas@eden.rutgers.edu) + * + * Chris Davis (cdavis@cois.on.ca) 03/27/1998 + * Added support for the intersil on the sun4/4200 + * + * Gleb Raiko (rajko@mech.math.msu.su) 08/18/1998 + * Support for MicroSPARC-IIep, PCI CPU. + * + * This file handles the Sparc specific time handling details. + * + * 1997-09-10 Updated NTP code according to technical memorandum Jan '96 + * "A Kernel Model for Precision Timekeeping" by Dave Mills + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "kernel.h" +#include "irq.h" + +static __cacheline_aligned_in_smp DEFINE_SEQLOCK(timer_cs_lock); +static __volatile__ u64 timer_cs_internal_counter = 0; +static char timer_cs_enabled = 0; + +static struct clock_event_device timer_ce; +static char timer_ce_enabled = 0; + +#ifdef CONFIG_SMP +DEFINE_PER_CPU(struct clock_event_device, sparc32_clockevent); +#endif + +DEFINE_SPINLOCK(rtc_lock); +EXPORT_SYMBOL(rtc_lock); + +unsigned long profile_pc(struct pt_regs *regs) +{ + extern char __copy_user_begin[], __copy_user_end[]; + extern char __bzero_begin[], __bzero_end[]; + + unsigned long pc = regs->pc; + + if (in_lock_functions(pc) || + (pc >= (unsigned long) __copy_user_begin && + pc < (unsigned long) __copy_user_end) || + (pc >= (unsigned long) __bzero_begin && + pc < (unsigned long) __bzero_end)) + pc = regs->u_regs[UREG_RETPC]; + return pc; +} + +EXPORT_SYMBOL(profile_pc); + +volatile u32 __iomem *master_l10_counter; + +irqreturn_t notrace timer_interrupt(int dummy, void *dev_id) +{ + if (timer_cs_enabled) { + write_seqlock(&timer_cs_lock); + timer_cs_internal_counter++; + sparc_config.clear_clock_irq(); + write_sequnlock(&timer_cs_lock); + } else { + sparc_config.clear_clock_irq(); + } + + if (timer_ce_enabled) + timer_ce.event_handler(&timer_ce); + + return IRQ_HANDLED; +} + +static int timer_ce_shutdown(struct clock_event_device *evt) +{ + timer_ce_enabled = 0; + smp_mb(); + return 0; +} + +static int timer_ce_set_periodic(struct clock_event_device *evt) +{ + timer_ce_enabled = 1; + smp_mb(); + return 0; +} + +static __init void setup_timer_ce(void) +{ + struct clock_event_device *ce = &timer_ce; + + BUG_ON(smp_processor_id() != boot_cpu_id); + + ce->name = "timer_ce"; + ce->rating = 100; + ce->features = CLOCK_EVT_FEAT_PERIODIC; + ce->set_state_shutdown = timer_ce_shutdown; + ce->set_state_periodic = timer_ce_set_periodic; + ce->tick_resume = timer_ce_set_periodic; + ce->cpumask = cpu_possible_mask; + ce->shift = 32; + ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC, + ce->shift); + clockevents_register_device(ce); +} + +static unsigned int sbus_cycles_offset(void) +{ + u32 val, offset; + + val = sbus_readl(master_l10_counter); + offset = (val >> TIMER_VALUE_SHIFT) & TIMER_VALUE_MASK; + + /* Limit hit? */ + if (val & TIMER_LIMIT_BIT) + offset += sparc_config.cs_period; + + return offset; +} + +static u64 timer_cs_read(struct clocksource *cs) +{ + unsigned int seq, offset; + u64 cycles; + + do { + seq = read_seqbegin(&timer_cs_lock); + + cycles = timer_cs_internal_counter; + offset = sparc_config.get_cycles_offset(); + } while (read_seqretry(&timer_cs_lock, seq)); + + /* Count absolute cycles */ + cycles *= sparc_config.cs_period; + cycles += offset; + + return cycles; +} + +static struct clocksource timer_cs = { + .name = "timer_cs", + .rating = 100, + .read = timer_cs_read, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, +}; + +static __init int setup_timer_cs(void) +{ + timer_cs_enabled = 1; + return clocksource_register_hz(&timer_cs, sparc_config.clock_rate); +} + +#ifdef CONFIG_SMP +static int percpu_ce_shutdown(struct clock_event_device *evt) +{ + int cpu = cpumask_first(evt->cpumask); + + sparc_config.load_profile_irq(cpu, 0); + return 0; +} + +static int percpu_ce_set_periodic(struct clock_event_device *evt) +{ + int cpu = cpumask_first(evt->cpumask); + + sparc_config.load_profile_irq(cpu, SBUS_CLOCK_RATE / HZ); + return 0; +} + +static int percpu_ce_set_next_event(unsigned long delta, + struct clock_event_device *evt) +{ + int cpu = cpumask_first(evt->cpumask); + unsigned int next = (unsigned int)delta; + + sparc_config.load_profile_irq(cpu, next); + return 0; +} + +void register_percpu_ce(int cpu) +{ + struct clock_event_device *ce = &per_cpu(sparc32_clockevent, cpu); + unsigned int features = CLOCK_EVT_FEAT_PERIODIC; + + if (sparc_config.features & FEAT_L14_ONESHOT) + features |= CLOCK_EVT_FEAT_ONESHOT; + + ce->name = "percpu_ce"; + ce->rating = 200; + ce->features = features; + ce->set_state_shutdown = percpu_ce_shutdown; + ce->set_state_periodic = percpu_ce_set_periodic; + ce->set_state_oneshot = percpu_ce_shutdown; + ce->set_next_event = percpu_ce_set_next_event; + ce->cpumask = cpumask_of(cpu); + ce->shift = 32; + ce->mult = div_sc(sparc_config.clock_rate, NSEC_PER_SEC, + ce->shift); + ce->max_delta_ns = clockevent_delta2ns(sparc_config.clock_rate, ce); + ce->max_delta_ticks = (unsigned long)sparc_config.clock_rate; + ce->min_delta_ns = clockevent_delta2ns(100, ce); + ce->min_delta_ticks = 100; + + clockevents_register_device(ce); +} +#endif + +static unsigned char mostek_read_byte(struct device *dev, u32 ofs) +{ + struct platform_device *pdev = to_platform_device(dev); + struct m48t59_plat_data *pdata = pdev->dev.platform_data; + + return readb(pdata->ioaddr + ofs); +} + +static void mostek_write_byte(struct device *dev, u32 ofs, u8 val) +{ + struct platform_device *pdev = to_platform_device(dev); + struct m48t59_plat_data *pdata = pdev->dev.platform_data; + + writeb(val, pdata->ioaddr + ofs); +} + +static struct m48t59_plat_data m48t59_data = { + .read_byte = mostek_read_byte, + .write_byte = mostek_write_byte, + .yy_offset = 68, +}; + +/* resource is set at runtime */ +static struct platform_device m48t59_rtc = { + .name = "rtc-m48t59", + .id = 0, + .num_resources = 1, + .dev = { + .platform_data = &m48t59_data, + }, +}; + +static int clock_probe(struct platform_device *op) +{ + struct device_node *dp = op->dev.of_node; + const char *model = of_get_property(dp, "model", NULL); + + if (!model) + return -ENODEV; + + /* Only the primary RTC has an address property */ + if (!of_property_present(dp, "address")) + return -ENODEV; + + m48t59_rtc.resource = &op->resource[0]; + if (!strcmp(model, "mk48t02")) { + /* Map the clock register io area read-only */ + m48t59_data.ioaddr = of_ioremap(&op->resource[0], 0, + 2048, "rtc-m48t59"); + m48t59_data.type = M48T59RTC_TYPE_M48T02; + } else if (!strcmp(model, "mk48t08")) { + m48t59_data.ioaddr = of_ioremap(&op->resource[0], 0, + 8192, "rtc-m48t59"); + m48t59_data.type = M48T59RTC_TYPE_M48T08; + } else + return -ENODEV; + + if (platform_device_register(&m48t59_rtc) < 0) + printk(KERN_ERR "Registering RTC device failed\n"); + + return 0; +} + +static const struct of_device_id clock_match[] = { + { + .name = "eeprom", + }, + {}, +}; + +static struct platform_driver clock_driver = { + .probe = clock_probe, + .driver = { + .name = "rtc", + .of_match_table = clock_match, + }, +}; + + +/* Probe for the mostek real time clock chip. */ +static int __init clock_init(void) +{ + return platform_driver_register(&clock_driver); +} +/* Must be after subsys_initcall() so that busses are probed. Must + * be before device_initcall() because things like the RTC driver + * need to see the clock registers. + */ +fs_initcall(clock_init); + +static void __init sparc32_late_time_init(void) +{ + if (sparc_config.features & FEAT_L10_CLOCKEVENT) + setup_timer_ce(); + if (sparc_config.features & FEAT_L10_CLOCKSOURCE) + setup_timer_cs(); +#ifdef CONFIG_SMP + register_percpu_ce(smp_processor_id()); +#endif +} + +static void __init sbus_time_init(void) +{ + sparc_config.get_cycles_offset = sbus_cycles_offset; + sparc_config.init_timers(); +} + +void __init time_init(void) +{ + sparc_config.features = 0; + late_time_init = sparc32_late_time_init; + + if (pcic_present()) + pci_time_init(); + else + sbus_time_init(); +} + diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c new file mode 100644 index 000000000..b1c54ab4b --- /dev/null +++ b/arch/sparc/kernel/time_64.c @@ -0,0 +1,901 @@ +// SPDX-License-Identifier: GPL-2.0 +/* time.c: UltraSparc timer and TOD clock support. + * + * Copyright (C) 1997, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1998 Eddie C. Dost (ecd@skynet.be) + * + * Based largely on code which is: + * + * Copyright (C) 1996 Thomas K. Dyas (tdyas@eden.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "entry.h" +#include "kernel.h" + +DEFINE_SPINLOCK(rtc_lock); + +#ifdef CONFIG_SMP +unsigned long profile_pc(struct pt_regs *regs) +{ + unsigned long pc = instruction_pointer(regs); + + if (in_lock_functions(pc)) + return regs->u_regs[UREG_RETPC]; + return pc; +} +EXPORT_SYMBOL(profile_pc); +#endif + +static void tick_disable_protection(void) +{ + /* Set things up so user can access tick register for profiling + * purposes. Also workaround BB_ERRATA_1 by doing a dummy + * read back of %tick after writing it. + */ + __asm__ __volatile__( + " ba,pt %%xcc, 1f\n" + " nop\n" + " .align 64\n" + "1: rd %%tick, %%g2\n" + " add %%g2, 6, %%g2\n" + " andn %%g2, %0, %%g2\n" + " wrpr %%g2, 0, %%tick\n" + " rdpr %%tick, %%g0" + : /* no outputs */ + : "r" (TICK_PRIV_BIT) + : "g2"); +} + +static void tick_disable_irq(void) +{ + __asm__ __volatile__( + " ba,pt %%xcc, 1f\n" + " nop\n" + " .align 64\n" + "1: wr %0, 0x0, %%tick_cmpr\n" + " rd %%tick_cmpr, %%g0" + : /* no outputs */ + : "r" (TICKCMP_IRQ_BIT)); +} + +static void tick_init_tick(void) +{ + tick_disable_protection(); + tick_disable_irq(); +} + +static unsigned long long tick_get_tick(void) +{ + unsigned long ret; + + __asm__ __volatile__("rd %%tick, %0\n\t" + "mov %0, %0" + : "=r" (ret)); + + return ret & ~TICK_PRIV_BIT; +} + +static int tick_add_compare(unsigned long adj) +{ + unsigned long orig_tick, new_tick, new_compare; + + __asm__ __volatile__("rd %%tick, %0" + : "=r" (orig_tick)); + + orig_tick &= ~TICKCMP_IRQ_BIT; + + /* Workaround for Spitfire Errata (#54 I think??), I discovered + * this via Sun BugID 4008234, mentioned in Solaris-2.5.1 patch + * number 103640. + * + * On Blackbird writes to %tick_cmpr can fail, the + * workaround seems to be to execute the wr instruction + * at the start of an I-cache line, and perform a dummy + * read back from %tick_cmpr right after writing to it. -DaveM + */ + __asm__ __volatile__("ba,pt %%xcc, 1f\n\t" + " add %1, %2, %0\n\t" + ".align 64\n" + "1:\n\t" + "wr %0, 0, %%tick_cmpr\n\t" + "rd %%tick_cmpr, %%g0\n\t" + : "=r" (new_compare) + : "r" (orig_tick), "r" (adj)); + + __asm__ __volatile__("rd %%tick, %0" + : "=r" (new_tick)); + new_tick &= ~TICKCMP_IRQ_BIT; + + return ((long)(new_tick - (orig_tick+adj))) > 0L; +} + +static unsigned long tick_add_tick(unsigned long adj) +{ + unsigned long new_tick; + + /* Also need to handle Blackbird bug here too. */ + __asm__ __volatile__("rd %%tick, %0\n\t" + "add %0, %1, %0\n\t" + "wrpr %0, 0, %%tick\n\t" + : "=&r" (new_tick) + : "r" (adj)); + + return new_tick; +} + +/* Searches for cpu clock frequency with given cpuid in OpenBoot tree */ +static unsigned long cpuid_to_freq(phandle node, int cpuid) +{ + bool is_cpu_node = false; + unsigned long freq = 0; + char type[128]; + + if (!node) + return freq; + + if (prom_getproperty(node, "device_type", type, sizeof(type)) != -1) + is_cpu_node = (strcmp(type, "cpu") == 0); + + /* try upa-portid then cpuid to get cpuid, see prom_64.c */ + if (is_cpu_node && (prom_getint(node, "upa-portid") == cpuid || + prom_getint(node, "cpuid") == cpuid)) + freq = prom_getintdefault(node, "clock-frequency", 0); + if (!freq) + freq = cpuid_to_freq(prom_getchild(node), cpuid); + if (!freq) + freq = cpuid_to_freq(prom_getsibling(node), cpuid); + + return freq; +} + +static unsigned long tick_get_frequency(void) +{ + return cpuid_to_freq(prom_root_node, hard_smp_processor_id()); +} + +static struct sparc64_tick_ops tick_operations __cacheline_aligned = { + .name = "tick", + .init_tick = tick_init_tick, + .disable_irq = tick_disable_irq, + .get_tick = tick_get_tick, + .add_tick = tick_add_tick, + .add_compare = tick_add_compare, + .get_frequency = tick_get_frequency, + .softint_mask = 1UL << 0, +}; + +struct sparc64_tick_ops *tick_ops __read_mostly = &tick_operations; +EXPORT_SYMBOL(tick_ops); + +static void stick_disable_irq(void) +{ + __asm__ __volatile__( + "wr %0, 0x0, %%asr25" + : /* no outputs */ + : "r" (TICKCMP_IRQ_BIT)); +} + +static void stick_init_tick(void) +{ + /* Writes to the %tick and %stick register are not + * allowed on sun4v. The Hypervisor controls that + * bit, per-strand. + */ + if (tlb_type != hypervisor) { + tick_disable_protection(); + tick_disable_irq(); + + /* Let the user get at STICK too. */ + __asm__ __volatile__( + " rd %%asr24, %%g2\n" + " andn %%g2, %0, %%g2\n" + " wr %%g2, 0, %%asr24" + : /* no outputs */ + : "r" (TICK_PRIV_BIT) + : "g1", "g2"); + } + + stick_disable_irq(); +} + +static unsigned long long stick_get_tick(void) +{ + unsigned long ret; + + __asm__ __volatile__("rd %%asr24, %0" + : "=r" (ret)); + + return ret & ~TICK_PRIV_BIT; +} + +static unsigned long stick_add_tick(unsigned long adj) +{ + unsigned long new_tick; + + __asm__ __volatile__("rd %%asr24, %0\n\t" + "add %0, %1, %0\n\t" + "wr %0, 0, %%asr24\n\t" + : "=&r" (new_tick) + : "r" (adj)); + + return new_tick; +} + +static int stick_add_compare(unsigned long adj) +{ + unsigned long orig_tick, new_tick; + + __asm__ __volatile__("rd %%asr24, %0" + : "=r" (orig_tick)); + orig_tick &= ~TICKCMP_IRQ_BIT; + + __asm__ __volatile__("wr %0, 0, %%asr25" + : /* no outputs */ + : "r" (orig_tick + adj)); + + __asm__ __volatile__("rd %%asr24, %0" + : "=r" (new_tick)); + new_tick &= ~TICKCMP_IRQ_BIT; + + return ((long)(new_tick - (orig_tick+adj))) > 0L; +} + +static unsigned long stick_get_frequency(void) +{ + return prom_getintdefault(prom_root_node, "stick-frequency", 0); +} + +static struct sparc64_tick_ops stick_operations __read_mostly = { + .name = "stick", + .init_tick = stick_init_tick, + .disable_irq = stick_disable_irq, + .get_tick = stick_get_tick, + .add_tick = stick_add_tick, + .add_compare = stick_add_compare, + .get_frequency = stick_get_frequency, + .softint_mask = 1UL << 16, +}; + +/* On Hummingbird the STICK/STICK_CMPR register is implemented + * in I/O space. There are two 64-bit registers each, the + * first holds the low 32-bits of the value and the second holds + * the high 32-bits. + * + * Since STICK is constantly updating, we have to access it carefully. + * + * The sequence we use to read is: + * 1) read high + * 2) read low + * 3) read high again, if it rolled re-read both low and high again. + * + * Writing STICK safely is also tricky: + * 1) write low to zero + * 2) write high + * 3) write low + */ +static unsigned long __hbird_read_stick(void) +{ + unsigned long ret, tmp1, tmp2, tmp3; + unsigned long addr = HBIRD_STICK_ADDR+8; + + __asm__ __volatile__("ldxa [%1] %5, %2\n" + "1:\n\t" + "sub %1, 0x8, %1\n\t" + "ldxa [%1] %5, %3\n\t" + "add %1, 0x8, %1\n\t" + "ldxa [%1] %5, %4\n\t" + "cmp %4, %2\n\t" + "bne,a,pn %%xcc, 1b\n\t" + " mov %4, %2\n\t" + "sllx %4, 32, %4\n\t" + "or %3, %4, %0\n\t" + : "=&r" (ret), "=&r" (addr), + "=&r" (tmp1), "=&r" (tmp2), "=&r" (tmp3) + : "i" (ASI_PHYS_BYPASS_EC_E), "1" (addr)); + + return ret; +} + +static void __hbird_write_stick(unsigned long val) +{ + unsigned long low = (val & 0xffffffffUL); + unsigned long high = (val >> 32UL); + unsigned long addr = HBIRD_STICK_ADDR; + + __asm__ __volatile__("stxa %%g0, [%0] %4\n\t" + "add %0, 0x8, %0\n\t" + "stxa %3, [%0] %4\n\t" + "sub %0, 0x8, %0\n\t" + "stxa %2, [%0] %4" + : "=&r" (addr) + : "0" (addr), "r" (low), "r" (high), + "i" (ASI_PHYS_BYPASS_EC_E)); +} + +static void __hbird_write_compare(unsigned long val) +{ + unsigned long low = (val & 0xffffffffUL); + unsigned long high = (val >> 32UL); + unsigned long addr = HBIRD_STICKCMP_ADDR + 0x8UL; + + __asm__ __volatile__("stxa %3, [%0] %4\n\t" + "sub %0, 0x8, %0\n\t" + "stxa %2, [%0] %4" + : "=&r" (addr) + : "0" (addr), "r" (low), "r" (high), + "i" (ASI_PHYS_BYPASS_EC_E)); +} + +static void hbtick_disable_irq(void) +{ + __hbird_write_compare(TICKCMP_IRQ_BIT); +} + +static void hbtick_init_tick(void) +{ + tick_disable_protection(); + + /* XXX This seems to be necessary to 'jumpstart' Hummingbird + * XXX into actually sending STICK interrupts. I think because + * XXX of how we store %tick_cmpr in head.S this somehow resets the + * XXX {TICK + STICK} interrupt mux. -DaveM + */ + __hbird_write_stick(__hbird_read_stick()); + + hbtick_disable_irq(); +} + +static unsigned long long hbtick_get_tick(void) +{ + return __hbird_read_stick() & ~TICK_PRIV_BIT; +} + +static unsigned long hbtick_add_tick(unsigned long adj) +{ + unsigned long val; + + val = __hbird_read_stick() + adj; + __hbird_write_stick(val); + + return val; +} + +static int hbtick_add_compare(unsigned long adj) +{ + unsigned long val = __hbird_read_stick(); + unsigned long val2; + + val &= ~TICKCMP_IRQ_BIT; + val += adj; + __hbird_write_compare(val); + + val2 = __hbird_read_stick() & ~TICKCMP_IRQ_BIT; + + return ((long)(val2 - val)) > 0L; +} + +static unsigned long hbtick_get_frequency(void) +{ + return prom_getintdefault(prom_root_node, "stick-frequency", 0); +} + +static struct sparc64_tick_ops hbtick_operations __read_mostly = { + .name = "hbtick", + .init_tick = hbtick_init_tick, + .disable_irq = hbtick_disable_irq, + .get_tick = hbtick_get_tick, + .add_tick = hbtick_add_tick, + .add_compare = hbtick_add_compare, + .get_frequency = hbtick_get_frequency, + .softint_mask = 1UL << 0, +}; + +unsigned long cmos_regs; +EXPORT_SYMBOL(cmos_regs); + +static struct resource rtc_cmos_resource; + +static struct platform_device rtc_cmos_device = { + .name = "rtc_cmos", + .id = -1, + .resource = &rtc_cmos_resource, + .num_resources = 1, +}; + +static int rtc_probe(struct platform_device *op) +{ + struct resource *r; + + printk(KERN_INFO "%pOF: RTC regs at 0x%llx\n", + op->dev.of_node, op->resource[0].start); + + /* The CMOS RTC driver only accepts IORESOURCE_IO, so cons + * up a fake resource so that the probe works for all cases. + * When the RTC is behind an ISA bus it will have IORESOURCE_IO + * already, whereas when it's behind EBUS is will be IORESOURCE_MEM. + */ + + r = &rtc_cmos_resource; + r->flags = IORESOURCE_IO; + r->name = op->resource[0].name; + r->start = op->resource[0].start; + r->end = op->resource[0].end; + + cmos_regs = op->resource[0].start; + return platform_device_register(&rtc_cmos_device); +} + +static const struct of_device_id rtc_match[] = { + { + .name = "rtc", + .compatible = "m5819", + }, + { + .name = "rtc", + .compatible = "isa-m5819p", + }, + { + .name = "rtc", + .compatible = "isa-m5823p", + }, + { + .name = "rtc", + .compatible = "ds1287", + }, + {}, +}; + +static struct platform_driver rtc_driver = { + .probe = rtc_probe, + .driver = { + .name = "rtc", + .of_match_table = rtc_match, + }, +}; + +static struct platform_device rtc_bq4802_device = { + .name = "rtc-bq4802", + .id = -1, + .num_resources = 1, +}; + +static int bq4802_probe(struct platform_device *op) +{ + + printk(KERN_INFO "%pOF: BQ4802 regs at 0x%llx\n", + op->dev.of_node, op->resource[0].start); + + rtc_bq4802_device.resource = &op->resource[0]; + return platform_device_register(&rtc_bq4802_device); +} + +static const struct of_device_id bq4802_match[] = { + { + .name = "rtc", + .compatible = "bq4802", + }, + {}, +}; + +static struct platform_driver bq4802_driver = { + .probe = bq4802_probe, + .driver = { + .name = "bq4802", + .of_match_table = bq4802_match, + }, +}; + +static unsigned char mostek_read_byte(struct device *dev, u32 ofs) +{ + struct platform_device *pdev = to_platform_device(dev); + void __iomem *regs = (void __iomem *) pdev->resource[0].start; + + return readb(regs + ofs); +} + +static void mostek_write_byte(struct device *dev, u32 ofs, u8 val) +{ + struct platform_device *pdev = to_platform_device(dev); + void __iomem *regs = (void __iomem *) pdev->resource[0].start; + + writeb(val, regs + ofs); +} + +static struct m48t59_plat_data m48t59_data = { + .read_byte = mostek_read_byte, + .write_byte = mostek_write_byte, + .yy_offset = 68, +}; + +static struct platform_device m48t59_rtc = { + .name = "rtc-m48t59", + .id = 0, + .num_resources = 1, + .dev = { + .platform_data = &m48t59_data, + }, +}; + +static int mostek_probe(struct platform_device *op) +{ + struct device_node *dp = op->dev.of_node; + + /* On an Enterprise system there can be multiple mostek clocks. + * We should only match the one that is on the central FHC bus. + */ + if (of_node_name_eq(dp->parent, "fhc") && + !of_node_name_eq(dp->parent->parent, "central")) + return -ENODEV; + + printk(KERN_INFO "%pOF: Mostek regs at 0x%llx\n", + dp, op->resource[0].start); + + m48t59_rtc.resource = &op->resource[0]; + return platform_device_register(&m48t59_rtc); +} + +static const struct of_device_id mostek_match[] = { + { + .name = "eeprom", + }, + {}, +}; + +static struct platform_driver mostek_driver = { + .probe = mostek_probe, + .driver = { + .name = "mostek", + .of_match_table = mostek_match, + }, +}; + +static struct platform_device rtc_sun4v_device = { + .name = "rtc-sun4v", + .id = -1, +}; + +static struct platform_device rtc_starfire_device = { + .name = "rtc-starfire", + .id = -1, +}; + +static int __init clock_init(void) +{ + if (this_is_starfire) + return platform_device_register(&rtc_starfire_device); + + if (tlb_type == hypervisor) + return platform_device_register(&rtc_sun4v_device); + + (void) platform_driver_register(&rtc_driver); + (void) platform_driver_register(&mostek_driver); + (void) platform_driver_register(&bq4802_driver); + + return 0; +} + +/* Must be after subsys_initcall() so that busses are probed. Must + * be before device_initcall() because things like the RTC driver + * need to see the clock registers. + */ +fs_initcall(clock_init); + +/* Return true if this is Hummingbird, aka Ultra-IIe */ +static bool is_hummingbird(void) +{ + unsigned long ver, manuf, impl; + + __asm__ __volatile__ ("rdpr %%ver, %0" + : "=&r" (ver)); + manuf = ((ver >> 48) & 0xffff); + impl = ((ver >> 32) & 0xffff); + + return (manuf == 0x17 && impl == 0x13); +} + +struct freq_table { + unsigned long clock_tick_ref; + unsigned int ref_freq; +}; +static DEFINE_PER_CPU(struct freq_table, sparc64_freq_table) = { 0, 0 }; + +unsigned long sparc64_get_clock_tick(unsigned int cpu) +{ + struct freq_table *ft = &per_cpu(sparc64_freq_table, cpu); + + if (ft->clock_tick_ref) + return ft->clock_tick_ref; + return cpu_data(cpu).clock_tick; +} +EXPORT_SYMBOL(sparc64_get_clock_tick); + +#ifdef CONFIG_CPU_FREQ + +static int sparc64_cpufreq_notifier(struct notifier_block *nb, unsigned long val, + void *data) +{ + struct cpufreq_freqs *freq = data; + unsigned int cpu; + struct freq_table *ft; + + for_each_cpu(cpu, freq->policy->cpus) { + ft = &per_cpu(sparc64_freq_table, cpu); + + if (!ft->ref_freq) { + ft->ref_freq = freq->old; + ft->clock_tick_ref = cpu_data(cpu).clock_tick; + } + + if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || + (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) { + cpu_data(cpu).clock_tick = + cpufreq_scale(ft->clock_tick_ref, ft->ref_freq, + freq->new); + } + } + + return 0; +} + +static struct notifier_block sparc64_cpufreq_notifier_block = { + .notifier_call = sparc64_cpufreq_notifier +}; + +static int __init register_sparc64_cpufreq_notifier(void) +{ + + cpufreq_register_notifier(&sparc64_cpufreq_notifier_block, + CPUFREQ_TRANSITION_NOTIFIER); + return 0; +} + +core_initcall(register_sparc64_cpufreq_notifier); + +#endif /* CONFIG_CPU_FREQ */ + +static int sparc64_next_event(unsigned long delta, + struct clock_event_device *evt) +{ + return tick_operations.add_compare(delta) ? -ETIME : 0; +} + +static int sparc64_timer_shutdown(struct clock_event_device *evt) +{ + tick_operations.disable_irq(); + return 0; +} + +static struct clock_event_device sparc64_clockevent = { + .features = CLOCK_EVT_FEAT_ONESHOT, + .set_state_shutdown = sparc64_timer_shutdown, + .set_next_event = sparc64_next_event, + .rating = 100, + .shift = 30, + .irq = -1, +}; +static DEFINE_PER_CPU(struct clock_event_device, sparc64_events); + +void __irq_entry timer_interrupt(int irq, struct pt_regs *regs) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + unsigned long tick_mask = tick_operations.softint_mask; + int cpu = smp_processor_id(); + struct clock_event_device *evt = &per_cpu(sparc64_events, cpu); + + clear_softint(tick_mask); + + irq_enter(); + + local_cpu_data().irq0_irqs++; + kstat_incr_irq_this_cpu(0); + + if (unlikely(!evt->event_handler)) { + printk(KERN_WARNING + "Spurious SPARC64 timer interrupt on cpu %d\n", cpu); + } else + evt->event_handler(evt); + + irq_exit(); + + set_irq_regs(old_regs); +} + +void setup_sparc64_timer(void) +{ + struct clock_event_device *sevt; + unsigned long pstate; + + /* Guarantee that the following sequences execute + * uninterrupted. + */ + __asm__ __volatile__("rdpr %%pstate, %0\n\t" + "wrpr %0, %1, %%pstate" + : "=r" (pstate) + : "i" (PSTATE_IE)); + + tick_operations.init_tick(); + + /* Restore PSTATE_IE. */ + __asm__ __volatile__("wrpr %0, 0x0, %%pstate" + : /* no outputs */ + : "r" (pstate)); + + sevt = this_cpu_ptr(&sparc64_events); + + memcpy(sevt, &sparc64_clockevent, sizeof(*sevt)); + sevt->cpumask = cpumask_of(smp_processor_id()); + + clockevents_register_device(sevt); +} + +#define SPARC64_NSEC_PER_CYC_SHIFT 10UL + +static struct clocksource clocksource_tick = { + .rating = 100, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, +}; + +static unsigned long tb_ticks_per_usec __read_mostly; + +void __delay(unsigned long loops) +{ + unsigned long bclock = get_tick(); + + while ((get_tick() - bclock) < loops) + ; +} +EXPORT_SYMBOL(__delay); + +void udelay(unsigned long usecs) +{ + __delay(tb_ticks_per_usec * usecs); +} +EXPORT_SYMBOL(udelay); + +static u64 clocksource_tick_read(struct clocksource *cs) +{ + return get_tick(); +} + +static void __init get_tick_patch(void) +{ + unsigned int *addr, *instr, i; + struct get_tick_patch *p; + + if (tlb_type == spitfire && is_hummingbird()) + return; + + for (p = &__get_tick_patch; p < &__get_tick_patch_end; p++) { + instr = (tlb_type == spitfire) ? p->tick : p->stick; + addr = (unsigned int *)(unsigned long)p->addr; + for (i = 0; i < GET_TICK_NINSTR; i++) { + addr[i] = instr[i]; + /* ensure that address is modified before flush */ + wmb(); + flushi(&addr[i]); + } + } +} + +static void __init init_tick_ops(struct sparc64_tick_ops *ops) +{ + unsigned long freq, quotient, tick; + + freq = ops->get_frequency(); + quotient = clocksource_hz2mult(freq, SPARC64_NSEC_PER_CYC_SHIFT); + tick = ops->get_tick(); + + ops->offset = (tick * quotient) >> SPARC64_NSEC_PER_CYC_SHIFT; + ops->ticks_per_nsec_quotient = quotient; + ops->frequency = freq; + tick_operations = *ops; + get_tick_patch(); +} + +void __init time_init_early(void) +{ + if (tlb_type == spitfire) { + if (is_hummingbird()) { + init_tick_ops(&hbtick_operations); + clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_NONE; + } else { + init_tick_ops(&tick_operations); + clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_TICK; + } + } else { + init_tick_ops(&stick_operations); + clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_STICK; + } +} + +void __init time_init(void) +{ + unsigned long freq; + + freq = tick_operations.frequency; + tb_ticks_per_usec = freq / USEC_PER_SEC; + + clocksource_tick.name = tick_operations.name; + clocksource_tick.read = clocksource_tick_read; + + clocksource_register_hz(&clocksource_tick, freq); + printk("clocksource: mult[%x] shift[%d]\n", + clocksource_tick.mult, clocksource_tick.shift); + + sparc64_clockevent.name = tick_operations.name; + clockevents_calc_mult_shift(&sparc64_clockevent, freq, 4); + + sparc64_clockevent.max_delta_ns = + clockevent_delta2ns(0x7fffffffffffffffUL, &sparc64_clockevent); + sparc64_clockevent.max_delta_ticks = 0x7fffffffffffffffUL; + sparc64_clockevent.min_delta_ns = + clockevent_delta2ns(0xF, &sparc64_clockevent); + sparc64_clockevent.min_delta_ticks = 0xF; + + printk("clockevent: mult[%x] shift[%d]\n", + sparc64_clockevent.mult, sparc64_clockevent.shift); + + setup_sparc64_timer(); +} + +unsigned long long sched_clock(void) +{ + unsigned long quotient = tick_operations.ticks_per_nsec_quotient; + unsigned long offset = tick_operations.offset; + + /* Use barrier so the compiler emits the loads first and overlaps load + * latency with reading tick, because reading %tick/%stick is a + * post-sync instruction that will flush and restart subsequent + * instructions after it commits. + */ + barrier(); + + return ((get_tick() * quotient) >> SPARC64_NSEC_PER_CYC_SHIFT) - offset; +} + +bool delay_read_timer(unsigned long *timer_val) +{ + *timer_val = get_tick(); + return true; +} diff --git a/arch/sparc/kernel/trampoline_32.S b/arch/sparc/kernel/trampoline_32.S new file mode 100644 index 000000000..82fafeeb3 --- /dev/null +++ b/arch/sparc/kernel/trampoline_32.S @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * trampoline.S: SMP cpu boot-up trampoline code. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include + + .globl sun4m_cpu_startup + .globl sun4d_cpu_startup + + .align 4 + +/* When we start up a cpu for the first time it enters this routine. + * This initializes the chip from whatever state the prom left it + * in and sets PIL in %psr to 15, no irqs. + */ + +sun4m_cpu_startup: +cpu1_startup: + sethi %hi(trapbase_cpu1), %g3 + b 1f + or %g3, %lo(trapbase_cpu1), %g3 + +cpu2_startup: + sethi %hi(trapbase_cpu2), %g3 + b 1f + or %g3, %lo(trapbase_cpu2), %g3 + +cpu3_startup: + sethi %hi(trapbase_cpu3), %g3 + b 1f + or %g3, %lo(trapbase_cpu3), %g3 + +1: + /* Set up a sane %psr -- PIL<0xf> S<0x1> PS<0x1> CWP<0x0> */ + set (PSR_PIL | PSR_S | PSR_PS), %g1 + wr %g1, 0x0, %psr ! traps off though + WRITE_PAUSE + + /* Our %wim is one behind CWP */ + mov 2, %g1 + wr %g1, 0x0, %wim + WRITE_PAUSE + + /* This identifies "this cpu". */ + wr %g3, 0x0, %tbr + WRITE_PAUSE + + /* Give ourselves a stack and curptr. */ + set current_set, %g5 + srl %g3, 10, %g4 + and %g4, 0xc, %g4 + ld [%g5 + %g4], %g6 + + sethi %hi(THREAD_SIZE - STACKFRAME_SZ), %sp + or %sp, %lo(THREAD_SIZE - STACKFRAME_SZ), %sp + add %g6, %sp, %sp + + /* Turn on traps (PSR_ET). */ + rd %psr, %g1 + wr %g1, PSR_ET, %psr ! traps on + WRITE_PAUSE + + /* Init our caches, etc. */ + set poke_srmmu, %g5 + ld [%g5], %g5 + call %g5 + nop + + /* Start this processor. */ + call smp_callin + nop + + b,a smp_panic + + .text + .align 4 + +smp_panic: + call cpu_panic + nop + +/* CPUID in bootbus can be found at PA 0xff0140000 */ +#define SUN4D_BOOTBUS_CPUID 0xf0140000 + + .align 4 + +sun4d_cpu_startup: + /* Set up a sane %psr -- PIL<0xf> S<0x1> PS<0x1> CWP<0x0> */ + set (PSR_PIL | PSR_S | PSR_PS), %g1 + wr %g1, 0x0, %psr ! traps off though + WRITE_PAUSE + + /* Our %wim is one behind CWP */ + mov 2, %g1 + wr %g1, 0x0, %wim + WRITE_PAUSE + + /* Set tbr - we use just one trap table. */ + set trapbase, %g1 + wr %g1, 0x0, %tbr + WRITE_PAUSE + + /* Get our CPU id out of bootbus */ + set SUN4D_BOOTBUS_CPUID, %g3 + lduba [%g3] ASI_M_CTL, %g3 + and %g3, 0xf8, %g3 + srl %g3, 3, %g1 + sta %g1, [%g0] ASI_M_VIKING_TMP1 + + /* Give ourselves a stack and curptr. */ + set current_set, %g5 + srl %g3, 1, %g4 + ld [%g5 + %g4], %g6 + + sethi %hi(THREAD_SIZE - STACKFRAME_SZ), %sp + or %sp, %lo(THREAD_SIZE - STACKFRAME_SZ), %sp + add %g6, %sp, %sp + + /* Turn on traps (PSR_ET). */ + rd %psr, %g1 + wr %g1, PSR_ET, %psr ! traps on + WRITE_PAUSE + + /* Init our caches, etc. */ + set poke_srmmu, %g5 + ld [%g5], %g5 + call %g5 + nop + + /* Start this processor. */ + call smp_callin + nop + + b,a smp_panic + + .align 4 + .global leon_smp_cpu_startup, smp_penguin_ctable + +leon_smp_cpu_startup: + + set smp_penguin_ctable,%g1 + ld [%g1+4],%g1 + srl %g1,4,%g1 + set 0x00000100,%g5 /* SRMMU_CTXTBL_PTR */ + sta %g1, [%g5] ASI_LEON_MMUREGS + + /* Set up a sane %psr -- PIL<0xf> S<0x1> PS<0x1> CWP<0x0> */ + set (PSR_PIL | PSR_S | PSR_PS), %g1 + wr %g1, 0x0, %psr ! traps off though + WRITE_PAUSE + + /* Our %wim is one behind CWP */ + mov 2, %g1 + wr %g1, 0x0, %wim + WRITE_PAUSE + + /* Set tbr - we use just one trap table. */ + set trapbase, %g1 + wr %g1, 0x0, %tbr + WRITE_PAUSE + + /* Get our CPU id */ + rd %asr17,%g3 + + /* Give ourselves a stack and curptr. */ + set current_set, %g5 + srl %g3, 28, %g4 + sll %g4, 2, %g4 + ld [%g5 + %g4], %g6 + + sethi %hi(THREAD_SIZE - STACKFRAME_SZ), %sp + or %sp, %lo(THREAD_SIZE - STACKFRAME_SZ), %sp + add %g6, %sp, %sp + + /* Turn on traps (PSR_ET). */ + rd %psr, %g1 + wr %g1, PSR_ET, %psr ! traps on + WRITE_PAUSE + + /* Init our caches, etc. */ + set poke_srmmu, %g5 + ld [%g5], %g5 + call %g5 + nop + + /* Start this processor. */ + call smp_callin + nop + + b,a smp_panic diff --git a/arch/sparc/kernel/trampoline_64.S b/arch/sparc/kernel/trampoline_64.S new file mode 100644 index 000000000..51bf1eb92 --- /dev/null +++ b/arch/sparc/kernel/trampoline_64.S @@ -0,0 +1,414 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * trampoline.S: Jump start slave processors on sparc64. + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + .data + .align 8 +call_method: + .asciz "call-method" + .align 8 +itlb_load: + .asciz "SUNW,itlb-load" + .align 8 +dtlb_load: + .asciz "SUNW,dtlb-load" + +#define TRAMP_STACK_SIZE 1024 + .align 16 +tramp_stack: + .skip TRAMP_STACK_SIZE + + .align 8 + .globl sparc64_cpu_startup, sparc64_cpu_startup_end +sparc64_cpu_startup: + BRANCH_IF_SUN4V(g1, niagara_startup) + BRANCH_IF_CHEETAH_BASE(g1, g5, cheetah_startup) + BRANCH_IF_CHEETAH_PLUS_OR_FOLLOWON(g1, g5, cheetah_plus_startup) + + ba,pt %xcc, spitfire_startup + nop + +cheetah_plus_startup: + /* Preserve OBP chosen DCU and DCR register settings. */ + ba,pt %xcc, cheetah_generic_startup + nop + +cheetah_startup: + mov DCR_BPE | DCR_RPE | DCR_SI | DCR_IFPOE | DCR_MS, %g1 + wr %g1, %asr18 + + sethi %uhi(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g5 + or %g5, %ulo(DCU_ME|DCU_RE|DCU_HPE|DCU_SPE|DCU_SL|DCU_WE), %g5 + sllx %g5, 32, %g5 + or %g5, DCU_DM | DCU_IM | DCU_DC | DCU_IC, %g5 + stxa %g5, [%g0] ASI_DCU_CONTROL_REG + membar #Sync + /* fallthru */ + +cheetah_generic_startup: + mov TSB_EXTENSION_P, %g3 + stxa %g0, [%g3] ASI_DMMU + stxa %g0, [%g3] ASI_IMMU + membar #Sync + + mov TSB_EXTENSION_S, %g3 + stxa %g0, [%g3] ASI_DMMU + membar #Sync + + mov TSB_EXTENSION_N, %g3 + stxa %g0, [%g3] ASI_DMMU + stxa %g0, [%g3] ASI_IMMU + membar #Sync + /* fallthru */ + +niagara_startup: + /* Disable STICK_INT interrupts. */ + sethi %hi(0x80000000), %g5 + sllx %g5, 32, %g5 + wr %g5, %asr25 + + ba,pt %xcc, startup_continue + nop + +spitfire_startup: + mov (LSU_CONTROL_IC | LSU_CONTROL_DC | LSU_CONTROL_IM | LSU_CONTROL_DM), %g1 + stxa %g1, [%g0] ASI_LSU_CONTROL + membar #Sync + +startup_continue: + mov %o0, %l0 + BRANCH_IF_SUN4V(g1, niagara_lock_tlb) + + sethi %hi(0x80000000), %g2 + sllx %g2, 32, %g2 + wr %g2, 0, %tick_cmpr + + /* Call OBP by hand to lock KERNBASE into i/d tlbs. + * We lock 'num_kernel_image_mappings' consequetive entries. + */ + sethi %hi(prom_entry_lock), %g2 +1: ldstub [%g2 + %lo(prom_entry_lock)], %g1 + brnz,pn %g1, 1b + nop + + /* Get onto temporary stack which will be in the locked + * kernel image. + */ + sethi %hi(tramp_stack), %g1 + or %g1, %lo(tramp_stack), %g1 + add %g1, TRAMP_STACK_SIZE, %g1 + sub %g1, STACKFRAME_SZ + STACK_BIAS + 256, %sp + flushw + + /* Setup the loop variables: + * %l3: VADDR base + * %l4: TTE base + * %l5: Loop iterator, iterates from 0 to 'num_kernel_image_mappings' + * %l6: Number of TTE entries to map + * %l7: Highest TTE entry number, we count down + */ + sethi %hi(KERNBASE), %l3 + sethi %hi(kern_locked_tte_data), %l4 + ldx [%l4 + %lo(kern_locked_tte_data)], %l4 + clr %l5 + sethi %hi(num_kernel_image_mappings), %l6 + lduw [%l6 + %lo(num_kernel_image_mappings)], %l6 + + mov 15, %l7 + BRANCH_IF_ANY_CHEETAH(g1,g5,2f) + + mov 63, %l7 +2: + +3: + /* Lock into I-MMU */ + sethi %hi(call_method), %g2 + or %g2, %lo(call_method), %g2 + stx %g2, [%sp + 2047 + 128 + 0x00] + mov 5, %g2 + stx %g2, [%sp + 2047 + 128 + 0x08] + mov 1, %g2 + stx %g2, [%sp + 2047 + 128 + 0x10] + sethi %hi(itlb_load), %g2 + or %g2, %lo(itlb_load), %g2 + stx %g2, [%sp + 2047 + 128 + 0x18] + sethi %hi(prom_mmu_ihandle_cache), %g2 + lduw [%g2 + %lo(prom_mmu_ihandle_cache)], %g2 + stx %g2, [%sp + 2047 + 128 + 0x20] + + /* Each TTE maps 4MB, convert index to offset. */ + sllx %l5, 22, %g1 + + add %l3, %g1, %g2 + stx %g2, [%sp + 2047 + 128 + 0x28] ! VADDR + add %l4, %g1, %g2 + stx %g2, [%sp + 2047 + 128 + 0x30] ! TTE + + /* TTE index is highest minus loop index. */ + sub %l7, %l5, %g2 + stx %g2, [%sp + 2047 + 128 + 0x38] + + sethi %hi(p1275buf), %g2 + or %g2, %lo(p1275buf), %g2 + ldx [%g2 + 0x08], %o1 + call %o1 + add %sp, (2047 + 128), %o0 + + /* Lock into D-MMU */ + sethi %hi(call_method), %g2 + or %g2, %lo(call_method), %g2 + stx %g2, [%sp + 2047 + 128 + 0x00] + mov 5, %g2 + stx %g2, [%sp + 2047 + 128 + 0x08] + mov 1, %g2 + stx %g2, [%sp + 2047 + 128 + 0x10] + sethi %hi(dtlb_load), %g2 + or %g2, %lo(dtlb_load), %g2 + stx %g2, [%sp + 2047 + 128 + 0x18] + sethi %hi(prom_mmu_ihandle_cache), %g2 + lduw [%g2 + %lo(prom_mmu_ihandle_cache)], %g2 + stx %g2, [%sp + 2047 + 128 + 0x20] + + /* Each TTE maps 4MB, convert index to offset. */ + sllx %l5, 22, %g1 + + add %l3, %g1, %g2 + stx %g2, [%sp + 2047 + 128 + 0x28] ! VADDR + add %l4, %g1, %g2 + stx %g2, [%sp + 2047 + 128 + 0x30] ! TTE + + /* TTE index is highest minus loop index. */ + sub %l7, %l5, %g2 + stx %g2, [%sp + 2047 + 128 + 0x38] + + sethi %hi(p1275buf), %g2 + or %g2, %lo(p1275buf), %g2 + ldx [%g2 + 0x08], %o1 + call %o1 + add %sp, (2047 + 128), %o0 + + add %l5, 1, %l5 + cmp %l5, %l6 + bne,pt %xcc, 3b + nop + + sethi %hi(prom_entry_lock), %g2 + stb %g0, [%g2 + %lo(prom_entry_lock)] + + ba,pt %xcc, after_lock_tlb + nop + +niagara_lock_tlb: + sethi %hi(KERNBASE), %l3 + sethi %hi(kern_locked_tte_data), %l4 + ldx [%l4 + %lo(kern_locked_tte_data)], %l4 + clr %l5 + sethi %hi(num_kernel_image_mappings), %l6 + lduw [%l6 + %lo(num_kernel_image_mappings)], %l6 + +1: + mov HV_FAST_MMU_MAP_PERM_ADDR, %o5 + sllx %l5, 22, %g2 + add %l3, %g2, %o0 + clr %o1 + add %l4, %g2, %o2 + mov HV_MMU_IMMU, %o3 + ta HV_FAST_TRAP + + mov HV_FAST_MMU_MAP_PERM_ADDR, %o5 + sllx %l5, 22, %g2 + add %l3, %g2, %o0 + clr %o1 + add %l4, %g2, %o2 + mov HV_MMU_DMMU, %o3 + ta HV_FAST_TRAP + + add %l5, 1, %l5 + cmp %l5, %l6 + bne,pt %xcc, 1b + nop + +after_lock_tlb: + wrpr %g0, (PSTATE_PRIV | PSTATE_PEF), %pstate + wr %g0, 0, %fprs + + wr %g0, ASI_P, %asi + + mov PRIMARY_CONTEXT, %g7 + +661: stxa %g0, [%g7] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g0, [%g7] ASI_MMU + .previous + + membar #Sync + mov SECONDARY_CONTEXT, %g7 + +661: stxa %g0, [%g7] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g0, [%g7] ASI_MMU + .previous + + membar #Sync + + /* Everything we do here, until we properly take over the + * trap table, must be done with extreme care. We cannot + * make any references to %g6 (current thread pointer), + * %g4 (current task pointer), or %g5 (base of current cpu's + * per-cpu area) until we properly take over the trap table + * from the firmware and hypervisor. + * + * Get onto temporary stack which is in the locked kernel image. + */ + sethi %hi(tramp_stack), %g1 + or %g1, %lo(tramp_stack), %g1 + add %g1, TRAMP_STACK_SIZE, %g1 + sub %g1, STACKFRAME_SZ + STACK_BIAS + 256, %sp + mov 0, %fp + + /* Put garbage in these registers to trap any access to them. */ + set 0xdeadbeef, %g4 + set 0xdeadbeef, %g5 + set 0xdeadbeef, %g6 + + call init_irqwork_curcpu + nop + + sethi %hi(tlb_type), %g3 + lduw [%g3 + %lo(tlb_type)], %g2 + cmp %g2, 3 + bne,pt %icc, 1f + nop + + call hard_smp_processor_id + nop + + call sun4v_register_mondo_queues + nop + +1: call init_cur_cpu_trap + ldx [%l0], %o0 + + /* Start using proper page size encodings in ctx register. */ + sethi %hi(sparc64_kern_pri_context), %g3 + ldx [%g3 + %lo(sparc64_kern_pri_context)], %g2 + mov PRIMARY_CONTEXT, %g1 + +661: stxa %g2, [%g1] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g1] ASI_MMU + .previous + + membar #Sync + + wrpr %g0, 0, %wstate + + sethi %hi(prom_entry_lock), %g2 +1: ldstub [%g2 + %lo(prom_entry_lock)], %g1 + brnz,pn %g1, 1b + nop + + /* As a hack, put &init_thread_union into %g6. + * prom_world() loads from here to restore the %asi + * register. + */ + sethi %hi(init_thread_union), %g6 + or %g6, %lo(init_thread_union), %g6 + + sethi %hi(is_sun4v), %o0 + lduw [%o0 + %lo(is_sun4v)], %o0 + brz,pt %o0, 2f + nop + + TRAP_LOAD_TRAP_BLOCK(%g2, %g3) + add %g2, TRAP_PER_CPU_FAULT_INFO, %g2 + stxa %g2, [%g0] ASI_SCRATCHPAD + + /* Compute physical address: + * + * paddr = kern_base + (mmfsa_vaddr - KERNBASE) + */ + sethi %hi(KERNBASE), %g3 + sub %g2, %g3, %g2 + sethi %hi(kern_base), %g3 + ldx [%g3 + %lo(kern_base)], %g3 + add %g2, %g3, %o1 + sethi %hi(sparc64_ttable_tl0), %o0 + + set prom_set_trap_table_name, %g2 + stx %g2, [%sp + 2047 + 128 + 0x00] + mov 2, %g2 + stx %g2, [%sp + 2047 + 128 + 0x08] + mov 0, %g2 + stx %g2, [%sp + 2047 + 128 + 0x10] + stx %o0, [%sp + 2047 + 128 + 0x18] + stx %o1, [%sp + 2047 + 128 + 0x20] + sethi %hi(p1275buf), %g2 + or %g2, %lo(p1275buf), %g2 + ldx [%g2 + 0x08], %o1 + call %o1 + add %sp, (2047 + 128), %o0 + + ba,pt %xcc, 3f + nop + +2: sethi %hi(sparc64_ttable_tl0), %o0 + set prom_set_trap_table_name, %g2 + stx %g2, [%sp + 2047 + 128 + 0x00] + mov 1, %g2 + stx %g2, [%sp + 2047 + 128 + 0x08] + mov 0, %g2 + stx %g2, [%sp + 2047 + 128 + 0x10] + stx %o0, [%sp + 2047 + 128 + 0x18] + sethi %hi(p1275buf), %g2 + or %g2, %lo(p1275buf), %g2 + ldx [%g2 + 0x08], %o1 + call %o1 + add %sp, (2047 + 128), %o0 + +3: sethi %hi(prom_entry_lock), %g2 + stb %g0, [%g2 + %lo(prom_entry_lock)] + + ldx [%l0], %g6 + ldx [%g6 + TI_TASK], %g4 + + mov 1, %g5 + sllx %g5, THREAD_SHIFT, %g5 + sub %g5, (STACKFRAME_SZ + STACK_BIAS), %g5 + add %g6, %g5, %sp + + rdpr %pstate, %o1 + or %o1, PSTATE_IE, %o1 + wrpr %o1, 0, %pstate + + call smp_callin + nop + + call cpu_panic + nop +1: b,a,pt %xcc, 1b + + .align 8 +sparc64_cpu_startup_end: diff --git a/arch/sparc/kernel/traps_32.c b/arch/sparc/kernel/traps_32.c new file mode 100644 index 000000000..bb149f6cc --- /dev/null +++ b/arch/sparc/kernel/traps_32.c @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/sparc/kernel/traps.c + * + * Copyright 1995, 2008 David S. Miller (davem@davemloft.net) + * Copyright 2000 Jakub Jelinek (jakub@redhat.com) + */ + +/* + * I hate traps on the sparc, grrr... + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "entry.h" +#include "kernel.h" + +/* #define TRAP_DEBUG */ + +static void instruction_dump(unsigned long *pc) +{ + int i; + + if((((unsigned long) pc) & 3)) + return; + + for(i = -3; i < 6; i++) + printk("%c%08lx%c",i?' ':'<',pc[i],i?' ':'>'); + printk("\n"); +} + +#define __SAVE __asm__ __volatile__("save %sp, -0x40, %sp\n\t") +#define __RESTORE __asm__ __volatile__("restore %g0, %g0, %g0\n\t") + +void __noreturn die_if_kernel(char *str, struct pt_regs *regs) +{ + static int die_counter; + int count = 0; + + /* Amuse the user. */ + printk( +" \\|/ ____ \\|/\n" +" \"@'/ ,. \\`@\"\n" +" /_| \\__/ |_\\\n" +" \\__U_/\n"); + + printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter); + show_regs(regs); + add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); + + __SAVE; __SAVE; __SAVE; __SAVE; + __SAVE; __SAVE; __SAVE; __SAVE; + __RESTORE; __RESTORE; __RESTORE; __RESTORE; + __RESTORE; __RESTORE; __RESTORE; __RESTORE; + + { + struct reg_window32 *rw = (struct reg_window32 *)regs->u_regs[UREG_FP]; + + /* Stop the back trace when we hit userland or we + * find some badly aligned kernel stack. Set an upper + * bound in case our stack is trashed and we loop. + */ + while(rw && + count++ < 30 && + (((unsigned long) rw) >= PAGE_OFFSET) && + !(((unsigned long) rw) & 0x7)) { + printk("Caller[%08lx]: %pS\n", rw->ins[7], + (void *) rw->ins[7]); + rw = (struct reg_window32 *)rw->ins[6]; + } + } + printk("Instruction DUMP:"); + instruction_dump ((unsigned long *) regs->pc); + make_task_dead((regs->psr & PSR_PS) ? SIGKILL : SIGSEGV); +} + +void do_hw_interrupt(struct pt_regs *regs, unsigned long type) +{ + if(type < 0x80) { + /* Sun OS's puke from bad traps, Linux survives! */ + printk("Unimplemented Sparc TRAP, type = %02lx\n", type); + die_if_kernel("Whee... Hello Mr. Penguin", regs); + } + + if(regs->psr & PSR_PS) + die_if_kernel("Kernel bad trap", regs); + + force_sig_fault_trapno(SIGILL, ILL_ILLTRP, + (void __user *)regs->pc, type - 0x80); +} + +void do_illegal_instruction(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ + if(psr & PSR_PS) + die_if_kernel("Kernel illegal instruction", regs); +#ifdef TRAP_DEBUG + printk("Ill instr. at pc=%08lx instruction is %08lx\n", + regs->pc, *(unsigned long *)regs->pc); +#endif + + send_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)pc, current); +} + +void do_priv_instruction(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ + if(psr & PSR_PS) + die_if_kernel("Penguin instruction from Penguin mode??!?!", regs); + send_sig_fault(SIGILL, ILL_PRVOPC, (void __user *)pc, current); +} + +/* XXX User may want to be allowed to do this. XXX */ + +void do_memaccess_unaligned(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ + if(regs->psr & PSR_PS) { + printk("KERNEL MNA at pc %08lx npc %08lx called by %08lx\n", pc, npc, + regs->u_regs[UREG_RETPC]); + die_if_kernel("BOGUS", regs); + /* die_if_kernel("Kernel MNA access", regs); */ + } +#if 0 + show_regs (regs); + instruction_dump ((unsigned long *) regs->pc); + printk ("do_MNA!\n"); +#endif + send_sig_fault(SIGBUS, BUS_ADRALN, + /* FIXME: Should dig out mna address */ (void *)0, + current); +} + +static unsigned long init_fsr = 0x0UL; +static unsigned long init_fregs[32] __attribute__ ((aligned (8))) = + { ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, + ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, + ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, + ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL, ~0UL }; + +void do_fpd_trap(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ + /* Sanity check... */ + if(psr & PSR_PS) + die_if_kernel("Kernel gets FloatingPenguinUnit disabled trap", regs); + + put_psr(get_psr() | PSR_EF); /* Allow FPU ops. */ + regs->psr |= PSR_EF; +#ifndef CONFIG_SMP + if(last_task_used_math == current) + return; + if(last_task_used_math) { + /* Other processes fpu state, save away */ + struct task_struct *fptask = last_task_used_math; + fpsave(&fptask->thread.float_regs[0], &fptask->thread.fsr, + &fptask->thread.fpqueue[0], &fptask->thread.fpqdepth); + } + last_task_used_math = current; + if(used_math()) { + fpload(¤t->thread.float_regs[0], ¤t->thread.fsr); + } else { + /* Set initial sane state. */ + fpload(&init_fregs[0], &init_fsr); + set_used_math(); + } +#else + if(!used_math()) { + fpload(&init_fregs[0], &init_fsr); + set_used_math(); + } else { + fpload(¤t->thread.float_regs[0], ¤t->thread.fsr); + } + set_thread_flag(TIF_USEDFPU); +#endif +} + +static unsigned long fake_regs[32] __attribute__ ((aligned (8))); +static unsigned long fake_fsr; +static unsigned long fake_queue[32] __attribute__ ((aligned (8))); +static unsigned long fake_depth; + +void do_fpe_trap(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ + static int calls; + unsigned long fsr; + int ret = 0; + int code; +#ifndef CONFIG_SMP + struct task_struct *fpt = last_task_used_math; +#else + struct task_struct *fpt = current; +#endif + put_psr(get_psr() | PSR_EF); + /* If nobody owns the fpu right now, just clear the + * error into our fake static buffer and hope it don't + * happen again. Thank you crashme... + */ +#ifndef CONFIG_SMP + if(!fpt) { +#else + if (!test_tsk_thread_flag(fpt, TIF_USEDFPU)) { +#endif + fpsave(&fake_regs[0], &fake_fsr, &fake_queue[0], &fake_depth); + regs->psr &= ~PSR_EF; + return; + } + fpsave(&fpt->thread.float_regs[0], &fpt->thread.fsr, + &fpt->thread.fpqueue[0], &fpt->thread.fpqdepth); +#ifdef DEBUG_FPU + printk("Hmm, FP exception, fsr was %016lx\n", fpt->thread.fsr); +#endif + + switch ((fpt->thread.fsr & 0x1c000)) { + /* switch on the contents of the ftt [floating point trap type] field */ +#ifdef DEBUG_FPU + case (1 << 14): + printk("IEEE_754_exception\n"); + break; +#endif + case (2 << 14): /* unfinished_FPop (underflow & co) */ + case (3 << 14): /* unimplemented_FPop (quad stuff, maybe sqrt) */ + ret = do_mathemu(regs, fpt); + break; +#ifdef DEBUG_FPU + case (4 << 14): + printk("sequence_error (OS bug...)\n"); + break; + case (5 << 14): + printk("hardware_error (uhoh!)\n"); + break; + case (6 << 14): + printk("invalid_fp_register (user error)\n"); + break; +#endif /* DEBUG_FPU */ + } + /* If we successfully emulated the FPop, we pretend the trap never happened :-> */ + if (ret) { + fpload(¤t->thread.float_regs[0], ¤t->thread.fsr); + return; + } + /* nope, better SIGFPE the offending process... */ + +#ifdef CONFIG_SMP + clear_tsk_thread_flag(fpt, TIF_USEDFPU); +#endif + if(psr & PSR_PS) { + /* The first fsr store/load we tried trapped, + * the second one will not (we hope). + */ + printk("WARNING: FPU exception from kernel mode. at pc=%08lx\n", + regs->pc); + regs->pc = regs->npc; + regs->npc += 4; + calls++; + if(calls > 2) + die_if_kernel("Too many Penguin-FPU traps from kernel mode", + regs); + return; + } + + fsr = fpt->thread.fsr; + code = FPE_FLTUNK; + if ((fsr & 0x1c000) == (1 << 14)) { + if (fsr & 0x10) + code = FPE_FLTINV; + else if (fsr & 0x08) + code = FPE_FLTOVF; + else if (fsr & 0x04) + code = FPE_FLTUND; + else if (fsr & 0x02) + code = FPE_FLTDIV; + else if (fsr & 0x01) + code = FPE_FLTRES; + } + send_sig_fault(SIGFPE, code, (void __user *)pc, fpt); +#ifndef CONFIG_SMP + last_task_used_math = NULL; +#endif + regs->psr &= ~PSR_EF; + if(calls > 0) + calls=0; +} + +void handle_tag_overflow(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ + if(psr & PSR_PS) + die_if_kernel("Penguin overflow trap from kernel mode", regs); + send_sig_fault(SIGEMT, EMT_TAGOVF, (void __user *)pc, current); +} + +void handle_watchpoint(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ +#ifdef TRAP_DEBUG + printk("Watchpoint detected at PC %08lx NPC %08lx PSR %08lx\n", + pc, npc, psr); +#endif + if(psr & PSR_PS) + panic("Tell me what a watchpoint trap is, and I'll then deal " + "with such a beast..."); +} + +void handle_reg_access(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ +#ifdef TRAP_DEBUG + printk("Register Access Exception at PC %08lx NPC %08lx PSR %08lx\n", + pc, npc, psr); +#endif + force_sig_fault(SIGBUS, BUS_OBJERR, (void __user *)pc); +} + +void handle_cp_disabled(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ + send_sig_fault(SIGILL, ILL_COPROC, (void __user *)pc, current); +} + +void handle_cp_exception(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ +#ifdef TRAP_DEBUG + printk("Co-Processor Exception at PC %08lx NPC %08lx PSR %08lx\n", + pc, npc, psr); +#endif + send_sig_fault(SIGILL, ILL_COPROC, (void __user *)pc, current); +} + +void handle_hw_divzero(struct pt_regs *regs, unsigned long pc, unsigned long npc, + unsigned long psr) +{ + send_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)pc, current); +} + +#ifdef CONFIG_DEBUG_BUGVERBOSE +void do_BUG(const char *file, int line) +{ + // bust_spinlocks(1); XXX Not in our original BUG() + printk("kernel BUG at %s:%d!\n", file, line); +} +EXPORT_SYMBOL(do_BUG); +#endif + +/* Since we have our mappings set up, on multiprocessors we can spin them + * up here so that timer interrupts work during initialization. + */ + +void trap_init(void) +{ + extern void thread_info_offsets_are_bolixed_pete(void); + + /* Force linker to barf if mismatched */ + if (TI_UWINMASK != offsetof(struct thread_info, uwinmask) || + TI_TASK != offsetof(struct thread_info, task) || + TI_FLAGS != offsetof(struct thread_info, flags) || + TI_CPU != offsetof(struct thread_info, cpu) || + TI_PREEMPT != offsetof(struct thread_info, preempt_count) || + TI_SOFTIRQ != offsetof(struct thread_info, softirq_count) || + TI_HARDIRQ != offsetof(struct thread_info, hardirq_count) || + TI_KSP != offsetof(struct thread_info, ksp) || + TI_KPC != offsetof(struct thread_info, kpc) || + TI_KPSR != offsetof(struct thread_info, kpsr) || + TI_KWIM != offsetof(struct thread_info, kwim) || + TI_REG_WINDOW != offsetof(struct thread_info, reg_window) || + TI_RWIN_SPTRS != offsetof(struct thread_info, rwbuf_stkptrs) || + TI_W_SAVED != offsetof(struct thread_info, w_saved)) + thread_info_offsets_are_bolixed_pete(); + + /* Attach to the address space of init_task. */ + mmgrab(&init_mm); + current->active_mm = &init_mm; + + /* NOTE: Other cpus have this done as they are started + * up on SMP. + */ +} diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c new file mode 100644 index 000000000..28cb0d66a --- /dev/null +++ b/arch/sparc/kernel/traps_64.c @@ -0,0 +1,2925 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* arch/sparc64/kernel/traps.c + * + * Copyright (C) 1995,1997,2008,2009,2012 David S. Miller (davem@davemloft.net) + * Copyright (C) 1997,1999,2000 Jakub Jelinek (jakub@redhat.com) + */ + +/* + * I like traps on v9, :)))) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "entry.h" +#include "kernel.h" +#include "kstack.h" + +/* When an irrecoverable trap occurs at tl > 0, the trap entry + * code logs the trap state registers at every level in the trap + * stack. It is found at (pt_regs + sizeof(pt_regs)) and the layout + * is as follows: + */ +struct tl1_traplog { + struct { + unsigned long tstate; + unsigned long tpc; + unsigned long tnpc; + unsigned long tt; + } trapstack[4]; + unsigned long tl; +}; + +static void dump_tl1_traplog(struct tl1_traplog *p) +{ + int i, limit; + + printk(KERN_EMERG "TRAPLOG: Error at trap level 0x%lx, " + "dumping track stack.\n", p->tl); + + limit = (tlb_type == hypervisor) ? 2 : 4; + for (i = 0; i < limit; i++) { + printk(KERN_EMERG + "TRAPLOG: Trap level %d TSTATE[%016lx] TPC[%016lx] " + "TNPC[%016lx] TT[%lx]\n", + i + 1, + p->trapstack[i].tstate, p->trapstack[i].tpc, + p->trapstack[i].tnpc, p->trapstack[i].tt); + printk("TRAPLOG: TPC<%pS>\n", (void *) p->trapstack[i].tpc); + } +} + +void bad_trap(struct pt_regs *regs, long lvl) +{ + char buffer[36]; + + if (notify_die(DIE_TRAP, "bad trap", regs, + 0, lvl, SIGTRAP) == NOTIFY_STOP) + return; + + if (lvl < 0x100) { + sprintf(buffer, "Bad hw trap %lx at tl0\n", lvl); + die_if_kernel(buffer, regs); + } + + lvl -= 0x100; + if (regs->tstate & TSTATE_PRIV) { + sprintf(buffer, "Kernel bad sw trap %lx", lvl); + die_if_kernel(buffer, regs); + } + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + force_sig_fault_trapno(SIGILL, ILL_ILLTRP, + (void __user *)regs->tpc, lvl); +} + +void bad_trap_tl1(struct pt_regs *regs, long lvl) +{ + char buffer[36]; + + if (notify_die(DIE_TRAP_TL1, "bad trap tl1", regs, + 0, lvl, SIGTRAP) == NOTIFY_STOP) + return; + + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + + sprintf (buffer, "Bad trap %lx at tl>0", lvl); + die_if_kernel (buffer, regs); +} + +#ifdef CONFIG_DEBUG_BUGVERBOSE +void do_BUG(const char *file, int line) +{ + bust_spinlocks(1); + printk("kernel BUG at %s:%d!\n", file, line); +} +EXPORT_SYMBOL(do_BUG); +#endif + +static DEFINE_SPINLOCK(dimm_handler_lock); +static dimm_printer_t dimm_handler; + +static int sprintf_dimm(int synd_code, unsigned long paddr, char *buf, int buflen) +{ + unsigned long flags; + int ret = -ENODEV; + + spin_lock_irqsave(&dimm_handler_lock, flags); + if (dimm_handler) { + ret = dimm_handler(synd_code, paddr, buf, buflen); + } else if (tlb_type == spitfire) { + if (prom_getunumber(synd_code, paddr, buf, buflen) == -1) + ret = -EINVAL; + else + ret = 0; + } else + ret = -ENODEV; + spin_unlock_irqrestore(&dimm_handler_lock, flags); + + return ret; +} + +int register_dimm_printer(dimm_printer_t func) +{ + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&dimm_handler_lock, flags); + if (!dimm_handler) + dimm_handler = func; + else + ret = -EEXIST; + spin_unlock_irqrestore(&dimm_handler_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(register_dimm_printer); + +void unregister_dimm_printer(dimm_printer_t func) +{ + unsigned long flags; + + spin_lock_irqsave(&dimm_handler_lock, flags); + if (dimm_handler == func) + dimm_handler = NULL; + spin_unlock_irqrestore(&dimm_handler_lock, flags); +} +EXPORT_SYMBOL_GPL(unregister_dimm_printer); + +void spitfire_insn_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) +{ + enum ctx_state prev_state = exception_enter(); + + if (notify_die(DIE_TRAP, "instruction access exception", regs, + 0, 0x8, SIGTRAP) == NOTIFY_STOP) + goto out; + + if (regs->tstate & TSTATE_PRIV) { + printk("spitfire_insn_access_exception: SFSR[%016lx] " + "SFAR[%016lx], going.\n", sfsr, sfar); + die_if_kernel("Iax", regs); + } + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)regs->tpc); +out: + exception_exit(prev_state); +} + +void spitfire_insn_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) +{ + if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs, + 0, 0x8, SIGTRAP) == NOTIFY_STOP) + return; + + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + spitfire_insn_access_exception(regs, sfsr, sfar); +} + +void sun4v_insn_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) +{ + unsigned short type = (type_ctx >> 16); + unsigned short ctx = (type_ctx & 0xffff); + + if (notify_die(DIE_TRAP, "instruction access exception", regs, + 0, 0x8, SIGTRAP) == NOTIFY_STOP) + return; + + if (regs->tstate & TSTATE_PRIV) { + printk("sun4v_insn_access_exception: ADDR[%016lx] " + "CTX[%04x] TYPE[%04x], going.\n", + addr, ctx, type); + die_if_kernel("Iax", regs); + } + + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *) addr); +} + +void sun4v_insn_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) +{ + if (notify_die(DIE_TRAP_TL1, "instruction access exception tl1", regs, + 0, 0x8, SIGTRAP) == NOTIFY_STOP) + return; + + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + sun4v_insn_access_exception(regs, addr, type_ctx); +} + +static bool is_no_fault_exception(struct pt_regs *regs) +{ + unsigned char asi; + u32 insn; + + if (get_user(insn, (u32 __user *)regs->tpc) == -EFAULT) + return false; + + /* + * Must do a little instruction decoding here in order to + * decide on a course of action. The bits of interest are: + * insn[31:30] = op, where 3 indicates the load/store group + * insn[24:19] = op3, which identifies individual opcodes + * insn[13] indicates an immediate offset + * op3[4]=1 identifies alternate space instructions + * op3[5:4]=3 identifies floating point instructions + * op3[2]=1 identifies stores + * See "Opcode Maps" in the appendix of any Sparc V9 + * architecture spec for full details. + */ + if ((insn & 0xc0800000) == 0xc0800000) { /* op=3, op3[4]=1 */ + if (insn & 0x2000) /* immediate offset */ + asi = (regs->tstate >> 24); /* saved %asi */ + else + asi = (insn >> 5); /* immediate asi */ + if ((asi & 0xf6) == ASI_PNF) { + if (insn & 0x200000) /* op3[2], stores */ + return false; + if (insn & 0x1000000) /* op3[5:4]=3 (fp) */ + handle_ldf_stq(insn, regs); + else + handle_ld_nf(insn, regs); + return true; + } + } + return false; +} + +void spitfire_data_access_exception(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) +{ + enum ctx_state prev_state = exception_enter(); + + if (notify_die(DIE_TRAP, "data access exception", regs, + 0, 0x30, SIGTRAP) == NOTIFY_STOP) + goto out; + + if (regs->tstate & TSTATE_PRIV) { + /* Test if this comes from uaccess places. */ + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + /* Ouch, somebody is trying VM hole tricks on us... */ +#ifdef DEBUG_EXCEPTIONS + printk("Exception: PC<%016lx> faddr\n", regs->tpc); + printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n", + regs->tpc, entry->fixup); +#endif + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + goto out; + } + /* Shit... */ + printk("spitfire_data_access_exception: SFSR[%016lx] " + "SFAR[%016lx], going.\n", sfsr, sfar); + die_if_kernel("Dax", regs); + } + + if (is_no_fault_exception(regs)) + return; + + force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)sfar); +out: + exception_exit(prev_state); +} + +void spitfire_data_access_exception_tl1(struct pt_regs *regs, unsigned long sfsr, unsigned long sfar) +{ + if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs, + 0, 0x30, SIGTRAP) == NOTIFY_STOP) + return; + + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + spitfire_data_access_exception(regs, sfsr, sfar); +} + +void sun4v_data_access_exception(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) +{ + unsigned short type = (type_ctx >> 16); + unsigned short ctx = (type_ctx & 0xffff); + + if (notify_die(DIE_TRAP, "data access exception", regs, + 0, 0x8, SIGTRAP) == NOTIFY_STOP) + return; + + if (regs->tstate & TSTATE_PRIV) { + /* Test if this comes from uaccess places. */ + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + /* Ouch, somebody is trying VM hole tricks on us... */ +#ifdef DEBUG_EXCEPTIONS + printk("Exception: PC<%016lx> faddr\n", regs->tpc); + printk("EX_TABLE: insn<%016lx> fixup<%016lx>\n", + regs->tpc, entry->fixup); +#endif + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return; + } + printk("sun4v_data_access_exception: ADDR[%016lx] " + "CTX[%04x] TYPE[%04x], going.\n", + addr, ctx, type); + die_if_kernel("Dax", regs); + } + + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + if (is_no_fault_exception(regs)) + return; + + /* MCD (Memory Corruption Detection) disabled trap (TT=0x19) in HV + * is vectored thorugh data access exception trap with fault type + * set to HV_FAULT_TYPE_MCD_DIS. Check for MCD disabled trap. + * Accessing an address with invalid ASI for the address, for + * example setting an ADI tag on an address with ASI_MCD_PRIMARY + * when TTE.mcd is not set for the VA, is also vectored into + * kerbel by HV as data access exception with fault type set to + * HV_FAULT_TYPE_INV_ASI. + */ + switch (type) { + case HV_FAULT_TYPE_INV_ASI: + force_sig_fault(SIGILL, ILL_ILLADR, (void __user *)addr); + break; + case HV_FAULT_TYPE_MCD_DIS: + force_sig_fault(SIGSEGV, SEGV_ACCADI, (void __user *)addr); + break; + default: + force_sig_fault(SIGSEGV, SEGV_MAPERR, (void __user *)addr); + break; + } +} + +void sun4v_data_access_exception_tl1(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) +{ + if (notify_die(DIE_TRAP_TL1, "data access exception tl1", regs, + 0, 0x8, SIGTRAP) == NOTIFY_STOP) + return; + + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + sun4v_data_access_exception(regs, addr, type_ctx); +} + +#ifdef CONFIG_PCI +#include "pci_impl.h" +#endif + +/* When access exceptions happen, we must do this. */ +static void spitfire_clean_and_reenable_l1_caches(void) +{ + unsigned long va; + + if (tlb_type != spitfire) + BUG(); + + /* Clean 'em. */ + for (va = 0; va < (PAGE_SIZE << 1); va += 32) { + spitfire_put_icache_tag(va, 0x0); + spitfire_put_dcache_tag(va, 0x0); + } + + /* Re-enable in LSU. */ + __asm__ __volatile__("flush %%g6\n\t" + "membar #Sync\n\t" + "stxa %0, [%%g0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (LSU_CONTROL_IC | LSU_CONTROL_DC | + LSU_CONTROL_IM | LSU_CONTROL_DM), + "i" (ASI_LSU_CONTROL) + : "memory"); +} + +static void spitfire_enable_estate_errors(void) +{ + __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (ESTATE_ERR_ALL), + "i" (ASI_ESTATE_ERROR_EN)); +} + +static char ecc_syndrome_table[] = { + 0x4c, 0x40, 0x41, 0x48, 0x42, 0x48, 0x48, 0x49, + 0x43, 0x48, 0x48, 0x49, 0x48, 0x49, 0x49, 0x4a, + 0x44, 0x48, 0x48, 0x20, 0x48, 0x39, 0x4b, 0x48, + 0x48, 0x25, 0x31, 0x48, 0x28, 0x48, 0x48, 0x2c, + 0x45, 0x48, 0x48, 0x21, 0x48, 0x3d, 0x04, 0x48, + 0x48, 0x4b, 0x35, 0x48, 0x2d, 0x48, 0x48, 0x29, + 0x48, 0x00, 0x01, 0x48, 0x0a, 0x48, 0x48, 0x4b, + 0x0f, 0x48, 0x48, 0x4b, 0x48, 0x49, 0x49, 0x48, + 0x46, 0x48, 0x48, 0x2a, 0x48, 0x3b, 0x27, 0x48, + 0x48, 0x4b, 0x33, 0x48, 0x22, 0x48, 0x48, 0x2e, + 0x48, 0x19, 0x1d, 0x48, 0x1b, 0x4a, 0x48, 0x4b, + 0x1f, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48, + 0x48, 0x4b, 0x24, 0x48, 0x07, 0x48, 0x48, 0x36, + 0x4b, 0x48, 0x48, 0x3e, 0x48, 0x30, 0x38, 0x48, + 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x16, 0x48, + 0x48, 0x12, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b, + 0x47, 0x48, 0x48, 0x2f, 0x48, 0x3f, 0x4b, 0x48, + 0x48, 0x06, 0x37, 0x48, 0x23, 0x48, 0x48, 0x2b, + 0x48, 0x05, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x32, + 0x26, 0x48, 0x48, 0x3a, 0x48, 0x34, 0x3c, 0x48, + 0x48, 0x11, 0x15, 0x48, 0x13, 0x4a, 0x48, 0x4b, + 0x17, 0x48, 0x4a, 0x4b, 0x48, 0x4b, 0x4b, 0x48, + 0x49, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x1e, 0x48, + 0x48, 0x1a, 0x4b, 0x48, 0x49, 0x48, 0x48, 0x4b, + 0x48, 0x08, 0x0d, 0x48, 0x02, 0x48, 0x48, 0x49, + 0x03, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x4b, 0x48, + 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x10, 0x48, + 0x48, 0x14, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b, + 0x49, 0x48, 0x48, 0x49, 0x48, 0x4b, 0x18, 0x48, + 0x48, 0x1c, 0x4b, 0x48, 0x4b, 0x48, 0x48, 0x4b, + 0x4a, 0x0c, 0x09, 0x48, 0x0e, 0x48, 0x48, 0x4b, + 0x0b, 0x48, 0x48, 0x4b, 0x48, 0x4b, 0x4b, 0x4a +}; + +static char *syndrome_unknown = ""; + +static void spitfire_log_udb_syndrome(unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long bit) +{ + unsigned short scode; + char memmod_str[64], *p; + + if (udbl & bit) { + scode = ecc_syndrome_table[udbl & 0xff]; + if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0) + p = syndrome_unknown; + else + p = memmod_str; + printk(KERN_WARNING "CPU[%d]: UDBL Syndrome[%x] " + "Memory Module \"%s\"\n", + smp_processor_id(), scode, p); + } + + if (udbh & bit) { + scode = ecc_syndrome_table[udbh & 0xff]; + if (sprintf_dimm(scode, afar, memmod_str, sizeof(memmod_str)) < 0) + p = syndrome_unknown; + else + p = memmod_str; + printk(KERN_WARNING "CPU[%d]: UDBH Syndrome[%x] " + "Memory Module \"%s\"\n", + smp_processor_id(), scode, p); + } + +} + +static void spitfire_cee_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, int tl1, struct pt_regs *regs) +{ + + printk(KERN_WARNING "CPU[%d]: Correctable ECC Error " + "AFSR[%lx] AFAR[%016lx] UDBL[%lx] UDBH[%lx] TL>1[%d]\n", + smp_processor_id(), afsr, afar, udbl, udbh, tl1); + + spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_CE); + + /* We always log it, even if someone is listening for this + * trap. + */ + notify_die(DIE_TRAP, "Correctable ECC Error", regs, + 0, TRAP_TYPE_CEE, SIGTRAP); + + /* The Correctable ECC Error trap does not disable I/D caches. So + * we only have to restore the ESTATE Error Enable register. + */ + spitfire_enable_estate_errors(); +} + +static void spitfire_ue_log(unsigned long afsr, unsigned long afar, unsigned long udbh, unsigned long udbl, unsigned long tt, int tl1, struct pt_regs *regs) +{ + printk(KERN_WARNING "CPU[%d]: Uncorrectable Error AFSR[%lx] " + "AFAR[%lx] UDBL[%lx] UDBH[%ld] TT[%lx] TL>1[%d]\n", + smp_processor_id(), afsr, afar, udbl, udbh, tt, tl1); + + /* XXX add more human friendly logging of the error status + * XXX as is implemented for cheetah + */ + + spitfire_log_udb_syndrome(afar, udbh, udbl, UDBE_UE); + + /* We always log it, even if someone is listening for this + * trap. + */ + notify_die(DIE_TRAP, "Uncorrectable Error", regs, + 0, tt, SIGTRAP); + + if (regs->tstate & TSTATE_PRIV) { + if (tl1) + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("UE", regs); + } + + /* XXX need more intelligent processing here, such as is implemented + * XXX for cheetah errors, in fact if the E-cache still holds the + * XXX line with bad parity this will loop + */ + + spitfire_clean_and_reenable_l1_caches(); + spitfire_enable_estate_errors(); + + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + force_sig_fault(SIGBUS, BUS_OBJERR, (void *)0); +} + +void spitfire_access_error(struct pt_regs *regs, unsigned long status_encoded, unsigned long afar) +{ + unsigned long afsr, tt, udbh, udbl; + int tl1; + + afsr = (status_encoded & SFSTAT_AFSR_MASK) >> SFSTAT_AFSR_SHIFT; + tt = (status_encoded & SFSTAT_TRAP_TYPE) >> SFSTAT_TRAP_TYPE_SHIFT; + tl1 = (status_encoded & SFSTAT_TL_GT_ONE) ? 1 : 0; + udbl = (status_encoded & SFSTAT_UDBL_MASK) >> SFSTAT_UDBL_SHIFT; + udbh = (status_encoded & SFSTAT_UDBH_MASK) >> SFSTAT_UDBH_SHIFT; + +#ifdef CONFIG_PCI + if (tt == TRAP_TYPE_DAE && + pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) { + spitfire_clean_and_reenable_l1_caches(); + spitfire_enable_estate_errors(); + + pci_poke_faulted = 1; + regs->tnpc = regs->tpc + 4; + return; + } +#endif + + if (afsr & SFAFSR_UE) + spitfire_ue_log(afsr, afar, udbh, udbl, tt, tl1, regs); + + if (tt == TRAP_TYPE_CEE) { + /* Handle the case where we took a CEE trap, but ACK'd + * only the UE state in the UDB error registers. + */ + if (afsr & SFAFSR_UE) { + if (udbh & UDBE_CE) { + __asm__ __volatile__( + "stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (udbh & UDBE_CE), + "r" (0x0), "i" (ASI_UDB_ERROR_W)); + } + if (udbl & UDBE_CE) { + __asm__ __volatile__( + "stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (udbl & UDBE_CE), + "r" (0x18), "i" (ASI_UDB_ERROR_W)); + } + } + + spitfire_cee_log(afsr, afar, udbh, udbl, tl1, regs); + } +} + +int cheetah_pcache_forced_on; + +void cheetah_enable_pcache(void) +{ + unsigned long dcr; + + printk("CHEETAH: Enabling P-Cache on cpu %d.\n", + smp_processor_id()); + + __asm__ __volatile__("ldxa [%%g0] %1, %0" + : "=r" (dcr) + : "i" (ASI_DCU_CONTROL_REG)); + dcr |= (DCU_PE | DCU_HPE | DCU_SPE | DCU_SL); + __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (dcr), "i" (ASI_DCU_CONTROL_REG)); +} + +/* Cheetah error trap handling. */ +static unsigned long ecache_flush_physbase; +static unsigned long ecache_flush_linesize; +static unsigned long ecache_flush_size; + +/* This table is ordered in priority of errors and matches the + * AFAR overwrite policy as well. + */ + +struct afsr_error_table { + unsigned long mask; + const char *name; +}; + +static const char CHAFSR_PERR_msg[] = + "System interface protocol error"; +static const char CHAFSR_IERR_msg[] = + "Internal processor error"; +static const char CHAFSR_ISAP_msg[] = + "System request parity error on incoming address"; +static const char CHAFSR_UCU_msg[] = + "Uncorrectable E-cache ECC error for ifetch/data"; +static const char CHAFSR_UCC_msg[] = + "SW Correctable E-cache ECC error for ifetch/data"; +static const char CHAFSR_UE_msg[] = + "Uncorrectable system bus data ECC error for read"; +static const char CHAFSR_EDU_msg[] = + "Uncorrectable E-cache ECC error for stmerge/blkld"; +static const char CHAFSR_EMU_msg[] = + "Uncorrectable system bus MTAG error"; +static const char CHAFSR_WDU_msg[] = + "Uncorrectable E-cache ECC error for writeback"; +static const char CHAFSR_CPU_msg[] = + "Uncorrectable ECC error for copyout"; +static const char CHAFSR_CE_msg[] = + "HW corrected system bus data ECC error for read"; +static const char CHAFSR_EDC_msg[] = + "HW corrected E-cache ECC error for stmerge/blkld"; +static const char CHAFSR_EMC_msg[] = + "HW corrected system bus MTAG ECC error"; +static const char CHAFSR_WDC_msg[] = + "HW corrected E-cache ECC error for writeback"; +static const char CHAFSR_CPC_msg[] = + "HW corrected ECC error for copyout"; +static const char CHAFSR_TO_msg[] = + "Unmapped error from system bus"; +static const char CHAFSR_BERR_msg[] = + "Bus error response from system bus"; +static const char CHAFSR_IVC_msg[] = + "HW corrected system bus data ECC error for ivec read"; +static const char CHAFSR_IVU_msg[] = + "Uncorrectable system bus data ECC error for ivec read"; +static struct afsr_error_table __cheetah_error_table[] = { + { CHAFSR_PERR, CHAFSR_PERR_msg }, + { CHAFSR_IERR, CHAFSR_IERR_msg }, + { CHAFSR_ISAP, CHAFSR_ISAP_msg }, + { CHAFSR_UCU, CHAFSR_UCU_msg }, + { CHAFSR_UCC, CHAFSR_UCC_msg }, + { CHAFSR_UE, CHAFSR_UE_msg }, + { CHAFSR_EDU, CHAFSR_EDU_msg }, + { CHAFSR_EMU, CHAFSR_EMU_msg }, + { CHAFSR_WDU, CHAFSR_WDU_msg }, + { CHAFSR_CPU, CHAFSR_CPU_msg }, + { CHAFSR_CE, CHAFSR_CE_msg }, + { CHAFSR_EDC, CHAFSR_EDC_msg }, + { CHAFSR_EMC, CHAFSR_EMC_msg }, + { CHAFSR_WDC, CHAFSR_WDC_msg }, + { CHAFSR_CPC, CHAFSR_CPC_msg }, + { CHAFSR_TO, CHAFSR_TO_msg }, + { CHAFSR_BERR, CHAFSR_BERR_msg }, + /* These two do not update the AFAR. */ + { CHAFSR_IVC, CHAFSR_IVC_msg }, + { CHAFSR_IVU, CHAFSR_IVU_msg }, + { 0, NULL }, +}; +static const char CHPAFSR_DTO_msg[] = + "System bus unmapped error for prefetch/storequeue-read"; +static const char CHPAFSR_DBERR_msg[] = + "System bus error for prefetch/storequeue-read"; +static const char CHPAFSR_THCE_msg[] = + "Hardware corrected E-cache Tag ECC error"; +static const char CHPAFSR_TSCE_msg[] = + "SW handled correctable E-cache Tag ECC error"; +static const char CHPAFSR_TUE_msg[] = + "Uncorrectable E-cache Tag ECC error"; +static const char CHPAFSR_DUE_msg[] = + "System bus uncorrectable data ECC error due to prefetch/store-fill"; +static struct afsr_error_table __cheetah_plus_error_table[] = { + { CHAFSR_PERR, CHAFSR_PERR_msg }, + { CHAFSR_IERR, CHAFSR_IERR_msg }, + { CHAFSR_ISAP, CHAFSR_ISAP_msg }, + { CHAFSR_UCU, CHAFSR_UCU_msg }, + { CHAFSR_UCC, CHAFSR_UCC_msg }, + { CHAFSR_UE, CHAFSR_UE_msg }, + { CHAFSR_EDU, CHAFSR_EDU_msg }, + { CHAFSR_EMU, CHAFSR_EMU_msg }, + { CHAFSR_WDU, CHAFSR_WDU_msg }, + { CHAFSR_CPU, CHAFSR_CPU_msg }, + { CHAFSR_CE, CHAFSR_CE_msg }, + { CHAFSR_EDC, CHAFSR_EDC_msg }, + { CHAFSR_EMC, CHAFSR_EMC_msg }, + { CHAFSR_WDC, CHAFSR_WDC_msg }, + { CHAFSR_CPC, CHAFSR_CPC_msg }, + { CHAFSR_TO, CHAFSR_TO_msg }, + { CHAFSR_BERR, CHAFSR_BERR_msg }, + { CHPAFSR_DTO, CHPAFSR_DTO_msg }, + { CHPAFSR_DBERR, CHPAFSR_DBERR_msg }, + { CHPAFSR_THCE, CHPAFSR_THCE_msg }, + { CHPAFSR_TSCE, CHPAFSR_TSCE_msg }, + { CHPAFSR_TUE, CHPAFSR_TUE_msg }, + { CHPAFSR_DUE, CHPAFSR_DUE_msg }, + /* These two do not update the AFAR. */ + { CHAFSR_IVC, CHAFSR_IVC_msg }, + { CHAFSR_IVU, CHAFSR_IVU_msg }, + { 0, NULL }, +}; +static const char JPAFSR_JETO_msg[] = + "System interface protocol error, hw timeout caused"; +static const char JPAFSR_SCE_msg[] = + "Parity error on system snoop results"; +static const char JPAFSR_JEIC_msg[] = + "System interface protocol error, illegal command detected"; +static const char JPAFSR_JEIT_msg[] = + "System interface protocol error, illegal ADTYPE detected"; +static const char JPAFSR_OM_msg[] = + "Out of range memory error has occurred"; +static const char JPAFSR_ETP_msg[] = + "Parity error on L2 cache tag SRAM"; +static const char JPAFSR_UMS_msg[] = + "Error due to unsupported store"; +static const char JPAFSR_RUE_msg[] = + "Uncorrectable ECC error from remote cache/memory"; +static const char JPAFSR_RCE_msg[] = + "Correctable ECC error from remote cache/memory"; +static const char JPAFSR_BP_msg[] = + "JBUS parity error on returned read data"; +static const char JPAFSR_WBP_msg[] = + "JBUS parity error on data for writeback or block store"; +static const char JPAFSR_FRC_msg[] = + "Foreign read to DRAM incurring correctable ECC error"; +static const char JPAFSR_FRU_msg[] = + "Foreign read to DRAM incurring uncorrectable ECC error"; +static struct afsr_error_table __jalapeno_error_table[] = { + { JPAFSR_JETO, JPAFSR_JETO_msg }, + { JPAFSR_SCE, JPAFSR_SCE_msg }, + { JPAFSR_JEIC, JPAFSR_JEIC_msg }, + { JPAFSR_JEIT, JPAFSR_JEIT_msg }, + { CHAFSR_PERR, CHAFSR_PERR_msg }, + { CHAFSR_IERR, CHAFSR_IERR_msg }, + { CHAFSR_ISAP, CHAFSR_ISAP_msg }, + { CHAFSR_UCU, CHAFSR_UCU_msg }, + { CHAFSR_UCC, CHAFSR_UCC_msg }, + { CHAFSR_UE, CHAFSR_UE_msg }, + { CHAFSR_EDU, CHAFSR_EDU_msg }, + { JPAFSR_OM, JPAFSR_OM_msg }, + { CHAFSR_WDU, CHAFSR_WDU_msg }, + { CHAFSR_CPU, CHAFSR_CPU_msg }, + { CHAFSR_CE, CHAFSR_CE_msg }, + { CHAFSR_EDC, CHAFSR_EDC_msg }, + { JPAFSR_ETP, JPAFSR_ETP_msg }, + { CHAFSR_WDC, CHAFSR_WDC_msg }, + { CHAFSR_CPC, CHAFSR_CPC_msg }, + { CHAFSR_TO, CHAFSR_TO_msg }, + { CHAFSR_BERR, CHAFSR_BERR_msg }, + { JPAFSR_UMS, JPAFSR_UMS_msg }, + { JPAFSR_RUE, JPAFSR_RUE_msg }, + { JPAFSR_RCE, JPAFSR_RCE_msg }, + { JPAFSR_BP, JPAFSR_BP_msg }, + { JPAFSR_WBP, JPAFSR_WBP_msg }, + { JPAFSR_FRC, JPAFSR_FRC_msg }, + { JPAFSR_FRU, JPAFSR_FRU_msg }, + /* These two do not update the AFAR. */ + { CHAFSR_IVU, CHAFSR_IVU_msg }, + { 0, NULL }, +}; +static struct afsr_error_table *cheetah_error_table; +static unsigned long cheetah_afsr_errors; + +struct cheetah_err_info *cheetah_error_log; + +static inline struct cheetah_err_info *cheetah_get_error_log(unsigned long afsr) +{ + struct cheetah_err_info *p; + int cpu = smp_processor_id(); + + if (!cheetah_error_log) + return NULL; + + p = cheetah_error_log + (cpu * 2); + if ((afsr & CHAFSR_TL1) != 0UL) + p++; + + return p; +} + +extern unsigned int tl0_icpe[], tl1_icpe[]; +extern unsigned int tl0_dcpe[], tl1_dcpe[]; +extern unsigned int tl0_fecc[], tl1_fecc[]; +extern unsigned int tl0_cee[], tl1_cee[]; +extern unsigned int tl0_iae[], tl1_iae[]; +extern unsigned int tl0_dae[], tl1_dae[]; +extern unsigned int cheetah_plus_icpe_trap_vector[], cheetah_plus_icpe_trap_vector_tl1[]; +extern unsigned int cheetah_plus_dcpe_trap_vector[], cheetah_plus_dcpe_trap_vector_tl1[]; +extern unsigned int cheetah_fecc_trap_vector[], cheetah_fecc_trap_vector_tl1[]; +extern unsigned int cheetah_cee_trap_vector[], cheetah_cee_trap_vector_tl1[]; +extern unsigned int cheetah_deferred_trap_vector[], cheetah_deferred_trap_vector_tl1[]; + +void __init cheetah_ecache_flush_init(void) +{ + unsigned long largest_size, smallest_linesize, order, ver; + int i, sz; + + /* Scan all cpu device tree nodes, note two values: + * 1) largest E-cache size + * 2) smallest E-cache line size + */ + largest_size = 0UL; + smallest_linesize = ~0UL; + + for (i = 0; i < NR_CPUS; i++) { + unsigned long val; + + val = cpu_data(i).ecache_size; + if (!val) + continue; + + if (val > largest_size) + largest_size = val; + + val = cpu_data(i).ecache_line_size; + if (val < smallest_linesize) + smallest_linesize = val; + + } + + if (largest_size == 0UL || smallest_linesize == ~0UL) { + prom_printf("cheetah_ecache_flush_init: Cannot probe cpu E-cache " + "parameters.\n"); + prom_halt(); + } + + ecache_flush_size = (2 * largest_size); + ecache_flush_linesize = smallest_linesize; + + ecache_flush_physbase = find_ecache_flush_span(ecache_flush_size); + + if (ecache_flush_physbase == ~0UL) { + prom_printf("cheetah_ecache_flush_init: Cannot find %ld byte " + "contiguous physical memory.\n", + ecache_flush_size); + prom_halt(); + } + + /* Now allocate error trap reporting scoreboard. */ + sz = NR_CPUS * (2 * sizeof(struct cheetah_err_info)); + for (order = 0; order < NR_PAGE_ORDERS; order++) { + if ((PAGE_SIZE << order) >= sz) + break; + } + cheetah_error_log = (struct cheetah_err_info *) + __get_free_pages(GFP_KERNEL, order); + if (!cheetah_error_log) { + prom_printf("cheetah_ecache_flush_init: Failed to allocate " + "error logging scoreboard (%d bytes).\n", sz); + prom_halt(); + } + memset(cheetah_error_log, 0, PAGE_SIZE << order); + + /* Mark all AFSRs as invalid so that the trap handler will + * log new new information there. + */ + for (i = 0; i < 2 * NR_CPUS; i++) + cheetah_error_log[i].afsr = CHAFSR_INVALID; + + __asm__ ("rdpr %%ver, %0" : "=r" (ver)); + if ((ver >> 32) == __JALAPENO_ID || + (ver >> 32) == __SERRANO_ID) { + cheetah_error_table = &__jalapeno_error_table[0]; + cheetah_afsr_errors = JPAFSR_ERRORS; + } else if ((ver >> 32) == 0x003e0015) { + cheetah_error_table = &__cheetah_plus_error_table[0]; + cheetah_afsr_errors = CHPAFSR_ERRORS; + } else { + cheetah_error_table = &__cheetah_error_table[0]; + cheetah_afsr_errors = CHAFSR_ERRORS; + } + + /* Now patch trap tables. */ + memcpy(tl0_fecc, cheetah_fecc_trap_vector, (8 * 4)); + memcpy(tl1_fecc, cheetah_fecc_trap_vector_tl1, (8 * 4)); + memcpy(tl0_cee, cheetah_cee_trap_vector, (8 * 4)); + memcpy(tl1_cee, cheetah_cee_trap_vector_tl1, (8 * 4)); + memcpy(tl0_iae, cheetah_deferred_trap_vector, (8 * 4)); + memcpy(tl1_iae, cheetah_deferred_trap_vector_tl1, (8 * 4)); + memcpy(tl0_dae, cheetah_deferred_trap_vector, (8 * 4)); + memcpy(tl1_dae, cheetah_deferred_trap_vector_tl1, (8 * 4)); + if (tlb_type == cheetah_plus) { + memcpy(tl0_dcpe, cheetah_plus_dcpe_trap_vector, (8 * 4)); + memcpy(tl1_dcpe, cheetah_plus_dcpe_trap_vector_tl1, (8 * 4)); + memcpy(tl0_icpe, cheetah_plus_icpe_trap_vector, (8 * 4)); + memcpy(tl1_icpe, cheetah_plus_icpe_trap_vector_tl1, (8 * 4)); + } + flushi(PAGE_OFFSET); +} + +static void cheetah_flush_ecache(void) +{ + unsigned long flush_base = ecache_flush_physbase; + unsigned long flush_linesize = ecache_flush_linesize; + unsigned long flush_size = ecache_flush_size; + + __asm__ __volatile__("1: subcc %0, %4, %0\n\t" + " bne,pt %%xcc, 1b\n\t" + " ldxa [%2 + %0] %3, %%g0\n\t" + : "=&r" (flush_size) + : "0" (flush_size), "r" (flush_base), + "i" (ASI_PHYS_USE_EC), "r" (flush_linesize)); +} + +static void cheetah_flush_ecache_line(unsigned long physaddr) +{ + unsigned long alias; + + physaddr &= ~(8UL - 1UL); + physaddr = (ecache_flush_physbase + + (physaddr & ((ecache_flush_size>>1UL) - 1UL))); + alias = physaddr + (ecache_flush_size >> 1UL); + __asm__ __volatile__("ldxa [%0] %2, %%g0\n\t" + "ldxa [%1] %2, %%g0\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (physaddr), "r" (alias), + "i" (ASI_PHYS_USE_EC)); +} + +/* Unfortunately, the diagnostic access to the I-cache tags we need to + * use to clear the thing interferes with I-cache coherency transactions. + * + * So we must only flush the I-cache when it is disabled. + */ +static void __cheetah_flush_icache(void) +{ + unsigned int icache_size, icache_line_size; + unsigned long addr; + + icache_size = local_cpu_data().icache_size; + icache_line_size = local_cpu_data().icache_line_size; + + /* Clear the valid bits in all the tags. */ + for (addr = 0; addr < icache_size; addr += icache_line_size) { + __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (addr | (2 << 3)), + "i" (ASI_IC_TAG)); + } +} + +static void cheetah_flush_icache(void) +{ + unsigned long dcu_save; + + /* Save current DCU, disable I-cache. */ + __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t" + "or %0, %2, %%g1\n\t" + "stxa %%g1, [%%g0] %1\n\t" + "membar #Sync" + : "=r" (dcu_save) + : "i" (ASI_DCU_CONTROL_REG), "i" (DCU_IC) + : "g1"); + + __cheetah_flush_icache(); + + /* Restore DCU register */ + __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (dcu_save), "i" (ASI_DCU_CONTROL_REG)); +} + +static void cheetah_flush_dcache(void) +{ + unsigned int dcache_size, dcache_line_size; + unsigned long addr; + + dcache_size = local_cpu_data().dcache_size; + dcache_line_size = local_cpu_data().dcache_line_size; + + for (addr = 0; addr < dcache_size; addr += dcache_line_size) { + __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (addr), "i" (ASI_DCACHE_TAG)); + } +} + +/* In order to make the even parity correct we must do two things. + * First, we clear DC_data_parity and set DC_utag to an appropriate value. + * Next, we clear out all 32-bytes of data for that line. Data of + * all-zero + tag parity value of zero == correct parity. + */ +static void cheetah_plus_zap_dcache_parity(void) +{ + unsigned int dcache_size, dcache_line_size; + unsigned long addr; + + dcache_size = local_cpu_data().dcache_size; + dcache_line_size = local_cpu_data().dcache_line_size; + + for (addr = 0; addr < dcache_size; addr += dcache_line_size) { + unsigned long tag = (addr >> 14); + unsigned long line; + + __asm__ __volatile__("membar #Sync\n\t" + "stxa %0, [%1] %2\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (tag), "r" (addr), + "i" (ASI_DCACHE_UTAG)); + for (line = addr; line < addr + dcache_line_size; line += 8) + __asm__ __volatile__("membar #Sync\n\t" + "stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (line), + "i" (ASI_DCACHE_DATA)); + } +} + +/* Conversion tables used to frob Cheetah AFSR syndrome values into + * something palatable to the memory controller driver get_unumber + * routine. + */ +#define MT0 137 +#define MT1 138 +#define MT2 139 +#define NONE 254 +#define MTC0 140 +#define MTC1 141 +#define MTC2 142 +#define MTC3 143 +#define C0 128 +#define C1 129 +#define C2 130 +#define C3 131 +#define C4 132 +#define C5 133 +#define C6 134 +#define C7 135 +#define C8 136 +#define M2 144 +#define M3 145 +#define M4 146 +#define M 147 +static unsigned char cheetah_ecc_syntab[] = { +/*00*/NONE, C0, C1, M2, C2, M2, M3, 47, C3, M2, M2, 53, M2, 41, 29, M, +/*01*/C4, M, M, 50, M2, 38, 25, M2, M2, 33, 24, M2, 11, M, M2, 16, +/*02*/C5, M, M, 46, M2, 37, 19, M2, M, 31, 32, M, 7, M2, M2, 10, +/*03*/M2, 40, 13, M2, 59, M, M2, 66, M, M2, M2, 0, M2, 67, 71, M, +/*04*/C6, M, M, 43, M, 36, 18, M, M2, 49, 15, M, 63, M2, M2, 6, +/*05*/M2, 44, 28, M2, M, M2, M2, 52, 68, M2, M2, 62, M2, M3, M3, M4, +/*06*/M2, 26, 106, M2, 64, M, M2, 2, 120, M, M2, M3, M, M3, M3, M4, +/*07*/116, M2, M2, M3, M2, M3, M, M4, M2, 58, 54, M2, M, M4, M4, M3, +/*08*/C7, M2, M, 42, M, 35, 17, M2, M, 45, 14, M2, 21, M2, M2, 5, +/*09*/M, 27, M, M, 99, M, M, 3, 114, M2, M2, 20, M2, M3, M3, M, +/*0a*/M2, 23, 113, M2, 112, M2, M, 51, 95, M, M2, M3, M2, M3, M3, M2, +/*0b*/103, M, M2, M3, M2, M3, M3, M4, M2, 48, M, M, 73, M2, M, M3, +/*0c*/M2, 22, 110, M2, 109, M2, M, 9, 108, M2, M, M3, M2, M3, M3, M, +/*0d*/102, M2, M, M, M2, M3, M3, M, M2, M3, M3, M2, M, M4, M, M3, +/*0e*/98, M, M2, M3, M2, M, M3, M4, M2, M3, M3, M4, M3, M, M, M, +/*0f*/M2, M3, M3, M, M3, M, M, M, 56, M4, M, M3, M4, M, M, M, +/*10*/C8, M, M2, 39, M, 34, 105, M2, M, 30, 104, M, 101, M, M, 4, +/*11*/M, M, 100, M, 83, M, M2, 12, 87, M, M, 57, M2, M, M3, M, +/*12*/M2, 97, 82, M2, 78, M2, M2, 1, 96, M, M, M, M, M, M3, M2, +/*13*/94, M, M2, M3, M2, M, M3, M, M2, M, 79, M, 69, M, M4, M, +/*14*/M2, 93, 92, M, 91, M, M2, 8, 90, M2, M2, M, M, M, M, M4, +/*15*/89, M, M, M3, M2, M3, M3, M, M, M, M3, M2, M3, M2, M, M3, +/*16*/86, M, M2, M3, M2, M, M3, M, M2, M, M3, M, M3, M, M, M3, +/*17*/M, M, M3, M2, M3, M2, M4, M, 60, M, M2, M3, M4, M, M, M2, +/*18*/M2, 88, 85, M2, 84, M, M2, 55, 81, M2, M2, M3, M2, M3, M3, M4, +/*19*/77, M, M, M, M2, M3, M, M, M2, M3, M3, M4, M3, M2, M, M, +/*1a*/74, M, M2, M3, M, M, M3, M, M, M, M3, M, M3, M, M4, M3, +/*1b*/M2, 70, 107, M4, 65, M2, M2, M, 127, M, M, M, M2, M3, M3, M, +/*1c*/80, M2, M2, 72, M, 119, 118, M, M2, 126, 76, M, 125, M, M4, M3, +/*1d*/M2, 115, 124, M, 75, M, M, M3, 61, M, M4, M, M4, M, M, M, +/*1e*/M, 123, 122, M4, 121, M4, M, M3, 117, M2, M2, M3, M4, M3, M, M, +/*1f*/111, M, M, M, M4, M3, M3, M, M, M, M3, M, M3, M2, M, M +}; +static unsigned char cheetah_mtag_syntab[] = { + NONE, MTC0, + MTC1, NONE, + MTC2, NONE, + NONE, MT0, + MTC3, NONE, + NONE, MT1, + NONE, MT2, + NONE, NONE +}; + +/* Return the highest priority error conditon mentioned. */ +static inline unsigned long cheetah_get_hipri(unsigned long afsr) +{ + unsigned long tmp = 0; + int i; + + for (i = 0; cheetah_error_table[i].mask; i++) { + if ((tmp = (afsr & cheetah_error_table[i].mask)) != 0UL) + return tmp; + } + return tmp; +} + +static const char *cheetah_get_string(unsigned long bit) +{ + int i; + + for (i = 0; cheetah_error_table[i].mask; i++) { + if ((bit & cheetah_error_table[i].mask) != 0UL) + return cheetah_error_table[i].name; + } + return "???"; +} + +static void cheetah_log_errors(struct pt_regs *regs, struct cheetah_err_info *info, + unsigned long afsr, unsigned long afar, int recoverable) +{ + unsigned long hipri; + char unum[256]; + + printk("%s" "ERROR(%d): Cheetah error trap taken afsr[%016lx] afar[%016lx] TL1(%d)\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + afsr, afar, + (afsr & CHAFSR_TL1) ? 1 : 0); + printk("%s" "ERROR(%d): TPC[%lx] TNPC[%lx] O7[%lx] TSTATE[%lx]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + regs->tpc, regs->tnpc, regs->u_regs[UREG_I7], regs->tstate); + printk("%s" "ERROR(%d): ", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id()); + printk("TPC<%pS>\n", (void *) regs->tpc); + printk("%s" "ERROR(%d): M_SYND(%lx), E_SYND(%lx)%s%s\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT, + (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT, + (afsr & CHAFSR_ME) ? ", Multiple Errors" : "", + (afsr & CHAFSR_PRIV) ? ", Privileged" : ""); + hipri = cheetah_get_hipri(afsr); + printk("%s" "ERROR(%d): Highest priority error (%016lx) \"%s\"\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + hipri, cheetah_get_string(hipri)); + + /* Try to get unumber if relevant. */ +#define ESYND_ERRORS (CHAFSR_IVC | CHAFSR_IVU | \ + CHAFSR_CPC | CHAFSR_CPU | \ + CHAFSR_UE | CHAFSR_CE | \ + CHAFSR_EDC | CHAFSR_EDU | \ + CHAFSR_UCC | CHAFSR_UCU | \ + CHAFSR_WDU | CHAFSR_WDC) +#define MSYND_ERRORS (CHAFSR_EMC | CHAFSR_EMU) + if (afsr & ESYND_ERRORS) { + int syndrome; + int ret; + + syndrome = (afsr & CHAFSR_E_SYNDROME) >> CHAFSR_E_SYNDROME_SHIFT; + syndrome = cheetah_ecc_syntab[syndrome]; + ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum)); + if (ret != -1) + printk("%s" "ERROR(%d): AFAR E-syndrome [%s]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), + smp_processor_id(), unum); + } else if (afsr & MSYND_ERRORS) { + int syndrome; + int ret; + + syndrome = (afsr & CHAFSR_M_SYNDROME) >> CHAFSR_M_SYNDROME_SHIFT; + syndrome = cheetah_mtag_syntab[syndrome]; + ret = sprintf_dimm(syndrome, afar, unum, sizeof(unum)); + if (ret != -1) + printk("%s" "ERROR(%d): AFAR M-syndrome [%s]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), + smp_processor_id(), unum); + } + + /* Now dump the cache snapshots. */ + printk("%s" "ERROR(%d): D-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + (int) info->dcache_index, + info->dcache_tag, + info->dcache_utag, + info->dcache_stag); + printk("%s" "ERROR(%d): D-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + info->dcache_data[0], + info->dcache_data[1], + info->dcache_data[2], + info->dcache_data[3]); + printk("%s" "ERROR(%d): I-cache idx[%x] tag[%016llx] utag[%016llx] stag[%016llx] " + "u[%016llx] l[%016llx]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + (int) info->icache_index, + info->icache_tag, + info->icache_utag, + info->icache_stag, + info->icache_upper, + info->icache_lower); + printk("%s" "ERROR(%d): I-cache INSN0[%016llx] INSN1[%016llx] INSN2[%016llx] INSN3[%016llx]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + info->icache_data[0], + info->icache_data[1], + info->icache_data[2], + info->icache_data[3]); + printk("%s" "ERROR(%d): I-cache INSN4[%016llx] INSN5[%016llx] INSN6[%016llx] INSN7[%016llx]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + info->icache_data[4], + info->icache_data[5], + info->icache_data[6], + info->icache_data[7]); + printk("%s" "ERROR(%d): E-cache idx[%x] tag[%016llx]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + (int) info->ecache_index, info->ecache_tag); + printk("%s" "ERROR(%d): E-cache data0[%016llx] data1[%016llx] data2[%016llx] data3[%016llx]\n", + (recoverable ? KERN_WARNING : KERN_CRIT), smp_processor_id(), + info->ecache_data[0], + info->ecache_data[1], + info->ecache_data[2], + info->ecache_data[3]); + + afsr = (afsr & ~hipri) & cheetah_afsr_errors; + while (afsr != 0UL) { + unsigned long bit = cheetah_get_hipri(afsr); + + printk("%s" "ERROR: Multiple-error (%016lx) \"%s\"\n", + (recoverable ? KERN_WARNING : KERN_CRIT), + bit, cheetah_get_string(bit)); + + afsr &= ~bit; + } + + if (!recoverable) + printk(KERN_CRIT "ERROR: This condition is not recoverable.\n"); +} + +static int cheetah_recheck_errors(struct cheetah_err_info *logp) +{ + unsigned long afsr, afar; + int ret = 0; + + __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t" + : "=r" (afsr) + : "i" (ASI_AFSR)); + if ((afsr & cheetah_afsr_errors) != 0) { + if (logp != NULL) { + __asm__ __volatile__("ldxa [%%g0] %1, %0\n\t" + : "=r" (afar) + : "i" (ASI_AFAR)); + logp->afsr = afsr; + logp->afar = afar; + } + ret = 1; + } + __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" + "membar #Sync\n\t" + : : "r" (afsr), "i" (ASI_AFSR)); + + return ret; +} + +void cheetah_fecc_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar) +{ + struct cheetah_err_info local_snapshot, *p; + int recoverable; + + /* Flush E-cache */ + cheetah_flush_ecache(); + + p = cheetah_get_error_log(afsr); + if (!p) { + prom_printf("ERROR: Early Fast-ECC error afsr[%016lx] afar[%016lx]\n", + afsr, afar); + prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n", + smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate); + prom_halt(); + } + + /* Grab snapshot of logged error. */ + memcpy(&local_snapshot, p, sizeof(local_snapshot)); + + /* If the current trap snapshot does not match what the + * trap handler passed along into our args, big trouble. + * In such a case, mark the local copy as invalid. + * + * Else, it matches and we mark the afsr in the non-local + * copy as invalid so we may log new error traps there. + */ + if (p->afsr != afsr || p->afar != afar) + local_snapshot.afsr = CHAFSR_INVALID; + else + p->afsr = CHAFSR_INVALID; + + cheetah_flush_icache(); + cheetah_flush_dcache(); + + /* Re-enable I-cache/D-cache */ + __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" + "or %%g1, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %0\n\t" + "membar #Sync" + : /* no outputs */ + : "i" (ASI_DCU_CONTROL_REG), + "i" (DCU_DC | DCU_IC) + : "g1"); + + /* Re-enable error reporting */ + __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" + "or %%g1, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %0\n\t" + "membar #Sync" + : /* no outputs */ + : "i" (ASI_ESTATE_ERROR_EN), + "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN) + : "g1"); + + /* Decide if we can continue after handling this trap and + * logging the error. + */ + recoverable = 1; + if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP)) + recoverable = 0; + + /* Re-check AFSR/AFAR. What we are looking for here is whether a new + * error was logged while we had error reporting traps disabled. + */ + if (cheetah_recheck_errors(&local_snapshot)) { + unsigned long new_afsr = local_snapshot.afsr; + + /* If we got a new asynchronous error, die... */ + if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU | + CHAFSR_WDU | CHAFSR_CPU | + CHAFSR_IVU | CHAFSR_UE | + CHAFSR_BERR | CHAFSR_TO)) + recoverable = 0; + } + + /* Log errors. */ + cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable); + + if (!recoverable) + panic("Irrecoverable Fast-ECC error trap.\n"); + + /* Flush E-cache to kick the error trap handlers out. */ + cheetah_flush_ecache(); +} + +/* Try to fix a correctable error by pushing the line out from + * the E-cache. Recheck error reporting registers to see if the + * problem is intermittent. + */ +static int cheetah_fix_ce(unsigned long physaddr) +{ + unsigned long orig_estate; + unsigned long alias1, alias2; + int ret; + + /* Make sure correctable error traps are disabled. */ + __asm__ __volatile__("ldxa [%%g0] %2, %0\n\t" + "andn %0, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %2\n\t" + "membar #Sync" + : "=&r" (orig_estate) + : "i" (ESTATE_ERROR_CEEN), + "i" (ASI_ESTATE_ERROR_EN) + : "g1"); + + /* We calculate alias addresses that will force the + * cache line in question out of the E-cache. Then + * we bring it back in with an atomic instruction so + * that we get it in some modified/exclusive state, + * then we displace it again to try and get proper ECC + * pushed back into the system. + */ + physaddr &= ~(8UL - 1UL); + alias1 = (ecache_flush_physbase + + (physaddr & ((ecache_flush_size >> 1) - 1))); + alias2 = alias1 + (ecache_flush_size >> 1); + __asm__ __volatile__("ldxa [%0] %3, %%g0\n\t" + "ldxa [%1] %3, %%g0\n\t" + "casxa [%2] %3, %%g0, %%g0\n\t" + "ldxa [%0] %3, %%g0\n\t" + "ldxa [%1] %3, %%g0\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (alias1), "r" (alias2), + "r" (physaddr), "i" (ASI_PHYS_USE_EC)); + + /* Did that trigger another error? */ + if (cheetah_recheck_errors(NULL)) { + /* Try one more time. */ + __asm__ __volatile__("ldxa [%0] %1, %%g0\n\t" + "membar #Sync" + : : "r" (physaddr), "i" (ASI_PHYS_USE_EC)); + if (cheetah_recheck_errors(NULL)) + ret = 2; + else + ret = 1; + } else { + /* No new error, intermittent problem. */ + ret = 0; + } + + /* Restore error enables. */ + __asm__ __volatile__("stxa %0, [%%g0] %1\n\t" + "membar #Sync" + : : "r" (orig_estate), "i" (ASI_ESTATE_ERROR_EN)); + + return ret; +} + +/* Return non-zero if PADDR is a valid physical memory address. */ +static int cheetah_check_main_memory(unsigned long paddr) +{ + unsigned long vaddr = PAGE_OFFSET + paddr; + + if (vaddr > (unsigned long) high_memory) + return 0; + + return kern_addr_valid(vaddr); +} + +void cheetah_cee_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar) +{ + struct cheetah_err_info local_snapshot, *p; + int recoverable, is_memory; + + p = cheetah_get_error_log(afsr); + if (!p) { + prom_printf("ERROR: Early CEE error afsr[%016lx] afar[%016lx]\n", + afsr, afar); + prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n", + smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate); + prom_halt(); + } + + /* Grab snapshot of logged error. */ + memcpy(&local_snapshot, p, sizeof(local_snapshot)); + + /* If the current trap snapshot does not match what the + * trap handler passed along into our args, big trouble. + * In such a case, mark the local copy as invalid. + * + * Else, it matches and we mark the afsr in the non-local + * copy as invalid so we may log new error traps there. + */ + if (p->afsr != afsr || p->afar != afar) + local_snapshot.afsr = CHAFSR_INVALID; + else + p->afsr = CHAFSR_INVALID; + + is_memory = cheetah_check_main_memory(afar); + + if (is_memory && (afsr & CHAFSR_CE) != 0UL) { + /* XXX Might want to log the results of this operation + * XXX somewhere... -DaveM + */ + cheetah_fix_ce(afar); + } + + { + int flush_all, flush_line; + + flush_all = flush_line = 0; + if ((afsr & CHAFSR_EDC) != 0UL) { + if ((afsr & cheetah_afsr_errors) == CHAFSR_EDC) + flush_line = 1; + else + flush_all = 1; + } else if ((afsr & CHAFSR_CPC) != 0UL) { + if ((afsr & cheetah_afsr_errors) == CHAFSR_CPC) + flush_line = 1; + else + flush_all = 1; + } + + /* Trap handler only disabled I-cache, flush it. */ + cheetah_flush_icache(); + + /* Re-enable I-cache */ + __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" + "or %%g1, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %0\n\t" + "membar #Sync" + : /* no outputs */ + : "i" (ASI_DCU_CONTROL_REG), + "i" (DCU_IC) + : "g1"); + + if (flush_all) + cheetah_flush_ecache(); + else if (flush_line) + cheetah_flush_ecache_line(afar); + } + + /* Re-enable error reporting */ + __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" + "or %%g1, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %0\n\t" + "membar #Sync" + : /* no outputs */ + : "i" (ASI_ESTATE_ERROR_EN), + "i" (ESTATE_ERROR_CEEN) + : "g1"); + + /* Decide if we can continue after handling this trap and + * logging the error. + */ + recoverable = 1; + if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP)) + recoverable = 0; + + /* Re-check AFSR/AFAR */ + (void) cheetah_recheck_errors(&local_snapshot); + + /* Log errors. */ + cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable); + + if (!recoverable) + panic("Irrecoverable Correctable-ECC error trap.\n"); +} + +void cheetah_deferred_handler(struct pt_regs *regs, unsigned long afsr, unsigned long afar) +{ + struct cheetah_err_info local_snapshot, *p; + int recoverable, is_memory; + +#ifdef CONFIG_PCI + /* Check for the special PCI poke sequence. */ + if (pci_poke_in_progress && pci_poke_cpu == smp_processor_id()) { + cheetah_flush_icache(); + cheetah_flush_dcache(); + + /* Re-enable I-cache/D-cache */ + __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" + "or %%g1, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %0\n\t" + "membar #Sync" + : /* no outputs */ + : "i" (ASI_DCU_CONTROL_REG), + "i" (DCU_DC | DCU_IC) + : "g1"); + + /* Re-enable error reporting */ + __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" + "or %%g1, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %0\n\t" + "membar #Sync" + : /* no outputs */ + : "i" (ASI_ESTATE_ERROR_EN), + "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN) + : "g1"); + + (void) cheetah_recheck_errors(NULL); + + pci_poke_faulted = 1; + regs->tpc += 4; + regs->tnpc = regs->tpc + 4; + return; + } +#endif + + p = cheetah_get_error_log(afsr); + if (!p) { + prom_printf("ERROR: Early deferred error afsr[%016lx] afar[%016lx]\n", + afsr, afar); + prom_printf("ERROR: CPU(%d) TPC[%016lx] TNPC[%016lx] TSTATE[%016lx]\n", + smp_processor_id(), regs->tpc, regs->tnpc, regs->tstate); + prom_halt(); + } + + /* Grab snapshot of logged error. */ + memcpy(&local_snapshot, p, sizeof(local_snapshot)); + + /* If the current trap snapshot does not match what the + * trap handler passed along into our args, big trouble. + * In such a case, mark the local copy as invalid. + * + * Else, it matches and we mark the afsr in the non-local + * copy as invalid so we may log new error traps there. + */ + if (p->afsr != afsr || p->afar != afar) + local_snapshot.afsr = CHAFSR_INVALID; + else + p->afsr = CHAFSR_INVALID; + + is_memory = cheetah_check_main_memory(afar); + + { + int flush_all, flush_line; + + flush_all = flush_line = 0; + if ((afsr & CHAFSR_EDU) != 0UL) { + if ((afsr & cheetah_afsr_errors) == CHAFSR_EDU) + flush_line = 1; + else + flush_all = 1; + } else if ((afsr & CHAFSR_BERR) != 0UL) { + if ((afsr & cheetah_afsr_errors) == CHAFSR_BERR) + flush_line = 1; + else + flush_all = 1; + } + + cheetah_flush_icache(); + cheetah_flush_dcache(); + + /* Re-enable I/D caches */ + __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" + "or %%g1, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %0\n\t" + "membar #Sync" + : /* no outputs */ + : "i" (ASI_DCU_CONTROL_REG), + "i" (DCU_IC | DCU_DC) + : "g1"); + + if (flush_all) + cheetah_flush_ecache(); + else if (flush_line) + cheetah_flush_ecache_line(afar); + } + + /* Re-enable error reporting */ + __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" + "or %%g1, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %0\n\t" + "membar #Sync" + : /* no outputs */ + : "i" (ASI_ESTATE_ERROR_EN), + "i" (ESTATE_ERROR_NCEEN | ESTATE_ERROR_CEEN) + : "g1"); + + /* Decide if we can continue after handling this trap and + * logging the error. + */ + recoverable = 1; + if (afsr & (CHAFSR_PERR | CHAFSR_IERR | CHAFSR_ISAP)) + recoverable = 0; + + /* Re-check AFSR/AFAR. What we are looking for here is whether a new + * error was logged while we had error reporting traps disabled. + */ + if (cheetah_recheck_errors(&local_snapshot)) { + unsigned long new_afsr = local_snapshot.afsr; + + /* If we got a new asynchronous error, die... */ + if (new_afsr & (CHAFSR_EMU | CHAFSR_EDU | + CHAFSR_WDU | CHAFSR_CPU | + CHAFSR_IVU | CHAFSR_UE | + CHAFSR_BERR | CHAFSR_TO)) + recoverable = 0; + } + + /* Log errors. */ + cheetah_log_errors(regs, &local_snapshot, afsr, afar, recoverable); + + /* "Recoverable" here means we try to yank the page from ever + * being newly used again. This depends upon a few things: + * 1) Must be main memory, and AFAR must be valid. + * 2) If we trapped from user, OK. + * 3) Else, if we trapped from kernel we must find exception + * table entry (ie. we have to have been accessing user + * space). + * + * If AFAR is not in main memory, or we trapped from kernel + * and cannot find an exception table entry, it is unacceptable + * to try and continue. + */ + if (recoverable && is_memory) { + if ((regs->tstate & TSTATE_PRIV) == 0UL) { + /* OK, usermode access. */ + recoverable = 1; + } else { + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + /* OK, kernel access to userspace. */ + recoverable = 1; + + } else { + /* BAD, privileged state is corrupted. */ + recoverable = 0; + } + + if (recoverable) { + if (pfn_valid(afar >> PAGE_SHIFT)) + get_page(pfn_to_page(afar >> PAGE_SHIFT)); + else + recoverable = 0; + + /* Only perform fixup if we still have a + * recoverable condition. + */ + if (recoverable) { + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + } + } + } + } else { + recoverable = 0; + } + + if (!recoverable) + panic("Irrecoverable deferred error trap.\n"); +} + +/* Handle a D/I cache parity error trap. TYPE is encoded as: + * + * Bit0: 0=dcache,1=icache + * Bit1: 0=recoverable,1=unrecoverable + * + * The hardware has disabled both the I-cache and D-cache in + * the %dcr register. + */ +void cheetah_plus_parity_error(int type, struct pt_regs *regs) +{ + if (type & 0x1) + __cheetah_flush_icache(); + else + cheetah_plus_zap_dcache_parity(); + cheetah_flush_dcache(); + + /* Re-enable I-cache/D-cache */ + __asm__ __volatile__("ldxa [%%g0] %0, %%g1\n\t" + "or %%g1, %1, %%g1\n\t" + "stxa %%g1, [%%g0] %0\n\t" + "membar #Sync" + : /* no outputs */ + : "i" (ASI_DCU_CONTROL_REG), + "i" (DCU_DC | DCU_IC) + : "g1"); + + if (type & 0x2) { + printk(KERN_EMERG "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n", + smp_processor_id(), + (type & 0x1) ? 'I' : 'D', + regs->tpc); + printk(KERN_EMERG "TPC<%pS>\n", (void *) regs->tpc); + panic("Irrecoverable Cheetah+ parity error."); + } + + printk(KERN_WARNING "CPU[%d]: Cheetah+ %c-cache parity error at TPC[%016lx]\n", + smp_processor_id(), + (type & 0x1) ? 'I' : 'D', + regs->tpc); + printk(KERN_WARNING "TPC<%pS>\n", (void *) regs->tpc); +} + +struct sun4v_error_entry { + /* Unique error handle */ +/*0x00*/u64 err_handle; + + /* %stick value at the time of the error */ +/*0x08*/u64 err_stick; + +/*0x10*/u8 reserved_1[3]; + + /* Error type */ +/*0x13*/u8 err_type; +#define SUN4V_ERR_TYPE_UNDEFINED 0 +#define SUN4V_ERR_TYPE_UNCORRECTED_RES 1 +#define SUN4V_ERR_TYPE_PRECISE_NONRES 2 +#define SUN4V_ERR_TYPE_DEFERRED_NONRES 3 +#define SUN4V_ERR_TYPE_SHUTDOWN_RQST 4 +#define SUN4V_ERR_TYPE_DUMP_CORE 5 +#define SUN4V_ERR_TYPE_SP_STATE_CHANGE 6 +#define SUN4V_ERR_TYPE_NUM 7 + + /* Error attributes */ +/*0x14*/u32 err_attrs; +#define SUN4V_ERR_ATTRS_PROCESSOR 0x00000001 +#define SUN4V_ERR_ATTRS_MEMORY 0x00000002 +#define SUN4V_ERR_ATTRS_PIO 0x00000004 +#define SUN4V_ERR_ATTRS_INT_REGISTERS 0x00000008 +#define SUN4V_ERR_ATTRS_FPU_REGISTERS 0x00000010 +#define SUN4V_ERR_ATTRS_SHUTDOWN_RQST 0x00000020 +#define SUN4V_ERR_ATTRS_ASR 0x00000040 +#define SUN4V_ERR_ATTRS_ASI 0x00000080 +#define SUN4V_ERR_ATTRS_PRIV_REG 0x00000100 +#define SUN4V_ERR_ATTRS_SPSTATE_MSK 0x00000600 +#define SUN4V_ERR_ATTRS_MCD 0x00000800 +#define SUN4V_ERR_ATTRS_SPSTATE_SHFT 9 +#define SUN4V_ERR_ATTRS_MODE_MSK 0x03000000 +#define SUN4V_ERR_ATTRS_MODE_SHFT 24 +#define SUN4V_ERR_ATTRS_RES_QUEUE_FULL 0x80000000 + +#define SUN4V_ERR_SPSTATE_FAULTED 0 +#define SUN4V_ERR_SPSTATE_AVAILABLE 1 +#define SUN4V_ERR_SPSTATE_NOT_PRESENT 2 + +#define SUN4V_ERR_MODE_USER 1 +#define SUN4V_ERR_MODE_PRIV 2 + + /* Real address of the memory region or PIO transaction */ +/*0x18*/u64 err_raddr; + + /* Size of the operation triggering the error, in bytes */ +/*0x20*/u32 err_size; + + /* ID of the CPU */ +/*0x24*/u16 err_cpu; + + /* Grace periof for shutdown, in seconds */ +/*0x26*/u16 err_secs; + + /* Value of the %asi register */ +/*0x28*/u8 err_asi; + +/*0x29*/u8 reserved_2; + + /* Value of the ASR register number */ +/*0x2a*/u16 err_asr; +#define SUN4V_ERR_ASR_VALID 0x8000 + +/*0x2c*/u32 reserved_3; +/*0x30*/u64 reserved_4; +/*0x38*/u64 reserved_5; +}; + +static atomic_t sun4v_resum_oflow_cnt = ATOMIC_INIT(0); +static atomic_t sun4v_nonresum_oflow_cnt = ATOMIC_INIT(0); + +static const char *sun4v_err_type_to_str(u8 type) +{ + static const char *types[SUN4V_ERR_TYPE_NUM] = { + "undefined", + "uncorrected resumable", + "precise nonresumable", + "deferred nonresumable", + "shutdown request", + "dump core", + "SP state change", + }; + + if (type < SUN4V_ERR_TYPE_NUM) + return types[type]; + + return "unknown"; +} + +static void sun4v_emit_err_attr_strings(u32 attrs) +{ + static const char *attr_names[] = { + "processor", + "memory", + "PIO", + "int-registers", + "fpu-registers", + "shutdown-request", + "ASR", + "ASI", + "priv-reg", + }; + static const char *sp_states[] = { + "sp-faulted", + "sp-available", + "sp-not-present", + "sp-state-reserved", + }; + static const char *modes[] = { + "mode-reserved0", + "user", + "priv", + "mode-reserved1", + }; + u32 sp_state, mode; + int i; + + for (i = 0; i < ARRAY_SIZE(attr_names); i++) { + if (attrs & (1U << i)) { + const char *s = attr_names[i]; + + pr_cont("%s ", s); + } + } + + sp_state = ((attrs & SUN4V_ERR_ATTRS_SPSTATE_MSK) >> + SUN4V_ERR_ATTRS_SPSTATE_SHFT); + pr_cont("%s ", sp_states[sp_state]); + + mode = ((attrs & SUN4V_ERR_ATTRS_MODE_MSK) >> + SUN4V_ERR_ATTRS_MODE_SHFT); + pr_cont("%s ", modes[mode]); + + if (attrs & SUN4V_ERR_ATTRS_RES_QUEUE_FULL) + pr_cont("res-queue-full "); +} + +/* When the report contains a real-address of "-1" it means that the + * hardware did not provide the address. So we compute the effective + * address of the load or store instruction at regs->tpc and report + * that. Usually when this happens it's a PIO and in such a case we + * are using physical addresses with bypass ASIs anyways, so what we + * report here is exactly what we want. + */ +static void sun4v_report_real_raddr(const char *pfx, struct pt_regs *regs) +{ + unsigned int insn; + u64 addr; + + if (!(regs->tstate & TSTATE_PRIV)) + return; + + insn = *(unsigned int *) regs->tpc; + + addr = compute_effective_address(regs, insn, 0); + + printk("%s: insn effective address [0x%016llx]\n", + pfx, addr); +} + +static void sun4v_log_error(struct pt_regs *regs, struct sun4v_error_entry *ent, + int cpu, const char *pfx, atomic_t *ocnt) +{ + u64 *raw_ptr = (u64 *) ent; + u32 attrs; + int cnt; + + printk("%s: Reporting on cpu %d\n", pfx, cpu); + printk("%s: TPC [0x%016lx] <%pS>\n", + pfx, regs->tpc, (void *) regs->tpc); + + printk("%s: RAW [%016llx:%016llx:%016llx:%016llx\n", + pfx, raw_ptr[0], raw_ptr[1], raw_ptr[2], raw_ptr[3]); + printk("%s: %016llx:%016llx:%016llx:%016llx]\n", + pfx, raw_ptr[4], raw_ptr[5], raw_ptr[6], raw_ptr[7]); + + printk("%s: handle [0x%016llx] stick [0x%016llx]\n", + pfx, ent->err_handle, ent->err_stick); + + printk("%s: type [%s]\n", pfx, sun4v_err_type_to_str(ent->err_type)); + + attrs = ent->err_attrs; + printk("%s: attrs [0x%08x] < ", pfx, attrs); + sun4v_emit_err_attr_strings(attrs); + pr_cont(">\n"); + + /* Various fields in the error report are only valid if + * certain attribute bits are set. + */ + if (attrs & (SUN4V_ERR_ATTRS_MEMORY | + SUN4V_ERR_ATTRS_PIO | + SUN4V_ERR_ATTRS_ASI)) { + printk("%s: raddr [0x%016llx]\n", pfx, ent->err_raddr); + + if (ent->err_raddr == ~(u64)0) + sun4v_report_real_raddr(pfx, regs); + } + + if (attrs & (SUN4V_ERR_ATTRS_MEMORY | SUN4V_ERR_ATTRS_ASI)) + printk("%s: size [0x%x]\n", pfx, ent->err_size); + + if (attrs & (SUN4V_ERR_ATTRS_PROCESSOR | + SUN4V_ERR_ATTRS_INT_REGISTERS | + SUN4V_ERR_ATTRS_FPU_REGISTERS | + SUN4V_ERR_ATTRS_PRIV_REG)) + printk("%s: cpu[%u]\n", pfx, ent->err_cpu); + + if (attrs & SUN4V_ERR_ATTRS_ASI) + printk("%s: asi [0x%02x]\n", pfx, ent->err_asi); + + if ((attrs & (SUN4V_ERR_ATTRS_INT_REGISTERS | + SUN4V_ERR_ATTRS_FPU_REGISTERS | + SUN4V_ERR_ATTRS_PRIV_REG)) && + (ent->err_asr & SUN4V_ERR_ASR_VALID) != 0) + printk("%s: reg [0x%04x]\n", + pfx, ent->err_asr & ~SUN4V_ERR_ASR_VALID); + + show_regs(regs); + + if ((cnt = atomic_read(ocnt)) != 0) { + atomic_set(ocnt, 0); + wmb(); + printk("%s: Queue overflowed %d times.\n", + pfx, cnt); + } +} + +/* Handle memory corruption detected error which is vectored in + * through resumable error trap. + */ +static void do_mcd_err(struct pt_regs *regs, struct sun4v_error_entry ent) +{ + if (notify_die(DIE_TRAP, "MCD error", regs, 0, 0x34, + SIGSEGV) == NOTIFY_STOP) + return; + + if (regs->tstate & TSTATE_PRIV) { + /* MCD exception could happen because the task was + * running a system call with MCD enabled and passed a + * non-versioned pointer or pointer with bad version + * tag to the system call. In such cases, hypervisor + * places the address of offending instruction in the + * resumable error report. This is a deferred error, + * so the read/write that caused the trap was potentially + * retired long time back and we may have no choice + * but to send SIGSEGV to the process. + */ + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + /* Looks like a bad syscall parameter */ +#ifdef DEBUG_EXCEPTIONS + pr_emerg("Exception: PC<%016lx> faddr\n", + regs->tpc); + pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n", + ent.err_raddr, entry->fixup); +#endif + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return; + } + } + + /* Send SIGSEGV to the userspace process with the right signal + * code + */ + force_sig_fault(SIGSEGV, SEGV_ADIDERR, (void __user *)ent.err_raddr); +} + +/* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate. + * Log the event and clear the first word of the entry. + */ +void sun4v_resum_error(struct pt_regs *regs, unsigned long offset) +{ + enum ctx_state prev_state = exception_enter(); + struct sun4v_error_entry *ent, local_copy; + struct trap_per_cpu *tb; + unsigned long paddr; + int cpu; + + cpu = get_cpu(); + + tb = &trap_block[cpu]; + paddr = tb->resum_kernel_buf_pa + offset; + ent = __va(paddr); + + memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry)); + + /* We have a local copy now, so release the entry. */ + ent->err_handle = 0; + wmb(); + + put_cpu(); + + if (local_copy.err_type == SUN4V_ERR_TYPE_SHUTDOWN_RQST) { + /* We should really take the seconds field of + * the error report and use it for the shutdown + * invocation, but for now do the same thing we + * do for a DS shutdown request. + */ + pr_info("Shutdown request, %u seconds...\n", + local_copy.err_secs); + orderly_poweroff(true); + goto out; + } + + /* If this is a memory corruption detected error vectored in + * by HV through resumable error trap, call the handler + */ + if (local_copy.err_attrs & SUN4V_ERR_ATTRS_MCD) { + do_mcd_err(regs, local_copy); + return; + } + + sun4v_log_error(regs, &local_copy, cpu, + KERN_ERR "RESUMABLE ERROR", + &sun4v_resum_oflow_cnt); +out: + exception_exit(prev_state); +} + +/* If we try to printk() we'll probably make matters worse, by trying + * to retake locks this cpu already holds or causing more errors. So + * just bump a counter, and we'll report these counter bumps above. + */ +void sun4v_resum_overflow(struct pt_regs *regs) +{ + atomic_inc(&sun4v_resum_oflow_cnt); +} + +/* Given a set of registers, get the virtual addressi that was being accessed + * by the faulting instructions at tpc. + */ +static unsigned long sun4v_get_vaddr(struct pt_regs *regs) +{ + unsigned int insn; + + if (!copy_from_user(&insn, (void __user *)regs->tpc, 4)) { + return compute_effective_address(regs, insn, + (insn >> 25) & 0x1f); + } + return 0; +} + +/* Attempt to handle non-resumable errors generated from userspace. + * Returns true if the signal was handled, false otherwise. + */ +static bool sun4v_nonresum_error_user_handled(struct pt_regs *regs, + struct sun4v_error_entry *ent) +{ + unsigned int attrs = ent->err_attrs; + + if (attrs & SUN4V_ERR_ATTRS_MEMORY) { + unsigned long addr = ent->err_raddr; + + if (addr == ~(u64)0) { + /* This seems highly unlikely to ever occur */ + pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory error detected in unknown location!\n"); + } else { + unsigned long page_cnt = DIV_ROUND_UP(ent->err_size, + PAGE_SIZE); + + /* Break the unfortunate news. */ + pr_emerg("SUN4V NON-RECOVERABLE ERROR: Memory failed at %016lX\n", + addr); + pr_emerg("SUN4V NON-RECOVERABLE ERROR: Claiming %lu ages.\n", + page_cnt); + + while (page_cnt-- > 0) { + if (pfn_valid(addr >> PAGE_SHIFT)) + get_page(pfn_to_page(addr >> PAGE_SHIFT)); + addr += PAGE_SIZE; + } + } + force_sig(SIGKILL); + + return true; + } + if (attrs & SUN4V_ERR_ATTRS_PIO) { + force_sig_fault(SIGBUS, BUS_ADRERR, + (void __user *)sun4v_get_vaddr(regs)); + return true; + } + + /* Default to doing nothing */ + return false; +} + +/* We run with %pil set to PIL_NORMAL_MAX and PSTATE_IE enabled in %pstate. + * Log the event, clear the first word of the entry, and die. + */ +void sun4v_nonresum_error(struct pt_regs *regs, unsigned long offset) +{ + struct sun4v_error_entry *ent, local_copy; + struct trap_per_cpu *tb; + unsigned long paddr; + int cpu; + + cpu = get_cpu(); + + tb = &trap_block[cpu]; + paddr = tb->nonresum_kernel_buf_pa + offset; + ent = __va(paddr); + + memcpy(&local_copy, ent, sizeof(struct sun4v_error_entry)); + + /* We have a local copy now, so release the entry. */ + ent->err_handle = 0; + wmb(); + + put_cpu(); + + if (!(regs->tstate & TSTATE_PRIV) && + sun4v_nonresum_error_user_handled(regs, &local_copy)) { + /* DON'T PANIC: This userspace error was handled. */ + return; + } + +#ifdef CONFIG_PCI + /* Check for the special PCI poke sequence. */ + if (pci_poke_in_progress && pci_poke_cpu == cpu) { + pci_poke_faulted = 1; + regs->tpc += 4; + regs->tnpc = regs->tpc + 4; + return; + } +#endif + + sun4v_log_error(regs, &local_copy, cpu, + KERN_EMERG "NON-RESUMABLE ERROR", + &sun4v_nonresum_oflow_cnt); + + panic("Non-resumable error."); +} + +/* If we try to printk() we'll probably make matters worse, by trying + * to retake locks this cpu already holds or causing more errors. So + * just bump a counter, and we'll report these counter bumps above. + */ +void sun4v_nonresum_overflow(struct pt_regs *regs) +{ + /* XXX Actually even this can make not that much sense. Perhaps + * XXX we should just pull the plug and panic directly from here? + */ + atomic_inc(&sun4v_nonresum_oflow_cnt); +} + +static void sun4v_tlb_error(struct pt_regs *regs) +{ + die_if_kernel("TLB/TSB error", regs); +} + +unsigned long sun4v_err_itlb_vaddr; +unsigned long sun4v_err_itlb_ctx; +unsigned long sun4v_err_itlb_pte; +unsigned long sun4v_err_itlb_error; + +void sun4v_itlb_error_report(struct pt_regs *regs, int tl) +{ + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + + printk(KERN_EMERG "SUN4V-ITLB: Error at TPC[%lx], tl %d\n", + regs->tpc, tl); + printk(KERN_EMERG "SUN4V-ITLB: TPC<%pS>\n", (void *) regs->tpc); + printk(KERN_EMERG "SUN4V-ITLB: O7[%lx]\n", regs->u_regs[UREG_I7]); + printk(KERN_EMERG "SUN4V-ITLB: O7<%pS>\n", + (void *) regs->u_regs[UREG_I7]); + printk(KERN_EMERG "SUN4V-ITLB: vaddr[%lx] ctx[%lx] " + "pte[%lx] error[%lx]\n", + sun4v_err_itlb_vaddr, sun4v_err_itlb_ctx, + sun4v_err_itlb_pte, sun4v_err_itlb_error); + + sun4v_tlb_error(regs); +} + +unsigned long sun4v_err_dtlb_vaddr; +unsigned long sun4v_err_dtlb_ctx; +unsigned long sun4v_err_dtlb_pte; +unsigned long sun4v_err_dtlb_error; + +void sun4v_dtlb_error_report(struct pt_regs *regs, int tl) +{ + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + + printk(KERN_EMERG "SUN4V-DTLB: Error at TPC[%lx], tl %d\n", + regs->tpc, tl); + printk(KERN_EMERG "SUN4V-DTLB: TPC<%pS>\n", (void *) regs->tpc); + printk(KERN_EMERG "SUN4V-DTLB: O7[%lx]\n", regs->u_regs[UREG_I7]); + printk(KERN_EMERG "SUN4V-DTLB: O7<%pS>\n", + (void *) regs->u_regs[UREG_I7]); + printk(KERN_EMERG "SUN4V-DTLB: vaddr[%lx] ctx[%lx] " + "pte[%lx] error[%lx]\n", + sun4v_err_dtlb_vaddr, sun4v_err_dtlb_ctx, + sun4v_err_dtlb_pte, sun4v_err_dtlb_error); + + sun4v_tlb_error(regs); +} + +void hypervisor_tlbop_error(unsigned long err, unsigned long op) +{ + printk(KERN_CRIT "SUN4V: TLB hv call error %lu for op %lu\n", + err, op); +} + +void hypervisor_tlbop_error_xcall(unsigned long err, unsigned long op) +{ + printk(KERN_CRIT "SUN4V: XCALL TLB hv call error %lu for op %lu\n", + err, op); +} + +static void do_fpe_common(struct pt_regs *regs) +{ + if (regs->tstate & TSTATE_PRIV) { + regs->tpc = regs->tnpc; + regs->tnpc += 4; + } else { + unsigned long fsr = current_thread_info()->xfsr[0]; + int code; + + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + code = FPE_FLTUNK; + if ((fsr & 0x1c000) == (1 << 14)) { + if (fsr & 0x10) + code = FPE_FLTINV; + else if (fsr & 0x08) + code = FPE_FLTOVF; + else if (fsr & 0x04) + code = FPE_FLTUND; + else if (fsr & 0x02) + code = FPE_FLTDIV; + else if (fsr & 0x01) + code = FPE_FLTRES; + } + force_sig_fault(SIGFPE, code, (void __user *)regs->tpc); + } +} + +void do_fpieee(struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + + if (notify_die(DIE_TRAP, "fpu exception ieee", regs, + 0, 0x24, SIGFPE) == NOTIFY_STOP) + goto out; + + do_fpe_common(regs); +out: + exception_exit(prev_state); +} + +void do_fpother(struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + struct fpustate *f = FPUSTATE; + int ret = 0; + + if (notify_die(DIE_TRAP, "fpu exception other", regs, + 0, 0x25, SIGFPE) == NOTIFY_STOP) + goto out; + + switch ((current_thread_info()->xfsr[0] & 0x1c000)) { + case (2 << 14): /* unfinished_FPop */ + case (3 << 14): /* unimplemented_FPop */ + ret = do_mathemu(regs, f, false); + break; + } + if (ret) + goto out; + do_fpe_common(regs); +out: + exception_exit(prev_state); +} + +void do_tof(struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + + if (notify_die(DIE_TRAP, "tagged arithmetic overflow", regs, + 0, 0x26, SIGEMT) == NOTIFY_STOP) + goto out; + + if (regs->tstate & TSTATE_PRIV) + die_if_kernel("Penguin overflow trap from kernel mode", regs); + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + force_sig_fault(SIGEMT, EMT_TAGOVF, (void __user *)regs->tpc); +out: + exception_exit(prev_state); +} + +void do_div0(struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + + if (notify_die(DIE_TRAP, "integer division by zero", regs, + 0, 0x28, SIGFPE) == NOTIFY_STOP) + goto out; + + if (regs->tstate & TSTATE_PRIV) + die_if_kernel("TL0: Kernel divide by zero.", regs); + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + force_sig_fault(SIGFPE, FPE_INTDIV, (void __user *)regs->tpc); +out: + exception_exit(prev_state); +} + +static void instruction_dump(unsigned int *pc) +{ + int i; + + if ((((unsigned long) pc) & 3)) + return; + + printk("Instruction DUMP:"); + for (i = -3; i < 6; i++) + printk("%c%08x%c",i?' ':'<',pc[i],i?' ':'>'); + printk("\n"); +} + +static void user_instruction_dump(unsigned int __user *pc) +{ + int i; + unsigned int buf[9]; + + if ((((unsigned long) pc) & 3)) + return; + + if (copy_from_user(buf, pc - 3, sizeof(buf))) + return; + + printk("Instruction DUMP:"); + for (i = 0; i < 9; i++) + printk("%c%08x%c",i==3?' ':'<',buf[i],i==3?' ':'>'); + printk("\n"); +} + +void show_stack(struct task_struct *tsk, unsigned long *_ksp, const char *loglvl) +{ + unsigned long fp, ksp; + struct thread_info *tp; + int count = 0; +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + int graph = 0; +#endif + + ksp = (unsigned long) _ksp; + if (!tsk) + tsk = current; + tp = task_thread_info(tsk); + if (ksp == 0UL) { + if (tsk == current) + asm("mov %%fp, %0" : "=r" (ksp)); + else + ksp = tp->ksp; + } + if (tp == current_thread_info()) + flushw_all(); + + fp = ksp + STACK_BIAS; + + printk("%sCall Trace:\n", loglvl); + do { + struct sparc_stackf *sf; + struct pt_regs *regs; + unsigned long pc; + + if (!kstack_valid(tp, fp)) + break; + sf = (struct sparc_stackf *) fp; + regs = (struct pt_regs *) (sf + 1); + + if (kstack_is_trap_frame(tp, regs)) { + if (!(regs->tstate & TSTATE_PRIV)) + break; + pc = regs->tpc; + fp = regs->u_regs[UREG_I6] + STACK_BIAS; + } else { + pc = sf->callers_pc; + fp = (unsigned long)sf->fp + STACK_BIAS; + } + + print_ip_sym(loglvl, pc); +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + if ((pc + 8UL) == (unsigned long) &return_to_handler) { + struct ftrace_ret_stack *ret_stack; + ret_stack = ftrace_graph_get_ret_stack(tsk, graph); + if (ret_stack) { + pc = ret_stack->ret; + print_ip_sym(loglvl, pc); + graph++; + } + } +#endif + } while (++count < 16); +} + +static inline struct reg_window *kernel_stack_up(struct reg_window *rw) +{ + unsigned long fp = rw->ins[6]; + + if (!fp) + return NULL; + + return (struct reg_window *) (fp + STACK_BIAS); +} + +void __noreturn die_if_kernel(char *str, struct pt_regs *regs) +{ + static int die_counter; + int count = 0; + + /* Amuse the user. */ + printk( +" \\|/ ____ \\|/\n" +" \"@'/ .. \\`@\"\n" +" /_| \\__/ |_\\\n" +" \\__U_/\n"); + + printk("%s(%d): %s [#%d]\n", current->comm, task_pid_nr(current), str, ++die_counter); + notify_die(DIE_OOPS, str, regs, 0, 255, SIGSEGV); + __asm__ __volatile__("flushw"); + show_regs(regs); + add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE); + if (regs->tstate & TSTATE_PRIV) { + struct thread_info *tp = current_thread_info(); + struct reg_window *rw = (struct reg_window *) + (regs->u_regs[UREG_FP] + STACK_BIAS); + + /* Stop the back trace when we hit userland or we + * find some badly aligned kernel stack. + */ + while (rw && + count++ < 30 && + kstack_valid(tp, (unsigned long) rw)) { + printk("Caller[%016lx]: %pS\n", rw->ins[7], + (void *) rw->ins[7]); + + rw = kernel_stack_up(rw); + } + instruction_dump ((unsigned int *) regs->tpc); + } else { + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + user_instruction_dump ((unsigned int __user *) regs->tpc); + } + if (panic_on_oops) + panic("Fatal exception"); + make_task_dead((regs->tstate & TSTATE_PRIV)? SIGKILL : SIGSEGV); +} +EXPORT_SYMBOL(die_if_kernel); + +#define VIS_OPCODE_MASK ((0x3 << 30) | (0x3f << 19)) +#define VIS_OPCODE_VAL ((0x2 << 30) | (0x36 << 19)) + +void do_illegal_instruction(struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + unsigned long pc = regs->tpc; + unsigned long tstate = regs->tstate; + u32 insn; + + if (notify_die(DIE_TRAP, "illegal instruction", regs, + 0, 0x10, SIGILL) == NOTIFY_STOP) + goto out; + + if (tstate & TSTATE_PRIV) + die_if_kernel("Kernel illegal instruction", regs); + if (test_thread_flag(TIF_32BIT)) + pc = (u32)pc; + if (get_user(insn, (u32 __user *) pc) != -EFAULT) { + if ((insn & 0xc1ffc000) == 0x81700000) /* POPC */ { + if (handle_popc(insn, regs)) + goto out; + } else if ((insn & 0xc1580000) == 0xc1100000) /* LDQ/STQ */ { + if (handle_ldf_stq(insn, regs)) + goto out; + } else if (tlb_type == hypervisor) { + if ((insn & VIS_OPCODE_MASK) == VIS_OPCODE_VAL) { + if (!vis_emul(regs, insn)) + goto out; + } else { + struct fpustate *f = FPUSTATE; + + /* On UltraSPARC T2 and later, FPU insns which + * are not implemented in HW signal an illegal + * instruction trap and do not set the FP Trap + * Trap in the %fsr to unimplemented_FPop. + */ + if (do_mathemu(regs, f, true)) + goto out; + } + } + } + force_sig_fault(SIGILL, ILL_ILLOPC, (void __user *)pc); +out: + exception_exit(prev_state); +} + +void mem_address_unaligned(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) +{ + enum ctx_state prev_state = exception_enter(); + + if (notify_die(DIE_TRAP, "memory address unaligned", regs, + 0, 0x34, SIGSEGV) == NOTIFY_STOP) + goto out; + + if (regs->tstate & TSTATE_PRIV) { + kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); + goto out; + } + if (is_no_fault_exception(regs)) + return; + + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *)sfar); +out: + exception_exit(prev_state); +} + +void sun4v_do_mna(struct pt_regs *regs, unsigned long addr, unsigned long type_ctx) +{ + if (notify_die(DIE_TRAP, "memory address unaligned", regs, + 0, 0x34, SIGSEGV) == NOTIFY_STOP) + return; + + if (regs->tstate & TSTATE_PRIV) { + kernel_unaligned_trap(regs, *((unsigned int *)regs->tpc)); + return; + } + if (is_no_fault_exception(regs)) + return; + + force_sig_fault(SIGBUS, BUS_ADRALN, (void __user *) addr); +} + +/* sun4v_mem_corrupt_detect_precise() - Handle precise exception on an ADI + * tag mismatch. + * + * ADI version tag mismatch on a load from memory always results in a + * precise exception. Tag mismatch on a store to memory will result in + * precise exception if MCDPER or PMCDPER is set to 1. + */ +void sun4v_mem_corrupt_detect_precise(struct pt_regs *regs, unsigned long addr, + unsigned long context) +{ + if (notify_die(DIE_TRAP, "memory corruption precise exception", regs, + 0, 0x8, SIGSEGV) == NOTIFY_STOP) + return; + + if (regs->tstate & TSTATE_PRIV) { + /* MCD exception could happen because the task was running + * a system call with MCD enabled and passed a non-versioned + * pointer or pointer with bad version tag to the system + * call. + */ + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + /* Looks like a bad syscall parameter */ +#ifdef DEBUG_EXCEPTIONS + pr_emerg("Exception: PC<%016lx> faddr\n", + regs->tpc); + pr_emerg("EX_TABLE: insn<%016lx> fixup<%016lx>\n", + regs->tpc, entry->fixup); +#endif + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return; + } + pr_emerg("%s: ADDR[%016lx] CTX[%lx], going.\n", + __func__, addr, context); + die_if_kernel("MCD precise", regs); + } + + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + force_sig_fault(SIGSEGV, SEGV_ADIPERR, (void __user *)addr); +} + +void do_privop(struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + + if (notify_die(DIE_TRAP, "privileged operation", regs, + 0, 0x11, SIGILL) == NOTIFY_STOP) + goto out; + + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } + force_sig_fault(SIGILL, ILL_PRVOPC, (void __user *)regs->tpc); +out: + exception_exit(prev_state); +} + +void do_privact(struct pt_regs *regs) +{ + do_privop(regs); +} + +/* Trap level 1 stuff or other traps we should never see... */ +void do_cee(struct pt_regs *regs) +{ + exception_enter(); + die_if_kernel("TL0: Cache Error Exception", regs); +} + +void do_div0_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: DIV0 Exception", regs); +} + +void do_fpieee_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: FPU IEEE Exception", regs); +} + +void do_fpother_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: FPU Other Exception", regs); +} + +void do_ill_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: Illegal Instruction Exception", regs); +} + +void do_irq_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: IRQ Exception", regs); +} + +void do_lddfmna_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: LDDF Exception", regs); +} + +void do_stdfmna_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: STDF Exception", regs); +} + +void do_paw(struct pt_regs *regs) +{ + exception_enter(); + die_if_kernel("TL0: Phys Watchpoint Exception", regs); +} + +void do_paw_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: Phys Watchpoint Exception", regs); +} + +void do_vaw(struct pt_regs *regs) +{ + exception_enter(); + die_if_kernel("TL0: Virt Watchpoint Exception", regs); +} + +void do_vaw_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: Virt Watchpoint Exception", regs); +} + +void do_tof_tl1(struct pt_regs *regs) +{ + exception_enter(); + dump_tl1_traplog((struct tl1_traplog *)(regs + 1)); + die_if_kernel("TL1: Tag Overflow Exception", regs); +} + +void do_getpsr(struct pt_regs *regs) +{ + regs->u_regs[UREG_I0] = tstate_to_psr(regs->tstate); + regs->tpc = regs->tnpc; + regs->tnpc += 4; + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } +} + +u64 cpu_mondo_counter[NR_CPUS] = {0}; +struct trap_per_cpu trap_block[NR_CPUS]; +EXPORT_SYMBOL(trap_block); + +/* This can get invoked before sched_init() so play it super safe + * and use hard_smp_processor_id(). + */ +void notrace init_cur_cpu_trap(struct thread_info *t) +{ + int cpu = hard_smp_processor_id(); + struct trap_per_cpu *p = &trap_block[cpu]; + + p->thread = t; + p->pgd_paddr = 0; +} + +extern void thread_info_offsets_are_bolixed_dave(void); +extern void trap_per_cpu_offsets_are_bolixed_dave(void); +extern void tsb_config_offsets_are_bolixed_dave(void); + +/* Only invoked on boot processor. */ +void __init trap_init(void) +{ + /* Compile time sanity check. */ + BUILD_BUG_ON(TI_TASK != offsetof(struct thread_info, task) || + TI_FLAGS != offsetof(struct thread_info, flags) || + TI_CPU != offsetof(struct thread_info, cpu) || + TI_FPSAVED != offsetof(struct thread_info, fpsaved) || + TI_KSP != offsetof(struct thread_info, ksp) || + TI_FAULT_ADDR != offsetof(struct thread_info, + fault_address) || + TI_KREGS != offsetof(struct thread_info, kregs) || + TI_UTRAPS != offsetof(struct thread_info, utraps) || + TI_REG_WINDOW != offsetof(struct thread_info, + reg_window) || + TI_RWIN_SPTRS != offsetof(struct thread_info, + rwbuf_stkptrs) || + TI_GSR != offsetof(struct thread_info, gsr) || + TI_XFSR != offsetof(struct thread_info, xfsr) || + TI_PRE_COUNT != offsetof(struct thread_info, + preempt_count) || + TI_NEW_CHILD != offsetof(struct thread_info, new_child) || + TI_KUNA_REGS != offsetof(struct thread_info, + kern_una_regs) || + TI_KUNA_INSN != offsetof(struct thread_info, + kern_una_insn) || + TI_FPREGS != offsetof(struct thread_info, fpregs) || + (TI_FPREGS & (64 - 1))); + + BUILD_BUG_ON(TRAP_PER_CPU_THREAD != offsetof(struct trap_per_cpu, + thread) || + (TRAP_PER_CPU_PGD_PADDR != + offsetof(struct trap_per_cpu, pgd_paddr)) || + (TRAP_PER_CPU_CPU_MONDO_PA != + offsetof(struct trap_per_cpu, cpu_mondo_pa)) || + (TRAP_PER_CPU_DEV_MONDO_PA != + offsetof(struct trap_per_cpu, dev_mondo_pa)) || + (TRAP_PER_CPU_RESUM_MONDO_PA != + offsetof(struct trap_per_cpu, resum_mondo_pa)) || + (TRAP_PER_CPU_RESUM_KBUF_PA != + offsetof(struct trap_per_cpu, resum_kernel_buf_pa)) || + (TRAP_PER_CPU_NONRESUM_MONDO_PA != + offsetof(struct trap_per_cpu, nonresum_mondo_pa)) || + (TRAP_PER_CPU_NONRESUM_KBUF_PA != + offsetof(struct trap_per_cpu, nonresum_kernel_buf_pa)) || + (TRAP_PER_CPU_FAULT_INFO != + offsetof(struct trap_per_cpu, fault_info)) || + (TRAP_PER_CPU_CPU_MONDO_BLOCK_PA != + offsetof(struct trap_per_cpu, cpu_mondo_block_pa)) || + (TRAP_PER_CPU_CPU_LIST_PA != + offsetof(struct trap_per_cpu, cpu_list_pa)) || + (TRAP_PER_CPU_TSB_HUGE != + offsetof(struct trap_per_cpu, tsb_huge)) || + (TRAP_PER_CPU_TSB_HUGE_TEMP != + offsetof(struct trap_per_cpu, tsb_huge_temp)) || + (TRAP_PER_CPU_IRQ_WORKLIST_PA != + offsetof(struct trap_per_cpu, irq_worklist_pa)) || + (TRAP_PER_CPU_CPU_MONDO_QMASK != + offsetof(struct trap_per_cpu, cpu_mondo_qmask)) || + (TRAP_PER_CPU_DEV_MONDO_QMASK != + offsetof(struct trap_per_cpu, dev_mondo_qmask)) || + (TRAP_PER_CPU_RESUM_QMASK != + offsetof(struct trap_per_cpu, resum_qmask)) || + (TRAP_PER_CPU_NONRESUM_QMASK != + offsetof(struct trap_per_cpu, nonresum_qmask)) || + (TRAP_PER_CPU_PER_CPU_BASE != + offsetof(struct trap_per_cpu, __per_cpu_base))); + + BUILD_BUG_ON((TSB_CONFIG_TSB != + offsetof(struct tsb_config, tsb)) || + (TSB_CONFIG_RSS_LIMIT != + offsetof(struct tsb_config, tsb_rss_limit)) || + (TSB_CONFIG_NENTRIES != + offsetof(struct tsb_config, tsb_nentries)) || + (TSB_CONFIG_REG_VAL != + offsetof(struct tsb_config, tsb_reg_val)) || + (TSB_CONFIG_MAP_VADDR != + offsetof(struct tsb_config, tsb_map_vaddr)) || + (TSB_CONFIG_MAP_PTE != + offsetof(struct tsb_config, tsb_map_pte))); + + /* Attach to the address space of init_task. On SMP we + * do this in smp.c:smp_callin for other cpus. + */ + mmgrab(&init_mm); + current->active_mm = &init_mm; +} diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S new file mode 100644 index 000000000..eaed39ce8 --- /dev/null +++ b/arch/sparc/kernel/tsb.S @@ -0,0 +1,591 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* tsb.S: Sparc64 TSB table handling. + * + * Copyright (C) 2006 David S. Miller + */ + + +#include +#include +#include +#include +#include + + .text + .align 32 + + /* Invoked from TLB miss handler, we are in the + * MMU global registers and they are setup like + * this: + * + * %g1: TSB entry pointer + * %g2: available temporary + * %g3: FAULT_CODE_{D,I}TLB + * %g4: available temporary + * %g5: available temporary + * %g6: TAG TARGET + * %g7: available temporary, will be loaded by us with + * the physical address base of the linux page + * tables for the current address space + */ +tsb_miss_dtlb: + mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_DMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 + ba,pt %xcc, tsb_miss_page_table_walk + sllx %g4, PAGE_SHIFT, %g4 + +tsb_miss_itlb: + mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_IMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 + ba,pt %xcc, tsb_miss_page_table_walk + sllx %g4, PAGE_SHIFT, %g4 + + /* At this point we have: + * %g1 -- PAGE_SIZE TSB entry address + * %g3 -- FAULT_CODE_{D,I}TLB + * %g4 -- missing virtual address + * %g6 -- TAG TARGET (vaddr >> 22) + */ +tsb_miss_page_table_walk: + TRAP_LOAD_TRAP_BLOCK(%g7, %g5) + + /* Before committing to a full page table walk, + * check the huge page TSB. + */ +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + +661: ldx [%g7 + TRAP_PER_CPU_TSB_HUGE], %g5 + nop + .section .sun4v_2insn_patch, "ax" + .word 661b + mov SCRATCHPAD_UTSBREG2, %g5 + ldxa [%g5] ASI_SCRATCHPAD, %g5 + .previous + + cmp %g5, -1 + be,pt %xcc, 80f + nop + + /* We need an aligned pair of registers containing 2 values + * which can be easily rematerialized. %g6 and %g7 foot the + * bill just nicely. We'll save %g6 away into %g2 for the + * huge page TSB TAG comparison. + * + * Perform a huge page TSB lookup. + */ + mov %g6, %g2 + and %g5, 0x7, %g6 + mov 512, %g7 + andn %g5, 0x7, %g5 + sllx %g7, %g6, %g7 + srlx %g4, REAL_HPAGE_SHIFT, %g6 + sub %g7, 1, %g7 + and %g6, %g7, %g6 + sllx %g6, 4, %g6 + add %g5, %g6, %g5 + + TSB_LOAD_QUAD(%g5, %g6) + cmp %g6, %g2 + be,a,pt %xcc, tsb_tlb_reload + mov %g7, %g5 + + /* No match, remember the huge page TSB entry address, + * and restore %g6 and %g7. + */ + TRAP_LOAD_TRAP_BLOCK(%g7, %g6) + srlx %g4, 22, %g6 +80: stx %g5, [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP] + +#endif + + ldx [%g7 + TRAP_PER_CPU_PGD_PADDR], %g7 + + /* At this point we have: + * %g1 -- TSB entry address + * %g3 -- FAULT_CODE_{D,I}TLB + * %g4 -- missing virtual address + * %g6 -- TAG TARGET (vaddr >> 22) + * %g7 -- page table physical address + * + * We know that both the base PAGE_SIZE TSB and the HPAGE_SIZE + * TSB both lack a matching entry. + */ +tsb_miss_page_table_walk_sun4v_fastpath: + USER_PGTABLE_WALK_TL1(%g4, %g7, %g5, %g2, tsb_do_fault) + + /* Valid PTE is now in %g5. */ + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + sethi %uhi(_PAGE_PMD_HUGE | _PAGE_PUD_HUGE), %g7 + sllx %g7, 32, %g7 + + andcc %g5, %g7, %g0 + be,pt %xcc, 60f + nop + + /* It is a huge page, use huge page TSB entry address we + * calculated above. If the huge page TSB has not been + * allocated, setup a trap stack and call hugetlb_setup() + * to do so, then return from the trap to replay the TLB + * miss. + * + * This is necessary to handle the case of transparent huge + * pages where we don't really have a non-atomic context + * in which to allocate the hugepage TSB hash table. When + * the 'mm' faults in the hugepage for the first time, we + * thus handle it here. This also makes sure that we can + * allocate the TSB hash table on the correct NUMA node. + */ + TRAP_LOAD_TRAP_BLOCK(%g7, %g2) + ldx [%g7 + TRAP_PER_CPU_TSB_HUGE_TEMP], %g1 + cmp %g1, -1 + bne,pt %xcc, 60f + nop + +661: rdpr %pstate, %g5 + wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate + .section .sun4v_2insn_patch, "ax" + .word 661b + SET_GL(1) + nop + .previous + + rdpr %tl, %g7 + cmp %g7, 1 + bne,pn %xcc, winfix_trampoline + mov %g3, %g4 + ba,pt %xcc, etrap + rd %pc, %g7 + call hugetlb_setup + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + +60: +#endif + + /* At this point we have: + * %g1 -- TSB entry address + * %g3 -- FAULT_CODE_{D,I}TLB + * %g5 -- valid PTE + * %g6 -- TAG TARGET (vaddr >> 22) + */ +tsb_reload: + TSB_LOCK_TAG(%g1, %g2, %g7) + TSB_WRITE(%g1, %g5, %g6) + + /* Finally, load TLB and return from trap. */ +tsb_tlb_reload: + cmp %g3, FAULT_CODE_DTLB + bne,pn %xcc, tsb_itlb_load + nop + +tsb_dtlb_load: + +661: stxa %g5, [%g0] ASI_DTLB_DATA_IN + retry + .section .sun4v_2insn_patch, "ax" + .word 661b + nop + nop + .previous + + /* For sun4v the ASI_DTLB_DATA_IN store and the retry + * instruction get nop'd out and we get here to branch + * to the sun4v tlb load code. The registers are setup + * as follows: + * + * %g4: vaddr + * %g5: PTE + * %g6: TAG + * + * The sun4v TLB load wants the PTE in %g3 so we fix that + * up here. + */ + ba,pt %xcc, sun4v_dtlb_load + mov %g5, %g3 + +tsb_itlb_load: + /* Executable bit must be set. */ +661: sethi %hi(_PAGE_EXEC_4U), %g4 + andcc %g5, %g4, %g0 + .section .sun4v_2insn_patch, "ax" + .word 661b + andcc %g5, _PAGE_EXEC_4V, %g0 + nop + .previous + + be,pn %xcc, tsb_do_fault + nop + +661: stxa %g5, [%g0] ASI_ITLB_DATA_IN + retry + .section .sun4v_2insn_patch, "ax" + .word 661b + nop + nop + .previous + + /* For sun4v the ASI_ITLB_DATA_IN store and the retry + * instruction get nop'd out and we get here to branch + * to the sun4v tlb load code. The registers are setup + * as follows: + * + * %g4: vaddr + * %g5: PTE + * %g6: TAG + * + * The sun4v TLB load wants the PTE in %g3 so we fix that + * up here. + */ + ba,pt %xcc, sun4v_itlb_load + mov %g5, %g3 + + /* No valid entry in the page tables, do full fault + * processing. + */ + + .globl tsb_do_fault +tsb_do_fault: + cmp %g3, FAULT_CODE_DTLB + +661: rdpr %pstate, %g5 + wrpr %g5, PSTATE_AG | PSTATE_MG, %pstate + .section .sun4v_2insn_patch, "ax" + .word 661b + SET_GL(1) + ldxa [%g0] ASI_SCRATCHPAD, %g4 + .previous + + bne,pn %xcc, tsb_do_itlb_fault + nop + +tsb_do_dtlb_fault: + rdpr %tl, %g3 + cmp %g3, 1 + +661: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_DMMU, %g5 + .section .sun4v_2insn_patch, "ax" + .word 661b + ldx [%g4 + HV_FAULT_D_ADDR_OFFSET], %g5 + nop + .previous + + /* Clear context ID bits. */ + srlx %g5, PAGE_SHIFT, %g5 + sllx %g5, PAGE_SHIFT, %g5 + + be,pt %xcc, sparc64_realfault_common + mov FAULT_CODE_DTLB, %g4 + ba,pt %xcc, winfix_trampoline + nop + +tsb_do_itlb_fault: + rdpr %tpc, %g5 + ba,pt %xcc, sparc64_realfault_common + mov FAULT_CODE_ITLB, %g4 + + .globl sparc64_realfault_common +sparc64_realfault_common: + /* fault code in %g4, fault address in %g5, etrap will + * preserve these two values in %l4 and %l5 respectively + */ + ba,pt %xcc, etrap ! Save trap state +1: rd %pc, %g7 ! ... + stb %l4, [%g6 + TI_FAULT_CODE] ! Save fault code + stx %l5, [%g6 + TI_FAULT_ADDR] ! Save fault address + call do_sparc64_fault ! Call fault handler + add %sp, PTREGS_OFF, %o0 ! Compute pt_regs arg + ba,pt %xcc, rtrap ! Restore cpu state + nop ! Delay slot (fill me) + +winfix_trampoline: + rdpr %tpc, %g3 ! Prepare winfixup TNPC + or %g3, 0x7c, %g3 ! Compute branch offset + wrpr %g3, %tnpc ! Write it into TNPC + done ! Trap return + + /* Insert an entry into the TSB. + * + * %o0: TSB entry pointer (virt or phys address) + * %o1: tag + * %o2: pte + */ + .align 32 + .globl __tsb_insert +__tsb_insert: + rdpr %pstate, %o5 + wrpr %o5, PSTATE_IE, %pstate + TSB_LOCK_TAG(%o0, %g2, %g3) + TSB_WRITE(%o0, %o2, %o1) + wrpr %o5, %pstate + retl + nop + .size __tsb_insert, .-__tsb_insert + + /* Flush the given TSB entry if it has the matching + * tag. + * + * %o0: TSB entry pointer (virt or phys address) + * %o1: tag + */ + .align 32 + .globl tsb_flush + .type tsb_flush,#function +tsb_flush: + sethi %hi(TSB_TAG_LOCK_HIGH), %g2 +1: TSB_LOAD_TAG(%o0, %g1) + srlx %g1, 32, %o3 + andcc %o3, %g2, %g0 + bne,pn %icc, 1b + nop + cmp %g1, %o1 + mov 1, %o3 + bne,pt %xcc, 2f + sllx %o3, TSB_TAG_INVALID_BIT, %o3 + TSB_CAS_TAG(%o0, %g1, %o3) + cmp %g1, %o3 + bne,pn %xcc, 1b + nop +2: retl + nop + .size tsb_flush, .-tsb_flush + + /* Reload MMU related context switch state at + * schedule() time. + * + * %o0: page table physical address + * %o1: TSB base config pointer + * %o2: TSB huge config pointer, or NULL if none + * %o3: Hypervisor TSB descriptor physical address + * %o4: Secondary context to load, if non-zero + * + * We have to run this whole thing with interrupts + * disabled so that the current cpu doesn't change + * due to preemption. + */ + .align 32 + .globl __tsb_context_switch + .type __tsb_context_switch,#function +__tsb_context_switch: + rdpr %pstate, %g1 + wrpr %g1, PSTATE_IE, %pstate + + brz,pn %o4, 1f + mov SECONDARY_CONTEXT, %o5 + +661: stxa %o4, [%o5] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %o4, [%o5] ASI_MMU + .previous + flush %g6 + +1: + TRAP_LOAD_TRAP_BLOCK(%g2, %g3) + + stx %o0, [%g2 + TRAP_PER_CPU_PGD_PADDR] + + ldx [%o1 + TSB_CONFIG_REG_VAL], %o0 + brz,pt %o2, 1f + mov -1, %g3 + + ldx [%o2 + TSB_CONFIG_REG_VAL], %g3 + +1: stx %g3, [%g2 + TRAP_PER_CPU_TSB_HUGE] + + sethi %hi(tlb_type), %g2 + lduw [%g2 + %lo(tlb_type)], %g2 + cmp %g2, 3 + bne,pt %icc, 50f + nop + + /* Hypervisor TSB switch. */ + mov SCRATCHPAD_UTSBREG1, %o5 + stxa %o0, [%o5] ASI_SCRATCHPAD + mov SCRATCHPAD_UTSBREG2, %o5 + stxa %g3, [%o5] ASI_SCRATCHPAD + + mov 2, %o0 + cmp %g3, -1 + move %xcc, 1, %o0 + + mov HV_FAST_MMU_TSB_CTXNON0, %o5 + mov %o3, %o1 + ta HV_FAST_TRAP + + /* Finish up. */ + ba,pt %xcc, 9f + nop + + /* SUN4U TSB switch. */ +50: mov TSB_REG, %o5 + stxa %o0, [%o5] ASI_DMMU + membar #Sync + stxa %o0, [%o5] ASI_IMMU + membar #Sync + +2: ldx [%o1 + TSB_CONFIG_MAP_VADDR], %o4 + brz %o4, 9f + ldx [%o1 + TSB_CONFIG_MAP_PTE], %o5 + + sethi %hi(sparc64_highest_unlocked_tlb_ent), %g2 + mov TLB_TAG_ACCESS, %g3 + lduw [%g2 + %lo(sparc64_highest_unlocked_tlb_ent)], %g2 + stxa %o4, [%g3] ASI_DMMU + membar #Sync + sllx %g2, 3, %g2 + stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS + membar #Sync + + brz,pt %o2, 9f + nop + + ldx [%o2 + TSB_CONFIG_MAP_VADDR], %o4 + ldx [%o2 + TSB_CONFIG_MAP_PTE], %o5 + mov TLB_TAG_ACCESS, %g3 + stxa %o4, [%g3] ASI_DMMU + membar #Sync + sub %g2, (1 << 3), %g2 + stxa %o5, [%g2] ASI_DTLB_DATA_ACCESS + membar #Sync + +9: + wrpr %g1, %pstate + + retl + nop + .size __tsb_context_switch, .-__tsb_context_switch + +#define TSB_PASS_BITS ((1 << TSB_TAG_LOCK_BIT) | \ + (1 << TSB_TAG_INVALID_BIT)) + + .align 32 + .globl copy_tsb + .type copy_tsb,#function +copy_tsb: /* %o0=old_tsb_base, %o1=old_tsb_size + * %o2=new_tsb_base, %o3=new_tsb_size + * %o4=page_size_shift + */ + sethi %uhi(TSB_PASS_BITS), %g7 + srlx %o3, 4, %o3 + add %o0, %o1, %o1 /* end of old tsb */ + sllx %g7, 32, %g7 + sub %o3, 1, %o3 /* %o3 == new tsb hash mask */ + + mov %o4, %g1 /* page_size_shift */ + +661: prefetcha [%o0] ASI_N, #one_read + .section .tsb_phys_patch, "ax" + .word 661b + prefetcha [%o0] ASI_PHYS_USE_EC, #one_read + .previous + +90: andcc %o0, (64 - 1), %g0 + bne 1f + add %o0, 64, %o5 + +661: prefetcha [%o5] ASI_N, #one_read + .section .tsb_phys_patch, "ax" + .word 661b + prefetcha [%o5] ASI_PHYS_USE_EC, #one_read + .previous + +1: TSB_LOAD_QUAD(%o0, %g2) /* %g2/%g3 == TSB entry */ + andcc %g2, %g7, %g0 /* LOCK or INVALID set? */ + bne,pn %xcc, 80f /* Skip it */ + sllx %g2, 22, %o4 /* TAG --> VADDR */ + + /* This can definitely be computed faster... */ + srlx %o0, 4, %o5 /* Build index */ + and %o5, 511, %o5 /* Mask index */ + sllx %o5, %g1, %o5 /* Put into vaddr position */ + or %o4, %o5, %o4 /* Full VADDR. */ + srlx %o4, %g1, %o4 /* Shift down to create index */ + and %o4, %o3, %o4 /* Mask with new_tsb_nents-1 */ + sllx %o4, 4, %o4 /* Shift back up into tsb ent offset */ + TSB_STORE(%o2 + %o4, %g2) /* Store TAG */ + add %o4, 0x8, %o4 /* Advance to TTE */ + TSB_STORE(%o2 + %o4, %g3) /* Store TTE */ + +80: add %o0, 16, %o0 + cmp %o0, %o1 + bne,pt %xcc, 90b + nop + + retl + nop + .size copy_tsb, .-copy_tsb + + /* Set the invalid bit in all TSB entries. */ + .align 32 + .globl tsb_init + .type tsb_init,#function +tsb_init: /* %o0 = TSB vaddr, %o1 = size in bytes */ + prefetch [%o0 + 0x000], #n_writes + mov 1, %g1 + prefetch [%o0 + 0x040], #n_writes + sllx %g1, TSB_TAG_INVALID_BIT, %g1 + prefetch [%o0 + 0x080], #n_writes +1: prefetch [%o0 + 0x0c0], #n_writes + stx %g1, [%o0 + 0x00] + stx %g1, [%o0 + 0x10] + stx %g1, [%o0 + 0x20] + stx %g1, [%o0 + 0x30] + prefetch [%o0 + 0x100], #n_writes + stx %g1, [%o0 + 0x40] + stx %g1, [%o0 + 0x50] + stx %g1, [%o0 + 0x60] + stx %g1, [%o0 + 0x70] + prefetch [%o0 + 0x140], #n_writes + stx %g1, [%o0 + 0x80] + stx %g1, [%o0 + 0x90] + stx %g1, [%o0 + 0xa0] + stx %g1, [%o0 + 0xb0] + prefetch [%o0 + 0x180], #n_writes + stx %g1, [%o0 + 0xc0] + stx %g1, [%o0 + 0xd0] + stx %g1, [%o0 + 0xe0] + stx %g1, [%o0 + 0xf0] + subcc %o1, 0x100, %o1 + bne,pt %xcc, 1b + add %o0, 0x100, %o0 + retl + nop + nop + nop + .size tsb_init, .-tsb_init + + .globl NGtsb_init + .type NGtsb_init,#function +NGtsb_init: + rd %asi, %g2 + mov 1, %g1 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + sllx %g1, TSB_TAG_INVALID_BIT, %g1 +1: stxa %g1, [%o0 + 0x00] %asi + stxa %g1, [%o0 + 0x10] %asi + stxa %g1, [%o0 + 0x20] %asi + stxa %g1, [%o0 + 0x30] %asi + stxa %g1, [%o0 + 0x40] %asi + stxa %g1, [%o0 + 0x50] %asi + stxa %g1, [%o0 + 0x60] %asi + stxa %g1, [%o0 + 0x70] %asi + stxa %g1, [%o0 + 0x80] %asi + stxa %g1, [%o0 + 0x90] %asi + stxa %g1, [%o0 + 0xa0] %asi + stxa %g1, [%o0 + 0xb0] %asi + stxa %g1, [%o0 + 0xc0] %asi + stxa %g1, [%o0 + 0xd0] %asi + stxa %g1, [%o0 + 0xe0] %asi + stxa %g1, [%o0 + 0xf0] %asi + subcc %o1, 0x100, %o1 + bne,pt %xcc, 1b + add %o0, 0x100, %o0 + membar #Sync + retl + wr %g2, 0x0, %asi + .size NGtsb_init, .-NGtsb_init diff --git a/arch/sparc/kernel/ttable_32.S b/arch/sparc/kernel/ttable_32.S new file mode 100644 index 000000000..e79fd786f --- /dev/null +++ b/arch/sparc/kernel/ttable_32.S @@ -0,0 +1,418 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* The Sparc trap table, bootloader gives us control at _start. */ + __HEAD + + .globl _start +_start: + + .globl _stext +_stext: + + .globl trapbase +trapbase: + +#ifdef CONFIG_SMP +trapbase_cpu0: +#endif +/* We get control passed to us here at t_zero. */ +t_zero: b gokernel; nop; nop; nop; +t_tflt: SRMMU_TFAULT /* Inst. Access Exception */ +t_bins: TRAP_ENTRY(0x2, bad_instruction) /* Illegal Instruction */ +t_pins: TRAP_ENTRY(0x3, priv_instruction) /* Privileged Instruction */ +t_fpd: TRAP_ENTRY(0x4, fpd_trap_handler) /* Floating Point Disabled */ +t_wovf: WINDOW_SPILL /* Window Overflow */ +t_wunf: WINDOW_FILL /* Window Underflow */ +t_mna: TRAP_ENTRY(0x7, mna_handler) /* Memory Address Not Aligned */ +t_fpe: TRAP_ENTRY(0x8, fpe_trap_handler) /* Floating Point Exception */ +t_dflt: SRMMU_DFAULT /* Data Miss Exception */ +t_tio: TRAP_ENTRY(0xa, do_tag_overflow) /* Tagged Instruction Ovrflw */ +t_wpt: TRAP_ENTRY(0xb, do_watchpoint) /* Watchpoint Detected */ +t_badc: BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) +t_irq1: TRAP_ENTRY_INTERRUPT(1) /* IRQ Software/SBUS Level 1 */ +t_irq2: TRAP_ENTRY_INTERRUPT(2) /* IRQ SBUS Level 2 */ +t_irq3: TRAP_ENTRY_INTERRUPT(3) /* IRQ SCSI/DMA/SBUS Level 3 */ +t_irq4: TRAP_ENTRY_INTERRUPT(4) /* IRQ Software Level 4 */ +t_irq5: TRAP_ENTRY_INTERRUPT(5) /* IRQ SBUS/Ethernet Level 5 */ +t_irq6: TRAP_ENTRY_INTERRUPT(6) /* IRQ Software Level 6 */ +t_irq7: TRAP_ENTRY_INTERRUPT(7) /* IRQ Video/SBUS Level 5 */ +t_irq8: TRAP_ENTRY_INTERRUPT(8) /* IRQ SBUS Level 6 */ +t_irq9: TRAP_ENTRY_INTERRUPT(9) /* IRQ SBUS Level 7 */ +t_irq10:TRAP_ENTRY_INTERRUPT(10) /* IRQ Timer #1 (one we use) */ +t_irq11:TRAP_ENTRY_INTERRUPT(11) /* IRQ Floppy Intr. */ +t_irq12:TRAP_ENTRY_INTERRUPT(12) /* IRQ Zilog serial chip */ +t_irq13:TRAP_ENTRY_INTERRUPT(13) /* IRQ Audio Intr. */ +t_irq14:TRAP_ENTRY_INTERRUPT(14) /* IRQ Timer #2 */ + + .globl t_nmi +t_nmi: TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) + +t_racc: TRAP_ENTRY(0x20, do_reg_access) /* General Register Access Error */ +t_iacce:BAD_TRAP(0x21) /* Instr Access Error */ +t_bad22:BAD_TRAP(0x22) + BAD_TRAP(0x23) +t_cpdis:TRAP_ENTRY(0x24, do_cp_disabled) /* Co-Processor Disabled */ +t_uflsh:SKIP_TRAP(0x25, unimp_flush) /* Unimplemented FLUSH inst. */ +t_bad26:BAD_TRAP(0x26) BAD_TRAP(0x27) +t_cpexc:TRAP_ENTRY(0x28, do_cp_exception) /* Co-Processor Exception */ +t_dacce:SRMMU_DFAULT /* Data Access Error */ +t_hwdz: TRAP_ENTRY(0x2a, do_hw_divzero) /* Division by zero, you lose... */ +t_dserr:BAD_TRAP(0x2b) /* Data Store Error */ +t_daccm:BAD_TRAP(0x2c) /* Data Access MMU-Miss */ +t_bad2d:BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) +t_bad32:BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) +t_bad37:BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) +t_iaccm:BAD_TRAP(0x3c) /* Instr Access MMU-Miss */ +t_bad3d:BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) BAD_TRAP(0x41) +t_bad42:BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) BAD_TRAP(0x46) +t_bad47:BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) BAD_TRAP(0x4b) +t_bad4c:BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) BAD_TRAP(0x50) +t_bad51:BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) +t_bad56:BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) +t_bad5b:BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) +t_bad60:BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) +t_bad65:BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) +t_bad6a:BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) +t_bad6f:BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) +t_bad74:BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) +t_bad79:BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) +t_bad7e:BAD_TRAP(0x7e) BAD_TRAP(0x7f) +t_bad80:BAD_TRAP(0x80) /* SunOS System Call */ +t_sbkpt:BREAKPOINT_TRAP /* Software Breakpoint/KGDB */ +t_divz: TRAP_ENTRY(0x82, do_hw_divzero) /* Divide by zero trap */ +t_flwin:TRAP_ENTRY(0x83, do_flush_windows) /* Flush Windows Trap */ +t_clwin:BAD_TRAP(0x84) /* Clean Windows Trap */ +t_rchk: BAD_TRAP(0x85) /* Range Check */ +t_funal:BAD_TRAP(0x86) /* Fix Unaligned Access Trap */ +t_iovf: BAD_TRAP(0x87) /* Integer Overflow Trap */ +t_bad88:BAD_TRAP(0x88) /* Slowaris System Call */ +t_bad89:BAD_TRAP(0x89) /* Net-B.S. System Call */ +t_bad8a:BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) BAD_TRAP(0x8d) BAD_TRAP(0x8e) +t_bad8f:BAD_TRAP(0x8f) +t_linux:LINUX_SYSCALL_TRAP /* Linux System Call */ +t_bad91:BAD_TRAP(0x91) BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) BAD_TRAP(0x95) +t_bad96:BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) BAD_TRAP(0x9a) +t_bad9b:BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) BAD_TRAP(0x9f) +t_getcc:GETCC_TRAP /* Get Condition Codes */ +t_setcc:SETCC_TRAP /* Set Condition Codes */ +t_getpsr:GETPSR_TRAP /* Get PSR Register */ +t_bada3:BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) +t_bada7:BAD_TRAP(0xa7) +t_bada8:BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) +t_badac:BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) +t_badb1:BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) +t_badb6:BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) +t_badbb:BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) +t_badc0:BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) +t_badc5:BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) +t_badca:BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) +t_badcf:BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) +t_badd4:BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) +t_badd9:BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) +t_badde:BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) +t_bade3:BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) +t_bade8:BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) +t_baded:BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) +t_badf2:BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) +t_badf7:BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) +t_badfc:BAD_TRAP(0xfc) +t_kgdb: KGDB_TRAP(0xfd) +dbtrap: BAD_TRAP(0xfe) /* Debugger/PROM breakpoint #1 */ +dbtrap2:BAD_TRAP(0xff) /* Debugger/PROM breakpoint #2 */ + + .globl end_traptable +end_traptable: + +#ifdef CONFIG_SMP + /* Trap tables for the other cpus. */ + .globl trapbase_cpu1, trapbase_cpu2, trapbase_cpu3 +trapbase_cpu1: + BAD_TRAP(0x0) + SRMMU_TFAULT + TRAP_ENTRY(0x2, bad_instruction) + TRAP_ENTRY(0x3, priv_instruction) + TRAP_ENTRY(0x4, fpd_trap_handler) + WINDOW_SPILL + WINDOW_FILL + TRAP_ENTRY(0x7, mna_handler) + TRAP_ENTRY(0x8, fpe_trap_handler) + SRMMU_DFAULT + TRAP_ENTRY(0xa, do_tag_overflow) + TRAP_ENTRY(0xb, do_watchpoint) + BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) + TRAP_ENTRY_INTERRUPT(1) TRAP_ENTRY_INTERRUPT(2) + TRAP_ENTRY_INTERRUPT(3) TRAP_ENTRY_INTERRUPT(4) + TRAP_ENTRY_INTERRUPT(5) TRAP_ENTRY_INTERRUPT(6) + TRAP_ENTRY_INTERRUPT(7) TRAP_ENTRY_INTERRUPT(8) + TRAP_ENTRY_INTERRUPT(9) TRAP_ENTRY_INTERRUPT(10) + TRAP_ENTRY_INTERRUPT(11) TRAP_ENTRY_INTERRUPT(12) + TRAP_ENTRY_INTERRUPT(13) TRAP_ENTRY_INTERRUPT(14) + TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) + TRAP_ENTRY(0x20, do_reg_access) + BAD_TRAP(0x21) + BAD_TRAP(0x22) + BAD_TRAP(0x23) + TRAP_ENTRY(0x24, do_cp_disabled) + SKIP_TRAP(0x25, unimp_flush) + BAD_TRAP(0x26) + BAD_TRAP(0x27) + TRAP_ENTRY(0x28, do_cp_exception) + SRMMU_DFAULT + TRAP_ENTRY(0x2a, do_hw_divzero) + BAD_TRAP(0x2b) + BAD_TRAP(0x2c) + BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) + BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) + BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) + BAD_TRAP(0x3c) BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) + BAD_TRAP(0x41) BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) + BAD_TRAP(0x46) BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) + BAD_TRAP(0x4b) BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) + BAD_TRAP(0x50) + BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) + BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) + BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) + BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) + BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) + BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) + BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) + BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) + BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) + BAD_TRAP(0x7e) BAD_TRAP(0x7f) + BAD_TRAP(0x80) + BREAKPOINT_TRAP + TRAP_ENTRY(0x82, do_hw_divzero) + TRAP_ENTRY(0x83, do_flush_windows) + BAD_TRAP(0x84) BAD_TRAP(0x85) BAD_TRAP(0x86) + BAD_TRAP(0x87) BAD_TRAP(0x88) BAD_TRAP(0x89) + BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) + BAD_TRAP(0x8d) BAD_TRAP(0x8e) BAD_TRAP(0x8f) + LINUX_SYSCALL_TRAP BAD_TRAP(0x91) + BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) + BAD_TRAP(0x95) BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) + BAD_TRAP(0x9a) BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) + BAD_TRAP(0x9f) + GETCC_TRAP + SETCC_TRAP + GETPSR_TRAP + BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) + BAD_TRAP(0xa7) BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) + BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) + BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) + BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) + BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) + BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) + BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) + BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) + BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) + BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) + BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) + BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) + BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) + BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) + BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) + BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) + BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) + BAD_TRAP(0xfc) + KGDB_TRAP(0xfd) + BAD_TRAP(0xfe) + BAD_TRAP(0xff) + +trapbase_cpu2: + BAD_TRAP(0x0) + SRMMU_TFAULT + TRAP_ENTRY(0x2, bad_instruction) + TRAP_ENTRY(0x3, priv_instruction) + TRAP_ENTRY(0x4, fpd_trap_handler) + WINDOW_SPILL + WINDOW_FILL + TRAP_ENTRY(0x7, mna_handler) + TRAP_ENTRY(0x8, fpe_trap_handler) + SRMMU_DFAULT + TRAP_ENTRY(0xa, do_tag_overflow) + TRAP_ENTRY(0xb, do_watchpoint) + BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) + TRAP_ENTRY_INTERRUPT(1) + TRAP_ENTRY_INTERRUPT(2) + TRAP_ENTRY_INTERRUPT(3) + TRAP_ENTRY_INTERRUPT(4) + TRAP_ENTRY_INTERRUPT(5) + TRAP_ENTRY_INTERRUPT(6) + TRAP_ENTRY_INTERRUPT(7) + TRAP_ENTRY_INTERRUPT(8) + TRAP_ENTRY_INTERRUPT(9) + TRAP_ENTRY_INTERRUPT(10) + TRAP_ENTRY_INTERRUPT(11) + TRAP_ENTRY_INTERRUPT(12) + TRAP_ENTRY_INTERRUPT(13) + TRAP_ENTRY_INTERRUPT(14) + TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) + TRAP_ENTRY(0x20, do_reg_access) + BAD_TRAP(0x21) + BAD_TRAP(0x22) + BAD_TRAP(0x23) + TRAP_ENTRY(0x24, do_cp_disabled) + SKIP_TRAP(0x25, unimp_flush) + BAD_TRAP(0x26) + BAD_TRAP(0x27) + TRAP_ENTRY(0x28, do_cp_exception) + SRMMU_DFAULT + TRAP_ENTRY(0x2a, do_hw_divzero) + BAD_TRAP(0x2b) + BAD_TRAP(0x2c) + BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) + BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) + BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) + BAD_TRAP(0x3c) BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) + BAD_TRAP(0x41) BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) + BAD_TRAP(0x46) BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) + BAD_TRAP(0x4b) BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) + BAD_TRAP(0x50) + BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) + BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) + BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) + BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) + BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) + BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) + BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) + BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) + BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) + BAD_TRAP(0x7e) BAD_TRAP(0x7f) + BAD_TRAP(0x80) + BREAKPOINT_TRAP + TRAP_ENTRY(0x82, do_hw_divzero) + TRAP_ENTRY(0x83, do_flush_windows) + BAD_TRAP(0x84) + BAD_TRAP(0x85) + BAD_TRAP(0x86) BAD_TRAP(0x87) BAD_TRAP(0x88) + BAD_TRAP(0x89) BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) + BAD_TRAP(0x8d) BAD_TRAP(0x8e) BAD_TRAP(0x8f) + LINUX_SYSCALL_TRAP BAD_TRAP(0x91) + BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) + BAD_TRAP(0x95) BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) + BAD_TRAP(0x9a) BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) + BAD_TRAP(0x9f) + GETCC_TRAP + SETCC_TRAP + GETPSR_TRAP + BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) + BAD_TRAP(0xa7) BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) + BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) + BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) + BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) + BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) + BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) + BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) + BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) + BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) + BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) + BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) + BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) + BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) + BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) + BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) + BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) + BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) + BAD_TRAP(0xfc) + KGDB_TRAP(0xfd) + BAD_TRAP(0xfe) + BAD_TRAP(0xff) + +trapbase_cpu3: + BAD_TRAP(0x0) + SRMMU_TFAULT + TRAP_ENTRY(0x2, bad_instruction) + TRAP_ENTRY(0x3, priv_instruction) + TRAP_ENTRY(0x4, fpd_trap_handler) + WINDOW_SPILL + WINDOW_FILL + TRAP_ENTRY(0x7, mna_handler) + TRAP_ENTRY(0x8, fpe_trap_handler) + SRMMU_DFAULT + TRAP_ENTRY(0xa, do_tag_overflow) + TRAP_ENTRY(0xb, do_watchpoint) + BAD_TRAP(0xc) BAD_TRAP(0xd) BAD_TRAP(0xe) BAD_TRAP(0xf) BAD_TRAP(0x10) + TRAP_ENTRY_INTERRUPT(1) + TRAP_ENTRY_INTERRUPT(2) + TRAP_ENTRY_INTERRUPT(3) + TRAP_ENTRY_INTERRUPT(4) + TRAP_ENTRY_INTERRUPT(5) + TRAP_ENTRY_INTERRUPT(6) + TRAP_ENTRY_INTERRUPT(7) + TRAP_ENTRY_INTERRUPT(8) + TRAP_ENTRY_INTERRUPT(9) + TRAP_ENTRY_INTERRUPT(10) + TRAP_ENTRY_INTERRUPT(11) + TRAP_ENTRY_INTERRUPT(12) + TRAP_ENTRY_INTERRUPT(13) + TRAP_ENTRY_INTERRUPT(14) + TRAP_ENTRY(0x1f, linux_trap_ipi15_sun4m) + TRAP_ENTRY(0x20, do_reg_access) + BAD_TRAP(0x21) + BAD_TRAP(0x22) + BAD_TRAP(0x23) + TRAP_ENTRY(0x24, do_cp_disabled) + SKIP_TRAP(0x25, unimp_flush) + BAD_TRAP(0x26) + BAD_TRAP(0x27) + TRAP_ENTRY(0x28, do_cp_exception) + SRMMU_DFAULT + TRAP_ENTRY(0x2a, do_hw_divzero) + BAD_TRAP(0x2b) BAD_TRAP(0x2c) + BAD_TRAP(0x2d) BAD_TRAP(0x2e) BAD_TRAP(0x2f) BAD_TRAP(0x30) BAD_TRAP(0x31) + BAD_TRAP(0x32) BAD_TRAP(0x33) BAD_TRAP(0x34) BAD_TRAP(0x35) BAD_TRAP(0x36) + BAD_TRAP(0x37) BAD_TRAP(0x38) BAD_TRAP(0x39) BAD_TRAP(0x3a) BAD_TRAP(0x3b) + BAD_TRAP(0x3c) BAD_TRAP(0x3d) BAD_TRAP(0x3e) BAD_TRAP(0x3f) BAD_TRAP(0x40) + BAD_TRAP(0x41) BAD_TRAP(0x42) BAD_TRAP(0x43) BAD_TRAP(0x44) BAD_TRAP(0x45) + BAD_TRAP(0x46) BAD_TRAP(0x47) BAD_TRAP(0x48) BAD_TRAP(0x49) BAD_TRAP(0x4a) + BAD_TRAP(0x4b) BAD_TRAP(0x4c) BAD_TRAP(0x4d) BAD_TRAP(0x4e) BAD_TRAP(0x4f) + BAD_TRAP(0x50) + BAD_TRAP(0x51) BAD_TRAP(0x52) BAD_TRAP(0x53) BAD_TRAP(0x54) BAD_TRAP(0x55) + BAD_TRAP(0x56) BAD_TRAP(0x57) BAD_TRAP(0x58) BAD_TRAP(0x59) BAD_TRAP(0x5a) + BAD_TRAP(0x5b) BAD_TRAP(0x5c) BAD_TRAP(0x5d) BAD_TRAP(0x5e) BAD_TRAP(0x5f) + BAD_TRAP(0x60) BAD_TRAP(0x61) BAD_TRAP(0x62) BAD_TRAP(0x63) BAD_TRAP(0x64) + BAD_TRAP(0x65) BAD_TRAP(0x66) BAD_TRAP(0x67) BAD_TRAP(0x68) BAD_TRAP(0x69) + BAD_TRAP(0x6a) BAD_TRAP(0x6b) BAD_TRAP(0x6c) BAD_TRAP(0x6d) BAD_TRAP(0x6e) + BAD_TRAP(0x6f) BAD_TRAP(0x70) BAD_TRAP(0x71) BAD_TRAP(0x72) BAD_TRAP(0x73) + BAD_TRAP(0x74) BAD_TRAP(0x75) BAD_TRAP(0x76) BAD_TRAP(0x77) BAD_TRAP(0x78) + BAD_TRAP(0x79) BAD_TRAP(0x7a) BAD_TRAP(0x7b) BAD_TRAP(0x7c) BAD_TRAP(0x7d) + BAD_TRAP(0x7e) BAD_TRAP(0x7f) + BAD_TRAP(0x80) + BREAKPOINT_TRAP + TRAP_ENTRY(0x82, do_hw_divzero) + TRAP_ENTRY(0x83, do_flush_windows) + BAD_TRAP(0x84) BAD_TRAP(0x85) + BAD_TRAP(0x86) BAD_TRAP(0x87) BAD_TRAP(0x88) + BAD_TRAP(0x89) BAD_TRAP(0x8a) BAD_TRAP(0x8b) BAD_TRAP(0x8c) + BAD_TRAP(0x8d) BAD_TRAP(0x8e) BAD_TRAP(0x8f) + LINUX_SYSCALL_TRAP + BAD_TRAP(0x91) BAD_TRAP(0x92) BAD_TRAP(0x93) BAD_TRAP(0x94) + BAD_TRAP(0x95) BAD_TRAP(0x96) BAD_TRAP(0x97) BAD_TRAP(0x98) BAD_TRAP(0x99) + BAD_TRAP(0x9a) BAD_TRAP(0x9b) BAD_TRAP(0x9c) BAD_TRAP(0x9d) BAD_TRAP(0x9e) + BAD_TRAP(0x9f) + GETCC_TRAP + SETCC_TRAP + GETPSR_TRAP + BAD_TRAP(0xa3) BAD_TRAP(0xa4) BAD_TRAP(0xa5) BAD_TRAP(0xa6) + BAD_TRAP(0xa7) BAD_TRAP(0xa8) BAD_TRAP(0xa9) BAD_TRAP(0xaa) BAD_TRAP(0xab) + BAD_TRAP(0xac) BAD_TRAP(0xad) BAD_TRAP(0xae) BAD_TRAP(0xaf) BAD_TRAP(0xb0) + BAD_TRAP(0xb1) BAD_TRAP(0xb2) BAD_TRAP(0xb3) BAD_TRAP(0xb4) BAD_TRAP(0xb5) + BAD_TRAP(0xb6) BAD_TRAP(0xb7) BAD_TRAP(0xb8) BAD_TRAP(0xb9) BAD_TRAP(0xba) + BAD_TRAP(0xbb) BAD_TRAP(0xbc) BAD_TRAP(0xbd) BAD_TRAP(0xbe) BAD_TRAP(0xbf) + BAD_TRAP(0xc0) BAD_TRAP(0xc1) BAD_TRAP(0xc2) BAD_TRAP(0xc3) BAD_TRAP(0xc4) + BAD_TRAP(0xc5) BAD_TRAP(0xc6) BAD_TRAP(0xc7) BAD_TRAP(0xc8) BAD_TRAP(0xc9) + BAD_TRAP(0xca) BAD_TRAP(0xcb) BAD_TRAP(0xcc) BAD_TRAP(0xcd) BAD_TRAP(0xce) + BAD_TRAP(0xcf) BAD_TRAP(0xd0) BAD_TRAP(0xd1) BAD_TRAP(0xd2) BAD_TRAP(0xd3) + BAD_TRAP(0xd4) BAD_TRAP(0xd5) BAD_TRAP(0xd6) BAD_TRAP(0xd7) BAD_TRAP(0xd8) + BAD_TRAP(0xd9) BAD_TRAP(0xda) BAD_TRAP(0xdb) BAD_TRAP(0xdc) BAD_TRAP(0xdd) + BAD_TRAP(0xde) BAD_TRAP(0xdf) BAD_TRAP(0xe0) BAD_TRAP(0xe1) BAD_TRAP(0xe2) + BAD_TRAP(0xe3) BAD_TRAP(0xe4) BAD_TRAP(0xe5) BAD_TRAP(0xe6) BAD_TRAP(0xe7) + BAD_TRAP(0xe8) BAD_TRAP(0xe9) BAD_TRAP(0xea) BAD_TRAP(0xeb) BAD_TRAP(0xec) + BAD_TRAP(0xed) BAD_TRAP(0xee) BAD_TRAP(0xef) BAD_TRAP(0xf0) BAD_TRAP(0xf1) + BAD_TRAP(0xf2) BAD_TRAP(0xf3) BAD_TRAP(0xf4) BAD_TRAP(0xf5) BAD_TRAP(0xf6) + BAD_TRAP(0xf7) BAD_TRAP(0xf8) BAD_TRAP(0xf9) BAD_TRAP(0xfa) BAD_TRAP(0xfb) + BAD_TRAP(0xfc) + KGDB_TRAP(0xfd) + BAD_TRAP(0xfe) + BAD_TRAP(0xff) + +#endif diff --git a/arch/sparc/kernel/ttable_64.S b/arch/sparc/kernel/ttable_64.S new file mode 100644 index 000000000..86e737e59 --- /dev/null +++ b/arch/sparc/kernel/ttable_64.S @@ -0,0 +1,275 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* ttable.S: Sparc V9 Trap Table(s) with SpitFire/Cheetah/SUN4V extensions. + * + * Copyright (C) 1996, 2001, 2006 David S. Miller (davem@davemloft.net) + */ + + + .globl sparc64_ttable_tl0, sparc64_ttable_tl1 + .globl tl0_icpe, tl1_icpe + .globl tl0_dcpe, tl1_dcpe + .globl tl0_fecc, tl1_fecc + .globl tl0_cee, tl1_cee + .globl tl0_iae, tl1_iae + .globl tl0_dae, tl1_dae + +sparc64_ttable_tl0: +tl0_resv000: BOOT_KERNEL BTRAP(0x1) BTRAP(0x2) BTRAP(0x3) +tl0_resv004: BTRAP(0x4) BTRAP(0x5) BTRAP(0x6) BTRAP(0x7) +tl0_iax: membar #Sync + TRAP_NOSAVE_7INSNS(__spitfire_insn_access_exception) +tl0_itsb_4v: SUN4V_ITSB_MISS +tl0_iae: membar #Sync + TRAP_NOSAVE_7INSNS(__spitfire_access_error) +tl0_resv00b: BTRAP(0xb) BTRAP(0xc) BTRAP(0xd) BTRAP(0xe) BTRAP(0xf) +tl0_ill: membar #Sync + TRAP_7INSNS(do_illegal_instruction) +tl0_privop: TRAP(do_privop) +tl0_resv012: BTRAP(0x12) BTRAP(0x13) BTRAP(0x14) BTRAP(0x15) BTRAP(0x16) BTRAP(0x17) +tl0_resv018: BTRAP(0x18) BTRAP(0x19) +tl0_mcd: SUN4V_MCD_PRECISE +tl0_resv01b: BTRAP(0x1b) +tl0_resv01c: BTRAP(0x1c) BTRAP(0x1d) BTRAP(0x1e) BTRAP(0x1f) +tl0_fpdis: TRAP_NOSAVE(do_fpdis) +tl0_fpieee: TRAP_SAVEFPU(do_fpieee) +tl0_fpother: TRAP_NOSAVE(do_fpother_check_fitos) +tl0_tof: TRAP(do_tof) +tl0_cwin: CLEAN_WINDOW +tl0_div0: TRAP(do_div0) +tl0_resv029: BTRAP(0x29) BTRAP(0x2a) BTRAP(0x2b) BTRAP(0x2c) BTRAP(0x2d) BTRAP(0x2e) +tl0_resv02f: BTRAP(0x2f) +tl0_dax: TRAP_NOSAVE(__spitfire_data_access_exception) +tl0_dtsb_4v: SUN4V_DTSB_MISS +tl0_dae: membar #Sync + TRAP_NOSAVE_7INSNS(__spitfire_access_error) +tl0_resv033: BTRAP(0x33) +tl0_mna: TRAP_NOSAVE(do_mna) +tl0_lddfmna: TRAP_NOSAVE(do_lddfmna) +tl0_stdfmna: TRAP_NOSAVE(do_stdfmna) +tl0_privact: TRAP_NOSAVE(__do_privact) +tl0_resv038: BTRAP(0x38) BTRAP(0x39) BTRAP(0x3a) BTRAP(0x3b) BTRAP(0x3c) BTRAP(0x3d) +tl0_resv03e: BTRAP(0x3e) BTRAP(0x3f) BTRAP(0x40) +#ifdef CONFIG_SMP +tl0_irq1: TRAP_IRQ(smp_call_function_client, 1) +tl0_irq2: TRAP_IRQ(smp_receive_signal_client, 2) +tl0_irq3: TRAP_IRQ(smp_penguin_jailcell, 3) +tl0_irq4: BTRAP(0x44) +#else +tl0_irq1: BTRAP(0x41) +tl0_irq2: BTRAP(0x42) +tl0_irq3: BTRAP(0x43) +tl0_irq4: BTRAP(0x44) +#endif +tl0_irq5: TRAP_IRQ(handler_irq, 5) +#ifdef CONFIG_SMP +tl0_irq6: TRAP_IRQ(smp_call_function_single_client, 6) +#else +tl0_irq6: BTRAP(0x46) +#endif +tl0_irq7: TRAP_IRQ(deferred_pcr_work_irq, 7) +#if defined(CONFIG_KGDB) && defined(CONFIG_SMP) +tl0_irq8: TRAP_IRQ(smp_kgdb_capture_client, 8) +#else +tl0_irq8: BTRAP(0x48) +#endif +tl0_irq9: BTRAP(0x49) +tl0_irq10: BTRAP(0x4a) BTRAP(0x4b) BTRAP(0x4c) BTRAP(0x4d) +tl0_irq14: TRAP_IRQ(timer_interrupt, 14) +tl0_irq15: TRAP_NMI_IRQ(perfctr_irq, 15) +tl0_resv050: BTRAP(0x50) BTRAP(0x51) BTRAP(0x52) BTRAP(0x53) BTRAP(0x54) BTRAP(0x55) +tl0_resv056: BTRAP(0x56) BTRAP(0x57) BTRAP(0x58) BTRAP(0x59) BTRAP(0x5a) BTRAP(0x5b) +tl0_resv05c: BTRAP(0x5c) BTRAP(0x5d) BTRAP(0x5e) BTRAP(0x5f) +tl0_ivec: TRAP_IVEC +tl0_paw: TRAP(do_paw) +tl0_vaw: TRAP(do_vaw) +tl0_cee: membar #Sync + TRAP_NOSAVE_7INSNS(__spitfire_cee_trap) +tl0_iamiss: +#include "itlb_miss.S" +tl0_damiss: +#include "dtlb_miss.S" +tl0_daprot: +#include "dtlb_prot.S" +tl0_fecc: BTRAP(0x70) /* Fast-ECC on Cheetah */ +tl0_dcpe: BTRAP(0x71) /* D-cache Parity Error on Cheetah+ */ +tl0_icpe: BTRAP(0x72) /* I-cache Parity Error on Cheetah+ */ +tl0_resv073: BTRAP(0x73) BTRAP(0x74) BTRAP(0x75) +tl0_resv076: BTRAP(0x76) BTRAP(0x77) BTRAP(0x78) BTRAP(0x79) BTRAP(0x7a) BTRAP(0x7b) +tl0_cpu_mondo: TRAP_NOSAVE(sun4v_cpu_mondo) +tl0_dev_mondo: TRAP_NOSAVE(sun4v_dev_mondo) +tl0_res_mondo: TRAP_NOSAVE(sun4v_res_mondo) +tl0_nres_mondo: TRAP_NOSAVE(sun4v_nonres_mondo) +tl0_s0n: SPILL_0_NORMAL +tl0_s1n: SPILL_1_NORMAL +tl0_s2n: SPILL_2_NORMAL +tl0_s3n: SPILL_0_NORMAL_ETRAP +tl0_s4n: SPILL_1_GENERIC_ETRAP +tl0_s5n: SPILL_1_GENERIC_ETRAP_FIXUP +tl0_s6n: SPILL_2_GENERIC_ETRAP +tl0_s7n: SPILL_2_GENERIC_ETRAP_FIXUP +tl0_s0o: SPILL_0_OTHER +tl0_s1o: SPILL_1_OTHER +tl0_s2o: SPILL_2_OTHER +tl0_s3o: SPILL_3_OTHER +tl0_s4o: SPILL_4_OTHER +tl0_s5o: SPILL_5_OTHER +tl0_s6o: SPILL_6_OTHER +tl0_s7o: SPILL_7_OTHER +tl0_f0n: FILL_0_NORMAL +tl0_f1n: FILL_1_NORMAL +tl0_f2n: FILL_2_NORMAL +tl0_f3n: FILL_3_NORMAL +tl0_f4n: FILL_4_NORMAL +tl0_f5n: FILL_0_NORMAL_RTRAP +tl0_f6n: FILL_1_GENERIC_RTRAP +tl0_f7n: FILL_2_GENERIC_RTRAP +tl0_f0o: FILL_0_OTHER +tl0_f1o: FILL_1_OTHER +tl0_f2o: FILL_2_OTHER +tl0_f3o: FILL_3_OTHER +tl0_f4o: FILL_4_OTHER +tl0_f5o: FILL_5_OTHER +tl0_f6o: FILL_6_OTHER +tl0_f7o: FILL_7_OTHER +tl0_resv100: BTRAP(0x100) +tl0_bkpt: BREAKPOINT_TRAP +tl0_divz: TRAP(do_div0) +tl0_flushw: FLUSH_WINDOW_TRAP +tl0_resv104: BTRAP(0x104) BTRAP(0x105) BTRAP(0x106) BTRAP(0x107) BTRAP(0x108) +tl0_resv109: BTRAP(0x109) BTRAP(0x10a) BTRAP(0x10b) BTRAP(0x10c) BTRAP(0x10d) +tl0_resv10e: BTRAP(0x10e) BTRAP(0x10f) +tl0_linux32: LINUX_32BIT_SYSCALL_TRAP +tl0_oldlinux64: LINUX_64BIT_SYSCALL_TRAP +tl0_resv112: TRAP_UTRAP(UT_TRAP_INSTRUCTION_18,0x112) TRAP_UTRAP(UT_TRAP_INSTRUCTION_19,0x113) +tl0_resv114: TRAP_UTRAP(UT_TRAP_INSTRUCTION_20,0x114) TRAP_UTRAP(UT_TRAP_INSTRUCTION_21,0x115) +tl0_resv116: TRAP_UTRAP(UT_TRAP_INSTRUCTION_22,0x116) TRAP_UTRAP(UT_TRAP_INSTRUCTION_23,0x117) +tl0_resv118: TRAP_UTRAP(UT_TRAP_INSTRUCTION_24,0x118) TRAP_UTRAP(UT_TRAP_INSTRUCTION_25,0x119) +tl0_resv11a: TRAP_UTRAP(UT_TRAP_INSTRUCTION_26,0x11a) TRAP_UTRAP(UT_TRAP_INSTRUCTION_27,0x11b) +tl0_resv11c: TRAP_UTRAP(UT_TRAP_INSTRUCTION_28,0x11c) TRAP_UTRAP(UT_TRAP_INSTRUCTION_29,0x11d) +tl0_resv11e: TRAP_UTRAP(UT_TRAP_INSTRUCTION_30,0x11e) TRAP_UTRAP(UT_TRAP_INSTRUCTION_31,0x11f) +tl0_getcc: GETCC_TRAP +tl0_setcc: SETCC_TRAP +tl0_getpsr: TRAP(do_getpsr) +tl0_resv123: BTRAP(0x123) BTRAP(0x124) BTRAP(0x125) BTRAP(0x126) BTRAP(0x127) +tl0_resv128: BTRAP(0x128) BTRAP(0x129) BTRAP(0x12a) BTRAP(0x12b) BTRAP(0x12c) +tl0_resv12d: BTRAP(0x12d) BTRAP(0x12e) BTRAP(0x12f) BTRAP(0x130) BTRAP(0x131) +tl0_resv132: BTRAP(0x132) BTRAP(0x133) BTRAP(0x134) BTRAP(0x135) BTRAP(0x136) +tl0_resv137: BTRAP(0x137) BTRAP(0x138) BTRAP(0x139) BTRAP(0x13a) BTRAP(0x13b) +tl0_resv13c: BTRAP(0x13c) BTRAP(0x13d) BTRAP(0x13e) BTRAP(0x13f) BTRAP(0x140) +tl0_resv141: BTRAP(0x141) BTRAP(0x142) BTRAP(0x143) BTRAP(0x144) BTRAP(0x145) +tl0_resv146: BTRAP(0x146) BTRAP(0x147) BTRAP(0x148) BTRAP(0x149) BTRAP(0x14a) +tl0_resv14b: BTRAP(0x14b) BTRAP(0x14c) BTRAP(0x14d) BTRAP(0x14e) BTRAP(0x14f) +tl0_resv150: BTRAP(0x150) BTRAP(0x151) BTRAP(0x152) BTRAP(0x153) BTRAP(0x154) +tl0_resv155: BTRAP(0x155) BTRAP(0x156) BTRAP(0x157) BTRAP(0x158) BTRAP(0x159) +tl0_resv15a: BTRAP(0x15a) BTRAP(0x15b) BTRAP(0x15c) BTRAP(0x15d) BTRAP(0x15e) +tl0_resv15f: BTRAP(0x15f) BTRAP(0x160) BTRAP(0x161) BTRAP(0x162) BTRAP(0x163) +tl0_resv164: BTRAP(0x164) BTRAP(0x165) BTRAP(0x166) BTRAP(0x167) BTRAP(0x168) +tl0_resv169: BTRAP(0x169) BTRAP(0x16a) BTRAP(0x16b) BTRAP(0x16c) +tl0_linux64: LINUX_64BIT_SYSCALL_TRAP +tl0_gsctx: TRAP(sparc64_get_context) TRAP(sparc64_set_context) +tl0_resv170: KPROBES_TRAP(0x170) KPROBES_TRAP(0x171) KGDB_TRAP(0x172) +tl0_resv173: UPROBES_TRAP(0x173) UPROBES_TRAP(0x174) BTRAP(0x175) BTRAP(0x176) BTRAP(0x177) +tl0_resv178: BTRAP(0x178) BTRAP(0x179) BTRAP(0x17a) BTRAP(0x17b) BTRAP(0x17c) +tl0_resv17d: BTRAP(0x17d) BTRAP(0x17e) BTRAP(0x17f) +#define BTRAPS(x) BTRAP(x) BTRAP(x+1) BTRAP(x+2) BTRAP(x+3) BTRAP(x+4) BTRAP(x+5) BTRAP(x+6) BTRAP(x+7) +tl0_resv180: BTRAPS(0x180) BTRAPS(0x188) +tl0_resv190: BTRAPS(0x190) BTRAPS(0x198) +tl0_resv1a0: BTRAPS(0x1a0) BTRAPS(0x1a8) +tl0_resv1b0: BTRAPS(0x1b0) BTRAPS(0x1b8) +tl0_resv1c0: BTRAPS(0x1c0) BTRAPS(0x1c8) +tl0_resv1d0: BTRAPS(0x1d0) BTRAPS(0x1d8) +tl0_resv1e0: BTRAPS(0x1e0) BTRAPS(0x1e8) +tl0_resv1f0: BTRAPS(0x1f0) BTRAPS(0x1f8) + +sparc64_ttable_tl1: +tl1_resv000: BOOT_KERNEL BTRAPTL1(0x1) BTRAPTL1(0x2) BTRAPTL1(0x3) +tl1_resv004: BTRAPTL1(0x4) BTRAPTL1(0x5) BTRAPTL1(0x6) BTRAPTL1(0x7) +tl1_iax: TRAP_NOSAVE(__spitfire_insn_access_exception_tl1) +tl1_itsb_4v: SUN4V_ITSB_MISS +tl1_iae: membar #Sync + TRAP_NOSAVE_7INSNS(__spitfire_access_error) +tl1_resv00b: BTRAPTL1(0xb) BTRAPTL1(0xc) BTRAPTL1(0xd) BTRAPTL1(0xe) BTRAPTL1(0xf) +tl1_ill: TRAPTL1(do_ill_tl1) +tl1_privop: BTRAPTL1(0x11) +tl1_resv012: BTRAPTL1(0x12) BTRAPTL1(0x13) BTRAPTL1(0x14) BTRAPTL1(0x15) +tl1_resv016: BTRAPTL1(0x16) BTRAPTL1(0x17) BTRAPTL1(0x18) BTRAPTL1(0x19) +tl1_resv01a: BTRAPTL1(0x1a) BTRAPTL1(0x1b) BTRAPTL1(0x1c) BTRAPTL1(0x1d) +tl1_resv01e: BTRAPTL1(0x1e) BTRAPTL1(0x1f) +tl1_fpdis: TRAP_NOSAVE(do_fpdis) +tl1_fpieee: TRAPTL1(do_fpieee_tl1) +tl1_fpother: TRAPTL1(do_fpother_tl1) +tl1_tof: TRAPTL1(do_tof_tl1) +tl1_cwin: CLEAN_WINDOW +tl1_div0: TRAPTL1(do_div0_tl1) +tl1_resv029: BTRAPTL1(0x29) BTRAPTL1(0x2a) BTRAPTL1(0x2b) BTRAPTL1(0x2c) +tl1_resv02d: BTRAPTL1(0x2d) BTRAPTL1(0x2e) BTRAPTL1(0x2f) +tl1_dax: TRAP_NOSAVE(__spitfire_data_access_exception_tl1) +tl1_dtsb_4v: SUN4V_DTSB_MISS +tl1_dae: membar #Sync + TRAP_NOSAVE_7INSNS(__spitfire_access_error) +tl1_resv033: BTRAPTL1(0x33) +tl1_mna: TRAP_NOSAVE(do_mna) +tl1_lddfmna: TRAPTL1(do_lddfmna_tl1) +tl1_stdfmna: TRAPTL1(do_stdfmna_tl1) +tl1_privact: BTRAPTL1(0x37) +tl1_resv038: BTRAPTL1(0x38) BTRAPTL1(0x39) BTRAPTL1(0x3a) BTRAPTL1(0x3b) +tl1_resv03c: BTRAPTL1(0x3c) BTRAPTL1(0x3d) BTRAPTL1(0x3e) BTRAPTL1(0x3f) +tl1_resv040: BTRAPTL1(0x40) +tl1_irq1: TRAP_IRQ(do_irq_tl1, 1) TRAP_IRQ(do_irq_tl1, 2) TRAP_IRQ(do_irq_tl1, 3) +tl1_irq4: TRAP_IRQ(do_irq_tl1, 4) TRAP_IRQ(do_irq_tl1, 5) TRAP_IRQ(do_irq_tl1, 6) +tl1_irq7: TRAP_IRQ(do_irq_tl1, 7) TRAP_IRQ(do_irq_tl1, 8) TRAP_IRQ(do_irq_tl1, 9) +tl1_irq10: TRAP_IRQ(do_irq_tl1, 10) TRAP_IRQ(do_irq_tl1, 11) +tl1_irq12: TRAP_IRQ(do_irq_tl1, 12) TRAP_IRQ(do_irq_tl1, 13) +tl1_irq14: TRAP_IRQ(do_irq_tl1, 14) TRAP_IRQ(do_irq_tl1, 15) +tl1_resv050: BTRAPTL1(0x50) BTRAPTL1(0x51) BTRAPTL1(0x52) BTRAPTL1(0x53) +tl1_resv054: BTRAPTL1(0x54) BTRAPTL1(0x55) BTRAPTL1(0x56) BTRAPTL1(0x57) +tl1_resv058: BTRAPTL1(0x58) BTRAPTL1(0x59) BTRAPTL1(0x5a) BTRAPTL1(0x5b) +tl1_resv05c: BTRAPTL1(0x5c) BTRAPTL1(0x5d) BTRAPTL1(0x5e) BTRAPTL1(0x5f) +tl1_ivec: TRAP_IVEC +tl1_paw: TRAPTL1(do_paw_tl1) +tl1_vaw: TRAPTL1(do_vaw_tl1) +tl1_cee: BTRAPTL1(0x63) +tl1_iamiss: BTRAPTL1(0x64) BTRAPTL1(0x65) BTRAPTL1(0x66) BTRAPTL1(0x67) +tl1_damiss: +#include "dtlb_miss.S" +tl1_daprot: +#include "dtlb_prot.S" +tl1_fecc: BTRAPTL1(0x70) /* Fast-ECC on Cheetah */ +tl1_dcpe: BTRAPTL1(0x71) /* D-cache Parity Error on Cheetah+ */ +tl1_icpe: BTRAPTL1(0x72) /* I-cache Parity Error on Cheetah+ */ +tl1_resv073: BTRAPTL1(0x73) +tl1_resv074: BTRAPTL1(0x74) BTRAPTL1(0x75) BTRAPTL1(0x76) BTRAPTL1(0x77) +tl1_resv078: BTRAPTL1(0x78) BTRAPTL1(0x79) BTRAPTL1(0x7a) BTRAPTL1(0x7b) +tl1_resv07c: BTRAPTL1(0x7c) BTRAPTL1(0x7d) BTRAPTL1(0x7e) BTRAPTL1(0x7f) +tl1_s0n: SPILL_0_NORMAL +tl1_s1n: SPILL_1_NORMAL +tl1_s2n: SPILL_2_NORMAL +tl1_s3n: SPILL_3_NORMAL +tl1_s4n: SPILL_4_NORMAL +tl1_s5n: SPILL_5_NORMAL +tl1_s6n: SPILL_6_NORMAL +tl1_s7n: SPILL_7_NORMAL +tl1_s0o: SPILL_0_OTHER +tl1_s1o: SPILL_1_OTHER +tl1_s2o: SPILL_2_OTHER +tl1_s3o: SPILL_3_OTHER +tl1_s4o: SPILL_4_OTHER +tl1_s5o: SPILL_5_OTHER +tl1_s6o: SPILL_6_OTHER +tl1_s7o: SPILL_7_OTHER +tl1_f0n: FILL_0_NORMAL +tl1_f1n: FILL_1_NORMAL +tl1_f2n: FILL_2_NORMAL +tl1_f3n: FILL_3_NORMAL +tl1_f4n: FILL_4_NORMAL +tl1_f5n: FILL_5_NORMAL +tl1_f6n: FILL_6_NORMAL +tl1_f7n: FILL_7_NORMAL +tl1_f0o: FILL_0_OTHER +tl1_f1o: FILL_1_OTHER +tl1_f2o: FILL_2_OTHER +tl1_f3o: FILL_3_OTHER +tl1_f4o: FILL_4_OTHER +tl1_f5o: FILL_5_OTHER +tl1_f6o: FILL_6_OTHER +tl1_f7o: FILL_7_OTHER diff --git a/arch/sparc/kernel/una_asm_32.S b/arch/sparc/kernel/una_asm_32.S new file mode 100644 index 000000000..f8bf83928 --- /dev/null +++ b/arch/sparc/kernel/una_asm_32.S @@ -0,0 +1,154 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* una_asm.S: Kernel unaligned trap assembler helpers. + * + * Copyright (C) 1996,2005,2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include + + .text + +retl_efault: + retl + mov -EFAULT, %o0 + + /* int __do_int_store(unsigned long *dst_addr, int size, + * unsigned long *src_val) + * + * %o0 = dest_addr + * %o1 = size + * %o2 = src_val + * + * Return '0' on success, -EFAULT on failure. + */ + .globl __do_int_store +__do_int_store: + ld [%o2], %g1 + cmp %o1, 2 + be 2f + cmp %o1, 4 + be 1f + srl %g1, 24, %g2 + srl %g1, 16, %g7 +4: stb %g2, [%o0] + srl %g1, 8, %g2 +5: stb %g7, [%o0 + 1] + ld [%o2 + 4], %g7 +6: stb %g2, [%o0 + 2] + srl %g7, 24, %g2 +7: stb %g1, [%o0 + 3] + srl %g7, 16, %g1 +8: stb %g2, [%o0 + 4] + srl %g7, 8, %g2 +9: stb %g1, [%o0 + 5] +10: stb %g2, [%o0 + 6] + b 0f +11: stb %g7, [%o0 + 7] +1: srl %g1, 16, %g7 +12: stb %g2, [%o0] + srl %g1, 8, %g2 +13: stb %g7, [%o0 + 1] +14: stb %g2, [%o0 + 2] + b 0f +15: stb %g1, [%o0 + 3] +2: srl %g1, 8, %g2 +16: stb %g2, [%o0] +17: stb %g1, [%o0 + 1] +0: retl + mov 0, %o0 + + .section __ex_table,#alloc + .word 4b, retl_efault + .word 5b, retl_efault + .word 6b, retl_efault + .word 7b, retl_efault + .word 8b, retl_efault + .word 9b, retl_efault + .word 10b, retl_efault + .word 11b, retl_efault + .word 12b, retl_efault + .word 13b, retl_efault + .word 14b, retl_efault + .word 15b, retl_efault + .word 16b, retl_efault + .word 17b, retl_efault + .previous + + /* int do_int_load(unsigned long *dest_reg, int size, + * unsigned long *saddr, int is_signed) + * + * %o0 = dest_reg + * %o1 = size + * %o2 = saddr + * %o3 = is_signed + * + * Return '0' on success, -EFAULT on failure. + */ + .globl do_int_load +do_int_load: + cmp %o1, 8 + be 9f + cmp %o1, 4 + be 6f +4: ldub [%o2], %g1 +5: ldub [%o2 + 1], %g2 + sll %g1, 8, %g1 + tst %o3 + be 3f + or %g1, %g2, %g1 + sll %g1, 16, %g1 + sra %g1, 16, %g1 +3: b 0f + st %g1, [%o0] +6: ldub [%o2 + 1], %g2 + sll %g1, 24, %g1 +7: ldub [%o2 + 2], %g7 + sll %g2, 16, %g2 +8: ldub [%o2 + 3], %g3 + sll %g7, 8, %g7 + or %g3, %g2, %g3 + or %g7, %g3, %g7 + or %g1, %g7, %g1 + b 0f + st %g1, [%o0] +9: ldub [%o2], %g1 +10: ldub [%o2 + 1], %g2 + sll %g1, 24, %g1 +11: ldub [%o2 + 2], %g7 + sll %g2, 16, %g2 +12: ldub [%o2 + 3], %g3 + sll %g7, 8, %g7 + or %g1, %g2, %g1 + or %g7, %g3, %g7 + or %g1, %g7, %g7 +13: ldub [%o2 + 4], %g1 + st %g7, [%o0] +14: ldub [%o2 + 5], %g2 + sll %g1, 24, %g1 +15: ldub [%o2 + 6], %g7 + sll %g2, 16, %g2 +16: ldub [%o2 + 7], %g3 + sll %g7, 8, %g7 + or %g1, %g2, %g1 + or %g7, %g3, %g7 + or %g1, %g7, %g7 + st %g7, [%o0 + 4] +0: retl + mov 0, %o0 + + .section __ex_table,#alloc + .word 4b, retl_efault + .word 5b, retl_efault + .word 6b, retl_efault + .word 7b, retl_efault + .word 8b, retl_efault + .word 9b, retl_efault + .word 10b, retl_efault + .word 11b, retl_efault + .word 12b, retl_efault + .word 13b, retl_efault + .word 14b, retl_efault + .word 15b, retl_efault + .word 16b, retl_efault + .previous diff --git a/arch/sparc/kernel/una_asm_64.S b/arch/sparc/kernel/una_asm_64.S new file mode 100644 index 000000000..e256f395e --- /dev/null +++ b/arch/sparc/kernel/una_asm_64.S @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* una_asm.S: Kernel unaligned trap assembler helpers. + * + * Copyright (C) 1996,2005 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + + .text + + .globl __do_int_store +__do_int_store: + rd %asi, %o4 + wr %o3, 0, %asi + mov %o2, %g3 + cmp %o1, 2 + be,pn %icc, 2f + cmp %o1, 4 + be,pt %icc, 1f + srlx %g3, 24, %g2 + srlx %g3, 56, %g1 + srlx %g3, 48, %g7 +4: stba %g1, [%o0] %asi + srlx %g3, 40, %g1 +5: stba %g7, [%o0 + 1] %asi + srlx %g3, 32, %g7 +6: stba %g1, [%o0 + 2] %asi +7: stba %g7, [%o0 + 3] %asi + srlx %g3, 16, %g1 +8: stba %g2, [%o0 + 4] %asi + srlx %g3, 8, %g7 +9: stba %g1, [%o0 + 5] %asi +10: stba %g7, [%o0 + 6] %asi + ba,pt %xcc, 0f +11: stba %g3, [%o0 + 7] %asi +1: srl %g3, 16, %g7 +12: stba %g2, [%o0] %asi + srl %g3, 8, %g2 +13: stba %g7, [%o0 + 1] %asi +14: stba %g2, [%o0 + 2] %asi + ba,pt %xcc, 0f +15: stba %g3, [%o0 + 3] %asi +2: srl %g3, 8, %g2 +16: stba %g2, [%o0] %asi +17: stba %g3, [%o0 + 1] %asi +0: + wr %o4, 0x0, %asi + retl + mov 0, %o0 + .size __do_int_store, .-__do_int_store + + .section __ex_table,"a" + .word 4b, __retl_efault + .word 5b, __retl_efault + .word 6b, __retl_efault + .word 7b, __retl_efault + .word 8b, __retl_efault + .word 9b, __retl_efault + .word 10b, __retl_efault + .word 11b, __retl_efault + .word 12b, __retl_efault + .word 13b, __retl_efault + .word 14b, __retl_efault + .word 15b, __retl_efault + .word 16b, __retl_efault + .word 17b, __retl_efault + .previous + + .globl do_int_load +do_int_load: + rd %asi, %o5 + wr %o4, 0, %asi + cmp %o1, 8 + bge,pn %icc, 9f + cmp %o1, 4 + be,pt %icc, 6f +4: lduba [%o2] %asi, %g2 +5: lduba [%o2 + 1] %asi, %g3 + sll %g2, 8, %g2 + brz,pt %o3, 3f + add %g2, %g3, %g2 + sllx %g2, 48, %g2 + srax %g2, 48, %g2 +3: ba,pt %xcc, 0f + stx %g2, [%o0] +6: lduba [%o2 + 1] %asi, %g3 + sll %g2, 24, %g2 +7: lduba [%o2 + 2] %asi, %g7 + sll %g3, 16, %g3 +8: lduba [%o2 + 3] %asi, %g1 + sll %g7, 8, %g7 + or %g2, %g3, %g2 + or %g7, %g1, %g7 + or %g2, %g7, %g2 + brnz,a,pt %o3, 3f + sra %g2, 0, %g2 +3: ba,pt %xcc, 0f + stx %g2, [%o0] +9: lduba [%o2] %asi, %g2 +10: lduba [%o2 + 1] %asi, %g3 + sllx %g2, 56, %g2 +11: lduba [%o2 + 2] %asi, %g7 + sllx %g3, 48, %g3 +12: lduba [%o2 + 3] %asi, %g1 + sllx %g7, 40, %g7 + sllx %g1, 32, %g1 + or %g2, %g3, %g2 + or %g7, %g1, %g7 +13: lduba [%o2 + 4] %asi, %g3 + or %g2, %g7, %g7 +14: lduba [%o2 + 5] %asi, %g1 + sllx %g3, 24, %g3 +15: lduba [%o2 + 6] %asi, %g2 + sllx %g1, 16, %g1 + or %g7, %g3, %g7 +16: lduba [%o2 + 7] %asi, %g3 + sllx %g2, 8, %g2 + or %g7, %g1, %g7 + or %g2, %g3, %g2 + or %g7, %g2, %g7 + cmp %o1, 8 + be,a,pt %icc, 0f + stx %g7, [%o0] + srlx %g7, 32, %g2 + sra %g7, 0, %g7 + stx %g2, [%o0] + stx %g7, [%o0 + 8] +0: + wr %o5, 0x0, %asi + retl + mov 0, %o0 + .size do_int_load, .-do_int_load + + .section __ex_table,"a" + .word 4b, __retl_efault + .word 5b, __retl_efault + .word 6b, __retl_efault + .word 7b, __retl_efault + .word 8b, __retl_efault + .word 9b, __retl_efault + .word 10b, __retl_efault + .word 11b, __retl_efault + .word 12b, __retl_efault + .word 13b, __retl_efault + .word 14b, __retl_efault + .word 15b, __retl_efault + .word 16b, __retl_efault + .previous diff --git a/arch/sparc/kernel/unaligned_32.c b/arch/sparc/kernel/unaligned_32.c new file mode 100644 index 000000000..455f0258c --- /dev/null +++ b/arch/sparc/kernel/unaligned_32.c @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * unaligned.c: Unaligned load/store trap handling with special + * cases for the kernel to do them more quickly. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "kernel.h" + +enum direction { + load, /* ld, ldd, ldh, ldsh */ + store, /* st, std, sth, stsh */ + both, /* Swap, ldstub, etc. */ + fpload, + fpstore, + invalid, +}; + +static inline enum direction decode_direction(unsigned int insn) +{ + unsigned long tmp = (insn >> 21) & 1; + + if(!tmp) + return load; + else { + if(((insn>>19)&0x3f) == 15) + return both; + else + return store; + } +} + +/* 8 = double-word, 4 = word, 2 = half-word */ +static inline int decode_access_size(unsigned int insn) +{ + insn = (insn >> 19) & 3; + + if(!insn) + return 4; + else if(insn == 3) + return 8; + else if(insn == 2) + return 2; + else { + printk("Impossible unaligned trap. insn=%08x\n", insn); + die_if_kernel("Byte sized unaligned access?!?!", current->thread.kregs); + return 4; /* just to keep gcc happy. */ + } +} + +/* 0x400000 = signed, 0 = unsigned */ +static inline int decode_signedness(unsigned int insn) +{ + return (insn & 0x400000); +} + +static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2, + unsigned int rd) +{ + if(rs2 >= 16 || rs1 >= 16 || rd >= 16) { + /* Wheee... */ + __asm__ __volatile__("save %sp, -0x40, %sp\n\t" + "save %sp, -0x40, %sp\n\t" + "save %sp, -0x40, %sp\n\t" + "save %sp, -0x40, %sp\n\t" + "save %sp, -0x40, %sp\n\t" + "save %sp, -0x40, %sp\n\t" + "save %sp, -0x40, %sp\n\t" + "restore; restore; restore; restore;\n\t" + "restore; restore; restore;\n\t"); + } +} + +static inline int sign_extend_imm13(int imm) +{ + return imm << 19 >> 19; +} + +static inline unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) +{ + struct reg_window32 *win; + + if(reg < 16) + return (!reg ? 0 : regs->u_regs[reg]); + + /* Ho hum, the slightly complicated case. */ + win = (struct reg_window32 *) regs->u_regs[UREG_FP]; + return win->locals[reg - 16]; /* yes, I know what this does... */ +} + +static inline unsigned long safe_fetch_reg(unsigned int reg, struct pt_regs *regs) +{ + struct reg_window32 __user *win; + unsigned long ret; + + if (reg < 16) + return (!reg ? 0 : regs->u_regs[reg]); + + /* Ho hum, the slightly complicated case. */ + win = (struct reg_window32 __user *) regs->u_regs[UREG_FP]; + + if ((unsigned long)win & 3) + return -1; + + if (get_user(ret, &win->locals[reg - 16])) + return -1; + + return ret; +} + +static inline unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs) +{ + struct reg_window32 *win; + + if(reg < 16) + return ®s->u_regs[reg]; + win = (struct reg_window32 *) regs->u_regs[UREG_FP]; + return &win->locals[reg - 16]; +} + +static unsigned long compute_effective_address(struct pt_regs *regs, + unsigned int insn) +{ + unsigned int rs1 = (insn >> 14) & 0x1f; + unsigned int rs2 = insn & 0x1f; + unsigned int rd = (insn >> 25) & 0x1f; + + if(insn & 0x2000) { + maybe_flush_windows(rs1, 0, rd); + return (fetch_reg(rs1, regs) + sign_extend_imm13(insn)); + } else { + maybe_flush_windows(rs1, rs2, rd); + return (fetch_reg(rs1, regs) + fetch_reg(rs2, regs)); + } +} + +unsigned long safe_compute_effective_address(struct pt_regs *regs, + unsigned int insn) +{ + unsigned int rs1 = (insn >> 14) & 0x1f; + unsigned int rs2 = insn & 0x1f; + unsigned int rd = (insn >> 25) & 0x1f; + + if(insn & 0x2000) { + maybe_flush_windows(rs1, 0, rd); + return (safe_fetch_reg(rs1, regs) + sign_extend_imm13(insn)); + } else { + maybe_flush_windows(rs1, rs2, rd); + return (safe_fetch_reg(rs1, regs) + safe_fetch_reg(rs2, regs)); + } +} + +/* This is just to make gcc think panic does return... */ +static void unaligned_panic(char *str) +{ + panic("%s", str); +} + +/* una_asm.S */ +extern int do_int_load(unsigned long *dest_reg, int size, + unsigned long *saddr, int is_signed); +extern int __do_int_store(unsigned long *dst_addr, int size, + unsigned long *src_val); + +static int do_int_store(int reg_num, int size, unsigned long *dst_addr, + struct pt_regs *regs) +{ + unsigned long zero[2] = { 0, 0 }; + unsigned long *src_val; + + if (reg_num) + src_val = fetch_reg_addr(reg_num, regs); + else { + src_val = &zero[0]; + if (size == 8) + zero[1] = fetch_reg(1, regs); + } + return __do_int_store(dst_addr, size, src_val); +} + +extern void smp_capture(void); +extern void smp_release(void); + +static inline void advance(struct pt_regs *regs) +{ + regs->pc = regs->npc; + regs->npc += 4; +} + +static inline int floating_point_load_or_store_p(unsigned int insn) +{ + return (insn >> 24) & 1; +} + +static inline int ok_for_kernel(unsigned int insn) +{ + return !floating_point_load_or_store_p(insn); +} + +static void kernel_mna_trap_fault(struct pt_regs *regs, unsigned int insn) +{ + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->pc); + if (!entry) { + unsigned long address = compute_effective_address(regs, insn); + if(address < PAGE_SIZE) { + printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference in mna handler"); + } else + printk(KERN_ALERT "Unable to handle kernel paging request in mna handler"); + printk(KERN_ALERT " at virtual address %08lx\n",address); + printk(KERN_ALERT "current->{mm,active_mm}->context = %08lx\n", + (current->mm ? current->mm->context : + current->active_mm->context)); + printk(KERN_ALERT "current->{mm,active_mm}->pgd = %08lx\n", + (current->mm ? (unsigned long) current->mm->pgd : + (unsigned long) current->active_mm->pgd)); + die_if_kernel("Oops", regs); + /* Not reached */ + } + regs->pc = entry->fixup; + regs->npc = regs->pc + 4; +} + +asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) +{ + enum direction dir = decode_direction(insn); + int size = decode_access_size(insn); + + if(!ok_for_kernel(insn) || dir == both) { + printk("Unsupported unaligned load/store trap for kernel at <%08lx>.\n", + regs->pc); + unaligned_panic("Wheee. Kernel does fpu/atomic unaligned load/store."); + } else { + unsigned long addr = compute_effective_address(regs, insn); + int err; + + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); + switch (dir) { + case load: + err = do_int_load(fetch_reg_addr(((insn>>25)&0x1f), + regs), + size, (unsigned long *) addr, + decode_signedness(insn)); + break; + + case store: + err = do_int_store(((insn>>25)&0x1f), size, + (unsigned long *) addr, regs); + break; + default: + panic("Impossible kernel unaligned trap."); + /* Not reached... */ + } + if (err) + kernel_mna_trap_fault(regs, insn); + else + advance(regs); + } +} + +asmlinkage void user_unaligned_trap(struct pt_regs *regs, unsigned int insn) +{ + send_sig_fault(SIGBUS, BUS_ADRALN, + (void __user *)safe_compute_effective_address(regs, insn), + current); +} diff --git a/arch/sparc/kernel/unaligned_64.c b/arch/sparc/kernel/unaligned_64.c new file mode 100644 index 000000000..23db2efda --- /dev/null +++ b/arch/sparc/kernel/unaligned_64.c @@ -0,0 +1,709 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * unaligned.c: Unaligned load/store trap handling with special + * cases for the kernel to do them more quickly. + * + * Copyright (C) 1996,2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "entry.h" +#include "kernel.h" + +enum direction { + load, /* ld, ldd, ldh, ldsh */ + store, /* st, std, sth, stsh */ + both, /* Swap, ldstub, cas, ... */ + fpld, + fpst, + invalid, +}; + +static inline enum direction decode_direction(unsigned int insn) +{ + unsigned long tmp = (insn >> 21) & 1; + + if (!tmp) + return load; + else { + switch ((insn>>19)&0xf) { + case 15: /* swap* */ + return both; + default: + return store; + } + } +} + +/* 16 = double-word, 8 = extra-word, 4 = word, 2 = half-word */ +static inline int decode_access_size(struct pt_regs *regs, unsigned int insn) +{ + unsigned int tmp; + + tmp = ((insn >> 19) & 0xf); + if (tmp == 11 || tmp == 14) /* ldx/stx */ + return 8; + tmp &= 3; + if (!tmp) + return 4; + else if (tmp == 3) + return 16; /* ldd/std - Although it is actually 8 */ + else if (tmp == 2) + return 2; + else { + printk("Impossible unaligned trap. insn=%08x\n", insn); + die_if_kernel("Byte sized unaligned access?!?!", regs); + + /* GCC should never warn that control reaches the end + * of this function without returning a value because + * die_if_kernel() is marked with attribute 'noreturn'. + * Alas, some versions do... + */ + + return 0; + } +} + +static inline int decode_asi(unsigned int insn, struct pt_regs *regs) +{ + if (insn & 0x800000) { + if (insn & 0x2000) + return (unsigned char)(regs->tstate >> 24); /* %asi */ + else + return (unsigned char)(insn >> 5); /* imm_asi */ + } else + return ASI_P; +} + +/* 0x400000 = signed, 0 = unsigned */ +static inline int decode_signedness(unsigned int insn) +{ + return (insn & 0x400000); +} + +static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2, + unsigned int rd, int from_kernel) +{ + if (rs2 >= 16 || rs1 >= 16 || rd >= 16) { + if (from_kernel != 0) + __asm__ __volatile__("flushw"); + else + flushw_user(); + } +} + +static inline long sign_extend_imm13(long imm) +{ + return imm << 51 >> 51; +} + +static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) +{ + unsigned long value, fp; + + if (reg < 16) + return (!reg ? 0 : regs->u_regs[reg]); + + fp = regs->u_regs[UREG_FP]; + + if (regs->tstate & TSTATE_PRIV) { + struct reg_window *win; + win = (struct reg_window *)(fp + STACK_BIAS); + value = win->locals[reg - 16]; + } else if (!test_thread_64bit_stack(fp)) { + struct reg_window32 __user *win32; + win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp)); + get_user(value, &win32->locals[reg - 16]); + } else { + struct reg_window __user *win; + win = (struct reg_window __user *)(fp + STACK_BIAS); + get_user(value, &win->locals[reg - 16]); + } + return value; +} + +static unsigned long *fetch_reg_addr(unsigned int reg, struct pt_regs *regs) +{ + unsigned long fp; + + if (reg < 16) + return ®s->u_regs[reg]; + + fp = regs->u_regs[UREG_FP]; + + if (regs->tstate & TSTATE_PRIV) { + struct reg_window *win; + win = (struct reg_window *)(fp + STACK_BIAS); + return &win->locals[reg - 16]; + } else if (!test_thread_64bit_stack(fp)) { + struct reg_window32 *win32; + win32 = (struct reg_window32 *)((unsigned long)((u32)fp)); + return (unsigned long *)&win32->locals[reg - 16]; + } else { + struct reg_window *win; + win = (struct reg_window *)(fp + STACK_BIAS); + return &win->locals[reg - 16]; + } +} + +unsigned long compute_effective_address(struct pt_regs *regs, + unsigned int insn, unsigned int rd) +{ + int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; + unsigned int rs1 = (insn >> 14) & 0x1f; + unsigned int rs2 = insn & 0x1f; + unsigned long addr; + + if (insn & 0x2000) { + maybe_flush_windows(rs1, 0, rd, from_kernel); + addr = (fetch_reg(rs1, regs) + sign_extend_imm13(insn)); + } else { + maybe_flush_windows(rs1, rs2, rd, from_kernel); + addr = (fetch_reg(rs1, regs) + fetch_reg(rs2, regs)); + } + + if (!from_kernel && test_thread_flag(TIF_32BIT)) + addr &= 0xffffffff; + + return addr; +} + +/* This is just to make gcc think die_if_kernel does return... */ +static void __used unaligned_panic(char *str, struct pt_regs *regs) +{ + die_if_kernel(str, regs); +} + +extern int do_int_load(unsigned long *dest_reg, int size, + unsigned long *saddr, int is_signed, int asi); + +extern int __do_int_store(unsigned long *dst_addr, int size, + unsigned long src_val, int asi); + +static inline int do_int_store(int reg_num, int size, unsigned long *dst_addr, + struct pt_regs *regs, int asi, int orig_asi) +{ + unsigned long zero = 0; + unsigned long *src_val_p = &zero; + unsigned long src_val; + + if (size == 16) { + size = 8; + zero = (((long)(reg_num ? + (unsigned int)fetch_reg(reg_num, regs) : 0)) << 32) | + (unsigned int)fetch_reg(reg_num + 1, regs); + } else if (reg_num) { + src_val_p = fetch_reg_addr(reg_num, regs); + } + src_val = *src_val_p; + if (unlikely(asi != orig_asi)) { + switch (size) { + case 2: + src_val = swab16(src_val); + break; + case 4: + src_val = swab32(src_val); + break; + case 8: + src_val = swab64(src_val); + break; + case 16: + default: + BUG(); + break; + } + } + return __do_int_store(dst_addr, size, src_val, asi); +} + +static inline void advance(struct pt_regs *regs) +{ + regs->tpc = regs->tnpc; + regs->tnpc += 4; + if (test_thread_flag(TIF_32BIT)) { + regs->tpc &= 0xffffffff; + regs->tnpc &= 0xffffffff; + } +} + +static inline int floating_point_load_or_store_p(unsigned int insn) +{ + return (insn >> 24) & 1; +} + +static inline int ok_for_kernel(unsigned int insn) +{ + return !floating_point_load_or_store_p(insn); +} + +static void kernel_mna_trap_fault(int fixup_tstate_asi) +{ + struct pt_regs *regs = current_thread_info()->kern_una_regs; + unsigned int insn = current_thread_info()->kern_una_insn; + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (!entry) { + unsigned long address; + + address = compute_effective_address(regs, insn, + ((insn >> 25) & 0x1f)); + if (address < PAGE_SIZE) { + printk(KERN_ALERT "Unable to handle kernel NULL " + "pointer dereference in mna handler"); + } else + printk(KERN_ALERT "Unable to handle kernel paging " + "request in mna handler"); + printk(KERN_ALERT " at virtual address %016lx\n",address); + printk(KERN_ALERT "current->{active_,}mm->context = %016lx\n", + (current->mm ? CTX_HWBITS(current->mm->context) : + CTX_HWBITS(current->active_mm->context))); + printk(KERN_ALERT "current->{active_,}mm->pgd = %016lx\n", + (current->mm ? (unsigned long) current->mm->pgd : + (unsigned long) current->active_mm->pgd)); + die_if_kernel("Oops", regs); + /* Not reached */ + } + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + + if (fixup_tstate_asi) { + regs->tstate &= ~TSTATE_ASI; + regs->tstate |= (ASI_AIUS << 24UL); + } +} + +static void log_unaligned(struct pt_regs *regs) +{ + static DEFINE_RATELIMIT_STATE(ratelimit, 5 * HZ, 5); + + if (__ratelimit(&ratelimit)) { + printk("Kernel unaligned access at TPC[%lx] %pS\n", + regs->tpc, (void *) regs->tpc); + } +} + +asmlinkage void kernel_unaligned_trap(struct pt_regs *regs, unsigned int insn) +{ + enum direction dir = decode_direction(insn); + int size = decode_access_size(regs, insn); + int orig_asi, asi; + + current_thread_info()->kern_una_regs = regs; + current_thread_info()->kern_una_insn = insn; + + orig_asi = asi = decode_asi(insn, regs); + + /* If this is a {get,put}_user() on an unaligned userspace pointer, + * just signal a fault and do not log the event. + */ + if (asi == ASI_AIUS) { + kernel_mna_trap_fault(0); + return; + } + + log_unaligned(regs); + + if (!ok_for_kernel(insn) || dir == both) { + printk("Unsupported unaligned load/store trap for kernel " + "at <%016lx>.\n", regs->tpc); + unaligned_panic("Kernel does fpu/atomic " + "unaligned load/store.", regs); + + kernel_mna_trap_fault(0); + } else { + unsigned long addr, *reg_addr; + int err; + + addr = compute_effective_address(regs, insn, + ((insn >> 25) & 0x1f)); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); + switch (asi) { + case ASI_NL: + case ASI_AIUPL: + case ASI_AIUSL: + case ASI_PL: + case ASI_SL: + case ASI_PNFL: + case ASI_SNFL: + asi &= ~0x08; + break; + } + switch (dir) { + case load: + reg_addr = fetch_reg_addr(((insn>>25)&0x1f), regs); + err = do_int_load(reg_addr, size, + (unsigned long *) addr, + decode_signedness(insn), asi); + if (likely(!err) && unlikely(asi != orig_asi)) { + unsigned long val_in = *reg_addr; + switch (size) { + case 2: + val_in = swab16(val_in); + break; + case 4: + val_in = swab32(val_in); + break; + case 8: + val_in = swab64(val_in); + break; + case 16: + default: + BUG(); + break; + } + *reg_addr = val_in; + } + break; + + case store: + err = do_int_store(((insn>>25)&0x1f), size, + (unsigned long *) addr, regs, + asi, orig_asi); + break; + + default: + panic("Impossible kernel unaligned trap."); + /* Not reached... */ + } + if (unlikely(err)) + kernel_mna_trap_fault(1); + else + advance(regs); + } +} + +int handle_popc(u32 insn, struct pt_regs *regs) +{ + int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; + int ret, rd = ((insn >> 25) & 0x1f); + u64 value; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + if (insn & 0x2000) { + maybe_flush_windows(0, 0, rd, from_kernel); + value = sign_extend_imm13(insn); + } else { + maybe_flush_windows(0, insn & 0x1f, rd, from_kernel); + value = fetch_reg(insn & 0x1f, regs); + } + ret = hweight64(value); + if (rd < 16) { + if (rd) + regs->u_regs[rd] = ret; + } else { + unsigned long fp = regs->u_regs[UREG_FP]; + + if (!test_thread_64bit_stack(fp)) { + struct reg_window32 __user *win32; + win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp)); + put_user(ret, &win32->locals[rd - 16]); + } else { + struct reg_window __user *win; + win = (struct reg_window __user *)(fp + STACK_BIAS); + put_user(ret, &win->locals[rd - 16]); + } + } + advance(regs); + return 1; +} + +extern void do_fpother(struct pt_regs *regs); +extern void do_privact(struct pt_regs *regs); +extern void sun4v_data_access_exception(struct pt_regs *regs, + unsigned long addr, + unsigned long type_ctx); + +int handle_ldf_stq(u32 insn, struct pt_regs *regs) +{ + unsigned long addr = compute_effective_address(regs, insn, 0); + int freg; + struct fpustate *f = FPUSTATE; + int asi = decode_asi(insn, regs); + int flag; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + + save_and_clear_fpu(); + current_thread_info()->xfsr[0] &= ~0x1c000; + if (insn & 0x200000) { + /* STQ */ + u64 first = 0, second = 0; + + freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); + flag = (freg < 32) ? FPRS_DL : FPRS_DU; + if (freg & 3) { + current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; + do_fpother(regs); + return 0; + } + if (current_thread_info()->fpsaved[0] & flag) { + first = *(u64 *)&f->regs[freg]; + second = *(u64 *)&f->regs[freg+2]; + } + if (asi < 0x80) { + do_privact(regs); + return 1; + } + switch (asi) { + case ASI_P: + case ASI_S: break; + case ASI_PL: + case ASI_SL: + { + /* Need to convert endians */ + u64 tmp = __swab64p(&first); + + first = __swab64p(&second); + second = tmp; + break; + } + default: + if (tlb_type == hypervisor) + sun4v_data_access_exception(regs, addr, 0); + else + spitfire_data_access_exception(regs, 0, addr); + return 1; + } + if (put_user (first >> 32, (u32 __user *)addr) || + __put_user ((u32)first, (u32 __user *)(addr + 4)) || + __put_user (second >> 32, (u32 __user *)(addr + 8)) || + __put_user ((u32)second, (u32 __user *)(addr + 12))) { + if (tlb_type == hypervisor) + sun4v_data_access_exception(regs, addr, 0); + else + spitfire_data_access_exception(regs, 0, addr); + return 1; + } + } else { + /* LDF, LDDF, LDQF */ + u32 data[4] __attribute__ ((aligned(8))); + int size, i; + int err; + + if (asi < 0x80) { + do_privact(regs); + return 1; + } else if (asi > ASI_SNFL) { + if (tlb_type == hypervisor) + sun4v_data_access_exception(regs, addr, 0); + else + spitfire_data_access_exception(regs, 0, addr); + return 1; + } + switch (insn & 0x180000) { + case 0x000000: size = 1; break; + case 0x100000: size = 4; break; + default: size = 2; break; + } + if (size == 1) + freg = (insn >> 25) & 0x1f; + else + freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); + flag = (freg < 32) ? FPRS_DL : FPRS_DU; + + for (i = 0; i < size; i++) + data[i] = 0; + + err = get_user (data[0], (u32 __user *) addr); + if (!err) { + for (i = 1; i < size; i++) + err |= __get_user (data[i], (u32 __user *)(addr + 4*i)); + } + if (err && !(asi & 0x2 /* NF */)) { + if (tlb_type == hypervisor) + sun4v_data_access_exception(regs, addr, 0); + else + spitfire_data_access_exception(regs, 0, addr); + return 1; + } + if (asi & 0x8) /* Little */ { + u64 tmp; + + switch (size) { + case 1: data[0] = le32_to_cpup(data + 0); break; + default:*(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 0)); + break; + case 4: tmp = le64_to_cpup((u64 *)(data + 0)); + *(u64 *)(data + 0) = le64_to_cpup((u64 *)(data + 2)); + *(u64 *)(data + 2) = tmp; + break; + } + } + if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) { + current_thread_info()->fpsaved[0] = FPRS_FEF; + current_thread_info()->gsr[0] = 0; + } + if (!(current_thread_info()->fpsaved[0] & flag)) { + if (freg < 32) + memset(f->regs, 0, 32*sizeof(u32)); + else + memset(f->regs+32, 0, 32*sizeof(u32)); + } + memcpy(f->regs + freg, data, size * 4); + current_thread_info()->fpsaved[0] |= flag; + } + advance(regs); + return 1; +} + +void handle_ld_nf(u32 insn, struct pt_regs *regs) +{ + int rd = ((insn >> 25) & 0x1f); + int from_kernel = (regs->tstate & TSTATE_PRIV) != 0; + unsigned long *reg; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + + maybe_flush_windows(0, 0, rd, from_kernel); + reg = fetch_reg_addr(rd, regs); + if (from_kernel || rd < 16) { + reg[0] = 0; + if ((insn & 0x780000) == 0x180000) + reg[1] = 0; + } else if (!test_thread_64bit_stack(regs->u_regs[UREG_FP])) { + put_user(0, (int __user *) reg); + if ((insn & 0x780000) == 0x180000) + put_user(0, ((int __user *) reg) + 1); + } else { + put_user(0, (unsigned long __user *) reg); + if ((insn & 0x780000) == 0x180000) + put_user(0, (unsigned long __user *) reg + 1); + } + advance(regs); +} + +void handle_lddfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) +{ + enum ctx_state prev_state = exception_enter(); + unsigned long pc = regs->tpc; + unsigned long tstate = regs->tstate; + u32 insn; + u64 value; + u8 freg; + int flag; + struct fpustate *f = FPUSTATE; + + if (tstate & TSTATE_PRIV) + die_if_kernel("lddfmna from kernel", regs); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar); + if (test_thread_flag(TIF_32BIT)) + pc = (u32)pc; + if (get_user(insn, (u32 __user *) pc) != -EFAULT) { + int asi = decode_asi(insn, regs); + u32 first, second; + int err; + + if ((asi > ASI_SNFL) || + (asi < ASI_P)) + goto daex; + first = second = 0; + err = get_user(first, (u32 __user *)sfar); + if (!err) + err = get_user(second, (u32 __user *)(sfar + 4)); + if (err) { + if (!(asi & 0x2)) + goto daex; + first = second = 0; + } + save_and_clear_fpu(); + freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); + value = (((u64)first) << 32) | second; + if (asi & 0x8) /* Little */ + value = __swab64p(&value); + flag = (freg < 32) ? FPRS_DL : FPRS_DU; + if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) { + current_thread_info()->fpsaved[0] = FPRS_FEF; + current_thread_info()->gsr[0] = 0; + } + if (!(current_thread_info()->fpsaved[0] & flag)) { + if (freg < 32) + memset(f->regs, 0, 32*sizeof(u32)); + else + memset(f->regs+32, 0, 32*sizeof(u32)); + } + *(u64 *)(f->regs + freg) = value; + current_thread_info()->fpsaved[0] |= flag; + } else { +daex: + if (tlb_type == hypervisor) + sun4v_data_access_exception(regs, sfar, sfsr); + else + spitfire_data_access_exception(regs, sfsr, sfar); + goto out; + } + advance(regs); +out: + exception_exit(prev_state); +} + +void handle_stdfmna(struct pt_regs *regs, unsigned long sfar, unsigned long sfsr) +{ + enum ctx_state prev_state = exception_enter(); + unsigned long pc = regs->tpc; + unsigned long tstate = regs->tstate; + u32 insn; + u64 value; + u8 freg; + int flag; + struct fpustate *f = FPUSTATE; + + if (tstate & TSTATE_PRIV) + die_if_kernel("stdfmna from kernel", regs); + perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, sfar); + if (test_thread_flag(TIF_32BIT)) + pc = (u32)pc; + if (get_user(insn, (u32 __user *) pc) != -EFAULT) { + int asi = decode_asi(insn, regs); + freg = ((insn >> 25) & 0x1e) | ((insn >> 20) & 0x20); + value = 0; + flag = (freg < 32) ? FPRS_DL : FPRS_DU; + if ((asi > ASI_SNFL) || + (asi < ASI_P)) + goto daex; + save_and_clear_fpu(); + if (current_thread_info()->fpsaved[0] & flag) + value = *(u64 *)&f->regs[freg]; + switch (asi) { + case ASI_P: + case ASI_S: break; + case ASI_PL: + case ASI_SL: + value = __swab64p(&value); break; + default: goto daex; + } + if (put_user (value >> 32, (u32 __user *) sfar) || + __put_user ((u32)value, (u32 __user *)(sfar + 4))) + goto daex; + } else { +daex: + if (tlb_type == hypervisor) + sun4v_data_access_exception(regs, sfar, sfsr); + else + spitfire_data_access_exception(regs, sfsr, sfar); + goto out; + } + advance(regs); +out: + exception_exit(prev_state); +} diff --git a/arch/sparc/kernel/uprobes.c b/arch/sparc/kernel/uprobes.c new file mode 100644 index 000000000..c8cac64e9 --- /dev/null +++ b/arch/sparc/kernel/uprobes.c @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * User-space Probes (UProbes) for sparc + * + * Copyright (C) 2013 Oracle Inc. + * + * Authors: + * Jose E. Marchesi + * Eric Saint Etienne + */ + +#include +#include +#include +#include +#include /* For struct task_struct */ +#include + +#include + +#include "kernel.h" + +/* Compute the address of the breakpoint instruction and return it. + * + * Note that uprobe_get_swbp_addr is defined as a weak symbol in + * kernel/events/uprobe.c. + */ +unsigned long uprobe_get_swbp_addr(struct pt_regs *regs) +{ + return instruction_pointer(regs); +} + +static void copy_to_page(struct page *page, unsigned long vaddr, + const void *src, int len) +{ + void *kaddr = kmap_atomic(page); + + memcpy(kaddr + (vaddr & ~PAGE_MASK), src, len); + kunmap_atomic(kaddr); +} + +/* Fill in the xol area with the probed instruction followed by the + * single-step trap. Some fixups in the copied instruction are + * performed at this point. + * + * Note that uprobe_xol_copy is defined as a weak symbol in + * kernel/events/uprobe.c. + */ +void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, + void *src, unsigned long len) +{ + const u32 stp_insn = UPROBE_STP_INSN; + u32 insn = *(u32 *) src; + + /* Branches annulling their delay slot must be fixed to not do + * so. Clearing the annul bit on these instructions we can be + * sure the single-step breakpoint in the XOL slot will be + * executed. + */ + + u32 op = (insn >> 30) & 0x3; + u32 op2 = (insn >> 22) & 0x7; + + if (op == 0 && + (op2 == 1 || op2 == 2 || op2 == 3 || op2 == 5 || op2 == 6) && + (insn & ANNUL_BIT) == ANNUL_BIT) + insn &= ~ANNUL_BIT; + + copy_to_page(page, vaddr, &insn, len); + copy_to_page(page, vaddr+len, &stp_insn, 4); +} + + +/* Instruction analysis/validity. + * + * This function returns 0 on success or a -ve number on error. + */ +int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, + struct mm_struct *mm, unsigned long addr) +{ + /* Any unsupported instruction? Then return -EINVAL */ + return 0; +} + +/* If INSN is a relative control transfer instruction, return the + * corrected branch destination value. + * + * Note that regs->tpc and regs->tnpc still hold the values of the + * program counters at the time of the single-step trap due to the + * execution of the UPROBE_STP_INSN at utask->xol_vaddr + 4. + * + */ +static unsigned long relbranch_fixup(u32 insn, struct uprobe_task *utask, + struct pt_regs *regs) +{ + /* Branch not taken, no mods necessary. */ + if (regs->tnpc == regs->tpc + 0x4UL) + return utask->autask.saved_tnpc + 0x4UL; + + /* The three cases are call, branch w/prediction, + * and traditional branch. + */ + if ((insn & 0xc0000000) == 0x40000000 || + (insn & 0xc1c00000) == 0x00400000 || + (insn & 0xc1c00000) == 0x00800000) { + unsigned long real_pc = (unsigned long) utask->vaddr; + unsigned long ixol_addr = utask->xol_vaddr; + + /* The instruction did all the work for us + * already, just apply the offset to the correct + * instruction location. + */ + return (real_pc + (regs->tnpc - ixol_addr)); + } + + /* It is jmpl or some other absolute PC modification instruction, + * leave NPC as-is. + */ + return regs->tnpc; +} + +/* If INSN is an instruction which writes its PC location + * into a destination register, fix that up. + */ +static int retpc_fixup(struct pt_regs *regs, u32 insn, + unsigned long real_pc) +{ + unsigned long *slot = NULL; + int rc = 0; + + /* Simplest case is 'call', which always uses %o7 */ + if ((insn & 0xc0000000) == 0x40000000) + slot = ®s->u_regs[UREG_I7]; + + /* 'jmpl' encodes the register inside of the opcode */ + if ((insn & 0xc1f80000) == 0x81c00000) { + unsigned long rd = ((insn >> 25) & 0x1f); + + if (rd <= 15) { + slot = ®s->u_regs[rd]; + } else { + unsigned long fp = regs->u_regs[UREG_FP]; + /* Hard case, it goes onto the stack. */ + flushw_all(); + + rd -= 16; + if (test_thread_64bit_stack(fp)) { + unsigned long __user *uslot = + (unsigned long __user *) (fp + STACK_BIAS) + rd; + rc = __put_user(real_pc, uslot); + } else { + unsigned int __user *uslot = (unsigned int + __user *) fp + rd; + rc = __put_user((u32) real_pc, uslot); + } + } + } + if (slot != NULL) + *slot = real_pc; + return rc; +} + +/* Single-stepping can be avoided for certain instructions: NOPs and + * instructions that can be emulated. This function determines + * whether the instruction where the uprobe is installed falls in one + * of these cases and emulates it. + * + * This function returns true if the single-stepping can be skipped, + * false otherwise. + */ +bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + /* We currently only emulate NOP instructions. + */ + + if (auprobe->ixol == (1 << 24)) { + regs->tnpc += 4; + regs->tpc += 4; + return true; + } + + return false; +} + +/* Prepare to execute out of line. At this point + * current->utask->xol_vaddr points to an allocated XOL slot properly + * initialized with the original instruction and the single-stepping + * trap instruction. + * + * This function returns 0 on success, any other number on error. + */ +int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + struct arch_uprobe_task *autask = ¤t->utask->autask; + + /* Save the current program counters so they can be restored + * later. + */ + autask->saved_tpc = regs->tpc; + autask->saved_tnpc = regs->tnpc; + + /* Adjust PC and NPC so the first instruction in the XOL slot + * will be executed by the user task. + */ + instruction_pointer_set(regs, utask->xol_vaddr); + + return 0; +} + +/* Prepare to resume execution after the single-step. Called after + * single-stepping. To avoid the SMP problems that can occur when we + * temporarily put back the original opcode to single-step, we + * single-stepped a copy of the instruction. + * + * This function returns 0 on success, any other number on error. + */ +int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + struct arch_uprobe_task *autask = &utask->autask; + u32 insn = auprobe->ixol; + int rc = 0; + + if (utask->state == UTASK_SSTEP_ACK) { + regs->tnpc = relbranch_fixup(insn, utask, regs); + regs->tpc = autask->saved_tnpc; + rc = retpc_fixup(regs, insn, (unsigned long) utask->vaddr); + } else { + regs->tnpc = utask->vaddr+4; + regs->tpc = autask->saved_tnpc+4; + } + return rc; +} + +/* Handler for uprobe traps. This is called from the traps table and + * triggers the proper die notification. + */ +asmlinkage void uprobe_trap(struct pt_regs *regs, + unsigned long trap_level) +{ + BUG_ON(trap_level != 0x173 && trap_level != 0x174); + + /* We are only interested in user-mode code. Uprobe traps + * shall not be present in kernel code. + */ + if (!user_mode(regs)) { + local_irq_enable(); + bad_trap(regs, trap_level); + return; + } + + /* trap_level == 0x173 --> ta 0x73 + * trap_level == 0x174 --> ta 0x74 + */ + if (notify_die((trap_level == 0x173) ? DIE_BPT : DIE_SSTEP, + (trap_level == 0x173) ? "bpt" : "sstep", + regs, 0, trap_level, SIGTRAP) != NOTIFY_STOP) + bad_trap(regs, trap_level); +} + +/* Callback routine for handling die notifications. +*/ +int arch_uprobe_exception_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + int ret = NOTIFY_DONE; + struct die_args *args = (struct die_args *)data; + + /* We are only interested in userspace traps */ + if (args->regs && !user_mode(args->regs)) + return NOTIFY_DONE; + + switch (val) { + case DIE_BPT: + if (uprobe_pre_sstep_notifier(args->regs)) + ret = NOTIFY_STOP; + break; + + case DIE_SSTEP: + if (uprobe_post_sstep_notifier(args->regs)) + ret = NOTIFY_STOP; + break; + + default: + break; + } + + return ret; +} + +/* This function gets called when a XOL instruction either gets + * trapped or the thread has a fatal signal, so reset the instruction + * pointer to its probed address. + */ +void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) +{ + struct uprobe_task *utask = current->utask; + + instruction_pointer_set(regs, utask->vaddr); +} + +/* If xol insn itself traps and generates a signal(Say, + * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped + * instruction jumps back to its own address. + */ +bool arch_uprobe_xol_was_trapped(struct task_struct *t) +{ + return false; +} + +unsigned long +arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, + struct pt_regs *regs) +{ + unsigned long orig_ret_vaddr = regs->u_regs[UREG_I7]; + + regs->u_regs[UREG_I7] = trampoline_vaddr-8; + + return orig_ret_vaddr + 8; +} diff --git a/arch/sparc/kernel/urtt_fill.S b/arch/sparc/kernel/urtt_fill.S new file mode 100644 index 000000000..e4cee7be5 --- /dev/null +++ b/arch/sparc/kernel/urtt_fill.S @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include + + .text + .align 8 + .globl user_rtt_fill_fixup_common +user_rtt_fill_fixup_common: + rdpr %cwp, %g1 + add %g1, 1, %g1 + wrpr %g1, 0x0, %cwp + + rdpr %wstate, %g2 + sll %g2, 3, %g2 + wrpr %g2, 0x0, %wstate + + /* We know %canrestore and %otherwin are both zero. */ + + sethi %hi(sparc64_kern_pri_context), %g2 + ldx [%g2 + %lo(sparc64_kern_pri_context)], %g2 + mov PRIMARY_CONTEXT, %g1 + +661: stxa %g2, [%g1] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g1] ASI_MMU + .previous + + sethi %hi(KERNBASE), %g1 + flush %g1 + + mov %g4, %l4 + mov %g5, %l5 + brnz,pn %g3, 1f + mov %g3, %l3 + + or %g4, FAULT_CODE_WINFIXUP, %g4 + stb %g4, [%g6 + TI_FAULT_CODE] + stx %g5, [%g6 + TI_FAULT_ADDR] +1: + mov %g6, %l1 + wrpr %g0, 0x0, %tl + +661: nop + .section .sun4v_1insn_patch, "ax" + .word 661b + SET_GL(0) + .previous + +661: wrpr %g0, RTRAP_PSTATE, %pstate + .section .sun_m7_1insn_patch, "ax" + .word 661b + /* Re-enable PSTATE.mcde to maintain ADI security */ + wrpr %g0, RTRAP_PSTATE|PSTATE_MCDE, %pstate + .previous + + mov %l1, %g6 + ldx [%g6 + TI_TASK], %g4 + LOAD_PER_CPU_BASE(%g5, %g6, %g1, %g2, %g3) + + brnz,pn %l3, 1f + nop + + call do_sparc64_fault + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + +1: cmp %g3, 2 + bne,pn %xcc, 2f + nop + + sethi %hi(tlb_type), %g1 + lduw [%g1 + %lo(tlb_type)], %g1 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + mov %l4, %o2 + call sun4v_do_mna + mov %l5, %o1 + ba,a,pt %xcc, rtrap +1: mov %l4, %o1 + mov %l5, %o2 + call mem_address_unaligned + nop + ba,a,pt %xcc, rtrap + +2: sethi %hi(tlb_type), %g1 + mov %l4, %o1 + lduw [%g1 + %lo(tlb_type)], %g1 + mov %l5, %o2 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + call sun4v_data_access_exception + nop + ba,a,pt %xcc, rtrap + nop + +1: call spitfire_data_access_exception + nop + ba,a,pt %xcc, rtrap diff --git a/arch/sparc/kernel/utrap.S b/arch/sparc/kernel/utrap.S new file mode 100644 index 000000000..7a2d9a9be --- /dev/null +++ b/arch/sparc/kernel/utrap.S @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + .globl utrap_trap + .type utrap_trap,#function +utrap_trap: /* %g3=handler,%g4=level */ + TRAP_LOAD_THREAD_REG(%g6, %g1) + ldx [%g6 + TI_UTRAPS], %g1 + brnz,pt %g1, invoke_utrap + nop + + ba,pt %xcc, etrap + rd %pc, %g7 + mov %l4, %o1 + call bad_trap + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + +invoke_utrap: + sllx %g3, 3, %g3 + ldx [%g1 + %g3], %g1 + save %sp, -128, %sp + rdpr %tstate, %l6 + rdpr %cwp, %l7 + andn %l6, TSTATE_CWP, %l6 + wrpr %l6, %l7, %tstate + rdpr %tpc, %l6 + rdpr %tnpc, %l7 + wrpr %g1, 0, %tnpc + done + .size utrap_trap,.-utrap_trap diff --git a/arch/sparc/kernel/vio.c b/arch/sparc/kernel/vio.c new file mode 100644 index 000000000..b7b06752a --- /dev/null +++ b/arch/sparc/kernel/vio.c @@ -0,0 +1,574 @@ +// SPDX-License-Identifier: GPL-2.0 +/* vio.c: Virtual I/O channel devices probing infrastructure. + * + * Copyright (c) 2003-2005 IBM Corp. + * Dave Engebretsen engebret@us.ibm.com + * Santiago Leon santil@us.ibm.com + * Hollis Blanchard + * Stephen Rothwell + * + * Adapted to sparc64 by David S. Miller davem@davemloft.net + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static const struct vio_device_id *vio_match_device( + const struct vio_device_id *matches, + const struct vio_dev *dev) +{ + const char *type, *compat; + int len; + + type = dev->type; + compat = dev->compat; + len = dev->compat_len; + + while (matches->type[0] || matches->compat[0]) { + int match = 1; + if (matches->type[0]) + match &= !strcmp(matches->type, type); + + if (matches->compat[0]) { + match &= len && + of_find_in_proplist(compat, matches->compat, len); + } + if (match) + return matches; + matches++; + } + return NULL; +} + +static int vio_hotplug(const struct device *dev, struct kobj_uevent_env *env) +{ + const struct vio_dev *vio_dev = to_vio_dev(dev); + + add_uevent_var(env, "MODALIAS=vio:T%sS%s", vio_dev->type, vio_dev->compat); + return 0; +} + +static int vio_bus_match(struct device *dev, const struct device_driver *drv) +{ + struct vio_dev *vio_dev = to_vio_dev(dev); + const struct vio_driver *vio_drv = to_vio_driver(drv); + const struct vio_device_id *matches = vio_drv->id_table; + + if (!matches) + return 0; + + return vio_match_device(matches, vio_dev) != NULL; +} + +static int vio_device_probe(struct device *dev) +{ + struct vio_dev *vdev = to_vio_dev(dev); + struct vio_driver *drv = to_vio_driver(dev->driver); + const struct vio_device_id *id; + + if (!drv->probe) + return -ENODEV; + + id = vio_match_device(drv->id_table, vdev); + if (!id) + return -ENODEV; + + /* alloc irqs (unless the driver specified not to) */ + if (!drv->no_irq) { + if (vdev->tx_irq == 0 && vdev->tx_ino != ~0UL) + vdev->tx_irq = sun4v_build_virq(vdev->cdev_handle, + vdev->tx_ino); + + if (vdev->rx_irq == 0 && vdev->rx_ino != ~0UL) + vdev->rx_irq = sun4v_build_virq(vdev->cdev_handle, + vdev->rx_ino); + } + + return drv->probe(vdev, id); +} + +static void vio_device_remove(struct device *dev) +{ + struct vio_dev *vdev = to_vio_dev(dev); + struct vio_driver *drv = to_vio_driver(dev->driver); + + if (drv->remove) { + /* + * Ideally, we would remove/deallocate tx/rx virqs + * here - however, there are currently no support + * routines to do so at the moment. TBD + */ + + drv->remove(vdev); + } +} + +static ssize_t devspec_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct vio_dev *vdev = to_vio_dev(dev); + const char *str = "none"; + + if (!strcmp(vdev->type, "vnet-port")) + str = "vnet"; + else if (!strcmp(vdev->type, "vdc-port")) + str = "vdisk"; + + return sysfs_emit(buf, "%s\n", str); +} +static DEVICE_ATTR_RO(devspec); + +static ssize_t type_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct vio_dev *vdev = to_vio_dev(dev); + return sysfs_emit(buf, "%s\n", vdev->type); +} +static DEVICE_ATTR_RO(type); + +static ssize_t modalias_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + const struct vio_dev *vdev = to_vio_dev(dev); + + return sysfs_emit(buf, "vio:T%sS%s\n", vdev->type, vdev->compat); +} +static DEVICE_ATTR_RO(modalias); + +static struct attribute *vio_dev_attrs[] = { + &dev_attr_devspec.attr, + &dev_attr_type.attr, + &dev_attr_modalias.attr, + NULL, + }; +ATTRIBUTE_GROUPS(vio_dev); + +static const struct bus_type vio_bus_type = { + .name = "vio", + .dev_groups = vio_dev_groups, + .uevent = vio_hotplug, + .match = vio_bus_match, + .probe = vio_device_probe, + .remove = vio_device_remove, +}; + +int __vio_register_driver(struct vio_driver *viodrv, struct module *owner, + const char *mod_name) +{ + viodrv->driver.bus = &vio_bus_type; + viodrv->driver.name = viodrv->name; + viodrv->driver.owner = owner; + viodrv->driver.mod_name = mod_name; + + return driver_register(&viodrv->driver); +} +EXPORT_SYMBOL(__vio_register_driver); + +void vio_unregister_driver(struct vio_driver *viodrv) +{ + driver_unregister(&viodrv->driver); +} +EXPORT_SYMBOL(vio_unregister_driver); + +static void vio_dev_release(struct device *dev) +{ + kfree(to_vio_dev(dev)); +} + +static ssize_t +show_pciobppath_attr(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct vio_dev *vdev; + struct device_node *dp; + + vdev = to_vio_dev(dev); + dp = vdev->dp; + + return sysfs_emit(buf, "%pOF\n", dp); +} + +static DEVICE_ATTR(obppath, S_IRUSR | S_IRGRP | S_IROTH, + show_pciobppath_attr, NULL); + +static struct device_node *cdev_node; + +static struct vio_dev *root_vdev; +static u64 cdev_cfg_handle; + +static const u64 *vio_cfg_handle(struct mdesc_handle *hp, u64 node) +{ + const u64 *cfg_handle = NULL; + u64 a; + + mdesc_for_each_arc(a, hp, node, MDESC_ARC_TYPE_BACK) { + u64 target; + + target = mdesc_arc_target(hp, a); + cfg_handle = mdesc_get_property(hp, target, + "cfg-handle", NULL); + if (cfg_handle) + break; + } + + return cfg_handle; +} + +/** + * vio_vdev_node() - Find VDEV node in MD + * @hp: Handle to the MD + * @vdev: Pointer to VDEV + * + * Find the node in the current MD which matches the given vio_dev. This + * must be done dynamically since the node value can change if the MD + * is updated. + * + * NOTE: the MD must be locked, using mdesc_grab(), when calling this routine + * + * Return: The VDEV node in MDESC + */ +u64 vio_vdev_node(struct mdesc_handle *hp, struct vio_dev *vdev) +{ + u64 node; + + if (vdev == NULL) + return MDESC_NODE_NULL; + + node = mdesc_get_node(hp, (const char *)vdev->node_name, + &vdev->md_node_info); + + return node; +} +EXPORT_SYMBOL(vio_vdev_node); + +static void vio_fill_channel_info(struct mdesc_handle *hp, u64 mp, + struct vio_dev *vdev) +{ + u64 a; + + vdev->tx_ino = ~0UL; + vdev->rx_ino = ~0UL; + vdev->channel_id = ~0UL; + mdesc_for_each_arc(a, hp, mp, MDESC_ARC_TYPE_FWD) { + const u64 *chan_id; + const u64 *irq; + u64 target; + + target = mdesc_arc_target(hp, a); + + irq = mdesc_get_property(hp, target, "tx-ino", NULL); + if (irq) + vdev->tx_ino = *irq; + + irq = mdesc_get_property(hp, target, "rx-ino", NULL); + if (irq) + vdev->rx_ino = *irq; + + chan_id = mdesc_get_property(hp, target, "id", NULL); + if (chan_id) + vdev->channel_id = *chan_id; + } + + vdev->cdev_handle = cdev_cfg_handle; +} + +int vio_set_intr(unsigned long dev_ino, int state) +{ + int err; + + err = sun4v_vintr_set_valid(cdev_cfg_handle, dev_ino, state); + return err; +} +EXPORT_SYMBOL(vio_set_intr); + +static struct vio_dev *vio_create_one(struct mdesc_handle *hp, u64 mp, + const char *node_name, + struct device *parent) +{ + const char *type, *compat; + struct device_node *dp; + struct vio_dev *vdev; + int err, tlen, clen; + const u64 *id, *cfg_handle; + + type = mdesc_get_property(hp, mp, "device-type", &tlen); + if (!type) { + type = mdesc_get_property(hp, mp, "name", &tlen); + if (!type) { + type = mdesc_node_name(hp, mp); + tlen = strlen(type) + 1; + } + } + if (tlen > VIO_MAX_TYPE_LEN || strlen(type) >= VIO_MAX_TYPE_LEN) { + printk(KERN_ERR "VIO: Type string [%s] is too long.\n", + type); + return NULL; + } + + id = mdesc_get_property(hp, mp, "id", NULL); + + cfg_handle = vio_cfg_handle(hp, mp); + + compat = mdesc_get_property(hp, mp, "device-type", &clen); + if (!compat) { + clen = 0; + } else if (clen > VIO_MAX_COMPAT_LEN) { + printk(KERN_ERR "VIO: Compat len %d for [%s] is too long.\n", + clen, type); + return NULL; + } + + vdev = kzalloc_obj(*vdev); + if (!vdev) { + printk(KERN_ERR "VIO: Could not allocate vio_dev\n"); + return NULL; + } + + vdev->mp = mp; + memcpy(vdev->type, type, tlen); + if (compat) + memcpy(vdev->compat, compat, clen); + else + memset(vdev->compat, 0, sizeof(vdev->compat)); + vdev->compat_len = clen; + + vdev->port_id = ~0UL; + vdev->tx_irq = 0; + vdev->rx_irq = 0; + + vio_fill_channel_info(hp, mp, vdev); + + if (!id) { + dev_set_name(&vdev->dev, "%s", type); + vdev->dev_no = ~(u64)0; + } else if (!cfg_handle) { + dev_set_name(&vdev->dev, "%s-%llu", type, *id); + vdev->dev_no = *id; + } else { + dev_set_name(&vdev->dev, "%s-%llu-%llu", type, + *cfg_handle, *id); + vdev->dev_no = *cfg_handle; + vdev->port_id = *id; + } + + vdev->dev.parent = parent; + vdev->dev.bus = &vio_bus_type; + vdev->dev.release = vio_dev_release; + + if (parent == NULL) { + dp = cdev_node; + } else if (to_vio_dev(parent) == root_vdev) { + for_each_child_of_node(cdev_node, dp) { + if (of_node_is_type(dp, type)) + break; + } + } else { + dp = to_vio_dev(parent)->dp; + } + vdev->dp = dp; + + /* + * node_name is NULL for the parent/channel-devices node and + * the parent doesn't require the MD node info. + */ + if (node_name != NULL) { + strscpy(vdev->node_name, node_name); + + err = mdesc_get_node_info(hp, mp, node_name, + &vdev->md_node_info); + if (err) { + pr_err("VIO: Could not get MD node info %s, err=%d\n", + dev_name(&vdev->dev), err); + kfree(vdev); + return NULL; + } + } + + pr_info("VIO: Adding device %s (tx_ino = %llx, rx_ino = %llx)\n", + dev_name(&vdev->dev), vdev->tx_ino, vdev->rx_ino); + + err = device_register(&vdev->dev); + if (err) { + printk(KERN_ERR "VIO: Could not register device %s, err=%d\n", + dev_name(&vdev->dev), err); + put_device(&vdev->dev); + return NULL; + } + if (vdev->dp) + err = sysfs_create_file(&vdev->dev.kobj, + &dev_attr_obppath.attr); + + return vdev; +} + +static void vio_add(struct mdesc_handle *hp, u64 node, + const char *node_name) +{ + (void) vio_create_one(hp, node, node_name, &root_vdev->dev); +} + +struct vio_remove_node_data { + struct mdesc_handle *hp; + u64 node; +}; + +static int vio_md_node_match(struct device *dev, const void *arg) +{ + struct vio_dev *vdev = to_vio_dev(dev); + const struct vio_remove_node_data *node_data; + u64 node; + + node_data = (const struct vio_remove_node_data *)arg; + + node = vio_vdev_node(node_data->hp, vdev); + + if (node == node_data->node) + return 1; + else + return 0; +} + +static void vio_remove(struct mdesc_handle *hp, u64 node, const char *node_name) +{ + struct vio_remove_node_data node_data; + struct device *dev; + + node_data.hp = hp; + node_data.node = node; + + dev = device_find_child(&root_vdev->dev, (void *)&node_data, + vio_md_node_match); + if (dev) { + printk(KERN_INFO "VIO: Removing device %s\n", dev_name(dev)); + + device_unregister(dev); + put_device(dev); + } else { + pr_err("VIO: %s node not found in MDESC\n", node_name); + } +} + +static struct mdesc_notifier_client vio_device_notifier = { + .add = vio_add, + .remove = vio_remove, + .node_name = "virtual-device-port", +}; + +/* We are only interested in domain service ports under the + * "domain-services" node. On control nodes there is another port + * under "openboot" that we should not mess with as aparently that is + * reserved exclusively for OBP use. + */ +static void vio_add_ds(struct mdesc_handle *hp, u64 node, + const char *node_name) +{ + int found; + u64 a; + + found = 0; + mdesc_for_each_arc(a, hp, node, MDESC_ARC_TYPE_BACK) { + u64 target = mdesc_arc_target(hp, a); + const char *name = mdesc_node_name(hp, target); + + if (!strcmp(name, "domain-services")) { + found = 1; + break; + } + } + + if (found) + (void) vio_create_one(hp, node, node_name, &root_vdev->dev); +} + +static struct mdesc_notifier_client vio_ds_notifier = { + .add = vio_add_ds, + .remove = vio_remove, + .node_name = "domain-services-port", +}; + +static const char *channel_devices_node = "channel-devices"; +static const char *channel_devices_compat = "SUNW,sun4v-channel-devices"; +static const char *cfg_handle_prop = "cfg-handle"; + +static int __init vio_init(void) +{ + struct mdesc_handle *hp; + const char *compat; + const u64 *cfg_handle; + int err, len; + u64 root; + + err = bus_register(&vio_bus_type); + if (err) { + printk(KERN_ERR "VIO: Could not register bus type err=%d\n", + err); + return err; + } + + hp = mdesc_grab(); + if (!hp) + return 0; + + root = mdesc_node_by_name(hp, MDESC_NODE_NULL, channel_devices_node); + if (root == MDESC_NODE_NULL) { + printk(KERN_INFO "VIO: No channel-devices MDESC node.\n"); + mdesc_release(hp); + return 0; + } + + cdev_node = of_find_node_by_name(NULL, "channel-devices"); + err = -ENODEV; + if (!cdev_node) { + printk(KERN_INFO "VIO: No channel-devices OBP node.\n"); + goto out_release; + } + + compat = mdesc_get_property(hp, root, "compatible", &len); + if (!compat) { + printk(KERN_ERR "VIO: Channel devices lacks compatible " + "property\n"); + goto out_release; + } + if (!of_find_in_proplist(compat, channel_devices_compat, len)) { + printk(KERN_ERR "VIO: Channel devices node lacks (%s) " + "compat entry.\n", channel_devices_compat); + goto out_release; + } + + cfg_handle = mdesc_get_property(hp, root, cfg_handle_prop, NULL); + if (!cfg_handle) { + printk(KERN_ERR "VIO: Channel devices lacks %s property\n", + cfg_handle_prop); + goto out_release; + } + + cdev_cfg_handle = *cfg_handle; + + root_vdev = vio_create_one(hp, root, NULL, NULL); + err = -ENODEV; + if (!root_vdev) { + printk(KERN_ERR "VIO: Could not create root device.\n"); + goto out_release; + } + + mdesc_register_notifier(&vio_device_notifier); + mdesc_register_notifier(&vio_ds_notifier); + + mdesc_release(hp); + + return err; + +out_release: + mdesc_release(hp); + return err; +} + +postcore_initcall(vio_init); diff --git a/arch/sparc/kernel/viohs.c b/arch/sparc/kernel/viohs.c new file mode 100644 index 000000000..8fb2e7ca5 --- /dev/null +++ b/arch/sparc/kernel/viohs.c @@ -0,0 +1,860 @@ +// SPDX-License-Identifier: GPL-2.0 +/* viohs.c: LDOM Virtual I/O handshake helper layer. + * + * Copyright (C) 2007 David S. Miller + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +int vio_ldc_send(struct vio_driver_state *vio, void *data, int len) +{ + int err, limit = 1000; + + err = -EINVAL; + while (limit-- > 0) { + err = ldc_write(vio->lp, data, len); + if (!err || (err != -EAGAIN)) + break; + udelay(1); + } + + return err; +} +EXPORT_SYMBOL(vio_ldc_send); + +static int send_ctrl(struct vio_driver_state *vio, + struct vio_msg_tag *tag, int len) +{ + tag->sid = vio_send_sid(vio); + return vio_ldc_send(vio, tag, len); +} + +static void init_tag(struct vio_msg_tag *tag, u8 type, u8 stype, u16 stype_env) +{ + tag->type = type; + tag->stype = stype; + tag->stype_env = stype_env; +} + +static int send_version(struct vio_driver_state *vio, u16 major, u16 minor) +{ + struct vio_ver_info pkt; + + vio->_local_sid = (u32) sched_clock(); + + memset(&pkt, 0, sizeof(pkt)); + init_tag(&pkt.tag, VIO_TYPE_CTRL, VIO_SUBTYPE_INFO, VIO_VER_INFO); + pkt.major = major; + pkt.minor = minor; + pkt.dev_class = vio->dev_class; + + viodbg(HS, "SEND VERSION INFO maj[%u] min[%u] devclass[%u]\n", + major, minor, vio->dev_class); + + return send_ctrl(vio, &pkt.tag, sizeof(pkt)); +} + +static int start_handshake(struct vio_driver_state *vio) +{ + int err; + + viodbg(HS, "START HANDSHAKE\n"); + + vio->hs_state = VIO_HS_INVALID; + + err = send_version(vio, + vio->ver_table[0].major, + vio->ver_table[0].minor); + if (err < 0) + return err; + + return 0; +} + +static void flush_rx_dring(struct vio_driver_state *vio) +{ + struct vio_dring_state *dr; + u64 ident; + + BUG_ON(!(vio->dr_state & VIO_DR_STATE_RXREG)); + + dr = &vio->drings[VIO_DRIVER_RX_RING]; + ident = dr->ident; + + BUG_ON(!vio->desc_buf); + kfree(vio->desc_buf); + vio->desc_buf = NULL; + + memset(dr, 0, sizeof(*dr)); + dr->ident = ident; +} + +void vio_link_state_change(struct vio_driver_state *vio, int event) +{ + if (event == LDC_EVENT_UP) { + vio->hs_state = VIO_HS_INVALID; + + switch (vio->dev_class) { + case VDEV_NETWORK: + case VDEV_NETWORK_SWITCH: + vio->dr_state = (VIO_DR_STATE_TXREQ | + VIO_DR_STATE_RXREQ); + break; + + case VDEV_DISK: + vio->dr_state = VIO_DR_STATE_TXREQ; + break; + case VDEV_DISK_SERVER: + vio->dr_state = VIO_DR_STATE_RXREQ; + break; + } + start_handshake(vio); + } else if (event == LDC_EVENT_RESET) { + vio->hs_state = VIO_HS_INVALID; + + if (vio->dr_state & VIO_DR_STATE_RXREG) + flush_rx_dring(vio); + + vio->dr_state = 0x00; + memset(&vio->ver, 0, sizeof(vio->ver)); + + ldc_disconnect(vio->lp); + } +} +EXPORT_SYMBOL(vio_link_state_change); + +static int handshake_failure(struct vio_driver_state *vio) +{ + struct vio_dring_state *dr; + + /* XXX Put policy here... Perhaps start a timer to fire + * XXX in 100 ms, which will bring the link up and retry + * XXX the handshake. + */ + + viodbg(HS, "HANDSHAKE FAILURE\n"); + + vio->dr_state &= ~(VIO_DR_STATE_TXREG | + VIO_DR_STATE_RXREG); + + dr = &vio->drings[VIO_DRIVER_RX_RING]; + memset(dr, 0, sizeof(*dr)); + + kfree(vio->desc_buf); + vio->desc_buf = NULL; + vio->desc_buf_len = 0; + + vio->hs_state = VIO_HS_INVALID; + + return -ECONNRESET; +} + +static int process_unknown(struct vio_driver_state *vio, void *arg) +{ + struct vio_msg_tag *pkt = arg; + + viodbg(HS, "UNKNOWN CONTROL [%02x:%02x:%04x:%08x]\n", + pkt->type, pkt->stype, pkt->stype_env, pkt->sid); + + printk(KERN_ERR "vio: ID[%lu] Resetting connection.\n", + vio->vdev->channel_id); + + ldc_disconnect(vio->lp); + + return -ECONNRESET; +} + +static int send_dreg(struct vio_driver_state *vio) +{ + struct vio_dring_state *dr = &vio->drings[VIO_DRIVER_TX_RING]; + union { + struct vio_dring_register pkt; + char all[sizeof(struct vio_dring_register) + + (sizeof(struct ldc_trans_cookie) * + VIO_MAX_RING_COOKIES)]; + } u; + size_t bytes = sizeof(struct vio_dring_register) + + (sizeof(struct ldc_trans_cookie) * + dr->ncookies); + int i; + + if (WARN_ON(bytes > sizeof(u))) + return -EINVAL; + + memset(&u, 0, bytes); + init_tag(&u.pkt.tag, VIO_TYPE_CTRL, VIO_SUBTYPE_INFO, VIO_DRING_REG); + u.pkt.dring_ident = 0; + u.pkt.num_descr = dr->num_entries; + u.pkt.descr_size = dr->entry_size; + u.pkt.options = VIO_TX_DRING; + u.pkt.num_cookies = dr->ncookies; + + viodbg(HS, "SEND DRING_REG INFO ndesc[%u] dsz[%u] opt[0x%x] " + "ncookies[%u]\n", + u.pkt.num_descr, u.pkt.descr_size, u.pkt.options, + u.pkt.num_cookies); + + for (i = 0; i < dr->ncookies; i++) { + u.pkt.cookies[i] = dr->cookies[i]; + + viodbg(HS, "DRING COOKIE(%d) [%016llx:%016llx]\n", + i, + (unsigned long long) u.pkt.cookies[i].cookie_addr, + (unsigned long long) u.pkt.cookies[i].cookie_size); + } + + return send_ctrl(vio, &u.pkt.tag, bytes); +} + +static int send_rdx(struct vio_driver_state *vio) +{ + struct vio_rdx pkt; + + memset(&pkt, 0, sizeof(pkt)); + + init_tag(&pkt.tag, VIO_TYPE_CTRL, VIO_SUBTYPE_INFO, VIO_RDX); + + viodbg(HS, "SEND RDX INFO\n"); + + return send_ctrl(vio, &pkt.tag, sizeof(pkt)); +} + +static int send_attr(struct vio_driver_state *vio) +{ + if (!vio->ops) + return -EINVAL; + + return vio->ops->send_attr(vio); +} + +static struct vio_version *find_by_major(struct vio_driver_state *vio, + u16 major) +{ + struct vio_version *ret = NULL; + int i; + + for (i = 0; i < vio->ver_table_entries; i++) { + struct vio_version *v = &vio->ver_table[i]; + if (v->major <= major) { + ret = v; + break; + } + } + return ret; +} + +static int process_ver_info(struct vio_driver_state *vio, + struct vio_ver_info *pkt) +{ + struct vio_version *vap; + int err; + + viodbg(HS, "GOT VERSION INFO maj[%u] min[%u] devclass[%u]\n", + pkt->major, pkt->minor, pkt->dev_class); + + if (vio->hs_state != VIO_HS_INVALID) { + /* XXX Perhaps invoke start_handshake? XXX */ + memset(&vio->ver, 0, sizeof(vio->ver)); + vio->hs_state = VIO_HS_INVALID; + } + + vap = find_by_major(vio, pkt->major); + + vio->_peer_sid = pkt->tag.sid; + + if (!vap) { + pkt->tag.stype = VIO_SUBTYPE_NACK; + pkt->major = 0; + pkt->minor = 0; + viodbg(HS, "SEND VERSION NACK maj[0] min[0]\n"); + err = send_ctrl(vio, &pkt->tag, sizeof(*pkt)); + } else if (vap->major != pkt->major) { + pkt->tag.stype = VIO_SUBTYPE_NACK; + pkt->major = vap->major; + pkt->minor = vap->minor; + viodbg(HS, "SEND VERSION NACK maj[%u] min[%u]\n", + pkt->major, pkt->minor); + err = send_ctrl(vio, &pkt->tag, sizeof(*pkt)); + } else { + struct vio_version ver = { + .major = pkt->major, + .minor = pkt->minor, + }; + if (ver.minor > vap->minor) + ver.minor = vap->minor; + pkt->minor = ver.minor; + pkt->tag.stype = VIO_SUBTYPE_ACK; + pkt->dev_class = vio->dev_class; + viodbg(HS, "SEND VERSION ACK maj[%u] min[%u]\n", + pkt->major, pkt->minor); + err = send_ctrl(vio, &pkt->tag, sizeof(*pkt)); + if (err > 0) { + vio->ver = ver; + vio->hs_state = VIO_HS_GOTVERS; + } + } + if (err < 0) + return handshake_failure(vio); + + return 0; +} + +static int process_ver_ack(struct vio_driver_state *vio, + struct vio_ver_info *pkt) +{ + viodbg(HS, "GOT VERSION ACK maj[%u] min[%u] devclass[%u]\n", + pkt->major, pkt->minor, pkt->dev_class); + + if (vio->hs_state & VIO_HS_GOTVERS) { + if (vio->ver.major != pkt->major || + vio->ver.minor != pkt->minor) { + pkt->tag.stype = VIO_SUBTYPE_NACK; + (void) send_ctrl(vio, &pkt->tag, sizeof(*pkt)); + return handshake_failure(vio); + } + } else { + vio->ver.major = pkt->major; + vio->ver.minor = pkt->minor; + vio->hs_state = VIO_HS_GOTVERS; + } + + switch (vio->dev_class) { + case VDEV_NETWORK: + case VDEV_DISK: + if (send_attr(vio) < 0) + return handshake_failure(vio); + break; + + default: + break; + } + + return 0; +} + +static int process_ver_nack(struct vio_driver_state *vio, + struct vio_ver_info *pkt) +{ + struct vio_version *nver; + + viodbg(HS, "GOT VERSION NACK maj[%u] min[%u] devclass[%u]\n", + pkt->major, pkt->minor, pkt->dev_class); + + if (pkt->major == 0 && pkt->minor == 0) + return handshake_failure(vio); + nver = find_by_major(vio, pkt->major); + if (!nver) + return handshake_failure(vio); + + if (send_version(vio, nver->major, nver->minor) < 0) + return handshake_failure(vio); + + return 0; +} + +static int process_ver(struct vio_driver_state *vio, struct vio_ver_info *pkt) +{ + switch (pkt->tag.stype) { + case VIO_SUBTYPE_INFO: + return process_ver_info(vio, pkt); + + case VIO_SUBTYPE_ACK: + return process_ver_ack(vio, pkt); + + case VIO_SUBTYPE_NACK: + return process_ver_nack(vio, pkt); + + default: + return handshake_failure(vio); + } +} + +static int process_attr(struct vio_driver_state *vio, void *pkt) +{ + int err; + + if (!(vio->hs_state & VIO_HS_GOTVERS)) + return handshake_failure(vio); + + if (!vio->ops) + return 0; + + err = vio->ops->handle_attr(vio, pkt); + if (err < 0) { + return handshake_failure(vio); + } else { + vio->hs_state |= VIO_HS_GOT_ATTR; + + if ((vio->dr_state & VIO_DR_STATE_TXREQ) && + !(vio->hs_state & VIO_HS_SENT_DREG)) { + if (send_dreg(vio) < 0) + return handshake_failure(vio); + + vio->hs_state |= VIO_HS_SENT_DREG; + } + } + + return 0; +} + +static int all_drings_registered(struct vio_driver_state *vio) +{ + int need_rx, need_tx; + + need_rx = (vio->dr_state & VIO_DR_STATE_RXREQ); + need_tx = (vio->dr_state & VIO_DR_STATE_TXREQ); + + if (need_rx && + !(vio->dr_state & VIO_DR_STATE_RXREG)) + return 0; + + if (need_tx && + !(vio->dr_state & VIO_DR_STATE_TXREG)) + return 0; + + return 1; +} + +static int process_dreg_info(struct vio_driver_state *vio, + struct vio_dring_register *pkt) +{ + struct vio_dring_state *dr; + int i; + + viodbg(HS, "GOT DRING_REG INFO ident[%llx] " + "ndesc[%u] dsz[%u] opt[0x%x] ncookies[%u]\n", + (unsigned long long) pkt->dring_ident, + pkt->num_descr, pkt->descr_size, pkt->options, + pkt->num_cookies); + + if (!(vio->dr_state & VIO_DR_STATE_RXREQ)) + goto send_nack; + + if (vio->dr_state & VIO_DR_STATE_RXREG) + goto send_nack; + + /* v1.6 and higher, ACK with desired, supported mode, or NACK */ + if (vio_version_after_eq(vio, 1, 6)) { + if (!(pkt->options & VIO_TX_DRING)) + goto send_nack; + pkt->options = VIO_TX_DRING; + } + + BUG_ON(vio->desc_buf); + + vio->desc_buf = kzalloc(pkt->descr_size, GFP_ATOMIC); + if (!vio->desc_buf) + goto send_nack; + + vio->desc_buf_len = pkt->descr_size; + + dr = &vio->drings[VIO_DRIVER_RX_RING]; + + dr->num_entries = pkt->num_descr; + dr->entry_size = pkt->descr_size; + dr->ncookies = pkt->num_cookies; + for (i = 0; i < dr->ncookies; i++) { + dr->cookies[i] = pkt->cookies[i]; + + viodbg(HS, "DRING COOKIE(%d) [%016llx:%016llx]\n", + i, + (unsigned long long) + pkt->cookies[i].cookie_addr, + (unsigned long long) + pkt->cookies[i].cookie_size); + } + + pkt->tag.stype = VIO_SUBTYPE_ACK; + pkt->dring_ident = ++dr->ident; + + viodbg(HS, "SEND DRING_REG ACK ident[%llx] " + "ndesc[%u] dsz[%u] opt[0x%x] ncookies[%u]\n", + (unsigned long long) pkt->dring_ident, + pkt->num_descr, pkt->descr_size, pkt->options, + pkt->num_cookies); + + if (send_ctrl(vio, &pkt->tag, struct_size(pkt, cookies, dr->ncookies)) < 0) + goto send_nack; + + vio->dr_state |= VIO_DR_STATE_RXREG; + + return 0; + +send_nack: + pkt->tag.stype = VIO_SUBTYPE_NACK; + viodbg(HS, "SEND DRING_REG NACK\n"); + (void) send_ctrl(vio, &pkt->tag, sizeof(*pkt)); + + return handshake_failure(vio); +} + +static int process_dreg_ack(struct vio_driver_state *vio, + struct vio_dring_register *pkt) +{ + struct vio_dring_state *dr; + + viodbg(HS, "GOT DRING_REG ACK ident[%llx] " + "ndesc[%u] dsz[%u] opt[0x%x] ncookies[%u]\n", + (unsigned long long) pkt->dring_ident, + pkt->num_descr, pkt->descr_size, pkt->options, + pkt->num_cookies); + + dr = &vio->drings[VIO_DRIVER_TX_RING]; + + if (!(vio->dr_state & VIO_DR_STATE_TXREQ)) + return handshake_failure(vio); + + dr->ident = pkt->dring_ident; + vio->dr_state |= VIO_DR_STATE_TXREG; + + if (all_drings_registered(vio)) { + if (send_rdx(vio) < 0) + return handshake_failure(vio); + vio->hs_state = VIO_HS_SENT_RDX; + } + return 0; +} + +static int process_dreg_nack(struct vio_driver_state *vio, + struct vio_dring_register *pkt) +{ + viodbg(HS, "GOT DRING_REG NACK ident[%llx] " + "ndesc[%u] dsz[%u] opt[0x%x] ncookies[%u]\n", + (unsigned long long) pkt->dring_ident, + pkt->num_descr, pkt->descr_size, pkt->options, + pkt->num_cookies); + + return handshake_failure(vio); +} + +static int process_dreg(struct vio_driver_state *vio, + struct vio_dring_register *pkt) +{ + if (!(vio->hs_state & VIO_HS_GOTVERS)) + return handshake_failure(vio); + + switch (pkt->tag.stype) { + case VIO_SUBTYPE_INFO: + return process_dreg_info(vio, pkt); + + case VIO_SUBTYPE_ACK: + return process_dreg_ack(vio, pkt); + + case VIO_SUBTYPE_NACK: + return process_dreg_nack(vio, pkt); + + default: + return handshake_failure(vio); + } +} + +static int process_dunreg(struct vio_driver_state *vio, + struct vio_dring_unregister *pkt) +{ + struct vio_dring_state *dr = &vio->drings[VIO_DRIVER_RX_RING]; + + viodbg(HS, "GOT DRING_UNREG\n"); + + if (pkt->dring_ident != dr->ident) + return 0; + + vio->dr_state &= ~VIO_DR_STATE_RXREG; + + memset(dr, 0, sizeof(*dr)); + + kfree(vio->desc_buf); + vio->desc_buf = NULL; + vio->desc_buf_len = 0; + + return 0; +} + +static int process_rdx_info(struct vio_driver_state *vio, struct vio_rdx *pkt) +{ + viodbg(HS, "GOT RDX INFO\n"); + + pkt->tag.stype = VIO_SUBTYPE_ACK; + viodbg(HS, "SEND RDX ACK\n"); + if (send_ctrl(vio, &pkt->tag, sizeof(*pkt)) < 0) + return handshake_failure(vio); + + vio->hs_state |= VIO_HS_SENT_RDX_ACK; + return 0; +} + +static int process_rdx_ack(struct vio_driver_state *vio, struct vio_rdx *pkt) +{ + viodbg(HS, "GOT RDX ACK\n"); + + if (!(vio->hs_state & VIO_HS_SENT_RDX)) + return handshake_failure(vio); + + vio->hs_state |= VIO_HS_GOT_RDX_ACK; + return 0; +} + +static int process_rdx_nack(struct vio_driver_state *vio, struct vio_rdx *pkt) +{ + viodbg(HS, "GOT RDX NACK\n"); + + return handshake_failure(vio); +} + +static int process_rdx(struct vio_driver_state *vio, struct vio_rdx *pkt) +{ + if (!all_drings_registered(vio)) + handshake_failure(vio); + + switch (pkt->tag.stype) { + case VIO_SUBTYPE_INFO: + return process_rdx_info(vio, pkt); + + case VIO_SUBTYPE_ACK: + return process_rdx_ack(vio, pkt); + + case VIO_SUBTYPE_NACK: + return process_rdx_nack(vio, pkt); + + default: + return handshake_failure(vio); + } +} + +int vio_control_pkt_engine(struct vio_driver_state *vio, void *pkt) +{ + struct vio_msg_tag *tag = pkt; + u8 prev_state = vio->hs_state; + int err; + + switch (tag->stype_env) { + case VIO_VER_INFO: + err = process_ver(vio, pkt); + break; + + case VIO_ATTR_INFO: + err = process_attr(vio, pkt); + break; + + case VIO_DRING_REG: + err = process_dreg(vio, pkt); + break; + + case VIO_DRING_UNREG: + err = process_dunreg(vio, pkt); + break; + + case VIO_RDX: + err = process_rdx(vio, pkt); + break; + + default: + err = process_unknown(vio, pkt); + break; + } + + if (!err && + vio->hs_state != prev_state && + (vio->hs_state & VIO_HS_COMPLETE)) { + if (vio->ops) + vio->ops->handshake_complete(vio); + } + + return err; +} +EXPORT_SYMBOL(vio_control_pkt_engine); + +void vio_conn_reset(struct vio_driver_state *vio) +{ +} +EXPORT_SYMBOL(vio_conn_reset); + +/* The issue is that the Solaris virtual disk server just mirrors the + * SID values it gets from the client peer. So we work around that + * here in vio_{validate,send}_sid() so that the drivers don't need + * to be aware of this crap. + */ +int vio_validate_sid(struct vio_driver_state *vio, struct vio_msg_tag *tp) +{ + u32 sid; + + /* Always let VERSION+INFO packets through unchecked, they + * define the new SID. + */ + if (tp->type == VIO_TYPE_CTRL && + tp->stype == VIO_SUBTYPE_INFO && + tp->stype_env == VIO_VER_INFO) + return 0; + + /* Ok, now figure out which SID to use. */ + switch (vio->dev_class) { + case VDEV_NETWORK: + case VDEV_NETWORK_SWITCH: + case VDEV_DISK_SERVER: + default: + sid = vio->_peer_sid; + break; + + case VDEV_DISK: + sid = vio->_local_sid; + break; + } + + if (sid == tp->sid) + return 0; + viodbg(DATA, "BAD SID tag->sid[%08x] peer_sid[%08x] local_sid[%08x]\n", + tp->sid, vio->_peer_sid, vio->_local_sid); + return -EINVAL; +} +EXPORT_SYMBOL(vio_validate_sid); + +u32 vio_send_sid(struct vio_driver_state *vio) +{ + switch (vio->dev_class) { + case VDEV_NETWORK: + case VDEV_NETWORK_SWITCH: + case VDEV_DISK: + default: + return vio->_local_sid; + + case VDEV_DISK_SERVER: + return vio->_peer_sid; + } +} +EXPORT_SYMBOL(vio_send_sid); + +int vio_ldc_alloc(struct vio_driver_state *vio, + struct ldc_channel_config *base_cfg, + void *event_arg) +{ + struct ldc_channel_config cfg = *base_cfg; + struct ldc_channel *lp; + + cfg.tx_irq = vio->vdev->tx_irq; + cfg.rx_irq = vio->vdev->rx_irq; + + lp = ldc_alloc(vio->vdev->channel_id, &cfg, event_arg, vio->name); + if (IS_ERR(lp)) + return PTR_ERR(lp); + + vio->lp = lp; + + return 0; +} +EXPORT_SYMBOL(vio_ldc_alloc); + +void vio_ldc_free(struct vio_driver_state *vio) +{ + ldc_free(vio->lp); + vio->lp = NULL; + + kfree(vio->desc_buf); + vio->desc_buf = NULL; + vio->desc_buf_len = 0; +} +EXPORT_SYMBOL(vio_ldc_free); + +void vio_port_up(struct vio_driver_state *vio) +{ + unsigned long flags; + int err, state; + + spin_lock_irqsave(&vio->lock, flags); + + state = ldc_state(vio->lp); + + err = 0; + if (state == LDC_STATE_INIT) { + err = ldc_bind(vio->lp); + if (err) + printk(KERN_WARNING "%s: Port %lu bind failed, " + "err=%d\n", + vio->name, vio->vdev->channel_id, err); + } + + if (!err) { + if (ldc_mode(vio->lp) == LDC_MODE_RAW) + ldc_set_state(vio->lp, LDC_STATE_CONNECTED); + else + err = ldc_connect(vio->lp); + + if (err) + printk(KERN_WARNING "%s: Port %lu connect failed, " + "err=%d\n", + vio->name, vio->vdev->channel_id, err); + } + if (err) { + unsigned long expires = jiffies + HZ; + + expires = round_jiffies(expires); + mod_timer(&vio->timer, expires); + } + + spin_unlock_irqrestore(&vio->lock, flags); +} +EXPORT_SYMBOL(vio_port_up); + +static void vio_port_timer(struct timer_list *t) +{ + struct vio_driver_state *vio = timer_container_of(vio, t, timer); + + vio_port_up(vio); +} + +int vio_driver_init(struct vio_driver_state *vio, struct vio_dev *vdev, + u8 dev_class, struct vio_version *ver_table, + int ver_table_size, struct vio_driver_ops *ops, + char *name) +{ + switch (dev_class) { + case VDEV_NETWORK: + case VDEV_NETWORK_SWITCH: + case VDEV_DISK: + case VDEV_DISK_SERVER: + case VDEV_CONSOLE_CON: + break; + + default: + return -EINVAL; + } + + if (dev_class == VDEV_NETWORK || + dev_class == VDEV_NETWORK_SWITCH || + dev_class == VDEV_DISK || + dev_class == VDEV_DISK_SERVER) { + if (!ops || !ops->send_attr || !ops->handle_attr || + !ops->handshake_complete) + return -EINVAL; + } + + if (!ver_table || ver_table_size < 0) + return -EINVAL; + + if (!name) + return -EINVAL; + + spin_lock_init(&vio->lock); + + vio->name = name; + + vio->dev_class = dev_class; + vio->vdev = vdev; + + vio->ver_table = ver_table; + vio->ver_table_entries = ver_table_size; + + vio->ops = ops; + + timer_setup(&vio->timer, vio_port_timer, 0); + + return 0; +} +EXPORT_SYMBOL(vio_driver_init); diff --git a/arch/sparc/kernel/visemul.c b/arch/sparc/kernel/visemul.c new file mode 100644 index 000000000..64ed80ed6 --- /dev/null +++ b/arch/sparc/kernel/visemul.c @@ -0,0 +1,899 @@ +// SPDX-License-Identifier: GPL-2.0 +/* visemul.c: Emulation of VIS instructions. + * + * Copyright (C) 2006 David S. Miller (davem@davemloft.net) + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/* OPF field of various VIS instructions. */ + +/* 000111011 - four 16-bit packs */ +#define FPACK16_OPF 0x03b + +/* 000111010 - two 32-bit packs */ +#define FPACK32_OPF 0x03a + +/* 000111101 - four 16-bit packs */ +#define FPACKFIX_OPF 0x03d + +/* 001001101 - four 16-bit expands */ +#define FEXPAND_OPF 0x04d + +/* 001001011 - two 32-bit merges */ +#define FPMERGE_OPF 0x04b + +/* 000110001 - 8-by-16-bit partitioned product */ +#define FMUL8x16_OPF 0x031 + +/* 000110011 - 8-by-16-bit upper alpha partitioned product */ +#define FMUL8x16AU_OPF 0x033 + +/* 000110101 - 8-by-16-bit lower alpha partitioned product */ +#define FMUL8x16AL_OPF 0x035 + +/* 000110110 - upper 8-by-16-bit partitioned product */ +#define FMUL8SUx16_OPF 0x036 + +/* 000110111 - lower 8-by-16-bit partitioned product */ +#define FMUL8ULx16_OPF 0x037 + +/* 000111000 - upper 8-by-16-bit partitioned product */ +#define FMULD8SUx16_OPF 0x038 + +/* 000111001 - lower unsigned 8-by-16-bit partitioned product */ +#define FMULD8ULx16_OPF 0x039 + +/* 000101000 - four 16-bit compare; set rd if src1 > src2 */ +#define FCMPGT16_OPF 0x028 + +/* 000101100 - two 32-bit compare; set rd if src1 > src2 */ +#define FCMPGT32_OPF 0x02c + +/* 000100000 - four 16-bit compare; set rd if src1 <= src2 */ +#define FCMPLE16_OPF 0x020 + +/* 000100100 - two 32-bit compare; set rd if src1 <= src2 */ +#define FCMPLE32_OPF 0x024 + +/* 000100010 - four 16-bit compare; set rd if src1 != src2 */ +#define FCMPNE16_OPF 0x022 + +/* 000100110 - two 32-bit compare; set rd if src1 != src2 */ +#define FCMPNE32_OPF 0x026 + +/* 000101010 - four 16-bit compare; set rd if src1 == src2 */ +#define FCMPEQ16_OPF 0x02a + +/* 000101110 - two 32-bit compare; set rd if src1 == src2 */ +#define FCMPEQ32_OPF 0x02e + +/* 000000000 - Eight 8-bit edge boundary processing */ +#define EDGE8_OPF 0x000 + +/* 000000001 - Eight 8-bit edge boundary processing, no CC */ +#define EDGE8N_OPF 0x001 + +/* 000000010 - Eight 8-bit edge boundary processing, little-endian */ +#define EDGE8L_OPF 0x002 + +/* 000000011 - Eight 8-bit edge boundary processing, little-endian, no CC */ +#define EDGE8LN_OPF 0x003 + +/* 000000100 - Four 16-bit edge boundary processing */ +#define EDGE16_OPF 0x004 + +/* 000000101 - Four 16-bit edge boundary processing, no CC */ +#define EDGE16N_OPF 0x005 + +/* 000000110 - Four 16-bit edge boundary processing, little-endian */ +#define EDGE16L_OPF 0x006 + +/* 000000111 - Four 16-bit edge boundary processing, little-endian, no CC */ +#define EDGE16LN_OPF 0x007 + +/* 000001000 - Two 32-bit edge boundary processing */ +#define EDGE32_OPF 0x008 + +/* 000001001 - Two 32-bit edge boundary processing, no CC */ +#define EDGE32N_OPF 0x009 + +/* 000001010 - Two 32-bit edge boundary processing, little-endian */ +#define EDGE32L_OPF 0x00a + +/* 000001011 - Two 32-bit edge boundary processing, little-endian, no CC */ +#define EDGE32LN_OPF 0x00b + +/* 000111110 - distance between 8 8-bit components */ +#define PDIST_OPF 0x03e + +/* 000010000 - convert 8-bit 3-D address to blocked byte address */ +#define ARRAY8_OPF 0x010 + +/* 000010010 - convert 16-bit 3-D address to blocked byte address */ +#define ARRAY16_OPF 0x012 + +/* 000010100 - convert 32-bit 3-D address to blocked byte address */ +#define ARRAY32_OPF 0x014 + +/* 000011001 - Set the GSR.MASK field in preparation for a BSHUFFLE */ +#define BMASK_OPF 0x019 + +/* 001001100 - Permute bytes as specified by GSR.MASK */ +#define BSHUFFLE_OPF 0x04c + +#define VIS_OPF_SHIFT 5 +#define VIS_OPF_MASK (0x1ff << VIS_OPF_SHIFT) + +#define RS1(INSN) (((INSN) >> 14) & 0x1f) +#define RS2(INSN) (((INSN) >> 0) & 0x1f) +#define RD(INSN) (((INSN) >> 25) & 0x1f) + +static inline void maybe_flush_windows(unsigned int rs1, unsigned int rs2, + unsigned int rd, int from_kernel) +{ + if (rs2 >= 16 || rs1 >= 16 || rd >= 16) { + if (from_kernel != 0) + __asm__ __volatile__("flushw"); + else + flushw_user(); + } +} + +static unsigned long fetch_reg(unsigned int reg, struct pt_regs *regs) +{ + unsigned long value, fp; + + if (reg < 16) + return (!reg ? 0 : regs->u_regs[reg]); + + fp = regs->u_regs[UREG_FP]; + + if (regs->tstate & TSTATE_PRIV) { + struct reg_window *win; + win = (struct reg_window *)(fp + STACK_BIAS); + value = win->locals[reg - 16]; + } else if (!test_thread_64bit_stack(fp)) { + struct reg_window32 __user *win32; + win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp)); + get_user(value, &win32->locals[reg - 16]); + } else { + struct reg_window __user *win; + win = (struct reg_window __user *)(fp + STACK_BIAS); + get_user(value, &win->locals[reg - 16]); + } + return value; +} + +static inline unsigned long __user *__fetch_reg_addr_user(unsigned int reg, + struct pt_regs *regs) +{ + unsigned long fp = regs->u_regs[UREG_FP]; + + BUG_ON(reg < 16); + BUG_ON(regs->tstate & TSTATE_PRIV); + + if (!test_thread_64bit_stack(fp)) { + struct reg_window32 __user *win32; + win32 = (struct reg_window32 __user *)((unsigned long)((u32)fp)); + return (unsigned long __user *)&win32->locals[reg - 16]; + } else { + struct reg_window __user *win; + win = (struct reg_window __user *)(fp + STACK_BIAS); + return &win->locals[reg - 16]; + } +} + +static inline unsigned long *__fetch_reg_addr_kern(unsigned int reg, + struct pt_regs *regs) +{ + BUG_ON(reg >= 16); + BUG_ON(regs->tstate & TSTATE_PRIV); + + return ®s->u_regs[reg]; +} + +static void store_reg(struct pt_regs *regs, unsigned long val, unsigned long rd) +{ + if (rd < 16) { + unsigned long *rd_kern = __fetch_reg_addr_kern(rd, regs); + + *rd_kern = val; + } else { + unsigned long __user *rd_user = __fetch_reg_addr_user(rd, regs); + + if (!test_thread_64bit_stack(regs->u_regs[UREG_FP])) + __put_user((u32)val, (u32 __user *)rd_user); + else + __put_user(val, rd_user); + } +} + +static inline unsigned long fpd_regval(struct fpustate *f, + unsigned int insn_regnum) +{ + insn_regnum = (((insn_regnum & 1) << 5) | + (insn_regnum & 0x1e)); + + return *(unsigned long *) &f->regs[insn_regnum]; +} + +static inline unsigned long *fpd_regaddr(struct fpustate *f, + unsigned int insn_regnum) +{ + insn_regnum = (((insn_regnum & 1) << 5) | + (insn_regnum & 0x1e)); + + return (unsigned long *) &f->regs[insn_regnum]; +} + +static inline unsigned int fps_regval(struct fpustate *f, + unsigned int insn_regnum) +{ + return f->regs[insn_regnum]; +} + +static inline unsigned int *fps_regaddr(struct fpustate *f, + unsigned int insn_regnum) +{ + return &f->regs[insn_regnum]; +} + +struct edge_tab { + u16 left, right; +}; +static struct edge_tab edge8_tab[8] = { + { 0xff, 0x80 }, + { 0x7f, 0xc0 }, + { 0x3f, 0xe0 }, + { 0x1f, 0xf0 }, + { 0x0f, 0xf8 }, + { 0x07, 0xfc }, + { 0x03, 0xfe }, + { 0x01, 0xff }, +}; +static struct edge_tab edge8_tab_l[8] = { + { 0xff, 0x01 }, + { 0xfe, 0x03 }, + { 0xfc, 0x07 }, + { 0xf8, 0x0f }, + { 0xf0, 0x1f }, + { 0xe0, 0x3f }, + { 0xc0, 0x7f }, + { 0x80, 0xff }, +}; +static struct edge_tab edge16_tab[4] = { + { 0xf, 0x8 }, + { 0x7, 0xc }, + { 0x3, 0xe }, + { 0x1, 0xf }, +}; +static struct edge_tab edge16_tab_l[4] = { + { 0xf, 0x1 }, + { 0xe, 0x3 }, + { 0xc, 0x7 }, + { 0x8, 0xf }, +}; +static struct edge_tab edge32_tab[2] = { + { 0x3, 0x2 }, + { 0x1, 0x3 }, +}; +static struct edge_tab edge32_tab_l[2] = { + { 0x3, 0x1 }, + { 0x2, 0x3 }, +}; + +static void edge(struct pt_regs *regs, unsigned int insn, unsigned int opf) +{ + unsigned long orig_rs1, rs1, orig_rs2, rs2, rd_val; + u16 left, right; + + maybe_flush_windows(RS1(insn), RS2(insn), RD(insn), 0); + orig_rs1 = rs1 = fetch_reg(RS1(insn), regs); + orig_rs2 = rs2 = fetch_reg(RS2(insn), regs); + + if (test_thread_flag(TIF_32BIT)) { + rs1 = rs1 & 0xffffffff; + rs2 = rs2 & 0xffffffff; + } + switch (opf) { + default: + case EDGE8_OPF: + case EDGE8N_OPF: + left = edge8_tab[rs1 & 0x7].left; + right = edge8_tab[rs2 & 0x7].right; + break; + case EDGE8L_OPF: + case EDGE8LN_OPF: + left = edge8_tab_l[rs1 & 0x7].left; + right = edge8_tab_l[rs2 & 0x7].right; + break; + + case EDGE16_OPF: + case EDGE16N_OPF: + left = edge16_tab[(rs1 >> 1) & 0x3].left; + right = edge16_tab[(rs2 >> 1) & 0x3].right; + break; + + case EDGE16L_OPF: + case EDGE16LN_OPF: + left = edge16_tab_l[(rs1 >> 1) & 0x3].left; + right = edge16_tab_l[(rs2 >> 1) & 0x3].right; + break; + + case EDGE32_OPF: + case EDGE32N_OPF: + left = edge32_tab[(rs1 >> 2) & 0x1].left; + right = edge32_tab[(rs2 >> 2) & 0x1].right; + break; + + case EDGE32L_OPF: + case EDGE32LN_OPF: + left = edge32_tab_l[(rs1 >> 2) & 0x1].left; + right = edge32_tab_l[(rs2 >> 2) & 0x1].right; + break; + } + + if ((rs1 & ~0x7UL) == (rs2 & ~0x7UL)) + rd_val = right & left; + else + rd_val = left; + + store_reg(regs, rd_val, RD(insn)); + + switch (opf) { + case EDGE8_OPF: + case EDGE8L_OPF: + case EDGE16_OPF: + case EDGE16L_OPF: + case EDGE32_OPF: + case EDGE32L_OPF: { + unsigned long ccr, tstate; + + __asm__ __volatile__("subcc %1, %2, %%g0\n\t" + "rd %%ccr, %0" + : "=r" (ccr) + : "r" (orig_rs1), "r" (orig_rs2) + : "cc"); + tstate = regs->tstate & ~(TSTATE_XCC | TSTATE_ICC); + regs->tstate = tstate | (ccr << 32UL); + } + } +} + +static void array(struct pt_regs *regs, unsigned int insn, unsigned int opf) +{ + unsigned long rs1, rs2, rd_val; + unsigned int bits, bits_mask; + + maybe_flush_windows(RS1(insn), RS2(insn), RD(insn), 0); + rs1 = fetch_reg(RS1(insn), regs); + rs2 = fetch_reg(RS2(insn), regs); + + bits = (rs2 > 5 ? 5 : rs2); + bits_mask = (1UL << bits) - 1UL; + + rd_val = ((((rs1 >> 11) & 0x3) << 0) | + (((rs1 >> 33) & 0x3) << 2) | + (((rs1 >> 55) & 0x1) << 4) | + (((rs1 >> 13) & 0xf) << 5) | + (((rs1 >> 35) & 0xf) << 9) | + (((rs1 >> 56) & 0xf) << 13) | + (((rs1 >> 17) & bits_mask) << 17) | + (((rs1 >> 39) & bits_mask) << (17 + bits)) | + (((rs1 >> 60) & 0xf) << (17 + (2*bits)))); + + switch (opf) { + case ARRAY16_OPF: + rd_val <<= 1; + break; + + case ARRAY32_OPF: + rd_val <<= 2; + } + + store_reg(regs, rd_val, RD(insn)); +} + +static void bmask(struct pt_regs *regs, unsigned int insn) +{ + unsigned long rs1, rs2, rd_val, gsr; + + maybe_flush_windows(RS1(insn), RS2(insn), RD(insn), 0); + rs1 = fetch_reg(RS1(insn), regs); + rs2 = fetch_reg(RS2(insn), regs); + rd_val = rs1 + rs2; + + store_reg(regs, rd_val, RD(insn)); + + gsr = current_thread_info()->gsr[0] & 0xffffffff; + gsr |= rd_val << 32UL; + current_thread_info()->gsr[0] = gsr; +} + +static void bshuffle(struct pt_regs *regs, unsigned int insn) +{ + struct fpustate *f = FPUSTATE; + unsigned long rs1, rs2, rd_val; + unsigned long bmask, i; + + bmask = current_thread_info()->gsr[0] >> 32UL; + + rs1 = fpd_regval(f, RS1(insn)); + rs2 = fpd_regval(f, RS2(insn)); + + rd_val = 0UL; + for (i = 0; i < 8; i++) { + unsigned long which = (bmask >> (i * 4)) & 0xf; + unsigned long byte; + + if (which < 8) + byte = (rs1 >> (which * 8)) & 0xff; + else + byte = (rs2 >> ((which-8)*8)) & 0xff; + rd_val |= (byte << (i * 8)); + } + + *fpd_regaddr(f, RD(insn)) = rd_val; +} + +static void pdist(struct pt_regs *regs, unsigned int insn) +{ + struct fpustate *f = FPUSTATE; + unsigned long rs1, rs2, *rd, rd_val; + unsigned long i; + + rs1 = fpd_regval(f, RS1(insn)); + rs2 = fpd_regval(f, RS2(insn)); + rd = fpd_regaddr(f, RD(insn)); + + rd_val = *rd; + + for (i = 0; i < 8; i++) { + s16 s1, s2; + + s1 = (rs1 >> (56 - (i * 8))) & 0xff; + s2 = (rs2 >> (56 - (i * 8))) & 0xff; + + /* Absolute value of difference. */ + s1 -= s2; + if (s1 < 0) + s1 = ~s1 + 1; + + rd_val += s1; + } + + *rd = rd_val; +} + +static void pformat(struct pt_regs *regs, unsigned int insn, unsigned int opf) +{ + struct fpustate *f = FPUSTATE; + unsigned long rs1, rs2, gsr, scale, rd_val; + + gsr = current_thread_info()->gsr[0]; + scale = (gsr >> 3) & (opf == FPACK16_OPF ? 0xf : 0x1f); + switch (opf) { + case FPACK16_OPF: { + unsigned long byte; + + rs2 = fpd_regval(f, RS2(insn)); + rd_val = 0; + for (byte = 0; byte < 4; byte++) { + unsigned int val; + s16 src = (rs2 >> (byte * 16UL)) & 0xffffUL; + int scaled = src << scale; + int from_fixed = scaled >> 7; + + val = ((from_fixed < 0) ? + 0 : + (from_fixed > 255) ? + 255 : from_fixed); + + rd_val |= (val << (8 * byte)); + } + *fps_regaddr(f, RD(insn)) = rd_val; + break; + } + + case FPACK32_OPF: { + unsigned long word; + + rs1 = fpd_regval(f, RS1(insn)); + rs2 = fpd_regval(f, RS2(insn)); + rd_val = (rs1 << 8) & ~(0x000000ff000000ffUL); + for (word = 0; word < 2; word++) { + unsigned long val; + s32 src = (rs2 >> (word * 32UL)); + s64 scaled = src << scale; + s64 from_fixed = scaled >> 23; + + val = ((from_fixed < 0) ? + 0 : + (from_fixed > 255) ? + 255 : from_fixed); + + rd_val |= (val << (32 * word)); + } + *fpd_regaddr(f, RD(insn)) = rd_val; + break; + } + + case FPACKFIX_OPF: { + unsigned long word; + + rs2 = fpd_regval(f, RS2(insn)); + + rd_val = 0; + for (word = 0; word < 2; word++) { + long val; + s32 src = (rs2 >> (word * 32UL)); + s64 scaled = src << scale; + s64 from_fixed = scaled >> 16; + + val = ((from_fixed < -32768) ? + -32768 : + (from_fixed > 32767) ? + 32767 : from_fixed); + + rd_val |= ((val & 0xffff) << (word * 16)); + } + *fps_regaddr(f, RD(insn)) = rd_val; + break; + } + + case FEXPAND_OPF: { + unsigned long byte; + + rs2 = fps_regval(f, RS2(insn)); + + rd_val = 0; + for (byte = 0; byte < 4; byte++) { + unsigned long val; + u8 src = (rs2 >> (byte * 8)) & 0xff; + + val = src << 4; + + rd_val |= (val << (byte * 16)); + } + *fpd_regaddr(f, RD(insn)) = rd_val; + break; + } + + case FPMERGE_OPF: { + rs1 = fps_regval(f, RS1(insn)); + rs2 = fps_regval(f, RS2(insn)); + + rd_val = (((rs2 & 0x000000ff) << 0) | + ((rs1 & 0x000000ff) << 8) | + ((rs2 & 0x0000ff00) << 8) | + ((rs1 & 0x0000ff00) << 16) | + ((rs2 & 0x00ff0000) << 16) | + ((rs1 & 0x00ff0000) << 24) | + ((rs2 & 0xff000000) << 24) | + ((rs1 & 0xff000000) << 32)); + *fpd_regaddr(f, RD(insn)) = rd_val; + break; + } + } +} + +static void pmul(struct pt_regs *regs, unsigned int insn, unsigned int opf) +{ + struct fpustate *f = FPUSTATE; + unsigned long rs1, rs2, rd_val; + + switch (opf) { + case FMUL8x16_OPF: { + unsigned long byte; + + rs1 = fps_regval(f, RS1(insn)); + rs2 = fpd_regval(f, RS2(insn)); + + rd_val = 0; + for (byte = 0; byte < 4; byte++) { + u16 src1 = (rs1 >> (byte * 8)) & 0x00ff; + s16 src2 = (rs2 >> (byte * 16)) & 0xffff; + u32 prod = src1 * src2; + u16 scaled = ((prod & 0x00ffff00) >> 8); + + /* Round up. */ + if (prod & 0x80) + scaled++; + rd_val |= ((scaled & 0xffffUL) << (byte * 16UL)); + } + + *fpd_regaddr(f, RD(insn)) = rd_val; + break; + } + + case FMUL8x16AU_OPF: + case FMUL8x16AL_OPF: { + unsigned long byte; + s16 src2; + + rs1 = fps_regval(f, RS1(insn)); + rs2 = fps_regval(f, RS2(insn)); + + rd_val = 0; + src2 = rs2 >> (opf == FMUL8x16AU_OPF ? 16 : 0); + for (byte = 0; byte < 4; byte++) { + u16 src1 = (rs1 >> (byte * 8)) & 0x00ff; + u32 prod = src1 * src2; + u16 scaled = ((prod & 0x00ffff00) >> 8); + + /* Round up. */ + if (prod & 0x80) + scaled++; + rd_val |= ((scaled & 0xffffUL) << (byte * 16UL)); + } + + *fpd_regaddr(f, RD(insn)) = rd_val; + break; + } + + case FMUL8SUx16_OPF: + case FMUL8ULx16_OPF: { + unsigned long byte, ushift; + + rs1 = fpd_regval(f, RS1(insn)); + rs2 = fpd_regval(f, RS2(insn)); + + rd_val = 0; + ushift = (opf == FMUL8SUx16_OPF) ? 8 : 0; + for (byte = 0; byte < 4; byte++) { + u16 src1; + s16 src2; + u32 prod; + u16 scaled; + + src1 = ((rs1 >> ((16 * byte) + ushift)) & 0x00ff); + src2 = ((rs2 >> (16 * byte)) & 0xffff); + prod = src1 * src2; + scaled = ((prod & 0x00ffff00) >> 8); + + /* Round up. */ + if (prod & 0x80) + scaled++; + rd_val |= ((scaled & 0xffffUL) << (byte * 16UL)); + } + + *fpd_regaddr(f, RD(insn)) = rd_val; + break; + } + + case FMULD8SUx16_OPF: + case FMULD8ULx16_OPF: { + unsigned long byte, ushift; + + rs1 = fps_regval(f, RS1(insn)); + rs2 = fps_regval(f, RS2(insn)); + + rd_val = 0; + ushift = (opf == FMULD8SUx16_OPF) ? 8 : 0; + for (byte = 0; byte < 2; byte++) { + u16 src1; + s16 src2; + u32 prod; + u16 scaled; + + src1 = ((rs1 >> ((16 * byte) + ushift)) & 0x00ff); + src2 = ((rs2 >> (16 * byte)) & 0xffff); + prod = src1 * src2; + scaled = ((prod & 0x00ffff00) >> 8); + + /* Round up. */ + if (prod & 0x80) + scaled++; + rd_val |= ((scaled & 0xffffUL) << + ((byte * 32UL) + 7UL)); + } + *fpd_regaddr(f, RD(insn)) = rd_val; + break; + } + } +} + +static void pcmp(struct pt_regs *regs, unsigned int insn, unsigned int opf) +{ + struct fpustate *f = FPUSTATE; + unsigned long rs1, rs2, rd_val, i; + + rs1 = fpd_regval(f, RS1(insn)); + rs2 = fpd_regval(f, RS2(insn)); + + rd_val = 0; + + switch (opf) { + case FCMPGT16_OPF: + for (i = 0; i < 4; i++) { + s16 a = (rs1 >> (i * 16)) & 0xffff; + s16 b = (rs2 >> (i * 16)) & 0xffff; + + if (a > b) + rd_val |= 8 >> i; + } + break; + + case FCMPGT32_OPF: + for (i = 0; i < 2; i++) { + s32 a = (rs1 >> (i * 32)) & 0xffffffff; + s32 b = (rs2 >> (i * 32)) & 0xffffffff; + + if (a > b) + rd_val |= 2 >> i; + } + break; + + case FCMPLE16_OPF: + for (i = 0; i < 4; i++) { + s16 a = (rs1 >> (i * 16)) & 0xffff; + s16 b = (rs2 >> (i * 16)) & 0xffff; + + if (a <= b) + rd_val |= 8 >> i; + } + break; + + case FCMPLE32_OPF: + for (i = 0; i < 2; i++) { + s32 a = (rs1 >> (i * 32)) & 0xffffffff; + s32 b = (rs2 >> (i * 32)) & 0xffffffff; + + if (a <= b) + rd_val |= 2 >> i; + } + break; + + case FCMPNE16_OPF: + for (i = 0; i < 4; i++) { + s16 a = (rs1 >> (i * 16)) & 0xffff; + s16 b = (rs2 >> (i * 16)) & 0xffff; + + if (a != b) + rd_val |= 8 >> i; + } + break; + + case FCMPNE32_OPF: + for (i = 0; i < 2; i++) { + s32 a = (rs1 >> (i * 32)) & 0xffffffff; + s32 b = (rs2 >> (i * 32)) & 0xffffffff; + + if (a != b) + rd_val |= 2 >> i; + } + break; + + case FCMPEQ16_OPF: + for (i = 0; i < 4; i++) { + s16 a = (rs1 >> (i * 16)) & 0xffff; + s16 b = (rs2 >> (i * 16)) & 0xffff; + + if (a == b) + rd_val |= 8 >> i; + } + break; + + case FCMPEQ32_OPF: + for (i = 0; i < 2; i++) { + s32 a = (rs1 >> (i * 32)) & 0xffffffff; + s32 b = (rs2 >> (i * 32)) & 0xffffffff; + + if (a == b) + rd_val |= 2 >> i; + } + break; + } + + maybe_flush_windows(0, 0, RD(insn), 0); + store_reg(regs, rd_val, RD(insn)); +} + +/* Emulate the VIS instructions which are not implemented in + * hardware on Niagara. + */ +int vis_emul(struct pt_regs *regs, unsigned int insn) +{ + unsigned long pc = regs->tpc; + unsigned int opf; + + BUG_ON(regs->tstate & TSTATE_PRIV); + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + + if (test_thread_flag(TIF_32BIT)) + pc = (u32)pc; + + if (get_user(insn, (u32 __user *) pc)) + return -EFAULT; + + save_and_clear_fpu(); + + opf = (insn & VIS_OPF_MASK) >> VIS_OPF_SHIFT; + switch (opf) { + default: + return -EINVAL; + + /* Pixel Formatting Instructions. */ + case FPACK16_OPF: + case FPACK32_OPF: + case FPACKFIX_OPF: + case FEXPAND_OPF: + case FPMERGE_OPF: + pformat(regs, insn, opf); + break; + + /* Partitioned Multiply Instructions */ + case FMUL8x16_OPF: + case FMUL8x16AU_OPF: + case FMUL8x16AL_OPF: + case FMUL8SUx16_OPF: + case FMUL8ULx16_OPF: + case FMULD8SUx16_OPF: + case FMULD8ULx16_OPF: + pmul(regs, insn, opf); + break; + + /* Pixel Compare Instructions */ + case FCMPGT16_OPF: + case FCMPGT32_OPF: + case FCMPLE16_OPF: + case FCMPLE32_OPF: + case FCMPNE16_OPF: + case FCMPNE32_OPF: + case FCMPEQ16_OPF: + case FCMPEQ32_OPF: + pcmp(regs, insn, opf); + break; + + /* Edge Handling Instructions */ + case EDGE8_OPF: + case EDGE8N_OPF: + case EDGE8L_OPF: + case EDGE8LN_OPF: + case EDGE16_OPF: + case EDGE16N_OPF: + case EDGE16L_OPF: + case EDGE16LN_OPF: + case EDGE32_OPF: + case EDGE32N_OPF: + case EDGE32L_OPF: + case EDGE32LN_OPF: + edge(regs, insn, opf); + break; + + /* Pixel Component Distance */ + case PDIST_OPF: + pdist(regs, insn); + break; + + /* Three-Dimensional Array Addressing Instructions */ + case ARRAY8_OPF: + case ARRAY16_OPF: + case ARRAY32_OPF: + array(regs, insn, opf); + break; + + /* Byte Mask and Shuffle Instructions */ + case BMASK_OPF: + bmask(regs, insn); + break; + + case BSHUFFLE_OPF: + bshuffle(regs, insn); + break; + } + + regs->tpc = regs->tnpc; + regs->tnpc += 4; + return 0; +} diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S new file mode 100644 index 000000000..7ea510d9b --- /dev/null +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* ld script for sparc32/sparc64 kernel */ + +#include + +#include +#include + +#ifdef CONFIG_SPARC32 +#define INITIAL_ADDRESS 0x10000 + SIZEOF_HEADERS +#define TEXTSTART 0xf0004000 + +#define SMP_CACHE_BYTES_SHIFT 5 + +#else +#define SMP_CACHE_BYTES_SHIFT 6 +#define INITIAL_ADDRESS 0x4000 +#define TEXTSTART 0x0000000000404000 + +#endif + +#define SMP_CACHE_BYTES (1 << SMP_CACHE_BYTES_SHIFT) + +#ifdef CONFIG_SPARC32 +OUTPUT_FORMAT("elf32-sparc", "elf32-sparc", "elf32-sparc") +OUTPUT_ARCH(sparc) +ENTRY(_start) +jiffies = jiffies_64 + 4; +#else +/* sparc64 */ +OUTPUT_FORMAT("elf64-sparc", "elf64-sparc", "elf64-sparc") +OUTPUT_ARCH(sparc:v9a) +ENTRY(_start) +jiffies = jiffies_64; +#endif + +#ifdef CONFIG_SPARC64 +ASSERT((swapper_tsb == 0x0000000000408000), "Error: sparc64 early assembler too large") +#endif + +SECTIONS +{ +#ifdef CONFIG_SPARC64 + swapper_pg_dir = 0x0000000000402000; +#endif + . = INITIAL_ADDRESS; + .text TEXTSTART : + { + _text = .; + HEAD_TEXT + ALIGN_FUNCTION(); +#ifdef CONFIG_SPARC64 + /* Match text section symbols in head_64.S first */ + *head_64.o(.text) +#endif + TEXT_TEXT + SCHED_TEXT + LOCK_TEXT + KPROBES_TEXT + IRQENTRY_TEXT + SOFTIRQENTRY_TEXT + *(.gnu.warning) + } = 0 + _etext = .; + + RO_DATA(PAGE_SIZE) + + /* Start of data section */ + _sdata = .; + + .data1 : { + *(.data1) + } + RW_DATA(SMP_CACHE_BYTES, 0, THREAD_SIZE) + + /* End of data section */ + _edata = .; + + .fixup : { + __start___fixup = .; + *(.fixup) + __stop___fixup = .; + } + EXCEPTION_TABLE(16) + + . = ALIGN(PAGE_SIZE); + __init_begin = ALIGN(PAGE_SIZE); + INIT_TEXT_SECTION(PAGE_SIZE) + __init_text_end = .; + INIT_DATA_SECTION(16) + + . = ALIGN(4); + .tsb_ldquad_phys_patch : { + __tsb_ldquad_phys_patch = .; + *(.tsb_ldquad_phys_patch) + __tsb_ldquad_phys_patch_end = .; + } + + .tsb_phys_patch : { + __tsb_phys_patch = .; + *(.tsb_phys_patch) + __tsb_phys_patch_end = .; + } + + .cpuid_patch : { + __cpuid_patch = .; + *(.cpuid_patch) + __cpuid_patch_end = .; + } + + .sun4v_1insn_patch : { + __sun4v_1insn_patch = .; + *(.sun4v_1insn_patch) + __sun4v_1insn_patch_end = .; + } + .sun4v_2insn_patch : { + __sun4v_2insn_patch = .; + *(.sun4v_2insn_patch) + __sun4v_2insn_patch_end = .; + } + .leon_1insn_patch : { + __leon_1insn_patch = .; + *(.leon_1insn_patch) + __leon_1insn_patch_end = .; + } + .swapper_tsb_phys_patch : { + __swapper_tsb_phys_patch = .; + *(.swapper_tsb_phys_patch) + __swapper_tsb_phys_patch_end = .; + } + .swapper_4m_tsb_phys_patch : { + __swapper_4m_tsb_phys_patch = .; + *(.swapper_4m_tsb_phys_patch) + __swapper_4m_tsb_phys_patch_end = .; + } + .popc_3insn_patch : { + __popc_3insn_patch = .; + *(.popc_3insn_patch) + __popc_3insn_patch_end = .; + } + .popc_6insn_patch : { + __popc_6insn_patch = .; + *(.popc_6insn_patch) + __popc_6insn_patch_end = .; + } + .pause_3insn_patch : { + __pause_3insn_patch = .; + *(.pause_3insn_patch) + __pause_3insn_patch_end = .; + } + .sun_m7_1insn_patch : { + __sun_m7_1insn_patch = .; + *(.sun_m7_1insn_patch) + __sun_m7_1insn_patch_end = .; + } + .sun_m7_2insn_patch : { + __sun_m7_2insn_patch = .; + *(.sun_m7_2insn_patch) + __sun_m7_2insn_patch_end = .; + } + .get_tick_patch : { + __get_tick_patch = .; + *(.get_tick_patch) + __get_tick_patch_end = .; + } + .pud_huge_patch : { + __pud_huge_patch = .; + *(.pud_huge_patch) + __pud_huge_patch_end = .; + } + .fast_win_ctrl_1insn_patch : { + __fast_win_ctrl_1insn_patch = .; + *(.fast_win_ctrl_1insn_patch) + __fast_win_ctrl_1insn_patch_end = .; + } + PERCPU_SECTION(SMP_CACHE_BYTES) + + . = ALIGN(PAGE_SIZE); + .exit.text : { + EXIT_TEXT + } + + .exit.data : { + EXIT_DATA + } + + . = ALIGN(PAGE_SIZE); + __init_end = .; + BSS_SECTION(0, 0, 0) + _end = . ; + + STABS_DEBUG + DWARF_DEBUG + MODINFO + ELF_DETAILS + + DISCARDS +} diff --git a/arch/sparc/kernel/windows.c b/arch/sparc/kernel/windows.c new file mode 100644 index 000000000..8f20862cc --- /dev/null +++ b/arch/sparc/kernel/windows.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: GPL-2.0 +/* windows.c: Routines to deal with register window management + * at the C-code level. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "kernel.h" + +/* Do save's until all user register windows are out of the cpu. */ +void flush_user_windows(void) +{ + register int ctr asm("g5"); + + ctr = 0; + __asm__ __volatile__( + "\n1:\n\t" + "ld [%%g6 + %2], %%g4\n\t" + "orcc %%g0, %%g4, %%g0\n\t" + "add %0, 1, %0\n\t" + "bne 1b\n\t" + " save %%sp, -64, %%sp\n" + "2:\n\t" + "subcc %0, 1, %0\n\t" + "bne 2b\n\t" + " restore %%g0, %%g0, %%g0\n" + : "=&r" (ctr) + : "0" (ctr), + "i" ((const unsigned long)TI_UWINMASK) + : "g4", "cc"); +} + +static inline void shift_window_buffer(int first_win, int last_win, struct thread_info *tp) +{ + int i; + + for(i = first_win; i < last_win; i++) { + tp->rwbuf_stkptrs[i] = tp->rwbuf_stkptrs[i+1]; + memcpy(&tp->reg_window[i], &tp->reg_window[i+1], sizeof(struct reg_window32)); + } +} + +/* Place as many of the user's current register windows + * on the stack that we can. Even if the %sp is unaligned + * we still copy the window there, the only case that we don't + * succeed is if the %sp points to a bum mapping altogether. + * setup_frame() and do_sigreturn() use this before shifting + * the user stack around. Future instruction and hardware + * bug workaround routines will need this functionality as + * well. + */ +void synchronize_user_stack(void) +{ + struct thread_info *tp = current_thread_info(); + int window; + + flush_user_windows(); + if(!tp->w_saved) + return; + + /* Ok, there is some dirty work to do. */ + for(window = tp->w_saved - 1; window >= 0; window--) { + unsigned long sp = tp->rwbuf_stkptrs[window]; + + /* Ok, let it rip. */ + if (copy_to_user((char __user *) sp, &tp->reg_window[window], + sizeof(struct reg_window32))) + continue; + + shift_window_buffer(window, tp->w_saved - 1, tp); + tp->w_saved--; + } +} + +#if 0 +/* An optimization. */ +static inline void copy_aligned_window(void *dest, const void *src) +{ + __asm__ __volatile__("ldd [%1], %%g2\n\t" + "ldd [%1 + 0x8], %%g4\n\t" + "std %%g2, [%0]\n\t" + "std %%g4, [%0 + 0x8]\n\t" + "ldd [%1 + 0x10], %%g2\n\t" + "ldd [%1 + 0x18], %%g4\n\t" + "std %%g2, [%0 + 0x10]\n\t" + "std %%g4, [%0 + 0x18]\n\t" + "ldd [%1 + 0x20], %%g2\n\t" + "ldd [%1 + 0x28], %%g4\n\t" + "std %%g2, [%0 + 0x20]\n\t" + "std %%g4, [%0 + 0x28]\n\t" + "ldd [%1 + 0x30], %%g2\n\t" + "ldd [%1 + 0x38], %%g4\n\t" + "std %%g2, [%0 + 0x30]\n\t" + "std %%g4, [%0 + 0x38]\n\t" : : + "r" (dest), "r" (src) : + "g2", "g3", "g4", "g5"); +} +#endif + +/* Try to push the windows in a threads window buffer to the + * user stack. Unaligned %sp's are not allowed here. + */ + +void try_to_clear_window_buffer(struct pt_regs *regs, int who) +{ + struct thread_info *tp = current_thread_info(); + int window; + + flush_user_windows(); + for(window = 0; window < tp->w_saved; window++) { + unsigned long sp = tp->rwbuf_stkptrs[window]; + + if ((sp & 7) || + copy_to_user((char __user *) sp, &tp->reg_window[window], + sizeof(struct reg_window32))) { + force_exit_sig(SIGILL); + return; + } + } + tp->w_saved = 0; +} diff --git a/arch/sparc/kernel/winfixup.S b/arch/sparc/kernel/winfixup.S new file mode 100644 index 000000000..448accee0 --- /dev/null +++ b/arch/sparc/kernel/winfixup.S @@ -0,0 +1,160 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* winfixup.S: Handle cases where user stack pointer is found to be bogus. + * + * Copyright (C) 1997, 2006 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include +#include + + .text + + /* It used to be the case that these register window fault + * handlers could run via the save and restore instructions + * done by the trap entry and exit code. They now do the + * window spill/fill by hand, so that case no longer can occur. + */ + + .align 32 +fill_fixup: + TRAP_LOAD_THREAD_REG(%g6, %g1) + rdpr %tstate, %g1 + and %g1, TSTATE_CWP, %g1 + or %g4, FAULT_CODE_WINFIXUP, %g4 + stb %g4, [%g6 + TI_FAULT_CODE] + stx %g5, [%g6 + TI_FAULT_ADDR] + wrpr %g1, %cwp + ba,pt %xcc, etrap + rd %pc, %g7 + call do_sparc64_fault + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + + /* Be very careful about usage of the trap globals here. + * You cannot touch %g5 as that has the fault information. + */ +spill_fixup: +spill_fixup_mna: +spill_fixup_dax: + TRAP_LOAD_THREAD_REG(%g6, %g1) + ldx [%g6 + TI_FLAGS], %g1 + andcc %sp, 0x1, %g0 + movne %icc, 0, %g1 + andcc %g1, _TIF_32BIT, %g0 + ldub [%g6 + TI_WSAVED], %g1 + sll %g1, 3, %g3 + add %g6, %g3, %g3 + stx %sp, [%g3 + TI_RWIN_SPTRS] + sll %g1, 7, %g3 + bne,pt %xcc, 1f + add %g6, %g3, %g3 + stx %l0, [%g3 + TI_REG_WINDOW + 0x00] + stx %l1, [%g3 + TI_REG_WINDOW + 0x08] + stx %l2, [%g3 + TI_REG_WINDOW + 0x10] + stx %l3, [%g3 + TI_REG_WINDOW + 0x18] + stx %l4, [%g3 + TI_REG_WINDOW + 0x20] + stx %l5, [%g3 + TI_REG_WINDOW + 0x28] + stx %l6, [%g3 + TI_REG_WINDOW + 0x30] + stx %l7, [%g3 + TI_REG_WINDOW + 0x38] + stx %i0, [%g3 + TI_REG_WINDOW + 0x40] + stx %i1, [%g3 + TI_REG_WINDOW + 0x48] + stx %i2, [%g3 + TI_REG_WINDOW + 0x50] + stx %i3, [%g3 + TI_REG_WINDOW + 0x58] + stx %i4, [%g3 + TI_REG_WINDOW + 0x60] + stx %i5, [%g3 + TI_REG_WINDOW + 0x68] + stx %i6, [%g3 + TI_REG_WINDOW + 0x70] + ba,pt %xcc, 2f + stx %i7, [%g3 + TI_REG_WINDOW + 0x78] +1: stw %l0, [%g3 + TI_REG_WINDOW + 0x00] + stw %l1, [%g3 + TI_REG_WINDOW + 0x04] + stw %l2, [%g3 + TI_REG_WINDOW + 0x08] + stw %l3, [%g3 + TI_REG_WINDOW + 0x0c] + stw %l4, [%g3 + TI_REG_WINDOW + 0x10] + stw %l5, [%g3 + TI_REG_WINDOW + 0x14] + stw %l6, [%g3 + TI_REG_WINDOW + 0x18] + stw %l7, [%g3 + TI_REG_WINDOW + 0x1c] + stw %i0, [%g3 + TI_REG_WINDOW + 0x20] + stw %i1, [%g3 + TI_REG_WINDOW + 0x24] + stw %i2, [%g3 + TI_REG_WINDOW + 0x28] + stw %i3, [%g3 + TI_REG_WINDOW + 0x2c] + stw %i4, [%g3 + TI_REG_WINDOW + 0x30] + stw %i5, [%g3 + TI_REG_WINDOW + 0x34] + stw %i6, [%g3 + TI_REG_WINDOW + 0x38] + stw %i7, [%g3 + TI_REG_WINDOW + 0x3c] +2: add %g1, 1, %g1 + stb %g1, [%g6 + TI_WSAVED] + rdpr %tstate, %g1 + andcc %g1, TSTATE_PRIV, %g0 + saved + be,pn %xcc, 1f + and %g1, TSTATE_CWP, %g1 + retry +1: mov FAULT_CODE_WRITE | FAULT_CODE_DTLB | FAULT_CODE_WINFIXUP, %g4 + stb %g4, [%g6 + TI_FAULT_CODE] + stx %g5, [%g6 + TI_FAULT_ADDR] + wrpr %g1, %cwp + ba,pt %xcc, etrap + rd %pc, %g7 + call do_sparc64_fault + add %sp, PTREGS_OFF, %o0 + ba,a,pt %xcc, rtrap + +winfix_mna: + andn %g3, 0x7f, %g3 + add %g3, 0x78, %g3 + wrpr %g3, %tnpc + done + +fill_fixup_mna: + rdpr %tstate, %g1 + and %g1, TSTATE_CWP, %g1 + wrpr %g1, %cwp + ba,pt %xcc, etrap + rd %pc, %g7 + sethi %hi(tlb_type), %g1 + lduw [%g1 + %lo(tlb_type)], %g1 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + mov %l4, %o2 + call sun4v_do_mna + mov %l5, %o1 + ba,a,pt %xcc, rtrap +1: mov %l4, %o1 + mov %l5, %o2 + call mem_address_unaligned + nop + ba,a,pt %xcc, rtrap + +winfix_dax: + andn %g3, 0x7f, %g3 + add %g3, 0x74, %g3 + wrpr %g3, %tnpc + done + +fill_fixup_dax: + rdpr %tstate, %g1 + and %g1, TSTATE_CWP, %g1 + wrpr %g1, %cwp + ba,pt %xcc, etrap + rd %pc, %g7 + sethi %hi(tlb_type), %g1 + mov %l4, %o1 + lduw [%g1 + %lo(tlb_type)], %g1 + mov %l5, %o2 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + call sun4v_data_access_exception + nop + ba,a,pt %xcc, rtrap + nop +1: call spitfire_data_access_exception + nop + ba,a,pt %xcc, rtrap + nop diff --git a/arch/sparc/kernel/wof.S b/arch/sparc/kernel/wof.S new file mode 100644 index 000000000..96a3a1124 --- /dev/null +++ b/arch/sparc/kernel/wof.S @@ -0,0 +1,366 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * wof.S: Sparc window overflow handler. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* WARNING: This routine is hairy and _very_ complicated, but it + * must be as fast as possible as it handles the allocation + * of register windows to the user and kernel. If you touch + * this code be _very_ careful as many other pieces of the + * kernel depend upon how this code behaves. You have been + * duly warned... + */ + +/* We define macro's for registers which have a fixed + * meaning throughout this entire routine. The 'T' in + * the comments mean that the register can only be + * accessed when in the 'trap' window, 'G' means + * accessible in any window. Do not change these registers + * after they have been set, until you are ready to return + * from the trap. + */ +#define t_psr l0 /* %psr at trap time T */ +#define t_pc l1 /* PC for trap return T */ +#define t_npc l2 /* NPC for trap return T */ +#define t_wim l3 /* %wim at trap time T */ +#define saved_g5 l5 /* Global save register T */ +#define saved_g6 l6 /* Global save register T */ +#define curptr g6 /* Gets set to 'current' then stays G */ + +/* Now registers whose values can change within the handler. */ +#define twin_tmp l4 /* Temp reg, only usable in trap window T */ +#define glob_tmp g5 /* Global temporary reg, usable anywhere G */ + + .text + .align 4 + /* BEGINNING OF PATCH INSTRUCTIONS */ + /* On a 7-window Sparc the boot code patches spnwin_* + * instructions with the following ones. + */ + .globl spnwin_patch1_7win, spnwin_patch2_7win, spnwin_patch3_7win +spnwin_patch1_7win: sll %t_wim, 6, %glob_tmp +spnwin_patch2_7win: and %glob_tmp, 0x7f, %glob_tmp +spnwin_patch3_7win: and %twin_tmp, 0x7f, %twin_tmp + /* END OF PATCH INSTRUCTIONS */ + + /* The trap entry point has done the following: + * + * rd %psr, %l0 + * rd %wim, %l3 + * b spill_window_entry + * andcc %l0, PSR_PS, %g0 + */ + + /* Datum current_thread_info->uwinmask contains at all times a bitmask + * where if any user windows are active, at least one bit will + * be set in to mask. If no user windows are active, the bitmask + * will be all zeroes. + */ + .globl spill_window_entry + .globl spnwin_patch1, spnwin_patch2, spnwin_patch3 +spill_window_entry: + /* LOCATION: Trap Window */ + + mov %g5, %saved_g5 ! save away global temp register + mov %g6, %saved_g6 ! save away 'current' ptr register + + /* Compute what the new %wim will be if we save the + * window properly in this trap handler. + * + * newwim = ((%wim>>1) | (%wim<<(nwindows - 1))); + */ + srl %t_wim, 0x1, %twin_tmp +spnwin_patch1: sll %t_wim, 7, %glob_tmp + or %glob_tmp, %twin_tmp, %glob_tmp +spnwin_patch2: and %glob_tmp, 0xff, %glob_tmp + + /* The trap entry point has set the condition codes + * up for us to see if this is from user or kernel. + * Get the load of 'curptr' out of the way. + */ + LOAD_CURRENT(curptr, twin_tmp) + + andcc %t_psr, PSR_PS, %g0 + be,a spwin_fromuser ! all user wins, branch + save %g0, %g0, %g0 ! Go where saving will occur + + /* See if any user windows are active in the set. */ + ld [%curptr + TI_UWINMASK], %twin_tmp ! grab win mask + orcc %g0, %twin_tmp, %g0 ! check for set bits + bne spwin_exist_uwins ! yep, there are some + andn %twin_tmp, %glob_tmp, %twin_tmp ! compute new uwinmask + + /* Save into the window which must be saved and do it. + * Basically if we are here, this means that we trapped + * from kernel mode with only kernel windows in the register + * file. + */ + save %g0, %g0, %g0 ! save into the window to stash away + wr %glob_tmp, 0x0, %wim ! set new %wim, this is safe now + +spwin_no_userwins_from_kernel: + /* LOCATION: Window to be saved */ + + STORE_WINDOW(sp) ! stash the window + restore %g0, %g0, %g0 ! go back into trap window + + /* LOCATION: Trap window */ + mov %saved_g5, %g5 ! restore %glob_tmp + mov %saved_g6, %g6 ! restore %curptr + wr %t_psr, 0x0, %psr ! restore condition codes in %psr + WRITE_PAUSE ! waste some time + jmp %t_pc ! Return from trap + rett %t_npc ! we are done + +spwin_exist_uwins: + /* LOCATION: Trap window */ + + /* Wow, user windows have to be dealt with, this is dirty + * and messy as all hell. And difficult to follow if you + * are approaching the infamous register window trap handling + * problem for the first time. DON'T LOOK! + * + * Note that how the execution path works out, the new %wim + * will be left for us in the global temporary register, + * %glob_tmp. We cannot set the new %wim first because we + * need to save into the appropriate window without inducing + * a trap (traps are off, we'd get a watchdog wheee)... + * But first, store the new user window mask calculated + * above. + */ + st %twin_tmp, [%curptr + TI_UWINMASK] + save %g0, %g0, %g0 ! Go to where the saving will occur + +spwin_fromuser: + /* LOCATION: Window to be saved */ + wr %glob_tmp, 0x0, %wim ! Now it is safe to set new %wim + + /* LOCATION: Window to be saved */ + + /* This instruction branches to a routine which will check + * to validity of the users stack pointer by whatever means + * are necessary. This means that this is architecture + * specific and thus this branch instruction will need to + * be patched at boot time once the machine type is known. + * This routine _shall not_ touch %curptr under any + * circumstances whatsoever! It will branch back to the + * label 'spwin_good_ustack' if the stack is ok but still + * needs to be dumped (SRMMU for instance will not need to + * do this) or 'spwin_finish_up' if the stack is ok and the + * registers have already been saved. If the stack is found + * to be bogus for some reason the routine shall branch to + * the label 'spwin_user_stack_is_bolixed' which will take + * care of things at that point. + */ + b spwin_srmmu_stackchk + andcc %sp, 0x7, %g0 + +spwin_good_ustack: + /* LOCATION: Window to be saved */ + + /* The users stack is ok and we can safely save it at + * %sp. + */ + STORE_WINDOW(sp) + +spwin_finish_up: + restore %g0, %g0, %g0 /* Back to trap window. */ + + /* LOCATION: Trap window */ + + /* We have spilled successfully, and we have properly stored + * the appropriate window onto the stack. + */ + + /* Restore saved globals */ + mov %saved_g5, %g5 + mov %saved_g6, %g6 + + wr %t_psr, 0x0, %psr + WRITE_PAUSE + jmp %t_pc + rett %t_npc + +spwin_user_stack_is_bolixed: + /* LOCATION: Window to be saved */ + + /* Wheee, user has trashed his/her stack. We have to decide + * how to proceed based upon whether we came from kernel mode + * or not. If we came from kernel mode, toss the window into + * a special buffer and proceed, the kernel _needs_ a window + * and we could be in an interrupt handler so timing is crucial. + * If we came from user land we build a full stack frame and call + * c-code to gun down the process. + */ + rd %psr, %glob_tmp + andcc %glob_tmp, PSR_PS, %g0 + bne spwin_bad_ustack_from_kernel + nop + + /* Oh well, throw this one window into the per-task window + * buffer, the first one. + */ + st %sp, [%curptr + TI_RWIN_SPTRS] + STORE_WINDOW(curptr + TI_REG_WINDOW) + restore %g0, %g0, %g0 + + /* LOCATION: Trap Window */ + + /* Back in the trap window, update winbuffer save count. */ + mov 1, %twin_tmp + st %twin_tmp, [%curptr + TI_W_SAVED] + + /* Compute new user window mask. What we are basically + * doing is taking two windows, the invalid one at trap + * time and the one we attempted to throw onto the users + * stack, and saying that everything else is an ok user + * window. umask = ((~(%t_wim | %wim)) & valid_wim_bits) + */ + rd %wim, %twin_tmp + or %twin_tmp, %t_wim, %twin_tmp + not %twin_tmp +spnwin_patch3: and %twin_tmp, 0xff, %twin_tmp ! patched on 7win Sparcs + st %twin_tmp, [%curptr + TI_UWINMASK] + +#define STACK_OFFSET (THREAD_SIZE - TRACEREG_SZ - STACKFRAME_SZ) + + sethi %hi(STACK_OFFSET), %sp + or %sp, %lo(STACK_OFFSET), %sp + add %curptr, %sp, %sp + + /* Restore the saved globals and build a pt_regs frame. */ + mov %saved_g5, %g5 + mov %saved_g6, %g6 + STORE_PT_ALL(sp, t_psr, t_pc, t_npc, g1) + + sethi %hi(STACK_OFFSET), %g6 + or %g6, %lo(STACK_OFFSET), %g6 + sub %sp, %g6, %g6 ! curptr + + /* Turn on traps and call c-code to deal with it. */ + wr %t_psr, PSR_ET, %psr + nop + call window_overflow_fault + nop + + /* Return from trap if C-code actually fixes things, if it + * doesn't then we never get this far as the process will + * be given the look of death from Commander Peanut. + */ + b ret_trap_entry + clr %l6 + +spwin_bad_ustack_from_kernel: + /* LOCATION: Window to be saved */ + + /* The kernel provoked a spill window trap, but the window we + * need to save is a user one and the process has trashed its + * stack pointer. We need to be quick, so we throw it into + * a per-process window buffer until we can properly handle + * this later on. + */ + SAVE_BOLIXED_USER_STACK(curptr, glob_tmp) + restore %g0, %g0, %g0 + + /* LOCATION: Trap window */ + + /* Restore globals, condition codes in the %psr and + * return from trap. Note, restoring %g6 when returning + * to kernel mode is not necessarily these days. ;-) + */ + mov %saved_g5, %g5 + mov %saved_g6, %g6 + + wr %t_psr, 0x0, %psr + WRITE_PAUSE + + jmp %t_pc + rett %t_npc + +/* Undefine the register macros which would only cause trouble + * if used below. This helps find 'stupid' coding errors that + * produce 'odd' behavior. The routines below are allowed to + * make usage of glob_tmp and t_psr so we leave them defined. + */ +#undef twin_tmp +#undef curptr +#undef t_pc +#undef t_npc +#undef t_wim +#undef saved_g5 +#undef saved_g6 + +/* Now come the per-architecture window overflow stack checking routines. + * As noted above %curptr cannot be touched by this routine at all. + */ + + /* This is a generic SRMMU routine. As far as I know this + * works for all current v8/srmmu implementations, we'll + * see... + */ + .globl spwin_srmmu_stackchk +spwin_srmmu_stackchk: + /* LOCATION: Window to be saved on the stack */ + + /* Because of SMP concerns and speed we play a trick. + * We disable fault traps in the MMU control register, + * Execute the stores, then check the fault registers + * to see what happens. I can hear Linus now + * "disgusting... broken hardware...". + * + * But first, check to see if the users stack has ended + * up in kernel vma, then we would succeed for the 'wrong' + * reason... ;( Note that the 'sethi' below assumes the + * kernel is page aligned, which should always be the case. + */ + /* Check results of callers andcc %sp, 0x7, %g0 */ + bne spwin_user_stack_is_bolixed + sethi %hi(PAGE_OFFSET), %glob_tmp + cmp %glob_tmp, %sp + bleu spwin_user_stack_is_bolixed + mov AC_M_SFSR, %glob_tmp + + /* Clear the fault status and turn on the no_fault bit. */ +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %g0) ! eat SFSR +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %g0) ! eat SFSR + +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %glob_tmp) ! read MMU control +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %glob_tmp) ! read MMU control + or %glob_tmp, 0x2, %glob_tmp ! or in no_fault bit +LEON_PI(sta %glob_tmp, [%g0] ASI_LEON_MMUREGS) ! set it +SUN_PI_(sta %glob_tmp, [%g0] ASI_M_MMUREGS) ! set it + + /* Dump the registers and cross fingers. */ + STORE_WINDOW(sp) + + /* Clear the no_fault bit and check the status. */ + andn %glob_tmp, 0x2, %glob_tmp +LEON_PI(sta %glob_tmp, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %glob_tmp, [%g0] ASI_M_MMUREGS) + + mov AC_M_SFAR, %glob_tmp +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %g0) +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %g0) + + mov AC_M_SFSR, %glob_tmp +LEON_PI(lda [%glob_tmp] ASI_LEON_MMUREGS, %glob_tmp) +SUN_PI_(lda [%glob_tmp] ASI_M_MMUREGS, %glob_tmp) + andcc %glob_tmp, 0x2, %g0 ! did we fault? + be,a spwin_finish_up + 0x4 ! cool beans, success + restore %g0, %g0, %g0 + + rd %psr, %glob_tmp + b spwin_user_stack_is_bolixed + 0x4 ! we faulted, ugh + nop diff --git a/arch/sparc/kernel/wuf.S b/arch/sparc/kernel/wuf.S new file mode 100644 index 000000000..1a4ca490e --- /dev/null +++ b/arch/sparc/kernel/wuf.S @@ -0,0 +1,313 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * wuf.S: Window underflow trap handler for the Sparc. + * + * Copyright (C) 1995 David S. Miller + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Just like the overflow handler we define macros for registers + * with fixed meanings in this routine. + */ +#define t_psr l0 +#define t_pc l1 +#define t_npc l2 +#define t_wim l3 +/* Don't touch the above registers or else you die horribly... */ + +/* Now macros for the available scratch registers in this routine. */ +#define twin_tmp1 l4 +#define twin_tmp2 l5 + +#define curptr g6 + + .text + .align 4 + + /* The trap entry point has executed the following: + * + * rd %psr, %l0 + * rd %wim, %l3 + * b fill_window_entry + * andcc %l0, PSR_PS, %g0 + */ + + /* Datum current_thread_info->uwinmask contains at all times a bitmask + * where if any user windows are active, at least one bit will + * be set in to mask. If no user windows are active, the bitmask + * will be all zeroes. + */ + + /* To get an idea of what has just happened to cause this + * trap take a look at this diagram: + * + * 1 2 3 4 <-- Window number + * ---------- + * T O W I <-- Symbolic name + * + * O == the window that execution was in when + * the restore was attempted + * + * T == the trap itself has save'd us into this + * window + * + * W == this window is the one which is now invalid + * and must be made valid plus loaded from the + * stack + * + * I == this window will be the invalid one when we + * are done and return from trap if successful + */ + + /* BEGINNING OF PATCH INSTRUCTIONS */ + + /* On 7-window Sparc the boot code patches fnwin_patch1 + * with the following instruction. + */ + .globl fnwin_patch1_7win, fnwin_patch2_7win +fnwin_patch1_7win: srl %t_wim, 6, %twin_tmp2 +fnwin_patch2_7win: and %twin_tmp1, 0x7f, %twin_tmp1 + /* END OF PATCH INSTRUCTIONS */ + + .globl fill_window_entry, fnwin_patch1, fnwin_patch2 +fill_window_entry: + /* LOCATION: Window 'T' */ + + /* Compute what the new %wim is going to be if we retrieve + * the proper window off of the stack. + */ + sll %t_wim, 1, %twin_tmp1 +fnwin_patch1: srl %t_wim, 7, %twin_tmp2 + or %twin_tmp1, %twin_tmp2, %twin_tmp1 +fnwin_patch2: and %twin_tmp1, 0xff, %twin_tmp1 + + wr %twin_tmp1, 0x0, %wim /* Make window 'I' invalid */ + + andcc %t_psr, PSR_PS, %g0 + be fwin_from_user + restore %g0, %g0, %g0 /* Restore to window 'O' */ + + /* Trapped from kernel, we trust that the kernel does not + * 'over restore' sorta speak and just grab the window + * from the stack and return. Easy enough. + */ +fwin_from_kernel: + /* LOCATION: Window 'O' */ + + restore %g0, %g0, %g0 + + /* LOCATION: Window 'W' */ + + LOAD_WINDOW(sp) /* Load it up */ + + /* Spin the wheel... */ + save %g0, %g0, %g0 + save %g0, %g0, %g0 + /* I'd like to buy a vowel please... */ + + /* LOCATION: Window 'T' */ + + /* Now preserve the condition codes in %psr, pause, and + * return from trap. This is the simplest case of all. + */ + wr %t_psr, 0x0, %psr + WRITE_PAUSE + + jmp %t_pc + rett %t_npc + +fwin_from_user: + /* LOCATION: Window 'O' */ + + restore %g0, %g0, %g0 /* Restore to window 'W' */ + + /* LOCATION: Window 'W' */ + + /* Branch to the stack validation routine */ + b srmmu_fwin_stackchk + andcc %sp, 0x7, %g0 + +#define STACK_OFFSET (THREAD_SIZE - TRACEREG_SZ - STACKFRAME_SZ) + +fwin_user_stack_is_bolixed: + /* LOCATION: Window 'W' */ + + /* Place a pt_regs frame on the kernel stack, save back + * to the trap window and call c-code to deal with this. + */ + LOAD_CURRENT(l4, l5) + + sethi %hi(STACK_OFFSET), %l5 + or %l5, %lo(STACK_OFFSET), %l5 + add %l4, %l5, %l5 + + /* Store globals into pt_regs frame. */ + STORE_PT_GLOBALS(l5) + STORE_PT_YREG(l5, g3) + + /* Save current in a global while we change windows. */ + mov %l4, %curptr + + save %g0, %g0, %g0 + + /* LOCATION: Window 'O' */ + + rd %psr, %g3 /* Read %psr in live user window */ + mov %fp, %g4 /* Save bogus frame pointer. */ + + save %g0, %g0, %g0 + + /* LOCATION: Window 'T' */ + + sethi %hi(STACK_OFFSET), %l5 + or %l5, %lo(STACK_OFFSET), %l5 + add %curptr, %l5, %sp + + /* Build rest of pt_regs. */ + STORE_PT_INS(sp) + STORE_PT_PRIV(sp, t_psr, t_pc, t_npc) + + /* re-set trap time %wim value */ + wr %t_wim, 0x0, %wim + + /* Fix users window mask and buffer save count. */ + mov 0x1, %g5 + sll %g5, %g3, %g5 + st %g5, [%curptr + TI_UWINMASK] ! one live user window still + st %g0, [%curptr + TI_W_SAVED] ! no windows in the buffer + + wr %t_psr, PSR_ET, %psr ! enable traps + nop + call window_underflow_fault + mov %g4, %o0 + + b ret_trap_entry + clr %l6 + +fwin_user_stack_is_ok: + /* LOCATION: Window 'W' */ + + /* The users stack area is kosher and mapped, load the + * window and fall through to the finish up routine. + */ + LOAD_WINDOW(sp) + + /* Round and round she goes... */ + save %g0, %g0, %g0 /* Save to window 'O' */ + save %g0, %g0, %g0 /* Save to window 'T' */ + /* Where she'll trap nobody knows... */ + + /* LOCATION: Window 'T' */ + +fwin_user_finish_up: + /* LOCATION: Window 'T' */ + + wr %t_psr, 0x0, %psr + WRITE_PAUSE + + jmp %t_pc + rett %t_npc + + /* Here come the architecture specific checks for stack. + * mappings. Note that unlike the window overflow handler + * we only need to check whether the user can read from + * the appropriate addresses. Also note that we are in + * an invalid window which will be loaded, and this means + * that until we actually load the window up we are free + * to use any of the local registers contained within. + * + * On success these routine branch to fwin_user_stack_is_ok + * if the area at %sp is user readable and the window still + * needs to be loaded, else fwin_user_finish_up if the + * routine has done the loading itself. On failure (bogus + * user stack) the routine shall branch to the label called + * fwin_user_stack_is_bolixed. + * + * Contrary to the arch-specific window overflow stack + * check routines in wof.S, these routines are free to use + * any of the local registers they want to as this window + * does not belong to anyone at this point, however the + * outs and ins are still verboten as they are part of + * 'someone elses' window possibly. + */ + + .globl srmmu_fwin_stackchk +srmmu_fwin_stackchk: + /* LOCATION: Window 'W' */ + + /* Caller did 'andcc %sp, 0x7, %g0' */ + bne fwin_user_stack_is_bolixed + sethi %hi(PAGE_OFFSET), %l5 + + /* Check if the users stack is in kernel vma, then our + * trial and error technique below would succeed for + * the 'wrong' reason. + */ + mov AC_M_SFSR, %l4 + cmp %l5, %sp + bleu fwin_user_stack_is_bolixed +LEON_PI( lda [%l4] ASI_LEON_MMUREGS, %g0) ! clear fault status +SUN_PI_( lda [%l4] ASI_M_MMUREGS, %g0) ! clear fault status + + /* The technique is, turn off faults on this processor, + * just let the load rip, then check the sfsr to see if + * a fault did occur. Then we turn on fault traps again + * and branch conditionally based upon what happened. + */ +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %l5) ! read mmu-ctrl reg +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %l5) ! read mmu-ctrl reg + or %l5, 0x2, %l5 ! turn on no-fault bit +LEON_PI(sta %l5, [%g0] ASI_LEON_MMUREGS) ! store it +SUN_PI_(sta %l5, [%g0] ASI_M_MMUREGS) ! store it + + /* Cross fingers and go for it. */ + LOAD_WINDOW(sp) + + /* A penny 'saved'... */ + save %g0, %g0, %g0 + save %g0, %g0, %g0 + /* Is a BADTRAP earned... */ + + /* LOCATION: Window 'T' */ + +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %twin_tmp1) ! load mmu-ctrl again +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %twin_tmp1) ! load mmu-ctrl again + andn %twin_tmp1, 0x2, %twin_tmp1 ! clear no-fault bit +LEON_PI(sta %twin_tmp1, [%g0] ASI_LEON_MMUREGS) ! store it +SUN_PI_(sta %twin_tmp1, [%g0] ASI_M_MMUREGS) ! store it + + mov AC_M_SFAR, %twin_tmp2 +LEON_PI(lda [%twin_tmp2] ASI_LEON_MMUREGS, %g0) ! read fault address +SUN_PI_(lda [%twin_tmp2] ASI_M_MMUREGS, %g0) ! read fault address + + mov AC_M_SFSR, %twin_tmp2 +LEON_PI(lda [%twin_tmp2] ASI_LEON_MMUREGS, %twin_tmp2) ! read fault status +SUN_PI_(lda [%twin_tmp2] ASI_M_MMUREGS, %twin_tmp2) ! read fault status + andcc %twin_tmp2, 0x2, %g0 ! did fault occur? + + bne 1f ! yep, cleanup + nop + + wr %t_psr, 0x0, %psr + nop + b fwin_user_finish_up + 0x4 + nop + + /* Did I ever tell you about my window lobotomy? + * anyways... fwin_user_stack_is_bolixed expects + * to be in window 'W' so make it happy or else + * we watchdog badly. + */ +1: + restore %g0, %g0, %g0 + b fwin_user_stack_is_bolixed ! oh well + restore %g0, %g0, %g0 diff --git a/arch/sparc/lib/GENbzero.S b/arch/sparc/lib/GENbzero.S new file mode 100644 index 000000000..63d618857 --- /dev/null +++ b/arch/sparc/lib/GENbzero.S @@ -0,0 +1,157 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* GENbzero.S: Generic sparc64 memset/clear_user. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ +#include + +#define EX_ST(x,y) \ +98: x,y; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, __retl_o1_asi;\ + .text; \ + .align 4; + + .align 32 + .text + + .globl GENmemset + .type GENmemset, #function +GENmemset: /* %o0=buf, %o1=pat, %o2=len */ + and %o1, 0xff, %o3 + mov %o2, %o1 + sllx %o3, 8, %g1 + or %g1, %o3, %o2 + sllx %o2, 16, %g1 + or %g1, %o2, %o2 + sllx %o2, 32, %g1 + ba,pt %xcc, 1f + or %g1, %o2, %o2 + + .globl GENbzero + .type GENbzero, #function +GENbzero: + clr %o2 +1: brz,pn %o1, GENbzero_return + mov %o0, %o3 + + /* %o5: saved %asi, restored at GENbzero_done + * %o4: store %asi to use + */ + rd %asi, %o5 + mov ASI_P, %o4 + wr %o4, 0x0, %asi + +GENbzero_from_clear_user: + cmp %o1, 15 + bl,pn %icc, GENbzero_tiny + andcc %o0, 0x7, %g1 + be,pt %xcc, 2f + mov 8, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: EX_ST(stba %o2, [%o0 + 0x00] %asi) + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %o0, 1, %o0 +2: cmp %o1, 128 + bl,pn %icc, GENbzero_medium + andcc %o0, (64 - 1), %g1 + be,pt %xcc, GENbzero_pre_loop + mov 64, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: EX_ST(stxa %o2, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %xcc, 1b + add %o0, 8, %o0 + +GENbzero_pre_loop: + andn %o1, (64 - 1), %g1 + sub %o1, %g1, %o1 +GENbzero_loop: + EX_ST(stxa %o2, [%o0 + 0x00] %asi) + EX_ST(stxa %o2, [%o0 + 0x08] %asi) + EX_ST(stxa %o2, [%o0 + 0x10] %asi) + EX_ST(stxa %o2, [%o0 + 0x18] %asi) + EX_ST(stxa %o2, [%o0 + 0x20] %asi) + EX_ST(stxa %o2, [%o0 + 0x28] %asi) + EX_ST(stxa %o2, [%o0 + 0x30] %asi) + EX_ST(stxa %o2, [%o0 + 0x38] %asi) + subcc %g1, 64, %g1 + bne,pt %xcc, GENbzero_loop + add %o0, 64, %o0 + + membar #Sync + wr %o4, 0x0, %asi + brz,pn %o1, GENbzero_done +GENbzero_medium: + andncc %o1, 0x7, %g1 + be,pn %xcc, 2f + sub %o1, %g1, %o1 +1: EX_ST(stxa %o2, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %xcc, 1b + add %o0, 8, %o0 +2: brz,pt %o1, GENbzero_done + nop + +GENbzero_tiny: +1: EX_ST(stba %o2, [%o0 + 0x00] %asi) + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 1, %o0 + + /* fallthrough */ + +GENbzero_done: + wr %o5, 0x0, %asi + +GENbzero_return: + retl + mov %o3, %o0 + .size GENbzero, .-GENbzero + .size GENmemset, .-GENmemset + + .globl GENclear_user + .type GENclear_user, #function +GENclear_user: /* %o0=buf, %o1=len */ + rd %asi, %o5 + brz,pn %o1, GENbzero_done + clr %o3 + cmp %o5, ASI_AIUS + bne,pn %icc, GENbzero + clr %o2 + ba,pt %xcc, GENbzero_from_clear_user + mov ASI_AIUS, %o4 + .size GENclear_user, .-GENclear_user + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define GEN_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl generic_patch_bzero + .type generic_patch_bzero,#function +generic_patch_bzero: + GEN_DO_PATCH(memset, GENmemset) + GEN_DO_PATCH(__bzero, GENbzero) + GEN_DO_PATCH(__clear_user, GENclear_user) + retl + nop + .size generic_patch_bzero,.-generic_patch_bzero diff --git a/arch/sparc/lib/GENcopy_from_user.S b/arch/sparc/lib/GENcopy_from_user.S new file mode 100644 index 000000000..6891a5678 --- /dev/null +++ b/arch/sparc/lib/GENcopy_from_user.S @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* GENcopy_from_user.S: Generic sparc64 copy from userspace. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME GENcopy_from_user +#define LOAD(type,addr,dest) type##a [addr] ASI_AIUS, dest +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "GENmemcpy.S" diff --git a/arch/sparc/lib/GENcopy_to_user.S b/arch/sparc/lib/GENcopy_to_user.S new file mode 100644 index 000000000..df75b532a --- /dev/null +++ b/arch/sparc/lib/GENcopy_to_user.S @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* GENcopy_to_user.S: Generic sparc64 copy to userspace. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME GENcopy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "GENmemcpy.S" diff --git a/arch/sparc/lib/GENmemcpy.S b/arch/sparc/lib/GENmemcpy.S new file mode 100644 index 000000000..114340a0d --- /dev/null +++ b/arch/sparc/lib/GENmemcpy.S @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* GENmemcpy.S: Generic sparc64 memcpy. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#ifdef __KERNEL__ +#include +#define GLOBAL_SPARE %g7 +#else +#define GLOBAL_SPARE %g5 +#endif + +#ifndef EX_LD +#define EX_LD(x,y) x +#endif + +#ifndef EX_ST +#define EX_ST(x,y) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME GENmemcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + + .register %g2,#scratch + .register %g3,#scratch + + .text + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +ENTRY(GEN_retl_o4_1) + add %o4, %o2, %o4 + retl + add %o4, 1, %o0 +ENDPROC(GEN_retl_o4_1) +ENTRY(GEN_retl_g1_8) + add %g1, %o2, %g1 + retl + add %g1, 8, %o0 +ENDPROC(GEN_retl_g1_8) +ENTRY(GEN_retl_o2_4) + retl + add %o2, 4, %o0 +ENDPROC(GEN_retl_o2_4) +ENTRY(GEN_retl_o2_1) + retl + add %o2, 1, %o0 +ENDPROC(GEN_retl_o2_1) +#endif + + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %XCC, 5 + PREAMBLE + mov %o0, GLOBAL_SPARE + + cmp %o2, 0 + be,pn %XCC, 85f + or %o0, %o1, %o3 + cmp %o2, 16 + blu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + xor %o0, %o1, %o4 + andcc %o4, 0x7, %g0 + bne,a,pn %XCC, 90f + sub %o0, %o1, %o3 + + and %o0, 0x7, %o4 + sub %o4, 0x8, %o4 + sub %g0, %o4, %o4 + sub %o2, %o4, %o2 +1: subcc %o4, 1, %o4 + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o4_1) + EX_ST(STORE(stb, %g1, %o0),GEN_retl_o4_1) + add %o1, 1, %o1 + bne,pt %XCC, 1b + add %o0, 1, %o0 + + andn %o2, 0x7, %g1 + sub %o2, %g1, %o2 +1: subcc %g1, 0x8, %g1 + EX_LD(LOAD(ldx, %o1, %g2),GEN_retl_g1_8) + EX_ST(STORE(stx, %g2, %o0),GEN_retl_g1_8) + add %o1, 0x8, %o1 + bne,pt %XCC, 1b + add %o0, 0x8, %o0 + + brz,pt %o2, 85f + sub %o0, %o1, %o3 + ba,a,pt %XCC, 90f + + .align 64 +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +1: + subcc %o2, 4, %o2 + EX_LD(LOAD(lduw, %o1, %g1),GEN_retl_o2_4) + EX_ST(STORE(stw, %g1, %o1 + %o3),GEN_retl_o2_4) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +85: retl + mov EX_RETVAL(GLOBAL_SPARE), %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + EX_LD(LOAD(ldub, %o1, %g1),GEN_retl_o2_1) + EX_ST(STORE(stb, %g1, %o1 + %o3),GEN_retl_o2_1) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov EX_RETVAL(GLOBAL_SPARE), %o0 + + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/GENpage.S b/arch/sparc/lib/GENpage.S new file mode 100644 index 000000000..c143c4d1d --- /dev/null +++ b/arch/sparc/lib/GENpage.S @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* GENpage.S: Generic clear and copy page. + * + * Copyright (C) 2007 (davem@davemloft.net) + */ +#include + + .text + .align 32 + +GENcopy_user_page: + set PAGE_SIZE, %g7 +1: ldx [%o1 + 0x00], %o2 + ldx [%o1 + 0x08], %o3 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + stx %o2, [%o0 + 0x00] + stx %o3, [%o0 + 0x08] + stx %o4, [%o0 + 0x10] + stx %o5, [%o0 + 0x18] + ldx [%o1 + 0x20], %o2 + ldx [%o1 + 0x28], %o3 + ldx [%o1 + 0x30], %o4 + ldx [%o1 + 0x38], %o5 + stx %o2, [%o0 + 0x20] + stx %o3, [%o0 + 0x28] + stx %o4, [%o0 + 0x30] + stx %o5, [%o0 + 0x38] + subcc %g7, 64, %g7 + add %o1, 64, %o1 + bne,pt %xcc, 1b + add %o0, 64, %o0 + retl + nop + +GENclear_page: +GENclear_user_page: + set PAGE_SIZE, %g7 +1: stx %g0, [%o0 + 0x00] + stx %g0, [%o0 + 0x08] + stx %g0, [%o0 + 0x10] + stx %g0, [%o0 + 0x18] + stx %g0, [%o0 + 0x20] + stx %g0, [%o0 + 0x28] + stx %g0, [%o0 + 0x30] + stx %g0, [%o0 + 0x38] + subcc %g7, 64, %g7 + bne,pt %xcc, 1b + add %o0, 64, %o0 + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define GEN_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl generic_patch_pageops + .type generic_patch_pageops,#function +generic_patch_pageops: + GEN_DO_PATCH(copy_user_page, GENcopy_user_page) + GEN_DO_PATCH(_clear_page, GENclear_page) + GEN_DO_PATCH(clear_user_page, GENclear_user_page) + retl + nop + .size generic_patch_pageops,.-generic_patch_pageops diff --git a/arch/sparc/lib/GENpatch.S b/arch/sparc/lib/GENpatch.S new file mode 100644 index 000000000..1ec1f02c8 --- /dev/null +++ b/arch/sparc/lib/GENpatch.S @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* GENpatch.S: Patch Ultra-I routines with generic variant. + * + * Copyright (C) 2007 David S. Miller + */ + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define GEN_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl generic_patch_copyops + .type generic_patch_copyops,#function +generic_patch_copyops: + GEN_DO_PATCH(memcpy, GENmemcpy) + GEN_DO_PATCH(raw_copy_from_user, GENcopy_from_user) + GEN_DO_PATCH(raw_copy_to_user, GENcopy_to_user) + retl + nop + .size generic_patch_copyops,.-generic_patch_copyops diff --git a/arch/sparc/lib/M7copy_from_user.S b/arch/sparc/lib/M7copy_from_user.S new file mode 100644 index 000000000..66464b3e3 --- /dev/null +++ b/arch/sparc/lib/M7copy_from_user.S @@ -0,0 +1,40 @@ +/* + * M7copy_from_user.S: SPARC M7 optimized copy from userspace. + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + + +#define EX_LD(x, y) \ +98: x; \ + .section __ex_table,"a"; \ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_LD_FP(x, y) \ +98: x; \ + .section __ex_table,"a"; \ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME M7copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "M7memcpy.S" diff --git a/arch/sparc/lib/M7copy_to_user.S b/arch/sparc/lib/M7copy_to_user.S new file mode 100644 index 000000000..a60ac467f --- /dev/null +++ b/arch/sparc/lib/M7copy_to_user.S @@ -0,0 +1,51 @@ +/* + * M7copy_to_user.S: SPARC M7 optimized copy to userspace. + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + + +#define EX_ST(x, y) \ +98: x; \ + .section __ex_table,"a"; \ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_ST_FP(x, y) \ +98: x; \ + .section __ex_table,"a"; \ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS +#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23 +#endif + +#define FUNC_NAME M7copy_to_user +#define STORE(type,src,addr) type##a src, [addr] %asi +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS +#define STORE_MRU_ASI ASI_ST_BLKINIT_MRU_S +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "M7memcpy.S" diff --git a/arch/sparc/lib/M7memcpy.S b/arch/sparc/lib/M7memcpy.S new file mode 100644 index 000000000..99357bfa8 --- /dev/null +++ b/arch/sparc/lib/M7memcpy.S @@ -0,0 +1,923 @@ +/* + * M7memcpy: Optimized SPARC M7 memcpy + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + + .file "M7memcpy.S" + +/* + * memcpy(s1, s2, len) + * + * Copy s2 to s1, always copy n bytes. + * Note: this C code does not work for overlapped copies. + * + * Fast assembler language version of the following C-program for memcpy + * which represents the `standard' for the C-library. + * + * void * + * memcpy(void *s, const void *s0, size_t n) + * { + * if (n != 0) { + * char *s1 = s; + * const char *s2 = s0; + * do { + * *s1++ = *s2++; + * } while (--n != 0); + * } + * return (s); + * } + * + * + * SPARC T7/M7 Flow : + * + * if (count < SMALL_MAX) { + * if count < SHORTCOPY (SHORTCOPY=3) + * copy bytes; exit with dst addr + * if src & dst aligned on word boundary but not long word boundary, + * copy with ldw/stw; branch to finish_up + * if src & dst aligned on long word boundary + * copy with ldx/stx; branch to finish_up + * if src & dst not aligned and length <= SHORTCHECK (SHORTCHECK=14) + * copy bytes; exit with dst addr + * move enough bytes to get src to word boundary + * if dst now on word boundary + * move_words: + * copy words; branch to finish_up + * if dst now on half word boundary + * load words, shift half words, store words; branch to finish_up + * if dst on byte 1 + * load words, shift 3 bytes, store words; branch to finish_up + * if dst on byte 3 + * load words, shift 1 byte, store words; branch to finish_up + * finish_up: + * copy bytes; exit with dst addr + * } else { More than SMALL_MAX bytes + * move bytes until dst is on long word boundary + * if( src is on long word boundary ) { + * if (count < MED_MAX) { + * finish_long: src/dst aligned on 8 bytes + * copy with ldx/stx in 8-way unrolled loop; + * copy final 0-63 bytes; exit with dst addr + * } else { src/dst aligned; count > MED_MAX + * align dst on 64 byte boundary; for main data movement: + * prefetch src data to L2 cache; let HW prefetch move data to L1 cache + * Use BIS (block initializing store) to avoid copying store cache + * lines from memory. But pre-store first element of each cache line + * ST_CHUNK lines in advance of the rest of that cache line. That + * gives time for replacement cache lines to be written back without + * excess STQ and Miss Buffer filling. Repeat until near the end, + * then finish up storing before going to finish_long. + * } + * } else { src/dst not aligned on 8 bytes + * if src is word aligned and count < MED_WMAX + * move words in 8-way unrolled loop + * move final 0-31 bytes; exit with dst addr + * if count < MED_UMAX + * use alignaddr/faligndata combined with ldd/std in 8-way + * unrolled loop to move data. + * go to unalign_done + * else + * setup alignaddr for faligndata instructions + * align dst on 64 byte boundary; prefetch src data to L1 cache + * loadx8, falign, block-store, prefetch loop + * (only use block-init-store when src/dst on 8 byte boundaries.) + * unalign_done: + * move remaining bytes for unaligned cases. exit with dst addr. + * } + * + */ + +#include +#include + +#if !defined(EX_LD) && !defined(EX_ST) +#define NON_USER_COPY +#endif + +#ifndef EX_LD +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x +#endif + +#ifndef EX_ST +#define EX_ST(x,y) x +#endif +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +/* + * ASI_BLK_INIT_QUAD_LDD_P/ASI_BLK_INIT_QUAD_LDD_S marks the cache + * line as "least recently used" which means if many threads are + * active, it has a high probability of being pushed out of the cache + * between the first initializing store and the final stores. + * Thus, we use ASI_ST_BLKINIT_MRU_P/ASI_ST_BLKINIT_MRU_S which + * marks the cache line as "most recently used" for all + * but the last cache line + */ +#ifndef STORE_ASI +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P +#else +#define STORE_ASI 0x80 /* ASI_P */ +#endif +#endif + +#ifndef STORE_MRU_ASI +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_MRU_ASI ASI_ST_BLKINIT_MRU_P +#else +#define STORE_MRU_ASI 0x80 /* ASI_P */ +#endif +#endif + +#ifndef STORE_INIT +#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI +#endif + +#ifndef STORE_INIT_MRU +#define STORE_INIT_MRU(src,addr) stxa src, [addr] STORE_MRU_ASI +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME M7memcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#define BLOCK_SIZE 64 +#define SHORTCOPY 3 +#define SHORTCHECK 14 +#define SHORT_LONG 64 /* max copy for short longword-aligned case */ + /* must be at least 64 */ +#define SMALL_MAX 128 +#define MED_UMAX 1024 /* max copy for medium un-aligned case */ +#define MED_WMAX 1024 /* max copy for medium word-aligned case */ +#define MED_MAX 1024 /* max copy for medium longword-aligned case */ +#define ST_CHUNK 24 /* ST_CHUNK - block of values for BIS Store */ +#define ALIGN_PRE 24 /* distance for aligned prefetch loop */ + + .register %g2,#scratch + + .section ".text" + .global FUNC_NAME + .type FUNC_NAME, #function + .align 16 +FUNC_NAME: + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %xcc, 5 + PREAMBLE + mov %o0, %g1 ! save %o0 + brz,pn %o2, .Lsmallx + cmp %o2, 3 + ble,pn %icc, .Ltiny_cp + cmp %o2, 19 + ble,pn %icc, .Lsmall_cp + or %o0, %o1, %g2 + cmp %o2, SMALL_MAX + bl,pn %icc, .Lmedium_cp + nop + +.Lmedium: + neg %o0, %o5 + andcc %o5, 7, %o5 ! bytes till DST 8 byte aligned + brz,pt %o5, .Ldst_aligned_on_8 + + ! %o5 has the bytes to be written in partial store. + sub %o2, %o5, %o2 + sub %o1, %o0, %o1 ! %o1 gets the difference +7: ! dst aligning loop + add %o1, %o0, %o4 + EX_LD(LOAD(ldub, %o4, %o4), memcpy_retl_o2_plus_o5) ! load one byte + subcc %o5, 1, %o5 + EX_ST(STORE(stb, %o4, %o0), memcpy_retl_o2_plus_o5_plus_1) + bgu,pt %xcc, 7b + add %o0, 1, %o0 ! advance dst + add %o1, %o0, %o1 ! restore %o1 +.Ldst_aligned_on_8: + andcc %o1, 7, %o5 + brnz,pt %o5, .Lsrc_dst_unaligned_on_8 + nop + +.Lsrc_dst_aligned_on_8: + ! check if we are copying MED_MAX or more bytes + set MED_MAX, %o3 + cmp %o2, %o3 ! limit to store buffer size + bgu,pn %xcc, .Llarge_align8_copy + nop + +/* + * Special case for handling when src and dest are both long word aligned + * and total data to move is less than MED_MAX bytes + */ +.Lmedlong: + subcc %o2, 63, %o2 ! adjust length to allow cc test + ble,pn %xcc, .Lmedl63 ! skip big loop if less than 64 bytes + nop +.Lmedl64: + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_63) ! load + subcc %o2, 64, %o2 ! decrement length count + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_63_64) ! and store + EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_63_56) ! a block of 64 + EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_63_56) + EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_63_48) + EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_63_48) + EX_LD(LOAD(ldx, %o1+24, %o3), memcpy_retl_o2_plus_63_40) + EX_ST(STORE(stx, %o3, %o0+24), memcpy_retl_o2_plus_63_40) + EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_63_32)! load and store + EX_ST(STORE(stx, %o4, %o0+32), memcpy_retl_o2_plus_63_32) + EX_LD(LOAD(ldx, %o1+40, %o3), memcpy_retl_o2_plus_63_24)! a block of 64 + add %o1, 64, %o1 ! increase src ptr by 64 + EX_ST(STORE(stx, %o3, %o0+40), memcpy_retl_o2_plus_63_24) + EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_63_16) + add %o0, 64, %o0 ! increase dst ptr by 64 + EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_63_16) + EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_63_8) + bgu,pt %xcc, .Lmedl64 ! repeat if at least 64 bytes left + EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_63_8) +.Lmedl63: + addcc %o2, 32, %o2 ! adjust remaining count + ble,pt %xcc, .Lmedl31 ! to skip if 31 or fewer bytes left + nop + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_31) ! load + sub %o2, 32, %o2 ! decrement length count + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_31_32) ! and store + EX_LD(LOAD(ldx, %o1+8, %o3), memcpy_retl_o2_plus_31_24) ! a block of 32 + add %o1, 32, %o1 ! increase src ptr by 32 + EX_ST(STORE(stx, %o3, %o0+8), memcpy_retl_o2_plus_31_24) + EX_LD(LOAD(ldx, %o1-16, %o4), memcpy_retl_o2_plus_31_16) + add %o0, 32, %o0 ! increase dst ptr by 32 + EX_ST(STORE(stx, %o4, %o0-16), memcpy_retl_o2_plus_31_16) + EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_31_8) + EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_31_8) +.Lmedl31: + addcc %o2, 16, %o2 ! adjust remaining count + ble,pt %xcc, .Lmedl15 ! skip if 15 or fewer bytes left + nop ! + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_15) + add %o1, 16, %o1 ! increase src ptr by 16 + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_15) + sub %o2, 16, %o2 ! decrease count by 16 + EX_LD(LOAD(ldx, %o1-8, %o3), memcpy_retl_o2_plus_15_8) + add %o0, 16, %o0 ! increase dst ptr by 16 + EX_ST(STORE(stx, %o3, %o0-8), memcpy_retl_o2_plus_15_8) +.Lmedl15: + addcc %o2, 15, %o2 ! restore count + bz,pt %xcc, .Lsmallx ! exit if finished + cmp %o2, 8 + blt,pt %xcc, .Lmedw7 ! skip if 7 or fewer bytes left + tst %o2 + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) ! load 8 bytes + add %o1, 8, %o1 ! increase src ptr by 8 + add %o0, 8, %o0 ! increase dst ptr by 8 + subcc %o2, 8, %o2 ! decrease count by 8 + bnz,pn %xcc, .Lmedw7 + EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) ! and store 8 + retl + mov EX_RETVAL(%g1), %o0 ! restore %o0 + + .align 16 +.Lsrc_dst_unaligned_on_8: + ! DST is 8-byte aligned, src is not +2: + andcc %o1, 0x3, %o5 ! test word alignment + bnz,pt %xcc, .Lunalignsetup ! branch to skip if not word aligned + nop + +/* + * Handle all cases where src and dest are aligned on word + * boundaries. Use unrolled loops for better performance. + * This option wins over standard large data move when + * source and destination is in cache for.Lmedium + * to short data moves. + */ + set MED_WMAX, %o3 + cmp %o2, %o3 ! limit to store buffer size + bge,pt %xcc, .Lunalignrejoin ! otherwise rejoin main loop + nop + + subcc %o2, 31, %o2 ! adjust length to allow cc test + ! for end of loop + ble,pt %xcc, .Lmedw31 ! skip big loop if less than 16 +.Lmedw32: + EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_31)! move a block of 32 + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_31) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_31) + subcc %o2, 32, %o2 ! decrement length count + EX_LD(LOAD(ld, %o1+8, %o4), memcpy_retl_o2_plus_31_24) + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1+12, %o4), memcpy_retl_o2_plus_31_24) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0+8), memcpy_retl_o2_plus_31_24) + add %o1, 32, %o1 ! increase src ptr by 32 + EX_LD(LOAD(ld, %o1-16, %o4), memcpy_retl_o2_plus_31_16) + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1-12, %o4), memcpy_retl_o2_plus_31_16) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0+16), memcpy_retl_o2_plus_31_16) + add %o0, 32, %o0 ! increase dst ptr by 32 + EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_31_8) + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_31_8) + or %o4, %o5, %o5 + bgu,pt %xcc, .Lmedw32 ! repeat if at least 32 bytes left + EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_31_8) +.Lmedw31: + addcc %o2, 31, %o2 ! restore count + + bz,pt %xcc, .Lsmallx ! exit if finished + nop + cmp %o2, 16 + blt,pt %xcc, .Lmedw15 + nop + EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2)! move a block of 16 bytes + sllx %o4, 32, %o5 + subcc %o2, 16, %o2 ! decrement length count + EX_LD(LOAD(ld, %o1+4, %o4), memcpy_retl_o2_plus_16) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_16) + add %o1, 16, %o1 ! increase src ptr by 16 + EX_LD(LOAD(ld, %o1-8, %o4), memcpy_retl_o2_plus_8) + add %o0, 16, %o0 ! increase dst ptr by 16 + sllx %o4, 32, %o5 + EX_LD(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_8) + or %o4, %o5, %o5 + EX_ST(STORE(stx, %o5, %o0-8), memcpy_retl_o2_plus_8) +.Lmedw15: + bz,pt %xcc, .Lsmallx ! exit if finished + cmp %o2, 8 + blt,pn %xcc, .Lmedw7 ! skip if 7 or fewer bytes left + tst %o2 + EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2) ! load 4 bytes + subcc %o2, 8, %o2 ! decrease count by 8 + EX_ST(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_8)! and store 4 bytes + add %o1, 8, %o1 ! increase src ptr by 8 + EX_LD(LOAD(ld, %o1-4, %o3), memcpy_retl_o2_plus_4) ! load 4 bytes + add %o0, 8, %o0 ! increase dst ptr by 8 + EX_ST(STORE(stw, %o3, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes + bz,pt %xcc, .Lsmallx ! exit if finished +.Lmedw7: ! count is ge 1, less than 8 + cmp %o2, 4 ! check for 4 bytes left + blt,pn %xcc, .Lsmallleft3 ! skip if 3 or fewer bytes left + nop ! + EX_LD(LOAD(ld, %o1, %o4), memcpy_retl_o2) ! load 4 bytes + add %o1, 4, %o1 ! increase src ptr by 4 + add %o0, 4, %o0 ! increase dst ptr by 4 + subcc %o2, 4, %o2 ! decrease count by 4 + bnz .Lsmallleft3 + EX_ST(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_4)! and store 4 bytes + retl + mov EX_RETVAL(%g1), %o0 + + .align 16 +.Llarge_align8_copy: ! Src and dst share 8 byte alignment + ! align dst to 64 byte boundary + andcc %o0, 0x3f, %o3 ! %o3 == 0 means dst is 64 byte aligned + brz,pn %o3, .Laligned_to_64 + andcc %o0, 8, %o3 ! odd long words to move? + brz,pt %o3, .Laligned_to_16 + nop + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) + sub %o2, 8, %o2 + add %o1, 8, %o1 ! increment src ptr + add %o0, 8, %o0 ! increment dst ptr + EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) +.Laligned_to_16: + andcc %o0, 16, %o3 ! pair of long words to move? + brz,pt %o3, .Laligned_to_32 + nop + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) + sub %o2, 16, %o2 + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_16) + add %o1, 16, %o1 ! increment src ptr + EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8) + add %o0, 16, %o0 ! increment dst ptr + EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) +.Laligned_to_32: + andcc %o0, 32, %o3 ! four long words to move? + brz,pt %o3, .Laligned_to_64 + nop + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2) + sub %o2, 32, %o2 + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_32) + EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_24) + EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_24) + EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_16) + EX_ST(STORE(stx, %o4, %o0+16), memcpy_retl_o2_plus_16) + add %o1, 32, %o1 ! increment src ptr + EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_8) + add %o0, 32, %o0 ! increment dst ptr + EX_ST(STORE(stx, %o4, %o0-8), memcpy_retl_o2_plus_8) +.Laligned_to_64: +! +! Using block init store (BIS) instructions to avoid fetching cache +! lines from memory. Use ST_CHUNK stores to first element of each cache +! line (similar to prefetching) to avoid overfilling STQ or miss buffers. +! Gives existing cache lines time to be moved out of L1/L2/L3 cache. +! Initial stores using MRU version of BIS to keep cache line in +! cache until we are ready to store final element of cache line. +! Then store last element using the LRU version of BIS. +! + andn %o2, 0x3f, %o5 ! %o5 is multiple of block size + and %o2, 0x3f, %o2 ! residue bytes in %o2 +! +! We use STORE_MRU_ASI for the first seven stores to each cache line +! followed by STORE_ASI (mark as LRU) for the last store. That +! mixed approach reduces the probability that the cache line is removed +! before we finish setting it, while minimizing the effects on +! other cached values during a large memcpy +! +! ST_CHUNK batches up initial BIS operations for several cache lines +! to allow multiple requests to not be blocked by overflowing the +! the store miss buffer. Then the matching stores for all those +! BIS operations are executed. +! + + sub %o0, 8, %o0 ! adjust %o0 for ASI alignment +.Lalign_loop: + cmp %o5, ST_CHUNK*64 + blu,pt %xcc, .Lalign_loop_fin + mov ST_CHUNK,%o3 +.Lalign_loop_start: + prefetch [%o1 + (ALIGN_PRE * BLOCK_SIZE)], 21 + subcc %o3, 1, %o3 + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5) + add %o1, 64, %o1 + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + bgu %xcc,.Lalign_loop_start + add %o0, 56, %o0 + + mov ST_CHUNK,%o3 + sllx %o3, 6, %o4 ! ST_CHUNK*64 + sub %o1, %o4, %o1 ! reset %o1 + sub %o0, %o4, %o0 ! reset %o0 + +.Lalign_loop_rest: + EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5) + add %o0, 16, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + subcc %o3, 1, %o3 + EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5) + add %o1, 64, %o1 + add %o0, 8, %o0 + EX_ST(STORE_INIT_MRU(%o4, %o0), memcpy_retl_o2_plus_o5) + add %o0, 8, %o0 + EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5) + sub %o5, 64, %o5 + bgu %xcc,.Lalign_loop_rest + ! mark cache line as LRU + EX_ST(STORE_INIT(%o4, %o0), memcpy_retl_o2_plus_o5_plus_64) + + cmp %o5, ST_CHUNK*64 + bgu,pt %xcc, .Lalign_loop_start + mov ST_CHUNK,%o3 + + cmp %o5, 0 + beq .Lalign_done + nop +.Lalign_loop_fin: + EX_LD(LOAD(ldx, %o1, %o4), memcpy_retl_o2_plus_o5) + EX_ST(STORE(stx, %o4, %o0+8), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+8, %o4), memcpy_retl_o2_plus_o5) + EX_ST(STORE(stx, %o4, %o0+8+8), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1+16, %o4), memcpy_retl_o2_plus_o5) + EX_ST(STORE(stx, %o4, %o0+8+16), memcpy_retl_o2_plus_o5) + subcc %o5, 64, %o5 + EX_LD(LOAD(ldx, %o1+24, %o4), memcpy_retl_o2_plus_o5_64) + EX_ST(STORE(stx, %o4, %o0+8+24), memcpy_retl_o2_plus_o5_64) + EX_LD(LOAD(ldx, %o1+32, %o4), memcpy_retl_o2_plus_o5_64) + EX_ST(STORE(stx, %o4, %o0+8+32), memcpy_retl_o2_plus_o5_64) + EX_LD(LOAD(ldx, %o1+40, %o4), memcpy_retl_o2_plus_o5_64) + EX_ST(STORE(stx, %o4, %o0+8+40), memcpy_retl_o2_plus_o5_64) + EX_LD(LOAD(ldx, %o1+48, %o4), memcpy_retl_o2_plus_o5_64) + add %o1, 64, %o1 + EX_ST(STORE(stx, %o4, %o0+8+48), memcpy_retl_o2_plus_o5_64) + add %o0, 64, %o0 + EX_LD(LOAD(ldx, %o1-8, %o4), memcpy_retl_o2_plus_o5_64) + bgu %xcc,.Lalign_loop_fin + EX_ST(STORE(stx, %o4, %o0), memcpy_retl_o2_plus_o5_64) + +.Lalign_done: + add %o0, 8, %o0 ! restore %o0 from ASI alignment + membar #StoreStore + sub %o2, 63, %o2 ! adjust length to allow cc test + ba .Lmedl63 ! in .Lmedl63 + nop + + .align 16 + ! Dst is on 8 byte boundary; src is not; remaining count > SMALL_MAX +.Lunalignsetup: +.Lunalignrejoin: + mov %g1, %o3 ! save %g1 as VISEntryHalf clobbers it +#ifdef NON_USER_COPY + VISEntryHalfFast(.Lmedium_vis_entry_fail_cp) +#else + VISEntryHalf +#endif + mov %o3, %g1 ! restore %g1 + + set MED_UMAX, %o3 + cmp %o2, %o3 ! check for.Lmedium unaligned limit + bge,pt %xcc,.Lunalign_large + prefetch [%o1 + (4 * BLOCK_SIZE)], 20 + andn %o2, 0x3f, %o5 ! %o5 is multiple of block size + and %o2, 0x3f, %o2 ! residue bytes in %o2 + cmp %o2, 8 ! Insure we do not load beyond + bgt .Lunalign_adjust ! end of source buffer + andn %o1, 0x7, %o4 ! %o4 has long word aligned src address + add %o2, 64, %o2 ! adjust to leave loop + sub %o5, 64, %o5 ! early if necessary +.Lunalign_adjust: + alignaddr %o1, %g0, %g0 ! generate %gsr + add %o1, %o5, %o1 ! advance %o1 to after blocks + EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5) +.Lunalign_loop: + EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5) + faligndata %f0, %f2, %f16 + EX_LD_FP(LOAD(ldd, %o4+16, %f4), memcpy_retl_o2_plus_o5) + subcc %o5, BLOCK_SIZE, %o5 + EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5_plus_64) + faligndata %f2, %f4, %f18 + EX_LD_FP(LOAD(ldd, %o4+24, %f6), memcpy_retl_o2_plus_o5_plus_56) + EX_ST_FP(STORE(std, %f18, %o0+8), memcpy_retl_o2_plus_o5_plus_56) + faligndata %f4, %f6, %f20 + EX_LD_FP(LOAD(ldd, %o4+32, %f8), memcpy_retl_o2_plus_o5_plus_48) + EX_ST_FP(STORE(std, %f20, %o0+16), memcpy_retl_o2_plus_o5_plus_48) + faligndata %f6, %f8, %f22 + EX_LD_FP(LOAD(ldd, %o4+40, %f10), memcpy_retl_o2_plus_o5_plus_40) + EX_ST_FP(STORE(std, %f22, %o0+24), memcpy_retl_o2_plus_o5_plus_40) + faligndata %f8, %f10, %f24 + EX_LD_FP(LOAD(ldd, %o4+48, %f12), memcpy_retl_o2_plus_o5_plus_32) + EX_ST_FP(STORE(std, %f24, %o0+32), memcpy_retl_o2_plus_o5_plus_32) + faligndata %f10, %f12, %f26 + EX_LD_FP(LOAD(ldd, %o4+56, %f14), memcpy_retl_o2_plus_o5_plus_24) + add %o4, BLOCK_SIZE, %o4 + EX_ST_FP(STORE(std, %f26, %o0+40), memcpy_retl_o2_plus_o5_plus_24) + faligndata %f12, %f14, %f28 + EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5_plus_16) + EX_ST_FP(STORE(std, %f28, %o0+48), memcpy_retl_o2_plus_o5_plus_16) + faligndata %f14, %f0, %f30 + EX_ST_FP(STORE(std, %f30, %o0+56), memcpy_retl_o2_plus_o5_plus_8) + add %o0, BLOCK_SIZE, %o0 + bgu,pt %xcc, .Lunalign_loop + prefetch [%o4 + (5 * BLOCK_SIZE)], 20 + ba .Lunalign_done + nop + +.Lunalign_large: + andcc %o0, 0x3f, %o3 ! is dst 64-byte block aligned? + bz %xcc, .Lunalignsrc + sub %o3, 64, %o3 ! %o3 will be multiple of 8 + neg %o3 ! bytes until dest is 64 byte aligned + sub %o2, %o3, %o2 ! update cnt with bytes to be moved + ! Move bytes according to source alignment + andcc %o1, 0x1, %o5 + bnz %xcc, .Lunalignbyte ! check for byte alignment + nop + andcc %o1, 2, %o5 ! check for half word alignment + bnz %xcc, .Lunalignhalf + nop + ! Src is word aligned +.Lunalignword: + EX_LD_FP(LOAD(ld, %o1, %o4), memcpy_retl_o2_plus_o3) ! load 4 bytes + add %o1, 8, %o1 ! increase src ptr by 8 + EX_ST_FP(STORE(stw, %o4, %o0), memcpy_retl_o2_plus_o3) ! and store 4 + subcc %o3, 8, %o3 ! decrease count by 8 + EX_LD_FP(LOAD(ld, %o1-4, %o4), memcpy_retl_o2_plus_o3_plus_4)! load 4 + add %o0, 8, %o0 ! increase dst ptr by 8 + bnz %xcc, .Lunalignword + EX_ST_FP(STORE(stw, %o4, %o0-4), memcpy_retl_o2_plus_o3_plus_4) + ba .Lunalignsrc + nop + + ! Src is half-word aligned +.Lunalignhalf: + EX_LD_FP(LOAD(lduh, %o1, %o4), memcpy_retl_o2_plus_o3) ! load 2 bytes + sllx %o4, 32, %o5 ! shift left + EX_LD_FP(LOAD(lduw, %o1+2, %o4), memcpy_retl_o2_plus_o3) + or %o4, %o5, %o5 + sllx %o5, 16, %o5 + EX_LD_FP(LOAD(lduh, %o1+6, %o4), memcpy_retl_o2_plus_o3) + or %o4, %o5, %o5 + EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3) + add %o1, 8, %o1 + subcc %o3, 8, %o3 + bnz %xcc, .Lunalignhalf + add %o0, 8, %o0 + ba .Lunalignsrc + nop + + ! Src is Byte aligned +.Lunalignbyte: + sub %o0, %o1, %o0 ! share pointer advance +.Lunalignbyte_loop: + EX_LD_FP(LOAD(ldub, %o1, %o4), memcpy_retl_o2_plus_o3) + sllx %o4, 56, %o5 + EX_LD_FP(LOAD(lduh, %o1+1, %o4), memcpy_retl_o2_plus_o3) + sllx %o4, 40, %o4 + or %o4, %o5, %o5 + EX_LD_FP(LOAD(lduh, %o1+3, %o4), memcpy_retl_o2_plus_o3) + sllx %o4, 24, %o4 + or %o4, %o5, %o5 + EX_LD_FP(LOAD(lduh, %o1+5, %o4), memcpy_retl_o2_plus_o3) + sllx %o4, 8, %o4 + or %o4, %o5, %o5 + EX_LD_FP(LOAD(ldub, %o1+7, %o4), memcpy_retl_o2_plus_o3) + or %o4, %o5, %o5 + add %o0, %o1, %o0 + EX_ST_FP(STORE(stx, %o5, %o0), memcpy_retl_o2_plus_o3) + sub %o0, %o1, %o0 + subcc %o3, 8, %o3 + bnz %xcc, .Lunalignbyte_loop + add %o1, 8, %o1 + add %o0,%o1, %o0 ! restore pointer + + ! Destination is now block (64 byte aligned) +.Lunalignsrc: + andn %o2, 0x3f, %o5 ! %o5 is multiple of block size + and %o2, 0x3f, %o2 ! residue bytes in %o2 + add %o2, 64, %o2 ! Insure we do not load beyond + sub %o5, 64, %o5 ! end of source buffer + + andn %o1, 0x7, %o4 ! %o4 has long word aligned src address + alignaddr %o1, %g0, %g0 ! generate %gsr + add %o1, %o5, %o1 ! advance %o1 to after blocks + + EX_LD_FP(LOAD(ldd, %o4, %f14), memcpy_retl_o2_plus_o5) + add %o4, 8, %o4 +.Lunalign_sloop: + EX_LD_FP(LOAD(ldd, %o4, %f16), memcpy_retl_o2_plus_o5) + faligndata %f14, %f16, %f0 + EX_LD_FP(LOAD(ldd, %o4+8, %f18), memcpy_retl_o2_plus_o5) + faligndata %f16, %f18, %f2 + EX_LD_FP(LOAD(ldd, %o4+16, %f20), memcpy_retl_o2_plus_o5) + faligndata %f18, %f20, %f4 + EX_ST_FP(STORE(std, %f0, %o0), memcpy_retl_o2_plus_o5) + subcc %o5, 64, %o5 + EX_LD_FP(LOAD(ldd, %o4+24, %f22), memcpy_retl_o2_plus_o5_plus_56) + faligndata %f20, %f22, %f6 + EX_ST_FP(STORE(std, %f2, %o0+8), memcpy_retl_o2_plus_o5_plus_56) + EX_LD_FP(LOAD(ldd, %o4+32, %f24), memcpy_retl_o2_plus_o5_plus_48) + faligndata %f22, %f24, %f8 + EX_ST_FP(STORE(std, %f4, %o0+16), memcpy_retl_o2_plus_o5_plus_48) + EX_LD_FP(LOAD(ldd, %o4+40, %f26), memcpy_retl_o2_plus_o5_plus_40) + faligndata %f24, %f26, %f10 + EX_ST_FP(STORE(std, %f6, %o0+24), memcpy_retl_o2_plus_o5_plus_40) + EX_LD_FP(LOAD(ldd, %o4+48, %f28), memcpy_retl_o2_plus_o5_plus_32) + faligndata %f26, %f28, %f12 + EX_ST_FP(STORE(std, %f8, %o0+32), memcpy_retl_o2_plus_o5_plus_32) + add %o4, 64, %o4 + EX_LD_FP(LOAD(ldd, %o4-8, %f30), memcpy_retl_o2_plus_o5_plus_24) + faligndata %f28, %f30, %f14 + EX_ST_FP(STORE(std, %f10, %o0+40), memcpy_retl_o2_plus_o5_plus_24) + EX_ST_FP(STORE(std, %f12, %o0+48), memcpy_retl_o2_plus_o5_plus_16) + add %o0, 64, %o0 + EX_ST_FP(STORE(std, %f14, %o0-8), memcpy_retl_o2_plus_o5_plus_8) + fsrc2 %f30, %f14 + bgu,pt %xcc, .Lunalign_sloop + prefetch [%o4 + (8 * BLOCK_SIZE)], 20 + +.Lunalign_done: + ! Handle trailing bytes, 64 to 127 + ! Dest long word aligned, Src not long word aligned + cmp %o2, 15 + bleu %xcc, .Lunalign_short + + andn %o2, 0x7, %o5 ! %o5 is multiple of 8 + and %o2, 0x7, %o2 ! residue bytes in %o2 + add %o2, 8, %o2 + sub %o5, 8, %o5 ! insure we do not load past end of src + andn %o1, 0x7, %o4 ! %o4 has long word aligned src address + add %o1, %o5, %o1 ! advance %o1 to after multiple of 8 + EX_LD_FP(LOAD(ldd, %o4, %f0), memcpy_retl_o2_plus_o5)! fetch partialword +.Lunalign_by8: + EX_LD_FP(LOAD(ldd, %o4+8, %f2), memcpy_retl_o2_plus_o5) + add %o4, 8, %o4 + faligndata %f0, %f2, %f16 + subcc %o5, 8, %o5 + EX_ST_FP(STORE(std, %f16, %o0), memcpy_retl_o2_plus_o5_plus_8) + fsrc2 %f2, %f0 + bgu,pt %xcc, .Lunalign_by8 + add %o0, 8, %o0 + +.Lunalign_short: +#ifdef NON_USER_COPY + VISExitHalfFast +#else + VISExitHalf +#endif + ba .Lsmallrest + nop + +/* + * This is a special case of nested memcpy. This can happen when kernel + * calls unaligned memcpy back to back without saving FP registers. We need + * traps(context switch) to save/restore FP registers. If the kernel calls + * memcpy without this trap sequence we will hit FP corruption. Let's use + * the normal integer load/store method in this case. + */ + +#ifdef NON_USER_COPY +.Lmedium_vis_entry_fail_cp: + or %o0, %o1, %g2 +#endif +.Lmedium_cp: + LOAD(prefetch, %o1 + 0x40, #n_reads_strong) + andcc %g2, 0x7, %g0 + bne,pn %xcc, .Lmedium_unaligned_cp + nop + +.Lmedium_noprefetch_cp: + andncc %o2, 0x20 - 1, %o5 + be,pn %xcc, 2f + sub %o2, %o5, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x10, %g7), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), memcpy_retl_o2_plus_o5) + add %o1, 0x20, %o1 + subcc %o5, 0x20, %o5 + EX_ST(STORE(stx, %o3, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32) + EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, %g7, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_16) + EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8) + bne,pt %xcc, 1b + add %o0, 0x20, %o0 +2: andcc %o2, 0x18, %o5 + be,pt %xcc, 3f + sub %o2, %o5, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5) + add %o1, 0x08, %o1 + add %o0, 0x08, %o0 + subcc %o5, 0x08, %o5 + bne,pt %xcc, 1b + EX_ST(STORE(stx, %o3, %o0 - 0x08), memcpy_retl_o2_plus_o5_plus_8) +3: brz,pt %o2, .Lexit_cp + cmp %o2, 0x04 + bl,pn %xcc, .Ltiny_cp + nop + EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2) + add %o1, 0x04, %o1 + add %o0, 0x04, %o0 + subcc %o2, 0x04, %o2 + bne,pn %xcc, .Ltiny_cp + EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_4) + ba,a,pt %xcc, .Lexit_cp + +.Lmedium_unaligned_cp: + /* First get dest 8 byte aligned. */ + sub %g0, %o0, %o3 + and %o3, 0x7, %o3 + brz,pt %o3, 2f + sub %o2, %o3, %o2 + +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_o3) + add %o1, 1, %o1 + subcc %o3, 1, %o3 + add %o0, 1, %o0 + bne,pt %xcc, 1b + EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_o3_plus_1) +2: + and %o1, 0x7, %o3 + brz,pn %o3, .Lmedium_noprefetch_cp + sll %o3, 3, %o3 + mov 64, %g2 + sub %g2, %o3, %g2 + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), memcpy_retl_o2) + sllx %o4, %o3, %o4 + andn %o2, 0x08 - 1, %o5 + sub %o2, %o5, %o2 + +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), memcpy_retl_o2_plus_o5) + add %o1, 0x08, %o1 + subcc %o5, 0x08, %o5 + srlx %g3, %g2, %g7 + or %g7, %o4, %g7 + EX_ST(STORE(stx, %g7, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_8) + add %o0, 0x08, %o0 + bne,pt %xcc, 1b + sllx %g3, %o3, %o4 + srl %o3, 3, %o3 + add %o1, %o3, %o1 + brz,pn %o2, .Lexit_cp + nop + ba,pt %xcc, .Lsmall_unaligned_cp + +.Ltiny_cp: + EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2) + subcc %o2, 1, %o2 + be,pn %xcc, .Lexit_cp + EX_ST(STORE(stb, %o3, %o0 + 0x00), memcpy_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x01, %o3), memcpy_retl_o2) + subcc %o2, 1, %o2 + be,pn %xcc, .Lexit_cp + EX_ST(STORE(stb, %o3, %o0 + 0x01), memcpy_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x02, %o3), memcpy_retl_o2) + ba,pt %xcc, .Lexit_cp + EX_ST(STORE(stb, %o3, %o0 + 0x02), memcpy_retl_o2) + +.Lsmall_cp: + andcc %g2, 0x3, %g0 + bne,pn %xcc, .Lsmall_unaligned_cp + andn %o2, 0x4 - 1, %o5 + sub %o2, %o5, %o2 +1: + EX_LD(LOAD(lduw, %o1 + 0x00, %o3), memcpy_retl_o2_plus_o5) + add %o1, 0x04, %o1 + subcc %o5, 0x04, %o5 + add %o0, 0x04, %o0 + bne,pt %xcc, 1b + EX_ST(STORE(stw, %o3, %o0 - 0x04), memcpy_retl_o2_plus_o5_plus_4) + brz,pt %o2, .Lexit_cp + nop + ba,a,pt %xcc, .Ltiny_cp + +.Lsmall_unaligned_cp: +1: EX_LD(LOAD(ldub, %o1 + 0x00, %o3), memcpy_retl_o2) + add %o1, 1, %o1 + add %o0, 1, %o0 + subcc %o2, 1, %o2 + bne,pt %xcc, 1b + EX_ST(STORE(stb, %o3, %o0 - 0x01), memcpy_retl_o2_plus_1) + ba,a,pt %xcc, .Lexit_cp + +.Lsmallrest: + tst %o2 + bz,pt %xcc, .Lsmallx + cmp %o2, 4 + blt,pn %xcc, .Lsmallleft3 + nop + sub %o2, 3, %o2 +.Lsmallnotalign4: + EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_3)! read byte + subcc %o2, 4, %o2 ! reduce count by 4 + EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_7)! write byte & repeat + EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2_plus_6)! for total of 4 + add %o1, 4, %o1 ! advance SRC by 4 + EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_6) + EX_LD(LOAD(ldub, %o1-2, %o3), memcpy_retl_o2_plus_5) + add %o0, 4, %o0 ! advance DST by 4 + EX_ST(STORE(stb, %o3, %o0-2), memcpy_retl_o2_plus_5) + EX_LD(LOAD(ldub, %o1-1, %o3), memcpy_retl_o2_plus_4) + bgu,pt %xcc, .Lsmallnotalign4 ! loop til 3 or fewer bytes remain + EX_ST(STORE(stb, %o3, %o0-1), memcpy_retl_o2_plus_4) + addcc %o2, 3, %o2 ! restore count + bz,pt %xcc, .Lsmallx +.Lsmallleft3: ! 1, 2, or 3 bytes remain + subcc %o2, 1, %o2 + EX_LD(LOAD(ldub, %o1, %o3), memcpy_retl_o2_plus_1) ! load one byte + bz,pt %xcc, .Lsmallx + EX_ST(STORE(stb, %o3, %o0), memcpy_retl_o2_plus_1) ! store one byte + EX_LD(LOAD(ldub, %o1+1, %o3), memcpy_retl_o2) ! load second byte + subcc %o2, 1, %o2 + bz,pt %xcc, .Lsmallx + EX_ST(STORE(stb, %o3, %o0+1), memcpy_retl_o2_plus_1)! store second byte + EX_LD(LOAD(ldub, %o1+2, %o3), memcpy_retl_o2) ! load third byte + EX_ST(STORE(stb, %o3, %o0+2), memcpy_retl_o2) ! store third byte +.Lsmallx: + retl + mov EX_RETVAL(%g1), %o0 +.Lsmallfin: + tst %o2 + bnz,pn %xcc, .Lsmallleft3 + nop + retl + mov EX_RETVAL(%g1), %o0 ! restore %o0 +.Lexit_cp: + retl + mov EX_RETVAL(%g1), %o0 + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/M7memset.S b/arch/sparc/lib/M7memset.S new file mode 100644 index 000000000..62ea91b3a --- /dev/null +++ b/arch/sparc/lib/M7memset.S @@ -0,0 +1,352 @@ +/* + * M7memset.S: SPARC M7 optimized memset. + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + +/* + * M7memset.S: M7 optimized memset. + * + * char *memset(sp, c, n) + * + * Set an array of n chars starting at sp to the character c. + * Return sp. + * + * Fast assembler language version of the following C-program for memset + * which represents the `standard' for the C-library. + * + * void * + * memset(void *sp1, int c, size_t n) + * { + * if (n != 0) { + * char *sp = sp1; + * do { + * *sp++ = (char)c; + * } while (--n != 0); + * } + * return (sp1); + * } + * + * The algorithm is as follows : + * + * For small 6 or fewer bytes stores, bytes will be stored. + * + * For less than 32 bytes stores, align the address on 4 byte boundary. + * Then store as many 4-byte chunks, followed by trailing bytes. + * + * For sizes greater than 32 bytes, align the address on 8 byte boundary. + * if (count >= 64) { + * store 8-bytes chunks to align the address on 64 byte boundary + * if (value to be set is zero && count >= MIN_ZERO) { + * Using BIS stores, set the first long word of each + * 64-byte cache line to zero which will also clear the + * other seven long words of the cache line. + * } + * else if (count >= MIN_LOOP) { + * Using BIS stores, set the first long word of each of + * ST_CHUNK cache lines (64 bytes each) before the main + * loop is entered. + * In the main loop, continue pre-setting the first long + * word of each cache line ST_CHUNK lines in advance while + * setting the other seven long words (56 bytes) of each + * cache line until fewer than ST_CHUNK*64 bytes remain. + * Then set the remaining seven long words of each cache + * line that has already had its first long word set. + * } + * store remaining data in 64-byte chunks until less than + * 64 bytes remain. + * } + * Store as many 8-byte chunks, followed by trailing bytes. + * + * BIS = Block Init Store + * Doing the advance store of the first element of the cache line + * initiates the displacement of a cache line while only using a single + * instruction in the pipeline. That avoids various pipeline delays, + * such as filling the miss buffer. The performance effect is + * similar to prefetching for normal stores. + * The special case for zero fills runs faster and uses fewer instruction + * cycles than the normal memset loop. + * + * We only use BIS for memset of greater than MIN_LOOP bytes because a sequence + * BIS stores must be followed by a membar #StoreStore. The benefit of + * the BIS store must be balanced against the cost of the membar operation. + */ + +/* + * ASI_STBI_P marks the cache line as "least recently used" + * which means if many threads are active, it has a high chance + * of being pushed out of the cache between the first initializing + * store and the final stores. + * Thus, we use ASI_STBIMRU_P which marks the cache line as + * "most recently used" for all but the last store to the cache line. + */ + +#include +#include + +#define ASI_STBI_P ASI_BLK_INIT_QUAD_LDD_P +#define ASI_STBIMRU_P ASI_ST_BLKINIT_MRU_P + + +#define ST_CHUNK 24 /* multiple of 4 due to loop unrolling */ +#define MIN_LOOP 16320 +#define MIN_ZERO 512 + + .section ".text" + .align 32 + +/* + * Define clear_page(dest) as memset(dest, 0, PAGE_SIZE) + * (can create a more optimized version later.) + */ + .globl M7clear_page + .globl M7clear_user_page +M7clear_page: /* clear_page(dest) */ +M7clear_user_page: + set PAGE_SIZE, %o1 + /* fall through into bzero code */ + + .size M7clear_page,.-M7clear_page + .size M7clear_user_page,.-M7clear_user_page + +/* + * Define bzero(dest, n) as memset(dest, 0, n) + * (can create a more optimized version later.) + */ + .globl M7bzero +M7bzero: /* bzero(dest, size) */ + mov %o1, %o2 + mov 0, %o1 + /* fall through into memset code */ + + .size M7bzero,.-M7bzero + + .global M7memset + .type M7memset, #function + .register %g3, #scratch +M7memset: + mov %o0, %o5 ! copy sp1 before using it + cmp %o2, 7 ! if small counts, just write bytes + bleu,pn %xcc, .wrchar + and %o1, 0xff, %o1 ! o1 is (char)c + + sll %o1, 8, %o3 + or %o1, %o3, %o1 ! now o1 has 2 bytes of c + sll %o1, 16, %o3 + cmp %o2, 32 + blu,pn %xcc, .wdalign + or %o1, %o3, %o1 ! now o1 has 4 bytes of c + + sllx %o1, 32, %o3 + or %o1, %o3, %o1 ! now o1 has 8 bytes of c + +.dbalign: + andcc %o5, 7, %o3 ! is sp1 aligned on a 8 byte bound? + bz,pt %xcc, .blkalign ! already long word aligned + sub %o3, 8, %o3 ! -(bytes till long word aligned) + + add %o2, %o3, %o2 ! update o2 with new count + ! Set -(%o3) bytes till sp1 long word aligned +1: stb %o1, [%o5] ! there is at least 1 byte to set + inccc %o3 ! byte clearing loop + bl,pt %xcc, 1b + inc %o5 + + ! Now sp1 is long word aligned (sp1 is found in %o5) +.blkalign: + cmp %o2, 64 ! check if there are 64 bytes to set + blu,pn %xcc, .wrshort + mov %o2, %o3 + + andcc %o5, 63, %o3 ! is sp1 block aligned? + bz,pt %xcc, .blkwr ! now block aligned + sub %o3, 64, %o3 ! o3 is -(bytes till block aligned) + add %o2, %o3, %o2 ! o2 is the remainder + + ! Store -(%o3) bytes till dst is block (64 byte) aligned. + ! Use long word stores. + ! Recall that dst is already long word aligned +1: + addcc %o3, 8, %o3 + stx %o1, [%o5] + bl,pt %xcc, 1b + add %o5, 8, %o5 + + ! Now sp1 is block aligned +.blkwr: + andn %o2, 63, %o4 ! calculate size of blocks in bytes + brz,pn %o1, .wrzero ! special case if c == 0 + and %o2, 63, %o3 ! %o3 = bytes left after blk stores. + + set MIN_LOOP, %g1 + cmp %o4, %g1 ! check there are enough bytes to set + blu,pn %xcc, .short_set ! to justify cost of membar + ! must be > pre-cleared lines + nop + + ! initial cache-clearing stores + ! get store pipeline moving + rd %asi, %g3 ! save %asi to be restored later + wr %g0, ASI_STBIMRU_P, %asi + + ! Primary memset loop for large memsets +.wr_loop: + sub %o5, 8, %o5 ! adjust %o5 for ASI store alignment + mov ST_CHUNK, %g1 +.wr_loop_start: + stxa %o1, [%o5+8]%asi + subcc %g1, 4, %g1 + stxa %o1, [%o5+8+64]%asi + add %o5, 256, %o5 + stxa %o1, [%o5+8-128]%asi + bgu %xcc, .wr_loop_start + stxa %o1, [%o5+8-64]%asi + + sub %o5, ST_CHUNK*64, %o5 ! reset %o5 + mov ST_CHUNK, %g1 + +.wr_loop_rest: + stxa %o1, [%o5+8+8]%asi + sub %o4, 64, %o4 + stxa %o1, [%o5+16+8]%asi + subcc %g1, 1, %g1 + stxa %o1, [%o5+24+8]%asi + stxa %o1, [%o5+32+8]%asi + stxa %o1, [%o5+40+8]%asi + add %o5, 64, %o5 + stxa %o1, [%o5-8]%asi + bgu %xcc, .wr_loop_rest + stxa %o1, [%o5]ASI_STBI_P + + ! If more than ST_CHUNK*64 bytes remain to set, continue + ! setting the first long word of each cache line in advance + ! to keep the store pipeline moving. + + cmp %o4, ST_CHUNK*64 + bge,pt %xcc, .wr_loop_start + mov ST_CHUNK, %g1 + + brz,a,pn %o4, .asi_done + add %o5, 8, %o5 ! restore %o5 offset + +.wr_loop_small: + stxa %o1, [%o5+8]%asi + stxa %o1, [%o5+8+8]%asi + stxa %o1, [%o5+16+8]%asi + stxa %o1, [%o5+24+8]%asi + stxa %o1, [%o5+32+8]%asi + subcc %o4, 64, %o4 + stxa %o1, [%o5+40+8]%asi + add %o5, 64, %o5 + stxa %o1, [%o5-8]%asi + bgu,pt %xcc, .wr_loop_small + stxa %o1, [%o5]ASI_STBI_P + + ba .asi_done + add %o5, 8, %o5 ! restore %o5 offset + + ! Special case loop for zero fill memsets + ! For each 64 byte cache line, single STBI to first element + ! clears line +.wrzero: + cmp %o4, MIN_ZERO ! check if enough bytes to set + ! to pay %asi + membar cost + blu %xcc, .short_set + nop + sub %o4, 256, %o4 + +.wrzero_loop: + mov 64, %g3 + stxa %o1, [%o5]ASI_STBI_P + subcc %o4, 256, %o4 + stxa %o1, [%o5+%g3]ASI_STBI_P + add %o5, 256, %o5 + sub %g3, 192, %g3 + stxa %o1, [%o5+%g3]ASI_STBI_P + add %g3, 64, %g3 + bge,pt %xcc, .wrzero_loop + stxa %o1, [%o5+%g3]ASI_STBI_P + add %o4, 256, %o4 + + brz,pn %o4, .bsi_done + nop + +.wrzero_small: + stxa %o1, [%o5]ASI_STBI_P + subcc %o4, 64, %o4 + bgu,pt %xcc, .wrzero_small + add %o5, 64, %o5 + ba,a .bsi_done + +.asi_done: + wr %g3, 0x0, %asi ! restored saved %asi +.bsi_done: + membar #StoreStore ! required by use of Block Store Init + +.short_set: + cmp %o4, 64 ! check if 64 bytes to set + blu %xcc, 5f + nop +4: ! set final blocks of 64 bytes + stx %o1, [%o5] + stx %o1, [%o5+8] + stx %o1, [%o5+16] + stx %o1, [%o5+24] + subcc %o4, 64, %o4 + stx %o1, [%o5+32] + stx %o1, [%o5+40] + add %o5, 64, %o5 + stx %o1, [%o5-16] + bgu,pt %xcc, 4b + stx %o1, [%o5-8] + +5: + ! Set the remaining long words +.wrshort: + subcc %o3, 8, %o3 ! Can we store any long words? + blu,pn %xcc, .wrchars + and %o2, 7, %o2 ! calc bytes left after long words +6: + subcc %o3, 8, %o3 + stx %o1, [%o5] ! store the long words + bgeu,pt %xcc, 6b + add %o5, 8, %o5 + +.wrchars: ! check for extra chars + brnz %o2, .wrfin + nop + retl + nop + +.wdalign: + andcc %o5, 3, %o3 ! is sp1 aligned on a word boundary + bz,pn %xcc, .wrword + andn %o2, 3, %o3 ! create word sized count in %o3 + + dec %o2 ! decrement count + stb %o1, [%o5] ! clear a byte + b .wdalign + inc %o5 ! next byte + +.wrword: + subcc %o3, 4, %o3 + st %o1, [%o5] ! 4-byte writing loop + bnz,pt %xcc, .wrword + add %o5, 4, %o5 + + and %o2, 3, %o2 ! leftover count, if any + +.wrchar: + ! Set the remaining bytes, if any + brz %o2, .exit + nop +.wrfin: + deccc %o2 + stb %o1, [%o5] + bgu,pt %xcc, .wrfin + inc %o5 +.exit: + retl ! %o0 was preserved + nop + + .size M7memset,.-M7memset diff --git a/arch/sparc/lib/M7patch.S b/arch/sparc/lib/M7patch.S new file mode 100644 index 000000000..9000b7bc5 --- /dev/null +++ b/arch/sparc/lib/M7patch.S @@ -0,0 +1,51 @@ +/* + * M7patch.S: Patch generic routines with M7 variant. + * + * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + */ + +#include + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + +ENTRY(m7_patch_copyops) + NG_DO_PATCH(memcpy, M7memcpy) + NG_DO_PATCH(raw_copy_from_user, M7copy_from_user) + NG_DO_PATCH(raw_copy_to_user, M7copy_to_user) + retl + nop +ENDPROC(m7_patch_copyops) + +ENTRY(m7_patch_bzero) + NG_DO_PATCH(memset, M7memset) + NG_DO_PATCH(__bzero, M7bzero) + NG_DO_PATCH(__clear_user, NGclear_user) + NG_DO_PATCH(tsb_init, NGtsb_init) + retl + nop +ENDPROC(m7_patch_bzero) + +ENTRY(m7_patch_pageops) + NG_DO_PATCH(copy_user_page, NG4copy_user_page) + NG_DO_PATCH(_clear_page, M7clear_page) + NG_DO_PATCH(clear_user_page, M7clear_user_page) + retl + nop +ENDPROC(m7_patch_pageops) diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile new file mode 100644 index 000000000..dd10cdd6f --- /dev/null +++ b/arch/sparc/lib/Makefile @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for Sparc library files.. +# + +asflags-y := -DST_DIV0=0x02 + +lib-$(CONFIG_SPARC32) += ashrdi3.o +lib-$(CONFIG_SPARC32) += memcpy.o memset.o +lib-y += strlen.o +lib-y += checksum_$(BITS).o +lib-$(CONFIG_SPARC32) += blockops.o +lib-y += memscan_$(BITS).o memcmp.o strncmp_$(BITS).o +lib-$(CONFIG_SPARC32) += divdi3.o udivdi3.o +lib-$(CONFIG_SPARC32) += copy_user.o locks.o +lib-$(CONFIG_SPARC64) += atomic_64.o +lib-$(CONFIG_SPARC32) += lshrdi3.o ashldi3.o +lib-$(CONFIG_SPARC32) += muldi3.o bitext.o +lib-$(CONFIG_SPARC64) += multi3.o +lib-$(CONFIG_SPARC64) += fls.o +lib-$(CONFIG_SPARC64) += fls64.o +lib-$(CONFIG_SPARC64) += NG4fls.o + +lib-$(CONFIG_SPARC64) += copy_page.o clear_page.o bzero.o +lib-$(CONFIG_SPARC64) += csum_copy.o csum_copy_from_user.o csum_copy_to_user.o +lib-$(CONFIG_SPARC64) += VISsave.o +lib-$(CONFIG_SPARC64) += bitops.o + +lib-$(CONFIG_SPARC64) += U1memcpy.o U1copy_from_user.o U1copy_to_user.o + +lib-$(CONFIG_SPARC64) += U3memcpy.o U3copy_from_user.o U3copy_to_user.o +lib-$(CONFIG_SPARC64) += U3patch.o + +lib-$(CONFIG_SPARC64) += NGmemcpy.o NGcopy_from_user.o NGcopy_to_user.o +lib-$(CONFIG_SPARC64) += NGpatch.o NGpage.o NGbzero.o + +lib-$(CONFIG_SPARC64) += NG2memcpy.o NG2copy_from_user.o NG2copy_to_user.o +lib-$(CONFIG_SPARC64) += NG2patch.o + +lib-$(CONFIG_SPARC64) += NG4memcpy.o NG4copy_from_user.o NG4copy_to_user.o +lib-$(CONFIG_SPARC64) += NG4patch.o NG4copy_page.o NG4clear_page.o NG4memset.o + +lib-$(CONFIG_SPARC64) += Memcpy_utils.o + +lib-$(CONFIG_SPARC64) += M7memcpy.o M7copy_from_user.o M7copy_to_user.o +lib-$(CONFIG_SPARC64) += M7patch.o M7memset.o + +lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o +lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o + +lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o +lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o hweight.o ffs.o + +obj-$(CONFIG_SPARC64) += iomap.o +obj-$(CONFIG_SPARC32) += atomic32.o +obj-$(CONFIG_SPARC64) += PeeCeeI.o diff --git a/arch/sparc/lib/Memcpy_utils.S b/arch/sparc/lib/Memcpy_utils.S new file mode 100644 index 000000000..207343367 --- /dev/null +++ b/arch/sparc/lib/Memcpy_utils.S @@ -0,0 +1,354 @@ +#ifndef __ASM_MEMCPY_UTILS +#define __ASM_MEMCPY_UTILS + +#include +#include +#include + +ENTRY(__restore_asi_fp) + VISExitHalf + retl + wr %g0, ASI_AIUS, %asi +ENDPROC(__restore_asi_fp) + +ENTRY(__restore_asi) + retl + wr %g0, ASI_AIUS, %asi +ENDPROC(__restore_asi) + +ENTRY(memcpy_retl_o2) + ba,pt %xcc, __restore_asi + mov %o2, %o0 +ENDPROC(memcpy_retl_o2) +ENTRY(memcpy_retl_o2_plus_1) + ba,pt %xcc, __restore_asi + add %o2, 1, %o0 +ENDPROC(memcpy_retl_o2_plus_1) +ENTRY(memcpy_retl_o2_plus_3) + ba,pt %xcc, __restore_asi + add %o2, 3, %o0 +ENDPROC(memcpy_retl_o2_plus_3) +ENTRY(memcpy_retl_o2_plus_4) + ba,pt %xcc, __restore_asi + add %o2, 4, %o0 +ENDPROC(memcpy_retl_o2_plus_4) +ENTRY(memcpy_retl_o2_plus_5) + ba,pt %xcc, __restore_asi + add %o2, 5, %o0 +ENDPROC(memcpy_retl_o2_plus_5) +ENTRY(memcpy_retl_o2_plus_6) + ba,pt %xcc, __restore_asi + add %o2, 6, %o0 +ENDPROC(memcpy_retl_o2_plus_6) +ENTRY(memcpy_retl_o2_plus_7) + ba,pt %xcc, __restore_asi + add %o2, 7, %o0 +ENDPROC(memcpy_retl_o2_plus_7) +ENTRY(memcpy_retl_o2_plus_8) + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(memcpy_retl_o2_plus_8) +ENTRY(memcpy_retl_o2_plus_15) + ba,pt %xcc, __restore_asi + add %o2, 15, %o0 +ENDPROC(memcpy_retl_o2_plus_15) +ENTRY(memcpy_retl_o2_plus_15_8) + add %o2, 15, %o2 + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(memcpy_retl_o2_plus_15_8) +ENTRY(memcpy_retl_o2_plus_16) + ba,pt %xcc, __restore_asi + add %o2, 16, %o0 +ENDPROC(memcpy_retl_o2_plus_16) +ENTRY(memcpy_retl_o2_plus_24) + ba,pt %xcc, __restore_asi + add %o2, 24, %o0 +ENDPROC(memcpy_retl_o2_plus_24) +ENTRY(memcpy_retl_o2_plus_31) + ba,pt %xcc, __restore_asi + add %o2, 31, %o0 +ENDPROC(memcpy_retl_o2_plus_31) +ENTRY(memcpy_retl_o2_plus_32) + ba,pt %xcc, __restore_asi + add %o2, 32, %o0 +ENDPROC(memcpy_retl_o2_plus_32) +ENTRY(memcpy_retl_o2_plus_31_32) + add %o2, 31, %o2 + ba,pt %xcc, __restore_asi + add %o2, 32, %o0 +ENDPROC(memcpy_retl_o2_plus_31_32) +ENTRY(memcpy_retl_o2_plus_31_24) + add %o2, 31, %o2 + ba,pt %xcc, __restore_asi + add %o2, 24, %o0 +ENDPROC(memcpy_retl_o2_plus_31_24) +ENTRY(memcpy_retl_o2_plus_31_16) + add %o2, 31, %o2 + ba,pt %xcc, __restore_asi + add %o2, 16, %o0 +ENDPROC(memcpy_retl_o2_plus_31_16) +ENTRY(memcpy_retl_o2_plus_31_8) + add %o2, 31, %o2 + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(memcpy_retl_o2_plus_31_8) +ENTRY(memcpy_retl_o2_plus_63) + ba,pt %xcc, __restore_asi + add %o2, 63, %o0 +ENDPROC(memcpy_retl_o2_plus_63) +ENTRY(memcpy_retl_o2_plus_63_64) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 64, %o0 +ENDPROC(memcpy_retl_o2_plus_63_64) +ENTRY(memcpy_retl_o2_plus_63_56) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 56, %o0 +ENDPROC(memcpy_retl_o2_plus_63_56) +ENTRY(memcpy_retl_o2_plus_63_48) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 48, %o0 +ENDPROC(memcpy_retl_o2_plus_63_48) +ENTRY(memcpy_retl_o2_plus_63_40) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 40, %o0 +ENDPROC(memcpy_retl_o2_plus_63_40) +ENTRY(memcpy_retl_o2_plus_63_32) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 32, %o0 +ENDPROC(memcpy_retl_o2_plus_63_32) +ENTRY(memcpy_retl_o2_plus_63_24) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 24, %o0 +ENDPROC(memcpy_retl_o2_plus_63_24) +ENTRY(memcpy_retl_o2_plus_63_16) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 16, %o0 +ENDPROC(memcpy_retl_o2_plus_63_16) +ENTRY(memcpy_retl_o2_plus_63_8) + add %o2, 63, %o2 + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(memcpy_retl_o2_plus_63_8) +ENTRY(memcpy_retl_o2_plus_o3) + ba,pt %xcc, __restore_asi + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3) +ENTRY(memcpy_retl_o2_plus_o3_plus_1) + add %o3, 1, %o3 + ba,pt %xcc, __restore_asi + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3_plus_1) +ENTRY(memcpy_retl_o2_plus_o5) + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5) +ENTRY(memcpy_retl_o2_plus_o5_plus_1) + add %o5, 1, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_1) +ENTRY(memcpy_retl_o2_plus_o5_plus_4) + add %o5, 4, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_4) +ENTRY(memcpy_retl_o2_plus_o5_plus_8) + add %o5, 8, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_8) +ENTRY(memcpy_retl_o2_plus_o5_plus_16) + add %o5, 16, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_16) +ENTRY(memcpy_retl_o2_plus_o5_plus_24) + add %o5, 24, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_24) +ENTRY(memcpy_retl_o2_plus_o5_plus_32) + add %o5, 32, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_32) +ENTRY(memcpy_retl_o2_plus_o5_64) + add %o5, 32, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_64) +ENTRY(memcpy_retl_o2_plus_g1) + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(memcpy_retl_o2_plus_g1) +ENTRY(memcpy_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(memcpy_retl_o2_plus_g1_plus_1) +ENTRY(memcpy_retl_o2_plus_g1_plus_8) + add %g1, 8, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(memcpy_retl_o2_plus_g1_plus_8) +ENTRY(memcpy_retl_o2_plus_o4) + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4) +ENTRY(memcpy_retl_o2_plus_o4_plus_8) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_8) +ENTRY(memcpy_retl_o2_plus_o4_plus_16) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_16) +ENTRY(memcpy_retl_o2_plus_o4_plus_24) + add %o4, 24, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_24) +ENTRY(memcpy_retl_o2_plus_o4_plus_32) + add %o4, 32, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_32) +ENTRY(memcpy_retl_o2_plus_o4_plus_40) + add %o4, 40, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_40) +ENTRY(memcpy_retl_o2_plus_o4_plus_48) + add %o4, 48, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_48) +ENTRY(memcpy_retl_o2_plus_o4_plus_56) + add %o4, 56, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_56) +ENTRY(memcpy_retl_o2_plus_o4_plus_64) + add %o4, 64, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_64) +ENTRY(memcpy_retl_o2_plus_o5_plus_64) + add %o5, 64, %o5 + ba,pt %xcc, __restore_asi + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_64) +ENTRY(memcpy_retl_o2_plus_o3_fp) + ba,pt %xcc, __restore_asi_fp + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3_fp) +ENTRY(memcpy_retl_o2_plus_o3_plus_1_fp) + add %o3, 1, %o3 + ba,pt %xcc, __restore_asi_fp + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3_plus_1_fp) +ENTRY(memcpy_retl_o2_plus_o3_plus_4_fp) + add %o3, 4, %o3 + ba,pt %xcc, __restore_asi_fp + add %o2, %o3, %o0 +ENDPROC(memcpy_retl_o2_plus_o3_plus_4_fp) +ENTRY(memcpy_retl_o2_plus_o4_fp) + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_8_fp) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_8_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_16_fp) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_16_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_24_fp) + add %o4, 24, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_24_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_32_fp) + add %o4, 32, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_32_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_40_fp) + add %o4, 40, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_40_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_48_fp) + add %o4, 48, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_48_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_56_fp) + add %o4, 56, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_56_fp) +ENTRY(memcpy_retl_o2_plus_o4_plus_64_fp) + add %o4, 64, %o4 + ba,pt %xcc, __restore_asi_fp + add %o2, %o4, %o0 +ENDPROC(memcpy_retl_o2_plus_o4_plus_64_fp) +ENTRY(memcpy_retl_o2_plus_o5_fp) + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_64_fp) + add %o5, 64, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_64_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_56_fp) + add %o5, 56, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_56_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_48_fp) + add %o5, 48, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_48_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_40_fp) + add %o5, 40, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_40_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_32_fp) + add %o5, 32, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_32_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_24_fp) + add %o5, 24, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_24_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_16_fp) + add %o5, 16, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_16_fp) +ENTRY(memcpy_retl_o2_plus_o5_plus_8_fp) + add %o5, 8, %o5 + ba,pt %xcc, __restore_asi_fp + add %o2, %o5, %o0 +ENDPROC(memcpy_retl_o2_plus_o5_plus_8_fp) + +#endif diff --git a/arch/sparc/lib/NG2copy_from_user.S b/arch/sparc/lib/NG2copy_from_user.S new file mode 100644 index 000000000..e57bc514f --- /dev/null +++ b/arch/sparc/lib/NG2copy_from_user.S @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG2copy_from_user.S: Niagara-2 optimized copy from userspace. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_LD_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#ifndef ASI_BLK_AIUS_4V +#define ASI_BLK_AIUS_4V 0x17 +#endif + +#define FUNC_NAME NG2copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS_4V, dest +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "NG2memcpy.S" diff --git a/arch/sparc/lib/NG2copy_to_user.S b/arch/sparc/lib/NG2copy_to_user.S new file mode 100644 index 000000000..367c0bf01 --- /dev/null +++ b/arch/sparc/lib/NG2copy_to_user.S @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG2copy_to_user.S: Niagara-2 optimized copy to userspace. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_ST_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#ifndef ASI_BLK_AIUS_4V +#define ASI_BLK_AIUS_4V 0x17 +#endif + +#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS +#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23 +#endif + +#define FUNC_NAME NG2copy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS_4V +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "NG2memcpy.S" diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S new file mode 100644 index 000000000..bcb21b3a8 --- /dev/null +++ b/arch/sparc/lib/NG2memcpy.S @@ -0,0 +1,594 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG2memcpy.S: Niagara-2 optimized memcpy. + * + * Copyright (C) 2007 David S. Miller (davem@davemloft.net) + */ + +#ifdef __KERNEL__ +#include +#include +#include +#define GLOBAL_SPARE %g7 +#else +#define ASI_PNF 0x82 +#define ASI_BLK_P 0xf0 +#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 +#define FPRS_FEF 0x04 +#ifdef MEMCPY_DEBUG +#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ + clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0; +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#else +#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#endif +#define GLOBAL_SPARE %g5 +#endif + +#ifndef STORE_ASI +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P +#else +#define STORE_ASI 0x80 /* ASI_P */ +#endif +#endif + +#ifndef EX_LD +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x +#endif + +#ifndef EX_ST +#define EX_ST(x,y) x +#endif +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef LOAD_BLK +#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest +#endif + +#ifndef STORE +#ifndef MEMCPY_DEBUG +#define STORE(type,src,addr) type src, [addr] +#else +#define STORE(type,src,addr) type##a src, [addr] 0x80 +#endif +#endif + +#ifndef STORE_BLK +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P +#endif + +#ifndef STORE_INIT +#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME NG2memcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + +#define FREG_FROB(x0, x1, x2, x3, x4, x5, x6, x7, x8) \ + faligndata %x0, %x1, %f0; \ + faligndata %x1, %x2, %f2; \ + faligndata %x2, %x3, %f4; \ + faligndata %x3, %x4, %f6; \ + faligndata %x4, %x5, %f8; \ + faligndata %x5, %x6, %f10; \ + faligndata %x6, %x7, %f12; \ + faligndata %x7, %x8, %f14; + +#define FREG_MOVE_1(x0) \ + fsrc2 %x0, %f0; +#define FREG_MOVE_2(x0, x1) \ + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; +#define FREG_MOVE_3(x0, x1, x2) \ + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; +#define FREG_MOVE_4(x0, x1, x2, x3) \ + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; +#define FREG_MOVE_5(x0, x1, x2, x3, x4) \ + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; +#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \ + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; \ + fsrc2 %x5, %f10; +#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \ + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; \ + fsrc2 %x5, %f10; \ + fsrc2 %x6, %f12; +#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \ + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; \ + fsrc2 %x5, %f10; \ + fsrc2 %x6, %f12; \ + fsrc2 %x7, %f14; +#define FREG_LOAD_1(base, x0) \ + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1) +#define FREG_LOAD_2(base, x0, x1) \ + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); +#define FREG_LOAD_3(base, x0, x1, x2) \ + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); +#define FREG_LOAD_4(base, x0, x1, x2, x3) \ + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); +#define FREG_LOAD_5(base, x0, x1, x2, x3, x4) \ + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); +#define FREG_LOAD_6(base, x0, x1, x2, x3, x4, x5) \ + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); +#define FREG_LOAD_7(base, x0, x1, x2, x3, x4, x5, x6) \ + EX_LD_FP(LOAD(ldd, base + 0x00, %x0), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x08, %x1), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x10, %x2), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x18, %x3), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x20, %x4), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x28, %x5), NG2_retl_o2_plus_g1); \ + EX_LD_FP(LOAD(ldd, base + 0x30, %x6), NG2_retl_o2_plus_g1); + + .register %g2,#scratch + .register %g3,#scratch + + .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_fp: + VISExitHalf +__restore_asi: + retl + wr %g0, ASI_AIUS, %asi +ENTRY(NG2_retl_o2) + ba,pt %xcc, __restore_asi + mov %o2, %o0 +ENDPROC(NG2_retl_o2) +ENTRY(NG2_retl_o2_plus_1) + ba,pt %xcc, __restore_asi + add %o2, 1, %o0 +ENDPROC(NG2_retl_o2_plus_1) +ENTRY(NG2_retl_o2_plus_4) + ba,pt %xcc, __restore_asi + add %o2, 4, %o0 +ENDPROC(NG2_retl_o2_plus_4) +ENTRY(NG2_retl_o2_plus_8) + ba,pt %xcc, __restore_asi + add %o2, 8, %o0 +ENDPROC(NG2_retl_o2_plus_8) +ENTRY(NG2_retl_o2_plus_o4_plus_1) + add %o4, 1, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_1) +ENTRY(NG2_retl_o2_plus_o4_plus_8) + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_8) +ENTRY(NG2_retl_o2_plus_o4_plus_16) + add %o4, 16, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_plus_o4_plus_16) +ENTRY(NG2_retl_o2_plus_g1_fp) + ba,pt %xcc, __restore_fp + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_fp) +ENTRY(NG2_retl_o2_plus_g1_plus_64_fp) + add %g1, 64, %g1 + ba,pt %xcc, __restore_fp + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_plus_64_fp) +ENTRY(NG2_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %o2, %g1, %o0 +ENDPROC(NG2_retl_o2_plus_g1_plus_1) +ENTRY(NG2_retl_o2_and_7_plus_o4) + and %o2, 7, %o2 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_and_7_plus_o4) +ENTRY(NG2_retl_o2_and_7_plus_o4_plus_8) + and %o2, 7, %o2 + add %o4, 8, %o4 + ba,pt %xcc, __restore_asi + add %o2, %o4, %o0 +ENDPROC(NG2_retl_o2_and_7_plus_o4_plus_8) +#endif + + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %xcc, 5 + PREAMBLE + mov %o0, %o3 + cmp %o2, 0 + be,pn %XCC, 85f + or %o0, %o1, GLOBAL_SPARE + cmp %o2, 16 + blu,a,pn %XCC, 80f + or GLOBAL_SPARE, %o2, GLOBAL_SPARE + + /* 2 blocks (128 bytes) is the minimum we can do the block + * copy with. We need to ensure that we'll iterate at least + * once in the block copy loop. At worst we'll need to align + * the destination to a 64-byte boundary which can chew up + * to (64 - 1) bytes from the length before we perform the + * block copy loop. + * + * However, the cut-off point, performance wise, is around + * 4 64-byte blocks. + */ + cmp %o2, (4 * 64) + blu,pt %XCC, 75f + andcc GLOBAL_SPARE, 0x7, %g0 + + /* %o0: dst + * %o1: src + * %o2: len (known to be >= 128) + * + * The block copy loops can use %o4, %g2, %g3 as + * temporaries while copying the data. %o5 must + * be preserved between VISEntryHalf and VISExitHalf + */ + + LOAD(prefetch, %o1 + 0x000, #one_read) + LOAD(prefetch, %o1 + 0x040, #one_read) + LOAD(prefetch, %o1 + 0x080, #one_read) + + /* Align destination on 64-byte boundary. */ + andcc %o0, (64 - 1), %o4 + be,pt %XCC, 2f + sub %o4, 64, %o4 + sub %g0, %o4, %o4 ! bytes to align dst + sub %o2, %o4, %o2 +1: subcc %o4, 1, %o4 + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_o4_plus_1) + EX_ST(STORE(stb, %g1, %o0), NG2_retl_o2_plus_o4_plus_1) + add %o1, 1, %o1 + bne,pt %XCC, 1b + add %o0, 1, %o0 + +2: + /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve + * o5 from here until we hit VISExitHalf. + */ + VISEntryHalf + + membar #Sync + alignaddr %o1, %g0, %g0 + + add %o1, (64 - 1), %o4 + andn %o4, (64 - 1), %o4 + andn %o2, (64 - 1), %g1 + sub %o2, %g1, %o2 + + and %o1, (64 - 1), %g2 + add %o1, %g1, %o1 + sub %o0, %o4, %g3 + brz,pt %g2, 190f + cmp %g2, 32 + blu,a 5f + cmp %g2, 16 + cmp %g2, 48 + blu,a 4f + cmp %g2, 40 + cmp %g2, 56 + blu 170f + nop + ba,a,pt %xcc, 180f + nop + +4: /* 32 <= low bits < 48 */ + blu 150f + nop + ba,a,pt %xcc, 160f + nop +5: /* 0 < low bits < 32 */ + blu,a 6f + cmp %g2, 8 + cmp %g2, 24 + blu 130f + nop + ba,a,pt %xcc, 140f + nop +6: /* 0 < low bits < 16 */ + bgeu 120f + nop + /* fall through for 0 < low bits < 8 */ +110: sub %o4, 64, %g2 + EX_LD_FP(LOAD_BLK(%g2, %f0), NG2_retl_o2_plus_g1) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) + FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f14, f16) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) + FREG_MOVE_8(f16, f18, f20, f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +120: sub %o4, 56, %g2 + FREG_LOAD_7(%g2, f0, f2, f4, f6, f8, f10, f12) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) + FREG_FROB(f0, f2, f4, f6, f8, f10, f12, f16, f18) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) + FREG_MOVE_7(f18, f20, f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +130: sub %o4, 48, %g2 + FREG_LOAD_6(%g2, f0, f2, f4, f6, f8, f10) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) + FREG_FROB(f0, f2, f4, f6, f8, f10, f16, f18, f20) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) + FREG_MOVE_6(f20, f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +140: sub %o4, 40, %g2 + FREG_LOAD_5(%g2, f0, f2, f4, f6, f8) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) + FREG_FROB(f0, f2, f4, f6, f8, f16, f18, f20, f22) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) + FREG_MOVE_5(f22, f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +150: sub %o4, 32, %g2 + FREG_LOAD_4(%g2, f0, f2, f4, f6) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) + FREG_FROB(f0, f2, f4, f6, f16, f18, f20, f22, f24) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) + FREG_MOVE_4(f24, f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +160: sub %o4, 24, %g2 + FREG_LOAD_3(%g2, f0, f2, f4) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) + FREG_FROB(f0, f2, f4, f16, f18, f20, f22, f24, f26) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) + FREG_MOVE_3(f26, f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +170: sub %o4, 16, %g2 + FREG_LOAD_2(%g2, f0, f2) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) + FREG_FROB(f0, f2, f16, f18, f20, f22, f24, f26, f28) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) + FREG_MOVE_2(f28, f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +180: sub %o4, 8, %g2 + FREG_LOAD_1(%g2, f0) +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + EX_LD_FP(LOAD_BLK(%o4, %f16), NG2_retl_o2_plus_g1) + FREG_FROB(f0, f16, f18, f20, f22, f24, f26, f28, f30) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1) + FREG_MOVE_1(f30) + subcc %g1, 64, %g1 + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + ba,pt %xcc, 195f + nop + +190: +1: EX_ST_FP(STORE_INIT(%g0, %o4 + %g3), NG2_retl_o2_plus_g1) + subcc %g1, 64, %g1 + EX_LD_FP(LOAD_BLK(%o4, %f0), NG2_retl_o2_plus_g1_plus_64) + EX_ST_FP(STORE_BLK(%f0, %o4 + %g3), NG2_retl_o2_plus_g1_plus_64) + add %o4, 64, %o4 + bne,pt %xcc, 1b + LOAD(prefetch, %o4 + 64, #one_read) + +195: + add %o4, %g3, %o0 + membar #Sync + + VISExitHalf + + /* %o2 contains any final bytes still needed to be copied + * over. If anything is left, we copy it one byte at a time. + */ + brz,pt %o2, 85f + sub %o0, %o1, GLOBAL_SPARE + ba,a,pt %XCC, 90f + nop + + .align 64 +75: /* 16 < len <= 64 */ + bne,pn %XCC, 75f + sub %o0, %o1, GLOBAL_SPARE + +72: + andn %o2, 0xf, %o4 + and %o2, 0xf, %o2 +1: subcc %o4, 0x10, %o4 + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_o4_plus_16) + add %o1, 0x08, %o1 + EX_LD(LOAD(ldx, %o1, %g1), NG2_retl_o2_plus_o4_plus_16) + sub %o1, 0x08, %o1 + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_16) + add %o1, 0x8, %o1 + EX_ST(STORE(stx, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_o4_plus_8) + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 +73: andcc %o2, 0x8, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x8, %o2 + EX_LD(LOAD(ldx, %o1, %o5), NG2_retl_o2_plus_8) + EX_ST(STORE(stx, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_8) + add %o1, 0x8, %o1 +1: andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x4, %o2 + EX_LD(LOAD(lduw, %o1, %o5), NG2_retl_o2_plus_4) + EX_ST(STORE(stw, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +75: + andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %o2, %g1, %o2 + +1: subcc %g1, 1, %g1 + EX_LD(LOAD(ldub, %o1, %o5), NG2_retl_o2_plus_g1_plus_1) + EX_ST(STORE(stb, %o5, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_g1_plus_1) + bgu,pt %icc, 1b + add %o1, 1, %o1 + +2: add %o1, GLOBAL_SPARE, %o0 + andcc %o1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %o2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %xcc, 73b + +8: mov 64, GLOBAL_SPARE + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1, %g2), NG2_retl_o2) + sub GLOBAL_SPARE, %g1, GLOBAL_SPARE + andn %o2, 0x7, %o4 + sllx %g2, %g1, %g2 +1: add %o1, 0x8, %o1 + EX_LD(LOAD(ldx, %o1, %g3), NG2_retl_o2_and_7_plus_o4) + subcc %o4, 0x8, %o4 + srlx %g3, GLOBAL_SPARE, %o5 + or %o5, %g2, %o5 + EX_ST(STORE(stx, %o5, %o0), NG2_retl_o2_and_7_plus_o4_plus_8) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %o2, 0x7, %o2 + be,pn %icc, 85f + add %o1, %g1, %o1 + ba,pt %xcc, 90f + sub %o0, %o1, GLOBAL_SPARE + + .align 64 +80: /* 0 < len <= 16 */ + andcc GLOBAL_SPARE, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, GLOBAL_SPARE + +1: + subcc %o2, 4, %o2 + EX_LD(LOAD(lduw, %o1, %g1), NG2_retl_o2_plus_4) + EX_ST(STORE(stw, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_4) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +85: retl + mov EX_RETVAL(%o3), %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + EX_LD(LOAD(ldub, %o1, %g1), NG2_retl_o2_plus_1) + EX_ST(STORE(stb, %g1, %o1 + GLOBAL_SPARE), NG2_retl_o2_plus_1) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov EX_RETVAL(%o3), %o0 + + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NG2patch.S b/arch/sparc/lib/NG2patch.S new file mode 100644 index 000000000..72431b244 --- /dev/null +++ b/arch/sparc/lib/NG2patch.S @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG2patch.S: Patch Ultra-I routines with Niagara-2 variant. + * + * Copyright (C) 2007 David S. Miller + */ + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara2_patch_copyops + .type niagara2_patch_copyops,#function +niagara2_patch_copyops: + NG_DO_PATCH(memcpy, NG2memcpy) + NG_DO_PATCH(raw_copy_from_user, NG2copy_from_user) + NG_DO_PATCH(raw_copy_to_user, NG2copy_to_user) + retl + nop + .size niagara2_patch_copyops,.-niagara2_patch_copyops diff --git a/arch/sparc/lib/NG4clear_page.S b/arch/sparc/lib/NG4clear_page.S new file mode 100644 index 000000000..d91d6b5f2 --- /dev/null +++ b/arch/sparc/lib/NG4clear_page.S @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG4copy_page.S: Niagara-4 optimized clear page. + * + * Copyright (C) 2012 (davem@davemloft.net) + */ + +#include +#include + + .text + + .register %g3, #scratch + + .align 32 + .globl NG4clear_page + .globl NG4clear_user_page +NG4clear_page: /* %o0=dest */ +NG4clear_user_page: /* %o0=dest, %o1=vaddr */ + set PAGE_SIZE, %g7 + mov 0x20, %g3 +1: stxa %g0, [%o0 + %g0] ASI_ST_BLKINIT_MRU_P + subcc %g7, 0x40, %g7 + stxa %g0, [%o0 + %g3] ASI_ST_BLKINIT_MRU_P + bne,pt %xcc, 1b + add %o0, 0x40, %o0 + membar #StoreLoad|#StoreStore + retl + nop + .size NG4clear_page,.-NG4clear_page + .size NG4clear_user_page,.-NG4clear_user_page diff --git a/arch/sparc/lib/NG4copy_from_user.S b/arch/sparc/lib/NG4copy_from_user.S new file mode 100644 index 000000000..0cac15a6d --- /dev/null +++ b/arch/sparc/lib/NG4copy_from_user.S @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG4copy_from_user.S: Niagara-4 optimized copy from userspace. + * + * Copyright (C) 2012 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x, y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_LD_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME NG4copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "NG4memcpy.S" diff --git a/arch/sparc/lib/NG4copy_page.S b/arch/sparc/lib/NG4copy_page.S new file mode 100644 index 000000000..581062f8b --- /dev/null +++ b/arch/sparc/lib/NG4copy_page.S @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG4copy_page.S: Niagara-4 optimized copy page. + * + * Copyright (C) 2012 (davem@davemloft.net) + */ + +#include +#include + + .text + .align 32 + + .register %g2, #scratch + .register %g3, #scratch + + .globl NG4copy_user_page +NG4copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ + prefetch [%o1 + 0x000], #n_reads_strong + prefetch [%o1 + 0x040], #n_reads_strong + prefetch [%o1 + 0x080], #n_reads_strong + prefetch [%o1 + 0x0c0], #n_reads_strong + set PAGE_SIZE, %g7 + prefetch [%o1 + 0x100], #n_reads_strong + prefetch [%o1 + 0x140], #n_reads_strong + prefetch [%o1 + 0x180], #n_reads_strong + prefetch [%o1 + 0x1c0], #n_reads_strong +1: + ldx [%o1 + 0x00], %o2 + subcc %g7, 0x40, %g7 + ldx [%o1 + 0x08], %o3 + ldx [%o1 + 0x10], %o4 + ldx [%o1 + 0x18], %o5 + ldx [%o1 + 0x20], %g1 + stxa %o2, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + ldx [%o1 + 0x28], %g2 + stxa %o3, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + ldx [%o1 + 0x30], %g3 + stxa %o4, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + ldx [%o1 + 0x38], %o2 + add %o1, 0x40, %o1 + stxa %o5, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + stxa %g1, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + stxa %g2, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + stxa %g3, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + stxa %o2, [%o0] ASI_ST_BLKINIT_MRU_P + add %o0, 0x08, %o0 + bne,pt %icc, 1b + prefetch [%o1 + 0x200], #n_reads_strong + retl + membar #StoreLoad | #StoreStore + .size NG4copy_user_page,.-NG4copy_user_page diff --git a/arch/sparc/lib/NG4copy_to_user.S b/arch/sparc/lib/NG4copy_to_user.S new file mode 100644 index 000000000..c5c9abb3c --- /dev/null +++ b/arch/sparc/lib/NG4copy_to_user.S @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG4copy_to_user.S: Niagara-4 optimized copy to userspace. + * + * Copyright (C) 2012 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_ST_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#ifndef ASI_BLK_INIT_QUAD_LDD_AIUS +#define ASI_BLK_INIT_QUAD_LDD_AIUS 0x23 +#endif + +#define FUNC_NAME NG4copy_to_user +#define STORE(type,src,addr) type##a src, [addr] %asi +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS +#define EX_RETVAL(x) 0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "NG4memcpy.S" diff --git a/arch/sparc/lib/NG4fls.S b/arch/sparc/lib/NG4fls.S new file mode 100644 index 000000000..2d0991e5b --- /dev/null +++ b/arch/sparc/lib/NG4fls.S @@ -0,0 +1,30 @@ +/* NG4fls.S: SPARC optimized fls and __fls for T4 and above. + * + * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + */ + +#include + +#define LZCNT_O0_G2 \ + .word 0x85b002e8 + + .text + .register %g2, #scratch + .register %g3, #scratch + +ENTRY(NG4fls) + LZCNT_O0_G2 !lzcnt %o0, %g2 + mov 64, %g3 + retl + sub %g3, %g2, %o0 +ENDPROC(NG4fls) + +ENTRY(__NG4fls) + brz,pn %o0, 1f + LZCNT_O0_G2 !lzcnt %o0, %g2 + mov 63, %g3 + sub %g3, %g2, %o0 +1: + retl + nop +ENDPROC(__NG4fls) diff --git a/arch/sparc/lib/NG4memcpy.S b/arch/sparc/lib/NG4memcpy.S new file mode 100644 index 000000000..df0ec1bd1 --- /dev/null +++ b/arch/sparc/lib/NG4memcpy.S @@ -0,0 +1,386 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG4memcpy.S: Niagara-4 optimized memcpy. + * + * Copyright (C) 2012 David S. Miller (davem@davemloft.net) + */ + +#ifdef __KERNEL__ +#include +#include +#include +#define GLOBAL_SPARE %g7 +#else +#define ASI_BLK_INIT_QUAD_LDD_P 0xe2 +#define FPRS_FEF 0x04 + +/* On T4 it is very expensive to access ASRs like %fprs and + * %asi, avoiding a read or a write can save ~50 cycles. + */ +#define FPU_ENTER \ + rd %fprs, %o5; \ + andcc %o5, FPRS_FEF, %g0; \ + be,a,pn %icc, 999f; \ + wr %g0, FPRS_FEF, %fprs; \ + 999: + +#ifdef MEMCPY_DEBUG +#define VISEntryHalf FPU_ENTER; \ + clr %g1; clr %g2; clr %g3; clr %g5; subcc %g0, %g0, %g0; +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#else +#define VISEntryHalf FPU_ENTER +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#endif + +#define GLOBAL_SPARE %g5 +#endif + +#ifndef STORE_ASI +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P +#else +#define STORE_ASI 0x80 /* ASI_P */ +#endif +#endif + +#if !defined(EX_LD) && !defined(EX_ST) +#define NON_USER_COPY +#endif + +#ifndef EX_LD +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x +#endif + +#ifndef EX_ST +#define EX_ST(x,y) x +#endif +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x +#endif + + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#ifndef MEMCPY_DEBUG +#define STORE(type,src,addr) type src, [addr] +#else +#define STORE(type,src,addr) type##a src, [addr] %asi +#endif +#endif + +#ifndef STORE_INIT +#define STORE_INIT(src,addr) stxa src, [addr] STORE_ASI +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME NG4memcpy +#endif +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + + .register %g2,#scratch + .register %g3,#scratch + + .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ +#ifdef MEMCPY_DEBUG + wr %g0, 0x80, %asi +#endif + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %XCC, 5 + PREAMBLE + mov %o0, %o3 + brz,pn %o2, .Lexit + cmp %o2, 3 + ble,pn %icc, .Ltiny + cmp %o2, 19 + ble,pn %icc, .Lsmall + or %o0, %o1, %g2 + cmp %o2, 128 + bl,pn %icc, .Lmedium + nop + +.Llarge:/* len >= 0x80 */ + /* First get dest 8 byte aligned. */ + sub %g0, %o0, %g1 + and %g1, 0x7, %g1 + brz,pt %g1, 51f + sub %o2, %g1, %o2 + + +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1) + add %o1, 1, %o1 + subcc %g1, 1, %g1 + add %o0, 1, %o0 + bne,pt %icc, 1b + EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_g1_plus_1) + +51: LOAD(prefetch, %o1 + 0x040, #n_reads_strong) + LOAD(prefetch, %o1 + 0x080, #n_reads_strong) + LOAD(prefetch, %o1 + 0x0c0, #n_reads_strong) + LOAD(prefetch, %o1 + 0x100, #n_reads_strong) + LOAD(prefetch, %o1 + 0x140, #n_reads_strong) + LOAD(prefetch, %o1 + 0x180, #n_reads_strong) + LOAD(prefetch, %o1 + 0x1c0, #n_reads_strong) + LOAD(prefetch, %o1 + 0x200, #n_reads_strong) + + /* Check if we can use the straight fully aligned + * loop, or we require the alignaddr/faligndata variant. + */ + andcc %o1, 0x7, %o5 + bne,pn %icc, .Llarge_src_unaligned + sub %g0, %o0, %g1 + + /* Legitimize the use of initializing stores by getting dest + * to be 64-byte aligned. + */ + and %g1, 0x3f, %g1 + brz,pt %g1, .Llarge_aligned + sub %o2, %g1, %o2 + +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1) + add %o1, 8, %o1 + subcc %g1, 8, %g1 + add %o0, 8, %o0 + bne,pt %icc, 1b + EX_ST(STORE(stx, %g2, %o0 - 0x08), memcpy_retl_o2_plus_g1_plus_8) + +.Llarge_aligned: + /* len >= 0x80 && src 8-byte aligned && dest 8-byte aligned */ + andn %o2, 0x3f, %o4 + sub %o2, %o4, %o2 + +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o4) + add %o1, 0x40, %o1 + EX_LD(LOAD(ldx, %o1 - 0x38, %g2), memcpy_retl_o2_plus_o4) + subcc %o4, 0x40, %o4 + EX_LD(LOAD(ldx, %o1 - 0x30, %g3), memcpy_retl_o2_plus_o4_plus_64) + EX_LD(LOAD(ldx, %o1 - 0x28, GLOBAL_SPARE), memcpy_retl_o2_plus_o4_plus_64) + EX_LD(LOAD(ldx, %o1 - 0x20, %o5), memcpy_retl_o2_plus_o4_plus_64) + EX_ST(STORE_INIT(%g1, %o0), memcpy_retl_o2_plus_o4_plus_64) + add %o0, 0x08, %o0 + EX_ST(STORE_INIT(%g2, %o0), memcpy_retl_o2_plus_o4_plus_56) + add %o0, 0x08, %o0 + EX_LD(LOAD(ldx, %o1 - 0x18, %g2), memcpy_retl_o2_plus_o4_plus_48) + EX_ST(STORE_INIT(%g3, %o0), memcpy_retl_o2_plus_o4_plus_48) + add %o0, 0x08, %o0 + EX_LD(LOAD(ldx, %o1 - 0x10, %g3), memcpy_retl_o2_plus_o4_plus_40) + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), memcpy_retl_o2_plus_o4_plus_40) + add %o0, 0x08, %o0 + EX_LD(LOAD(ldx, %o1 - 0x08, GLOBAL_SPARE), memcpy_retl_o2_plus_o4_plus_32) + EX_ST(STORE_INIT(%o5, %o0), memcpy_retl_o2_plus_o4_plus_32) + add %o0, 0x08, %o0 + EX_ST(STORE_INIT(%g2, %o0), memcpy_retl_o2_plus_o4_plus_24) + add %o0, 0x08, %o0 + EX_ST(STORE_INIT(%g3, %o0), memcpy_retl_o2_plus_o4_plus_16) + add %o0, 0x08, %o0 + EX_ST(STORE_INIT(GLOBAL_SPARE, %o0), memcpy_retl_o2_plus_o4_plus_8) + add %o0, 0x08, %o0 + bne,pt %icc, 1b + LOAD(prefetch, %o1 + 0x200, #n_reads_strong) + + membar #StoreLoad | #StoreStore + + brz,pn %o2, .Lexit + cmp %o2, 19 + ble,pn %icc, .Lsmall_unaligned + nop + ba,a,pt %icc, .Lmedium_noprefetch + +.Lexit: retl + mov EX_RETVAL(%o3), %o0 + +.Llarge_src_unaligned: +#ifdef NON_USER_COPY + VISEntryHalfFast(.Lmedium_vis_entry_fail) +#else + VISEntryHalf +#endif + andn %o2, 0x3f, %o4 + sub %o2, %o4, %o2 + alignaddr %o1, %g0, %g1 + add %o1, %o4, %o1 + EX_LD_FP(LOAD(ldd, %g1 + 0x00, %f0), memcpy_retl_o2_plus_o4) +1: EX_LD_FP(LOAD(ldd, %g1 + 0x08, %f2), memcpy_retl_o2_plus_o4) + subcc %o4, 0x40, %o4 + EX_LD_FP(LOAD(ldd, %g1 + 0x10, %f4), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x18, %f6), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x20, %f8), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x28, %f10), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x30, %f12), memcpy_retl_o2_plus_o4_plus_64) + EX_LD_FP(LOAD(ldd, %g1 + 0x38, %f14), memcpy_retl_o2_plus_o4_plus_64) + faligndata %f0, %f2, %f16 + EX_LD_FP(LOAD(ldd, %g1 + 0x40, %f0), memcpy_retl_o2_plus_o4_plus_64) + faligndata %f2, %f4, %f18 + add %g1, 0x40, %g1 + faligndata %f4, %f6, %f20 + faligndata %f6, %f8, %f22 + faligndata %f8, %f10, %f24 + faligndata %f10, %f12, %f26 + faligndata %f12, %f14, %f28 + faligndata %f14, %f0, %f30 + EX_ST_FP(STORE(std, %f16, %o0 + 0x00), memcpy_retl_o2_plus_o4_plus_64) + EX_ST_FP(STORE(std, %f18, %o0 + 0x08), memcpy_retl_o2_plus_o4_plus_56) + EX_ST_FP(STORE(std, %f20, %o0 + 0x10), memcpy_retl_o2_plus_o4_plus_48) + EX_ST_FP(STORE(std, %f22, %o0 + 0x18), memcpy_retl_o2_plus_o4_plus_40) + EX_ST_FP(STORE(std, %f24, %o0 + 0x20), memcpy_retl_o2_plus_o4_plus_32) + EX_ST_FP(STORE(std, %f26, %o0 + 0x28), memcpy_retl_o2_plus_o4_plus_24) + EX_ST_FP(STORE(std, %f28, %o0 + 0x30), memcpy_retl_o2_plus_o4_plus_16) + EX_ST_FP(STORE(std, %f30, %o0 + 0x38), memcpy_retl_o2_plus_o4_plus_8) + add %o0, 0x40, %o0 + bne,pt %icc, 1b + LOAD(prefetch, %g1 + 0x200, #n_reads_strong) +#ifdef NON_USER_COPY + VISExitHalfFast +#else + VISExitHalf +#endif + brz,pn %o2, .Lexit + cmp %o2, 19 + ble,pn %icc, .Lsmall_unaligned + nop + ba,a,pt %icc, .Lmedium_unaligned + +#ifdef NON_USER_COPY +.Lmedium_vis_entry_fail: + or %o0, %o1, %g2 +#endif +.Lmedium: + LOAD(prefetch, %o1 + 0x40, #n_reads_strong) + andcc %g2, 0x7, %g0 + bne,pn %icc, .Lmedium_unaligned + nop +.Lmedium_noprefetch: + andncc %o2, 0x20 - 1, %o5 + be,pn %icc, 2f + sub %o2, %o5, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x08, %g2), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x10, GLOBAL_SPARE), memcpy_retl_o2_plus_o5) + EX_LD(LOAD(ldx, %o1 + 0x18, %o4), memcpy_retl_o2_plus_o5) + add %o1, 0x20, %o1 + subcc %o5, 0x20, %o5 + EX_ST(STORE(stx, %g1, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_32) + EX_ST(STORE(stx, %g2, %o0 + 0x08), memcpy_retl_o2_plus_o5_plus_24) + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x10), memcpy_retl_o2_plus_o5_plus_16) + EX_ST(STORE(stx, %o4, %o0 + 0x18), memcpy_retl_o2_plus_o5_plus_8) + bne,pt %icc, 1b + add %o0, 0x20, %o0 +2: andcc %o2, 0x18, %o5 + be,pt %icc, 3f + sub %o2, %o5, %o2 + +1: EX_LD(LOAD(ldx, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o5) + add %o1, 0x08, %o1 + add %o0, 0x08, %o0 + subcc %o5, 0x08, %o5 + bne,pt %icc, 1b + EX_ST(STORE(stx, %g1, %o0 - 0x08), memcpy_retl_o2_plus_o5_plus_8) +3: brz,pt %o2, .Lexit + cmp %o2, 0x04 + bl,pn %icc, .Ltiny + nop + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), memcpy_retl_o2) + add %o1, 0x04, %o1 + add %o0, 0x04, %o0 + subcc %o2, 0x04, %o2 + bne,pn %icc, .Ltiny + EX_ST(STORE(stw, %g1, %o0 - 0x04), memcpy_retl_o2_plus_4) + ba,a,pt %icc, .Lexit +.Lmedium_unaligned: + /* First get dest 8 byte aligned. */ + sub %g0, %o0, %g1 + and %g1, 0x7, %g1 + brz,pt %g1, 2f + sub %o2, %g1, %o2 + +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g2), memcpy_retl_o2_plus_g1) + add %o1, 1, %o1 + subcc %g1, 1, %g1 + add %o0, 1, %o0 + bne,pt %icc, 1b + EX_ST(STORE(stb, %g2, %o0 - 0x01), memcpy_retl_o2_plus_g1_plus_1) +2: + and %o1, 0x7, %g1 + brz,pn %g1, .Lmedium_noprefetch + sll %g1, 3, %g1 + mov 64, %g2 + sub %g2, %g1, %g2 + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1 + 0x00, %o4), memcpy_retl_o2) + sllx %o4, %g1, %o4 + andn %o2, 0x08 - 1, %o5 + sub %o2, %o5, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x08, %g3), memcpy_retl_o2_plus_o5) + add %o1, 0x08, %o1 + subcc %o5, 0x08, %o5 + srlx %g3, %g2, GLOBAL_SPARE + or GLOBAL_SPARE, %o4, GLOBAL_SPARE + EX_ST(STORE(stx, GLOBAL_SPARE, %o0 + 0x00), memcpy_retl_o2_plus_o5_plus_8) + add %o0, 0x08, %o0 + bne,pt %icc, 1b + sllx %g3, %g1, %o4 + srl %g1, 3, %g1 + add %o1, %g1, %o1 + brz,pn %o2, .Lexit + nop + ba,pt %icc, .Lsmall_unaligned + +.Ltiny: + EX_LD(LOAD(ldub, %o1 + 0x00, %g1), memcpy_retl_o2) + subcc %o2, 1, %o2 + be,pn %icc, .Lexit + EX_ST(STORE(stb, %g1, %o0 + 0x00), memcpy_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x01, %g1), memcpy_retl_o2) + subcc %o2, 1, %o2 + be,pn %icc, .Lexit + EX_ST(STORE(stb, %g1, %o0 + 0x01), memcpy_retl_o2_plus_1) + EX_LD(LOAD(ldub, %o1 + 0x02, %g1), memcpy_retl_o2) + ba,pt %icc, .Lexit + EX_ST(STORE(stb, %g1, %o0 + 0x02), memcpy_retl_o2) + +.Lsmall: + andcc %g2, 0x3, %g0 + bne,pn %icc, .Lsmall_unaligned + andn %o2, 0x4 - 1, %o5 + sub %o2, %o5, %o2 +1: + EX_LD(LOAD(lduw, %o1 + 0x00, %g1), memcpy_retl_o2_plus_o5) + add %o1, 0x04, %o1 + subcc %o5, 0x04, %o5 + add %o0, 0x04, %o0 + bne,pt %icc, 1b + EX_ST(STORE(stw, %g1, %o0 - 0x04), memcpy_retl_o2_plus_o5_plus_4) + brz,pt %o2, .Lexit + nop + ba,a,pt %icc, .Ltiny + +.Lsmall_unaligned: +1: EX_LD(LOAD(ldub, %o1 + 0x00, %g1), memcpy_retl_o2) + add %o1, 1, %o1 + add %o0, 1, %o0 + subcc %o2, 1, %o2 + bne,pt %icc, 1b + EX_ST(STORE(stb, %g1, %o0 - 0x01), memcpy_retl_o2_plus_1) + ba,a,pt %icc, .Lexit + nop + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NG4memset.S b/arch/sparc/lib/NG4memset.S new file mode 100644 index 000000000..f81ee5419 --- /dev/null +++ b/arch/sparc/lib/NG4memset.S @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG4memset.S: Niagara-4 optimized memset/bzero. + * + * Copyright (C) 2012 David S. Miller (davem@davemloft.net) + */ + +#include + + .register %g2, #scratch + .register %g3, #scratch + + .text + .align 32 + .globl NG4memset +NG4memset: + andcc %o1, 0xff, %o4 + be,pt %icc, 1f + mov %o2, %o1 + sllx %o4, 8, %g1 + or %g1, %o4, %o2 + sllx %o2, 16, %g1 + or %g1, %o2, %o2 + sllx %o2, 32, %g1 + ba,pt %icc, 1f + or %g1, %o2, %o4 + .size NG4memset,.-NG4memset + + .align 32 + .globl NG4bzero +NG4bzero: + clr %o4 +1: cmp %o1, 16 + ble %icc, .Ltiny + mov %o0, %o3 + sub %g0, %o0, %g1 + and %g1, 0x7, %g1 + brz,pt %g1, .Laligned8 + sub %o1, %g1, %o1 +1: stb %o4, [%o0 + 0x00] + subcc %g1, 1, %g1 + bne,pt %icc, 1b + add %o0, 1, %o0 +.Laligned8: + cmp %o1, 64 + (64 - 8) + ble .Lmedium + sub %g0, %o0, %g1 + andcc %g1, (64 - 1), %g1 + brz,pn %g1, .Laligned64 + sub %o1, %g1, %o1 +1: stx %o4, [%o0 + 0x00] + subcc %g1, 8, %g1 + bne,pt %icc, 1b + add %o0, 0x8, %o0 +.Laligned64: + andn %o1, 64 - 1, %g1 + sub %o1, %g1, %o1 + brnz,pn %o4, .Lnon_bzero_loop + mov 0x20, %g2 +1: stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + subcc %g1, 0x40, %g1 + stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + bne,pt %icc, 1b + add %o0, 0x40, %o0 +.Lpostloop: + cmp %o1, 8 + bl,pn %icc, .Ltiny + membar #StoreStore|#StoreLoad +.Lmedium: + andn %o1, 0x7, %g1 + sub %o1, %g1, %o1 +1: stx %o4, [%o0 + 0x00] + subcc %g1, 0x8, %g1 + bne,pt %icc, 1b + add %o0, 0x08, %o0 + andcc %o1, 0x4, %g1 + be,pt %icc, .Ltiny + sub %o1, %g1, %o1 + stw %o4, [%o0 + 0x00] + add %o0, 0x4, %o0 +.Ltiny: + cmp %o1, 0 + be,pn %icc, .Lexit +1: subcc %o1, 1, %o1 + stb %o4, [%o0 + 0x00] + bne,pt %icc, 1b + add %o0, 1, %o0 +.Lexit: + retl + mov %o3, %o0 +.Lnon_bzero_loop: + mov 0x08, %g3 + mov 0x28, %o5 +1: stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + subcc %g1, 0x40, %g1 + stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %o5] ASI_BLK_INIT_QUAD_LDD_P + add %o0, 0x10, %o0 + stxa %o4, [%o0 + %g0] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %g2] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %g3] ASI_BLK_INIT_QUAD_LDD_P + stxa %o4, [%o0 + %o5] ASI_BLK_INIT_QUAD_LDD_P + bne,pt %icc, 1b + add %o0, 0x30, %o0 + ba,a,pt %icc, .Lpostloop + nop + .size NG4bzero,.-NG4bzero diff --git a/arch/sparc/lib/NG4patch.S b/arch/sparc/lib/NG4patch.S new file mode 100644 index 000000000..37866175c --- /dev/null +++ b/arch/sparc/lib/NG4patch.S @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NG4patch.S: Patch Ultra-I routines with Niagara-4 variant. + * + * Copyright (C) 2012 David S. Miller + */ + +#include + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara4_patch_copyops + .type niagara4_patch_copyops,#function +niagara4_patch_copyops: + NG_DO_PATCH(memcpy, NG4memcpy) + NG_DO_PATCH(raw_copy_from_user, NG4copy_from_user) + NG_DO_PATCH(raw_copy_to_user, NG4copy_to_user) + retl + nop + .size niagara4_patch_copyops,.-niagara4_patch_copyops + + .globl niagara4_patch_bzero + .type niagara4_patch_bzero,#function +niagara4_patch_bzero: + NG_DO_PATCH(memset, NG4memset) + NG_DO_PATCH(__bzero, NG4bzero) + NG_DO_PATCH(__clear_user, NGclear_user) + NG_DO_PATCH(tsb_init, NGtsb_init) + retl + nop + .size niagara4_patch_bzero,.-niagara4_patch_bzero + + .globl niagara4_patch_pageops + .type niagara4_patch_pageops,#function +niagara4_patch_pageops: + NG_DO_PATCH(copy_user_page, NG4copy_user_page) + NG_DO_PATCH(_clear_page, NG4clear_page) + NG_DO_PATCH(clear_user_page, NG4clear_user_page) + retl + nop + .size niagara4_patch_pageops,.-niagara4_patch_pageops + +ENTRY(niagara4_patch_fls) + NG_DO_PATCH(fls, NG4fls) + NG_DO_PATCH(__fls, __NG4fls) + retl + nop +ENDPROC(niagara4_patch_fls) diff --git a/arch/sparc/lib/NGbzero.S b/arch/sparc/lib/NGbzero.S new file mode 100644 index 000000000..19327614d --- /dev/null +++ b/arch/sparc/lib/NGbzero.S @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NGbzero.S: Niagara optimized memset/clear_user. + * + * Copyright (C) 2006 David S. Miller (davem@davemloft.net) + */ +#include + +#define EX_ST(x,y) \ +98: x,y; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, __retl_o1_asi;\ + .text; \ + .align 4; + + .text + + .globl NGmemset + .type NGmemset, #function +NGmemset: /* %o0=buf, %o1=pat, %o2=len */ + and %o1, 0xff, %o3 + mov %o2, %o1 + sllx %o3, 8, %g1 + or %g1, %o3, %o2 + sllx %o2, 16, %g1 + or %g1, %o2, %o2 + sllx %o2, 32, %g1 + ba,pt %xcc, 1f + or %g1, %o2, %o2 + + .globl NGbzero + .type NGbzero, #function +NGbzero: + clr %o2 +1: brz,pn %o1, NGbzero_return + mov %o0, %o3 + + /* %o5: saved %asi, restored at NGbzero_done + * %g7: store-init %asi to use + * %o4: non-store-init %asi to use + */ + rd %asi, %o5 + mov ASI_BLK_INIT_QUAD_LDD_P, %g7 + mov ASI_P, %o4 + wr %o4, 0x0, %asi + +NGbzero_from_clear_user: + cmp %o1, 15 + bl,pn %icc, NGbzero_tiny + andcc %o0, 0x7, %g1 + be,pt %xcc, 2f + mov 8, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: EX_ST(stba %o2, [%o0 + 0x00] %asi) + subcc %g1, 1, %g1 + bne,pt %xcc, 1b + add %o0, 1, %o0 +2: cmp %o1, 128 + bl,pn %icc, NGbzero_medium + andcc %o0, (64 - 1), %g1 + be,pt %xcc, NGbzero_pre_loop + mov 64, %g2 + sub %g2, %g1, %g1 + sub %o1, %g1, %o1 +1: EX_ST(stxa %o2, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %xcc, 1b + add %o0, 8, %o0 + +NGbzero_pre_loop: + wr %g7, 0x0, %asi + andn %o1, (64 - 1), %g1 + sub %o1, %g1, %o1 +NGbzero_loop: + EX_ST(stxa %o2, [%o0 + 0x00] %asi) + EX_ST(stxa %o2, [%o0 + 0x08] %asi) + EX_ST(stxa %o2, [%o0 + 0x10] %asi) + EX_ST(stxa %o2, [%o0 + 0x18] %asi) + EX_ST(stxa %o2, [%o0 + 0x20] %asi) + EX_ST(stxa %o2, [%o0 + 0x28] %asi) + EX_ST(stxa %o2, [%o0 + 0x30] %asi) + EX_ST(stxa %o2, [%o0 + 0x38] %asi) + subcc %g1, 64, %g1 + bne,pt %xcc, NGbzero_loop + add %o0, 64, %o0 + + membar #Sync + wr %o4, 0x0, %asi + brz,pn %o1, NGbzero_done +NGbzero_medium: + andncc %o1, 0x7, %g1 + be,pn %xcc, 2f + sub %o1, %g1, %o1 +1: EX_ST(stxa %o2, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %xcc, 1b + add %o0, 8, %o0 +2: brz,pt %o1, NGbzero_done + nop + +NGbzero_tiny: +1: EX_ST(stba %o2, [%o0 + 0x00] %asi) + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 1, %o0 + + /* fallthrough */ + +NGbzero_done: + wr %o5, 0x0, %asi + +NGbzero_return: + retl + mov %o3, %o0 + .size NGbzero, .-NGbzero + .size NGmemset, .-NGmemset + + .globl NGclear_user + .type NGclear_user, #function +NGclear_user: /* %o0=buf, %o1=len */ + rd %asi, %o5 + brz,pn %o1, NGbzero_done + clr %o3 + cmp %o5, ASI_AIUS + bne,pn %icc, NGbzero + clr %o2 + mov ASI_BLK_INIT_QUAD_LDD_AIUS, %g7 + ba,pt %xcc, NGbzero_from_clear_user + mov ASI_AIUS, %o4 + .size NGclear_user, .-NGclear_user + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara_patch_bzero + .type niagara_patch_bzero,#function +niagara_patch_bzero: + NG_DO_PATCH(memset, NGmemset) + NG_DO_PATCH(__bzero, NGbzero) + NG_DO_PATCH(__clear_user, NGclear_user) + NG_DO_PATCH(tsb_init, NGtsb_init) + retl + nop + .size niagara_patch_bzero,.-niagara_patch_bzero diff --git a/arch/sparc/lib/NGcopy_from_user.S b/arch/sparc/lib/NGcopy_from_user.S new file mode 100644 index 000000000..9abc49fcd --- /dev/null +++ b/arch/sparc/lib/NGcopy_from_user.S @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NGcopy_from_user.S: Niagara optimized copy from userspace. + * + * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME NGcopy_from_user +#define LOAD(type,addr,dest) type##a [addr] ASI_AIUS, dest +#define LOAD_TWIN(addr_reg,dest0,dest1) \ + ldda [addr_reg] ASI_BLK_INIT_QUAD_LDD_AIUS, dest0 +#define EX_RETVAL(x) %g0 + +#ifdef __KERNEL__ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "NGmemcpy.S" diff --git a/arch/sparc/lib/NGcopy_to_user.S b/arch/sparc/lib/NGcopy_to_user.S new file mode 100644 index 000000000..9cbe2f18e --- /dev/null +++ b/arch/sparc/lib/NGcopy_to_user.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NGcopy_to_user.S: Niagara optimized copy to userspace. + * + * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#ifndef ASI_AIUS +#define ASI_AIUS 0x11 +#endif + +#define FUNC_NAME NGcopy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_AIUS +#define EX_RETVAL(x) %g0 + +#ifdef __KERNEL__ + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop +#endif + +#include "NGmemcpy.S" diff --git a/arch/sparc/lib/NGmemcpy.S b/arch/sparc/lib/NGmemcpy.S new file mode 100644 index 000000000..bbd3ea0a6 --- /dev/null +++ b/arch/sparc/lib/NGmemcpy.S @@ -0,0 +1,516 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NGmemcpy.S: Niagara optimized memcpy. + * + * Copyright (C) 2006, 2007 David S. Miller (davem@davemloft.net) + */ + +#ifdef __KERNEL__ +#include +#include +#include +#define GLOBAL_SPARE %g7 +#define RESTORE_ASI(TMP) \ + wr %g0, ASI_AIUS, %asi +#else +#define GLOBAL_SPARE %g5 +#define RESTORE_ASI(TMP) \ + wr %g0, ASI_PNF, %asi +#endif + +#ifdef __sparc_v9__ +#define SAVE_AMOUNT 128 +#else +#define SAVE_AMOUNT 64 +#endif + +#ifndef STORE_ASI +#define STORE_ASI ASI_BLK_INIT_QUAD_LDD_P +#endif + +#ifndef EX_LD +#define EX_LD(x,y) x +#endif + +#ifndef EX_ST +#define EX_ST(x,y) x +#endif + +#ifndef LOAD +#ifndef MEMCPY_DEBUG +#define LOAD(type,addr,dest) type [addr], dest +#else +#define LOAD(type,addr,dest) type##a [addr] 0x80, dest +#endif +#endif + +#ifndef LOAD_TWIN +#define LOAD_TWIN(addr_reg,dest0,dest1) \ + ldda [addr_reg] ASI_BLK_INIT_QUAD_LDD_P, dest0 +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef STORE_INIT +#ifndef SIMULATE_NIAGARA_ON_NON_NIAGARA +#define STORE_INIT(src,addr) stxa src, [addr] %asi +#else +#define STORE_INIT(src,addr) stx src, [addr + 0x00] +#endif +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME NGmemcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + + .register %g2,#scratch + .register %g3,#scratch + + .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_asi: + wr %g0, ASI_AIUS, %asi + ret + restore +ENTRY(NG_ret_i2_plus_i4_plus_1) + ba,pt %xcc, __restore_asi + add %i2, %i5, %i0 +ENDPROC(NG_ret_i2_plus_i4_plus_1) +ENTRY(NG_ret_i2_plus_g1) + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1) +ENTRY(NG_ret_i2_plus_g1_minus_8) + sub %g1, 8, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_8) +ENTRY(NG_ret_i2_plus_g1_minus_16) + sub %g1, 16, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_16) +ENTRY(NG_ret_i2_plus_g1_minus_24) + sub %g1, 24, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_24) +ENTRY(NG_ret_i2_plus_g1_minus_32) + sub %g1, 32, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_32) +ENTRY(NG_ret_i2_plus_g1_minus_40) + sub %g1, 40, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_40) +ENTRY(NG_ret_i2_plus_g1_minus_48) + sub %g1, 48, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_48) +ENTRY(NG_ret_i2_plus_g1_minus_56) + sub %g1, 56, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_minus_56) +ENTRY(NG_ret_i2_plus_i4_plus_16) + add %i4, 16, %i4 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_plus_i4_plus_16) +ENTRY(NG_ret_i2_plus_i4_plus_8) + add %i4, 8, %i4 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_plus_i4_plus_8) +ENTRY(NG_ret_i2_plus_8) + ba,pt %xcc, __restore_asi + add %i2, 8, %i0 +ENDPROC(NG_ret_i2_plus_8) +ENTRY(NG_ret_i2_plus_4) + ba,pt %xcc, __restore_asi + add %i2, 4, %i0 +ENDPROC(NG_ret_i2_plus_4) +ENTRY(NG_ret_i2_plus_1) + ba,pt %xcc, __restore_asi + add %i2, 1, %i0 +ENDPROC(NG_ret_i2_plus_1) +ENTRY(NG_ret_i2_plus_g1_plus_1) + add %g1, 1, %g1 + ba,pt %xcc, __restore_asi + add %i2, %g1, %i0 +ENDPROC(NG_ret_i2_plus_g1_plus_1) +ENTRY(NG_ret_i2) + ba,pt %xcc, __restore_asi + mov %i2, %i0 +ENDPROC(NG_ret_i2) +ENTRY(NG_ret_i2_and_7_plus_i4) + and %i2, 7, %i2 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_and_7_plus_i4) +ENTRY(NG_ret_i2_and_7_plus_i4_plus_8) + and %i2, 7, %i2 + add %i4, 8, %i4 + ba,pt %xcc, __restore_asi + add %i2, %i4, %i0 +ENDPROC(NG_ret_i2_and_7_plus_i4) +#endif + + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %i0=dst, %i1=src, %i2=len */ + PREAMBLE + save %sp, -SAVE_AMOUNT, %sp + srlx %i2, 31, %g2 + cmp %g2, 0 + tne %xcc, 5 + mov %i0, %o0 + cmp %i2, 0 + be,pn %XCC, 85f + or %o0, %i1, %i3 + cmp %i2, 16 + blu,a,pn %XCC, 80f + or %i3, %i2, %i3 + + /* 2 blocks (128 bytes) is the minimum we can do the block + * copy with. We need to ensure that we'll iterate at least + * once in the block copy loop. At worst we'll need to align + * the destination to a 64-byte boundary which can chew up + * to (64 - 1) bytes from the length before we perform the + * block copy loop. + */ + cmp %i2, (2 * 64) + blu,pt %XCC, 70f + andcc %i3, 0x7, %g0 + + /* %o0: dst + * %i1: src + * %i2: len (known to be >= 128) + * + * The block copy loops will use %i4/%i5,%g2/%g3 as + * temporaries while copying the data. + */ + + LOAD(prefetch, %i1, #one_read) + wr %g0, STORE_ASI, %asi + + /* Align destination on 64-byte boundary. */ + andcc %o0, (64 - 1), %i4 + be,pt %XCC, 2f + sub %i4, 64, %i4 + sub %g0, %i4, %i4 ! bytes to align dst + sub %i2, %i4, %i2 +1: subcc %i4, 1, %i4 + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_i4_plus_1) + EX_ST(STORE(stb, %g1, %o0), NG_ret_i2_plus_i4_plus_1) + add %i1, 1, %i1 + bne,pt %XCC, 1b + add %o0, 1, %o0 + + /* If the source is on a 16-byte boundary we can do + * the direct block copy loop. If it is 8-byte aligned + * we can do the 16-byte loads offset by -8 bytes and the + * init stores offset by one register. + * + * If the source is not even 8-byte aligned, we need to do + * shifting and masking (basically integer faligndata). + * + * The careful bit with init stores is that if we store + * to any part of the cache line we have to store the whole + * cacheline else we can end up with corrupt L2 cache line + * contents. Since the loop works on 64-bytes of 64-byte + * aligned store data at a time, this is easy to ensure. + */ +2: + andcc %i1, (16 - 1), %i4 + andn %i2, (64 - 1), %g1 ! block copy loop iterator + be,pt %XCC, 50f + sub %i2, %g1, %i2 ! final sub-block copy bytes + + cmp %i4, 8 + be,pt %XCC, 10f + sub %i1, %i4, %i1 + + /* Neither 8-byte nor 16-byte aligned, shift and mask. */ + and %i4, 0x7, GLOBAL_SPARE + sll GLOBAL_SPARE, 3, GLOBAL_SPARE + mov 64, %i5 + EX_LD(LOAD_TWIN(%i1, %g2, %g3), NG_ret_i2_plus_g1) + sub %i5, GLOBAL_SPARE, %i5 + mov 16, %o4 + mov 32, %o5 + mov 48, %o7 + mov 64, %i3 + + bg,pn %XCC, 9f + nop + +#define MIX_THREE_WORDS(WORD1, WORD2, WORD3, PRE_SHIFT, POST_SHIFT, TMP) \ + sllx WORD1, POST_SHIFT, WORD1; \ + srlx WORD2, PRE_SHIFT, TMP; \ + sllx WORD2, POST_SHIFT, WORD2; \ + or WORD1, TMP, WORD1; \ + srlx WORD3, PRE_SHIFT, TMP; \ + or WORD2, TMP, WORD2; + +8: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) + MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) + LOAD(prefetch, %i1 + %i3, #one_read) + + EX_ST(STORE_INIT(%g2, %o0 + 0x00), NG_ret_i2_plus_g1) + EX_ST(STORE_INIT(%g3, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) + MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) + MIX_THREE_WORDS(%g2, %g3, %o2, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%g2, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%g3, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) + add %i1, 64, %i1 + MIX_THREE_WORDS(%o2, %o3, %g2, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) + + subcc %g1, 64, %g1 + bne,pt %XCC, 8b + add %o0, 64, %o0 + + ba,pt %XCC, 60f + add %i1, %i4, %i1 + +9: EX_LD(LOAD_TWIN(%i1 + %o4, %o2, %o3), NG_ret_i2_plus_g1) + MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) + LOAD(prefetch, %i1 + %i3, #one_read) + + EX_ST(STORE_INIT(%g3, %o0 + 0x00), NG_ret_i2_plus_g1) + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + + EX_LD(LOAD_TWIN(%i1 + %o5, %g2, %g3), NG_ret_i2_plus_g1_minus_16) + MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%g2, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1_minus_32) + MIX_THREE_WORDS(%g3, %o2, %o3, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%g3, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + + EX_LD(LOAD_TWIN(%i1 + %i3, %g2, %g3), NG_ret_i2_plus_g1_minus_48) + add %i1, 64, %i1 + MIX_THREE_WORDS(%o3, %g2, %g3, %i5, GLOBAL_SPARE, %o1) + + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%g2, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) + + subcc %g1, 64, %g1 + bne,pt %XCC, 9b + add %o0, 64, %o0 + + ba,pt %XCC, 60f + add %i1, %i4, %i1 + +10: /* Destination is 64-byte aligned, source was only 8-byte + * aligned but it has been subtracted by 8 and we perform + * one twin load ahead, then add 8 back into source when + * we finish the loop. + */ + EX_LD(LOAD_TWIN(%i1, %o4, %o5), NG_ret_i2_plus_g1) + mov 16, %o7 + mov 32, %g2 + mov 48, %g3 + mov 64, %o1 +1: EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) + LOAD(prefetch, %i1 + %o1, #one_read) + EX_ST(STORE_INIT(%o5, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line + EX_ST(STORE_INIT(%o2, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o4, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o5, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o2, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + EX_LD(LOAD_TWIN(%i1 + %o1, %o4, %o5), NG_ret_i2_plus_g1_minus_48) + add %i1, 64, %i1 + EX_ST(STORE_INIT(%o3, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o4, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) + subcc %g1, 64, %g1 + bne,pt %XCC, 1b + add %o0, 64, %o0 + + ba,pt %XCC, 60f + add %i1, 0x8, %i1 + +50: /* Destination is 64-byte aligned, and source is 16-byte + * aligned. + */ + mov 16, %o7 + mov 32, %g2 + mov 48, %g3 + mov 64, %o1 +1: EX_LD(LOAD_TWIN(%i1 + %g0, %o4, %o5), NG_ret_i2_plus_g1) + EX_LD(LOAD_TWIN(%i1 + %o7, %o2, %o3), NG_ret_i2_plus_g1) + LOAD(prefetch, %i1 + %o1, #one_read) + EX_ST(STORE_INIT(%o4, %o0 + 0x00), NG_ret_i2_plus_g1) ! initializes cache line + EX_ST(STORE_INIT(%o5, %o0 + 0x08), NG_ret_i2_plus_g1_minus_8) + EX_LD(LOAD_TWIN(%i1 + %g2, %o4, %o5), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o2, %o0 + 0x10), NG_ret_i2_plus_g1_minus_16) + EX_ST(STORE_INIT(%o3, %o0 + 0x18), NG_ret_i2_plus_g1_minus_24) + EX_LD(LOAD_TWIN(%i1 + %g3, %o2, %o3), NG_ret_i2_plus_g1_minus_32) + add %i1, 64, %i1 + EX_ST(STORE_INIT(%o4, %o0 + 0x20), NG_ret_i2_plus_g1_minus_32) + EX_ST(STORE_INIT(%o5, %o0 + 0x28), NG_ret_i2_plus_g1_minus_40) + EX_ST(STORE_INIT(%o2, %o0 + 0x30), NG_ret_i2_plus_g1_minus_48) + EX_ST(STORE_INIT(%o3, %o0 + 0x38), NG_ret_i2_plus_g1_minus_56) + subcc %g1, 64, %g1 + bne,pt %XCC, 1b + add %o0, 64, %o0 + /* fall through */ + +60: + membar #Sync + + /* %i2 contains any final bytes still needed to be copied + * over. If anything is left, we copy it one byte at a time. + */ + RESTORE_ASI(%i3) + brz,pt %i2, 85f + sub %o0, %i1, %i3 + ba,a,pt %XCC, 90f + nop + + .align 64 +70: /* 16 < len <= 64 */ + bne,pn %XCC, 75f + sub %o0, %i1, %i3 + +72: + andn %i2, 0xf, %i4 + and %i2, 0xf, %i2 +1: subcc %i4, 0x10, %i4 + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_i4_plus_16) + add %i1, 0x08, %i1 + EX_LD(LOAD(ldx, %i1, %g1), NG_ret_i2_plus_i4_plus_16) + sub %i1, 0x08, %i1 + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_i4_plus_16) + add %i1, 0x8, %i1 + EX_ST(STORE(stx, %g1, %i1 + %i3), NG_ret_i2_plus_i4_plus_8) + bgu,pt %XCC, 1b + add %i1, 0x8, %i1 +73: andcc %i2, 0x8, %g0 + be,pt %XCC, 1f + nop + sub %i2, 0x8, %i2 + EX_LD(LOAD(ldx, %i1, %o4), NG_ret_i2_plus_8) + EX_ST(STORE(stx, %o4, %i1 + %i3), NG_ret_i2_plus_8) + add %i1, 0x8, %i1 +1: andcc %i2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %i2, 0x4, %i2 + EX_LD(LOAD(lduw, %i1, %i5), NG_ret_i2_plus_4) + EX_ST(STORE(stw, %i5, %i1 + %i3), NG_ret_i2_plus_4) + add %i1, 0x4, %i1 +1: cmp %i2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +75: + andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %i2, %g1, %i2 + +1: subcc %g1, 1, %g1 + EX_LD(LOAD(ldub, %i1, %i5), NG_ret_i2_plus_g1_plus_1) + EX_ST(STORE(stb, %i5, %i1 + %i3), NG_ret_i2_plus_g1_plus_1) + bgu,pt %icc, 1b + add %i1, 1, %i1 + +2: add %i1, %i3, %o0 + andcc %i1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %i2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %xcc, 73b + +8: mov 64, %i3 + andn %i1, 0x7, %i1 + EX_LD(LOAD(ldx, %i1, %g2), NG_ret_i2) + sub %i3, %g1, %i3 + andn %i2, 0x7, %i4 + sllx %g2, %g1, %g2 +1: add %i1, 0x8, %i1 + EX_LD(LOAD(ldx, %i1, %g3), NG_ret_i2_and_7_plus_i4) + subcc %i4, 0x8, %i4 + srlx %g3, %i3, %i5 + or %i5, %g2, %i5 + EX_ST(STORE(stx, %i5, %o0), NG_ret_i2_and_7_plus_i4_plus_8) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %i2, 0x7, %i2 + be,pn %icc, 85f + add %i1, %g1, %i1 + ba,pt %xcc, 90f + sub %o0, %i1, %i3 + + .align 64 +80: /* 0 < len <= 16 */ + andcc %i3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %i1, %i3 + +1: + subcc %i2, 4, %i2 + EX_LD(LOAD(lduw, %i1, %g1), NG_ret_i2_plus_4) + EX_ST(STORE(stw, %g1, %i1 + %i3), NG_ret_i2_plus_4) + bgu,pt %XCC, 1b + add %i1, 4, %i1 + +85: ret + restore EX_RETVAL(%i0), %g0, %o0 + + .align 32 +90: + subcc %i2, 1, %i2 + EX_LD(LOAD(ldub, %i1, %g1), NG_ret_i2_plus_1) + EX_ST(STORE(stb, %g1, %i1 + %i3), NG_ret_i2_plus_1) + bgu,pt %XCC, 90b + add %i1, 1, %i1 + ret + restore EX_RETVAL(%i0), %g0, %o0 + + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/NGpage.S b/arch/sparc/lib/NGpage.S new file mode 100644 index 000000000..88fec7818 --- /dev/null +++ b/arch/sparc/lib/NGpage.S @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NGpage.S: Niagara optimize clear and copy page. + * + * Copyright (C) 2006 (davem@davemloft.net) + */ + +#include +#include + + .text + .align 32 + + /* This is heavily simplified from the sun4u variants + * because Niagara does not have any D-cache aliasing issues + * and also we don't need to use the FPU in order to implement + * an optimal page copy/clear. + */ + +NGcopy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ + save %sp, -192, %sp + rd %asi, %g3 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + set PAGE_SIZE, %g7 + prefetch [%i1 + 0x00], #one_read + prefetch [%i1 + 0x40], #one_read + +1: prefetch [%i1 + 0x80], #one_read + prefetch [%i1 + 0xc0], #one_read + ldda [%i1 + 0x00] %asi, %o2 + ldda [%i1 + 0x10] %asi, %o4 + ldda [%i1 + 0x20] %asi, %l2 + ldda [%i1 + 0x30] %asi, %l4 + stxa %o2, [%i0 + 0x00] %asi + stxa %o3, [%i0 + 0x08] %asi + stxa %o4, [%i0 + 0x10] %asi + stxa %o5, [%i0 + 0x18] %asi + stxa %l2, [%i0 + 0x20] %asi + stxa %l3, [%i0 + 0x28] %asi + stxa %l4, [%i0 + 0x30] %asi + stxa %l5, [%i0 + 0x38] %asi + ldda [%i1 + 0x40] %asi, %o2 + ldda [%i1 + 0x50] %asi, %o4 + ldda [%i1 + 0x60] %asi, %l2 + ldda [%i1 + 0x70] %asi, %l4 + stxa %o2, [%i0 + 0x40] %asi + stxa %o3, [%i0 + 0x48] %asi + stxa %o4, [%i0 + 0x50] %asi + stxa %o5, [%i0 + 0x58] %asi + stxa %l2, [%i0 + 0x60] %asi + stxa %l3, [%i0 + 0x68] %asi + stxa %l4, [%i0 + 0x70] %asi + stxa %l5, [%i0 + 0x78] %asi + add %i1, 128, %i1 + subcc %g7, 128, %g7 + bne,pt %xcc, 1b + add %i0, 128, %i0 + wr %g3, 0x0, %asi + membar #Sync + ret + restore + + .align 32 + .globl NGclear_page + .globl NGclear_user_page +NGclear_page: /* %o0=dest */ +NGclear_user_page: /* %o0=dest, %o1=vaddr */ + rd %asi, %g3 + wr %g0, ASI_BLK_INIT_QUAD_LDD_P, %asi + set PAGE_SIZE, %g7 + +1: stxa %g0, [%o0 + 0x00] %asi + stxa %g0, [%o0 + 0x08] %asi + stxa %g0, [%o0 + 0x10] %asi + stxa %g0, [%o0 + 0x18] %asi + stxa %g0, [%o0 + 0x20] %asi + stxa %g0, [%o0 + 0x28] %asi + stxa %g0, [%o0 + 0x30] %asi + stxa %g0, [%o0 + 0x38] %asi + stxa %g0, [%o0 + 0x40] %asi + stxa %g0, [%o0 + 0x48] %asi + stxa %g0, [%o0 + 0x50] %asi + stxa %g0, [%o0 + 0x58] %asi + stxa %g0, [%o0 + 0x60] %asi + stxa %g0, [%o0 + 0x68] %asi + stxa %g0, [%o0 + 0x70] %asi + stxa %g0, [%o0 + 0x78] %asi + stxa %g0, [%o0 + 0x80] %asi + stxa %g0, [%o0 + 0x88] %asi + stxa %g0, [%o0 + 0x90] %asi + stxa %g0, [%o0 + 0x98] %asi + stxa %g0, [%o0 + 0xa0] %asi + stxa %g0, [%o0 + 0xa8] %asi + stxa %g0, [%o0 + 0xb0] %asi + stxa %g0, [%o0 + 0xb8] %asi + stxa %g0, [%o0 + 0xc0] %asi + stxa %g0, [%o0 + 0xc8] %asi + stxa %g0, [%o0 + 0xd0] %asi + stxa %g0, [%o0 + 0xd8] %asi + stxa %g0, [%o0 + 0xe0] %asi + stxa %g0, [%o0 + 0xe8] %asi + stxa %g0, [%o0 + 0xf0] %asi + stxa %g0, [%o0 + 0xf8] %asi + subcc %g7, 256, %g7 + bne,pt %xcc, 1b + add %o0, 256, %o0 + wr %g3, 0x0, %asi + membar #Sync + retl + nop + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara_patch_pageops + .type niagara_patch_pageops,#function +niagara_patch_pageops: + NG_DO_PATCH(copy_user_page, NGcopy_user_page) + NG_DO_PATCH(_clear_page, NGclear_page) + NG_DO_PATCH(clear_user_page, NGclear_user_page) + retl + nop + .size niagara_patch_pageops,.-niagara_patch_pageops diff --git a/arch/sparc/lib/NGpatch.S b/arch/sparc/lib/NGpatch.S new file mode 100644 index 000000000..e9f843f10 --- /dev/null +++ b/arch/sparc/lib/NGpatch.S @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* NGpatch.S: Patch Ultra-I routines with Niagara variant. + * + * Copyright (C) 2006 David S. Miller + */ + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define NG_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl niagara_patch_copyops + .type niagara_patch_copyops,#function +niagara_patch_copyops: + NG_DO_PATCH(memcpy, NGmemcpy) + NG_DO_PATCH(raw_copy_from_user, NGcopy_from_user) + NG_DO_PATCH(raw_copy_to_user, NGcopy_to_user) + retl + nop + .size niagara_patch_copyops,.-niagara_patch_copyops diff --git a/arch/sparc/lib/PeeCeeI.c b/arch/sparc/lib/PeeCeeI.c new file mode 100644 index 000000000..cde4c9a51 --- /dev/null +++ b/arch/sparc/lib/PeeCeeI.c @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PeeCeeI.c: The emerging standard... + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#include + +#include +#include + +void outsb(unsigned long __addr, const void *src, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + const u8 *p = src; + + while (count--) + __raw_writeb(*p++, addr); +} +EXPORT_SYMBOL(outsb); + +void outsw(unsigned long __addr, const void *src, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + + while (count--) { + __raw_writew(*(u16 *)src, addr); + src += sizeof(u16); + } +} +EXPORT_SYMBOL(outsw); + +void outsl(unsigned long __addr, const void *src, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + u32 l, l2; + + if (!count) + return; + + switch (((unsigned long)src) & 0x3) { + case 0x0: + /* src is naturally aligned */ + while (count--) { + __raw_writel(*(u32 *)src, addr); + src += sizeof(u32); + } + break; + case 0x2: + /* 2-byte alignment */ + while (count--) { + l = (*(u16 *)src) << 16; + l |= *(u16 *)(src + sizeof(u16)); + __raw_writel(l, addr); + src += sizeof(u32); + } + break; + case 0x1: + /* Hold three bytes in l each time, grab a byte from l2 */ + l = (*(u8 *)src) << 24; + l |= (*(u16 *)(src + sizeof(u8))) << 8; + src += sizeof(u8) + sizeof(u16); + while (count--) { + l2 = *(u32 *)src; + l |= (l2 >> 24); + __raw_writel(l, addr); + l = l2 << 8; + src += sizeof(u32); + } + break; + case 0x3: + /* Hold a byte in l each time, grab 3 bytes from l2 */ + l = (*(u8 *)src) << 24; + src += sizeof(u8); + while (count--) { + l2 = *(u32 *)src; + l |= (l2 >> 8); + __raw_writel(l, addr); + l = l2 << 24; + src += sizeof(u32); + } + break; + } +} +EXPORT_SYMBOL(outsl); + +void insb(unsigned long __addr, void *dst, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + + if (count) { + u32 *pi; + u8 *pb = dst; + + while ((((unsigned long)pb) & 0x3) && count--) + *pb++ = __raw_readb(addr); + pi = (u32 *)pb; + while (count >= 4) { + u32 w; + + w = (__raw_readb(addr) << 24); + w |= (__raw_readb(addr) << 16); + w |= (__raw_readb(addr) << 8); + w |= (__raw_readb(addr) << 0); + *pi++ = w; + count -= 4; + } + pb = (u8 *)pi; + while (count--) + *pb++ = __raw_readb(addr); + } +} +EXPORT_SYMBOL(insb); + +void insw(unsigned long __addr, void *dst, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + + if (count) { + u16 *ps = dst; + u32 *pi; + + if (((unsigned long)ps) & 0x2) { + *ps++ = __raw_readw(addr); + count--; + } + pi = (u32 *)ps; + while (count >= 2) { + u32 w; + + w = __raw_readw(addr) << 16; + w |= __raw_readw(addr) << 0; + *pi++ = w; + count -= 2; + } + ps = (u16 *)pi; + if (count) + *ps = __raw_readw(addr); + } +} +EXPORT_SYMBOL(insw); + +void insl(unsigned long __addr, void *dst, unsigned long count) +{ + void __iomem *addr = (void __iomem *) __addr; + + if (count) { + if ((((unsigned long)dst) & 0x3) == 0) { + u32 *pi = dst; + while (count--) + *pi++ = __raw_readl(addr); + } else { + u32 l = 0, l2, *pi; + u16 *ps; + u8 *pb; + + switch (((unsigned long)dst) & 3) { + case 0x2: + ps = dst; + count -= 1; + l = __raw_readl(addr); + *ps++ = l; + pi = (u32 *)ps; + while (count--) { + l2 = __raw_readl(addr); + *pi++ = (l << 16) | (l2 >> 16); + l = l2; + } + ps = (u16 *)pi; + *ps = l; + break; + + case 0x1: + pb = dst; + count -= 1; + l = __raw_readl(addr); + *pb++ = l >> 24; + ps = (u16 *)pb; + *ps++ = ((l >> 8) & 0xffff); + pi = (u32 *)ps; + while (count--) { + l2 = __raw_readl(addr); + *pi++ = (l << 24) | (l2 >> 8); + l = l2; + } + pb = (u8 *)pi; + *pb = l; + break; + + case 0x3: + pb = (u8 *)dst; + count -= 1; + l = __raw_readl(addr); + *pb++ = l >> 24; + pi = (u32 *)pb; + while (count--) { + l2 = __raw_readl(addr); + *pi++ = (l << 8) | (l2 >> 24); + l = l2; + } + ps = (u16 *)pi; + *ps++ = ((l >> 8) & 0xffff); + pb = (u8 *)ps; + *pb = l; + break; + } + } + } +} +EXPORT_SYMBOL(insl); + diff --git a/arch/sparc/lib/U1copy_from_user.S b/arch/sparc/lib/U1copy_from_user.S new file mode 100644 index 000000000..bf08d1c78 --- /dev/null +++ b/arch/sparc/lib/U1copy_from_user.S @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* U1copy_from_user.S: UltraSparc-I/II/IIi/IIe optimized copy from userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#define EX_LD(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_LD_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define FUNC_NAME raw_copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_AIUS, dest +#define EX_RETVAL(x) 0 + + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop; \ + +#include "U1memcpy.S" diff --git a/arch/sparc/lib/U1copy_to_user.S b/arch/sparc/lib/U1copy_to_user.S new file mode 100644 index 000000000..15169851e --- /dev/null +++ b/arch/sparc/lib/U1copy_to_user.S @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* U1copy_to_user.S: UltraSparc-I/II/IIi/IIe optimized copy to userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#define EX_ST(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_ST_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define FUNC_NAME raw_copy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS +#define EX_RETVAL(x) 0 + + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop; \ + +#include "U1memcpy.S" diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S new file mode 100644 index 000000000..154fbd354 --- /dev/null +++ b/arch/sparc/lib/U1memcpy.S @@ -0,0 +1,686 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy. + * + * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com) + * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#ifdef __KERNEL__ +#include +#include +#include +#include +#define GLOBAL_SPARE g7 +#else +#define GLOBAL_SPARE g5 +#define ASI_BLK_P 0xf0 +#define FPRS_FEF 0x04 +#ifdef MEMCPY_DEBUG +#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ + clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0; +#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#else +#define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs +#define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#endif +#endif + +#ifndef EX_LD +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x +#endif + +#ifndef EX_ST +#define EX_ST(x,y) x +#endif +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef LOAD_BLK +#define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef STORE_BLK +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME memcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + +#define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9) \ + faligndata %f1, %f2, %f48; \ + faligndata %f2, %f3, %f50; \ + faligndata %f3, %f4, %f52; \ + faligndata %f4, %f5, %f54; \ + faligndata %f5, %f6, %f56; \ + faligndata %f6, %f7, %f58; \ + faligndata %f7, %f8, %f60; \ + faligndata %f8, %f9, %f62; + +#define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, jmptgt) \ + EX_LD_FP(LOAD_BLK(%src, %fdest), U1_gs_80_fp); \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ + add %src, 0x40, %src; \ + subcc %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE; \ + be,pn %xcc, jmptgt; \ + add %dest, 0x40, %dest; \ + +#define LOOP_CHUNK1(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f0, f48, branch_dest) +#define LOOP_CHUNK2(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f16, f48, branch_dest) +#define LOOP_CHUNK3(src, dest, branch_dest) \ + MAIN_LOOP_CHUNK(src, dest, f32, f48, branch_dest) + +#define DO_SYNC membar #Sync; +#define STORE_SYNC(dest, fsrc) \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_80_fp); \ + add %dest, 0x40, %dest; \ + DO_SYNC + +#define STORE_JUMP(dest, fsrc, target) \ + EX_ST_FP(STORE_BLK(%fsrc, %dest), U1_gs_40_fp); \ + add %dest, 0x40, %dest; \ + ba,pt %xcc, target; \ + nop; + +#define FINISH_VISCHUNK(dest, f0, f1) \ + subcc %g3, 8, %g3; \ + bl,pn %xcc, 95f; \ + faligndata %f0, %f1, %f48; \ + EX_ST_FP(STORE(std, %f48, %dest), U1_g3_8_fp); \ + add %dest, 8, %dest; + +#define UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ + subcc %g3, 8, %g3; \ + bl,pn %xcc, 95f; \ + fsrc2 %f0, %f1; + +#define UNEVEN_VISCHUNK(dest, f0, f1) \ + UNEVEN_VISCHUNK_LAST(dest, f0, f1) \ + ba,a,pt %xcc, 93f; + + .register %g2,#scratch + .register %g3,#scratch + + .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +ENTRY(U1_g1_1_fp) + VISExitHalf + add %g1, 1, %g1 + add %g1, %g2, %g1 + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_1_fp) +ENTRY(U1_g2_0_fp) + VISExitHalf + retl + add %g2, %o2, %o0 +ENDPROC(U1_g2_0_fp) +ENTRY(U1_g2_8_fp) + VISExitHalf + add %g2, 8, %g2 + retl + add %g2, %o2, %o0 +ENDPROC(U1_g2_8_fp) +ENTRY(U1_gs_0_fp) + VISExitHalf + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_0_fp) +ENTRY(U1_gs_80_fp) + VISExitHalf + add %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_80_fp) +ENTRY(U1_gs_40_fp) + VISExitHalf + add %GLOBAL_SPARE, 0x40, %GLOBAL_SPARE + add %GLOBAL_SPARE, %g3, %o0 + retl + add %o0, %o2, %o0 +ENDPROC(U1_gs_40_fp) +ENTRY(U1_g3_8_fp) + VISExitHalf + add %g3, 8, %g3 + retl + add %g3, %o2, %o0 +ENDPROC(U1_g3_8_fp) +ENTRY(U1_g3_16_fp) + VISExitHalf + add %g3, 16, %g3 + retl + add %g3, %o2, %o0 +ENDPROC(U1_g3_16_fp) +ENTRY(U1_o2_0_fp) + VISExitHalf + retl + mov %o2, %o0 +ENDPROC(U1_o2_0_fp) +ENTRY(U1_o2_1_fp) + VISExitHalf + retl + add %o2, 1, %o0 +ENDPROC(U1_o2_1_fp) +ENTRY(U1_gs_0) + VISExitHalf + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_0) +ENTRY(U1_gs_8) + VISExitHalf + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, 0x8, %o0 +ENDPROC(U1_gs_8) +ENTRY(U1_gs_10) + VISExitHalf + add %GLOBAL_SPARE, %o2, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, 0x10, %o0 +ENDPROC(U1_gs_10) +ENTRY(U1_o2_0) + retl + mov %o2, %o0 +ENDPROC(U1_o2_0) +ENTRY(U1_o2_8) + retl + add %o2, 8, %o0 +ENDPROC(U1_o2_8) +ENTRY(U1_o2_4) + retl + add %o2, 4, %o0 +ENDPROC(U1_o2_4) +ENTRY(U1_o2_1) + retl + add %o2, 1, %o0 +ENDPROC(U1_o2_1) +ENTRY(U1_g1_0) + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_0) +ENTRY(U1_g1_1) + add %g1, 1, %g1 + retl + add %g1, %o2, %o0 +ENDPROC(U1_g1_1) +ENTRY(U1_gs_0_o2_adj) + and %o2, 7, %o2 + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_0_o2_adj) +ENTRY(U1_gs_8_o2_adj) + and %o2, 7, %o2 + add %GLOBAL_SPARE, 8, %GLOBAL_SPARE + retl + add %GLOBAL_SPARE, %o2, %o0 +ENDPROC(U1_gs_8_o2_adj) +#endif + + .align 64 + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ + srlx %o2, 31, %g2 + cmp %g2, 0 + tne %xcc, 5 + PREAMBLE + mov %o0, %o4 + cmp %o2, 0 + be,pn %XCC, 85f + or %o0, %o1, %o3 + cmp %o2, 16 + blu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + cmp %o2, (5 * 64) + blu,pt %XCC, 70f + andcc %o3, 0x7, %g0 + + /* Clobbers o5/g1/g2/g3/g7/icc/xcc. */ + VISEntry + + /* Is 'dst' already aligned on an 64-byte boundary? */ + andcc %o0, 0x3f, %g2 + be,pt %XCC, 2f + + /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number + * of bytes to copy to make 'dst' 64-byte aligned. We pre- + * subtract this from 'len'. + */ + sub %o0, %o1, %GLOBAL_SPARE + sub %g2, 0x40, %g2 + sub %g0, %g2, %g2 + sub %o2, %g2, %o2 + andcc %g2, 0x7, %g1 + be,pt %icc, 2f + and %g2, 0x38, %g2 + +1: subcc %g1, 0x1, %g1 + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U1_g1_1_fp) + EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE), U1_g1_1_fp) + bgu,pt %XCC, 1b + add %o1, 0x1, %o1 + + add %o1, %GLOBAL_SPARE, %o0 + +2: cmp %g2, 0x0 + and %o1, 0x7, %g1 + be,pt %icc, 3f + alignaddr %o1, %g0, %o1 + + EX_LD_FP(LOAD(ldd, %o1, %f4), U1_g2_0_fp) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U1_g2_0_fp) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f4, %f6, %f0 + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) + be,pn %icc, 3f + add %o0, 0x8, %o0 + + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U1_g2_0_fp) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f6, %f4, %f0 + EX_ST_FP(STORE(std, %f0, %o0), U1_g2_8_fp) + bne,pt %icc, 1b + add %o0, 0x8, %o0 + + /* Destination is 64-byte aligned. */ +3: + membar #LoadStore | #StoreStore | #StoreLoad + + subcc %o2, 0x40, %GLOBAL_SPARE + add %o1, %g1, %g1 + andncc %GLOBAL_SPARE, (0x40 - 1), %GLOBAL_SPARE + srl %g1, 3, %g2 + sub %o2, %GLOBAL_SPARE, %g3 + andn %o1, (0x40 - 1), %o1 + and %g2, 7, %g2 + andncc %g3, 0x7, %g3 + fsrc2 %f0, %f2 + sub %g3, 0x8, %g3 + sub %o2, %GLOBAL_SPARE, %o2 + + add %g1, %GLOBAL_SPARE, %g1 + subcc %o2, %g3, %o2 + + EX_LD_FP(LOAD_BLK(%o1, %f0), U1_gs_0_fp) + add %o1, 0x40, %o1 + add %g1, %g3, %g1 + EX_LD_FP(LOAD_BLK(%o1, %f16), U1_gs_0_fp) + add %o1, 0x40, %o1 + sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE + EX_LD_FP(LOAD_BLK(%o1, %f32), U1_gs_80_fp) + add %o1, 0x40, %o1 + + /* There are 8 instances of the unrolled loop, + * one for each possible alignment of the + * source buffer. Each loop instance is 452 + * bytes. + */ + sll %g2, 3, %o3 + sub %o3, %g2, %o3 + sllx %o3, 4, %o3 + add %o3, %g2, %o3 + sllx %o3, 2, %g2 +1: rd %pc, %o3 + add %o3, %lo(1f - 1b), %o3 + jmpl %o3 + %g2, %g0 + nop + + .align 64 +1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) + LOOP_CHUNK1(o1, o0, 1f) + FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) + LOOP_CHUNK2(o1, o0, 2f) + FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) + LOOP_CHUNK3(o1, o0, 3f) + ba,pt %xcc, 1b+4 + faligndata %f0, %f2, %f48 +1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) + STORE_SYNC(o0, f48) + FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) + STORE_JUMP(o0, f48, 40f) +2: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) + STORE_SYNC(o0, f48) + FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) + STORE_JUMP(o0, f48, 48f) +3: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) + STORE_SYNC(o0, f48) + FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) + STORE_JUMP(o0, f48, 56f) + +1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) + LOOP_CHUNK1(o1, o0, 1f) + FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) + LOOP_CHUNK2(o1, o0, 2f) + FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) + LOOP_CHUNK3(o1, o0, 3f) + ba,pt %xcc, 1b+4 + faligndata %f2, %f4, %f48 +1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) + STORE_SYNC(o0, f48) + FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) + STORE_JUMP(o0, f48, 41f) +2: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) + STORE_SYNC(o0, f48) + FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) + STORE_JUMP(o0, f48, 49f) +3: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) + STORE_SYNC(o0, f48) + FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) + STORE_JUMP(o0, f48, 57f) + +1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) + LOOP_CHUNK1(o1, o0, 1f) + FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) + LOOP_CHUNK2(o1, o0, 2f) + FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) + LOOP_CHUNK3(o1, o0, 3f) + ba,pt %xcc, 1b+4 + faligndata %f4, %f6, %f48 +1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) + STORE_SYNC(o0, f48) + FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) + STORE_JUMP(o0, f48, 42f) +2: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) + STORE_SYNC(o0, f48) + FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) + STORE_JUMP(o0, f48, 50f) +3: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) + STORE_SYNC(o0, f48) + FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) + STORE_JUMP(o0, f48, 58f) + +1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) + LOOP_CHUNK1(o1, o0, 1f) + FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) + LOOP_CHUNK2(o1, o0, 2f) + FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) + LOOP_CHUNK3(o1, o0, 3f) + ba,pt %xcc, 1b+4 + faligndata %f6, %f8, %f48 +1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) + STORE_SYNC(o0, f48) + FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) + STORE_JUMP(o0, f48, 43f) +2: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) + STORE_SYNC(o0, f48) + FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) + STORE_JUMP(o0, f48, 51f) +3: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) + STORE_SYNC(o0, f48) + FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) + STORE_JUMP(o0, f48, 59f) + +1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) + LOOP_CHUNK1(o1, o0, 1f) + FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) + LOOP_CHUNK2(o1, o0, 2f) + FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) + LOOP_CHUNK3(o1, o0, 3f) + ba,pt %xcc, 1b+4 + faligndata %f8, %f10, %f48 +1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) + STORE_SYNC(o0, f48) + FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) + STORE_JUMP(o0, f48, 44f) +2: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) + STORE_SYNC(o0, f48) + FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) + STORE_JUMP(o0, f48, 52f) +3: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) + STORE_SYNC(o0, f48) + FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) + STORE_JUMP(o0, f48, 60f) + +1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) + LOOP_CHUNK1(o1, o0, 1f) + FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) + LOOP_CHUNK2(o1, o0, 2f) + FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) + LOOP_CHUNK3(o1, o0, 3f) + ba,pt %xcc, 1b+4 + faligndata %f10, %f12, %f48 +1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) + STORE_SYNC(o0, f48) + FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) + STORE_JUMP(o0, f48, 45f) +2: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) + STORE_SYNC(o0, f48) + FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) + STORE_JUMP(o0, f48, 53f) +3: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) + STORE_SYNC(o0, f48) + FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) + STORE_JUMP(o0, f48, 61f) + +1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) + LOOP_CHUNK1(o1, o0, 1f) + FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) + LOOP_CHUNK2(o1, o0, 2f) + FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) + LOOP_CHUNK3(o1, o0, 3f) + ba,pt %xcc, 1b+4 + faligndata %f12, %f14, %f48 +1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) + STORE_SYNC(o0, f48) + FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) + STORE_JUMP(o0, f48, 46f) +2: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) + STORE_SYNC(o0, f48) + FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) + STORE_JUMP(o0, f48, 54f) +3: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) + STORE_SYNC(o0, f48) + FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) + STORE_JUMP(o0, f48, 62f) + +1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) + LOOP_CHUNK1(o1, o0, 1f) + FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) + LOOP_CHUNK2(o1, o0, 2f) + FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) + LOOP_CHUNK3(o1, o0, 3f) + ba,pt %xcc, 1b+4 + faligndata %f14, %f16, %f48 +1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) + STORE_SYNC(o0, f48) + FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) + STORE_JUMP(o0, f48, 47f) +2: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) + STORE_SYNC(o0, f48) + FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) + STORE_JUMP(o0, f48, 55f) +3: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) + STORE_SYNC(o0, f48) + FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) + STORE_JUMP(o0, f48, 63f) + +40: FINISH_VISCHUNK(o0, f0, f2) +41: FINISH_VISCHUNK(o0, f2, f4) +42: FINISH_VISCHUNK(o0, f4, f6) +43: FINISH_VISCHUNK(o0, f6, f8) +44: FINISH_VISCHUNK(o0, f8, f10) +45: FINISH_VISCHUNK(o0, f10, f12) +46: FINISH_VISCHUNK(o0, f12, f14) +47: UNEVEN_VISCHUNK(o0, f14, f0) +48: FINISH_VISCHUNK(o0, f16, f18) +49: FINISH_VISCHUNK(o0, f18, f20) +50: FINISH_VISCHUNK(o0, f20, f22) +51: FINISH_VISCHUNK(o0, f22, f24) +52: FINISH_VISCHUNK(o0, f24, f26) +53: FINISH_VISCHUNK(o0, f26, f28) +54: FINISH_VISCHUNK(o0, f28, f30) +55: UNEVEN_VISCHUNK(o0, f30, f0) +56: FINISH_VISCHUNK(o0, f32, f34) +57: FINISH_VISCHUNK(o0, f34, f36) +58: FINISH_VISCHUNK(o0, f36, f38) +59: FINISH_VISCHUNK(o0, f38, f40) +60: FINISH_VISCHUNK(o0, f40, f42) +61: FINISH_VISCHUNK(o0, f42, f44) +62: FINISH_VISCHUNK(o0, f44, f46) +63: UNEVEN_VISCHUNK_LAST(o0, f46, f0) + +93: EX_LD_FP(LOAD(ldd, %o1, %f2), U1_g3_8_fp) + add %o1, 8, %o1 + subcc %g3, 8, %g3 + faligndata %f0, %f2, %f8 + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_16_fp) + bl,pn %xcc, 95f + add %o0, 8, %o0 + EX_LD_FP(LOAD(ldd, %o1, %f0), U1_g3_8_fp) + add %o1, 8, %o1 + subcc %g3, 8, %g3 + faligndata %f2, %f0, %f8 + EX_ST_FP(STORE(std, %f8, %o0), U1_g3_16_fp) + bge,pt %xcc, 93b + add %o0, 8, %o0 + +95: brz,pt %o2, 2f + mov %g1, %o1 + +1: EX_LD_FP(LOAD(ldub, %o1, %o3), U1_o2_0_fp) + add %o1, 1, %o1 + subcc %o2, 1, %o2 + EX_ST_FP(STORE(stb, %o3, %o0), U1_o2_1_fp) + bne,pt %xcc, 1b + add %o0, 1, %o0 + +2: membar #StoreLoad | #StoreStore + VISExit + retl + mov EX_RETVAL(%o4), %o0 + + .align 64 +70: /* 16 < len <= (5 * 64) */ + bne,pn %XCC, 75f + sub %o0, %o1, %o3 + +72: andn %o2, 0xf, %GLOBAL_SPARE + and %o2, 0xf, %o2 +1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U1_gs_0) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U1_gs_0) + subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_gs_10) + add %o1, 0x8, %o1 + EX_ST(STORE(stx, %g1, %o1 + %o3), U1_gs_8) + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 +73: andcc %o2, 0x8, %g0 + be,pt %XCC, 1f + nop + EX_LD(LOAD(ldx, %o1, %o5), U1_o2_0) + sub %o2, 0x8, %o2 + EX_ST(STORE(stx, %o5, %o1 + %o3), U1_o2_8) + add %o1, 0x8, %o1 +1: andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + EX_LD(LOAD(lduw, %o1, %o5), U1_o2_0) + sub %o2, 0x4, %o2 + EX_ST(STORE(stw, %o5, %o1 + %o3), U1_o2_4) + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +75: andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %o2, %g1, %o2 + +1: EX_LD(LOAD(ldub, %o1, %o5), U1_g1_0) + subcc %g1, 1, %g1 + EX_ST(STORE(stb, %o5, %o1 + %o3), U1_g1_1) + bgu,pt %icc, 1b + add %o1, 1, %o1 + +2: add %o1, %o3, %o0 + andcc %o1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %o2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %xcc, 73b + +8: mov 64, %o3 + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1, %g2), U1_o2_0) + sub %o3, %g1, %o3 + andn %o2, 0x7, %GLOBAL_SPARE + sllx %g2, %g1, %g2 +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U1_gs_0_o2_adj) + subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE + add %o1, 0x8, %o1 + srlx %g3, %o3, %o5 + or %o5, %g2, %o5 + EX_ST(STORE(stx, %o5, %o0), U1_gs_8_o2_adj) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %o2, 0x7, %o2 + be,pn %icc, 85f + add %o1, %g1, %o1 + ba,pt %xcc, 90f + sub %o0, %o1, %o3 + + .align 64 +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +1: EX_LD(LOAD(lduw, %o1, %g1), U1_o2_0) + subcc %o2, 4, %o2 + EX_ST(STORE(stw, %g1, %o1 + %o3), U1_o2_4) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +85: retl + mov EX_RETVAL(%o4), %o0 + + .align 32 +90: EX_LD(LOAD(ldub, %o1, %g1), U1_o2_0) + subcc %o2, 1, %o2 + EX_ST(STORE(stb, %g1, %o1 + %o3), U1_o2_1) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov EX_RETVAL(%o4), %o0 + + .size FUNC_NAME, .-FUNC_NAME +EXPORT_SYMBOL(FUNC_NAME) diff --git a/arch/sparc/lib/U3copy_from_user.S b/arch/sparc/lib/U3copy_from_user.S new file mode 100644 index 000000000..9c891e9ed --- /dev/null +++ b/arch/sparc/lib/U3copy_from_user.S @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* U3copy_from_user.S: UltraSparc-III optimized copy from userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#define EX_LD(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_LD_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + +#define FUNC_NAME U3copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest +#define EX_RETVAL(x) 0 + +#include "U3memcpy.S" diff --git a/arch/sparc/lib/U3copy_to_user.S b/arch/sparc/lib/U3copy_to_user.S new file mode 100644 index 000000000..da4246082 --- /dev/null +++ b/arch/sparc/lib/U3copy_to_user.S @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* U3copy_to_user.S: UltraSparc-III optimized copy to userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#define EX_ST(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y; \ + .text; \ + .align 4; + +#define EX_ST_FP(x,y) \ +98: x; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, y##_fp; \ + .text; \ + .align 4; + +#define FUNC_NAME U3copy_to_user +#define STORE(type,src,addr) type##a src, [addr] ASI_AIUS +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_AIUS +#define EX_RETVAL(x) 0 + + /* Writing to %asi is _expensive_ so we hardcode it. + * Reading %asi to check for KERNEL_DS is comparatively + * cheap. + */ +#define PREAMBLE \ + rd %asi, %g1; \ + cmp %g1, ASI_AIUS; \ + bne,pn %icc, raw_copy_in_user; \ + nop; \ + +#include "U3memcpy.S" diff --git a/arch/sparc/lib/U3memcpy.S b/arch/sparc/lib/U3memcpy.S new file mode 100644 index 000000000..bace3a18f --- /dev/null +++ b/arch/sparc/lib/U3memcpy.S @@ -0,0 +1,520 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* U3memcpy.S: UltraSparc-III optimized memcpy. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#ifdef __KERNEL__ +#include +#include +#include +#define GLOBAL_SPARE %g7 +#else +#define ASI_BLK_P 0xf0 +#define FPRS_FEF 0x04 +#ifdef MEMCPY_DEBUG +#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ + clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0; +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#else +#define VISEntryHalf rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs +#define VISExitHalf and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs +#endif +#define GLOBAL_SPARE %g5 +#endif + +#ifndef EX_LD +#define EX_LD(x,y) x +#endif +#ifndef EX_LD_FP +#define EX_LD_FP(x,y) x +#endif + +#ifndef EX_ST +#define EX_ST(x,y) x +#endif +#ifndef EX_ST_FP +#define EX_ST_FP(x,y) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef STORE_BLK +#define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME U3memcpy +#endif + +#ifndef PREAMBLE +#define PREAMBLE +#endif + +#ifndef XCC +#define XCC xcc +#endif + + .register %g2,#scratch + .register %g3,#scratch + + /* Special/non-trivial issues of this code: + * + * 1) %o5 is preserved from VISEntryHalf to VISExitHalf + * 2) Only low 32 FPU registers are used so that only the + * lower half of the FPU register set is dirtied by this + * code. This is especially important in the kernel. + * 3) This code never prefetches cachelines past the end + * of the source buffer. + */ + + .text +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +__restore_fp: + VISExitHalf + retl + nop +ENTRY(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) + add %g1, 1, %g1 + add %g2, %g1, %g2 + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_plus_g1_plus_1_fp) +ENTRY(U3_retl_o2_plus_g2_fp) + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_fp) +ENTRY(U3_retl_o2_plus_g2_plus_8_fp) + add %g2, 8, %g2 + ba,pt %xcc, __restore_fp + add %o2, %g2, %o0 +ENDPROC(U3_retl_o2_plus_g2_plus_8_fp) +ENTRY(U3_retl_o2) + retl + mov %o2, %o0 +ENDPROC(U3_retl_o2) +ENTRY(U3_retl_o2_plus_1) + retl + add %o2, 1, %o0 +ENDPROC(U3_retl_o2_plus_1) +ENTRY(U3_retl_o2_plus_4) + retl + add %o2, 4, %o0 +ENDPROC(U3_retl_o2_plus_4) +ENTRY(U3_retl_o2_plus_8) + retl + add %o2, 8, %o0 +ENDPROC(U3_retl_o2_plus_8) +ENTRY(U3_retl_o2_plus_g1_plus_1) + add %g1, 1, %g1 + retl + add %o2, %g1, %o0 +ENDPROC(U3_retl_o2_plus_g1_plus_1) +ENTRY(U3_retl_o2_fp) + ba,pt %xcc, __restore_fp + mov %o2, %o0 +ENDPROC(U3_retl_o2_fp) +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) + sll %o3, 6, %o3 + add %o3, 0x80, %o3 + ba,pt %xcc, __restore_fp + add %o2, %o3, %o0 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x80_fp) +ENTRY(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) + sll %o3, 6, %o3 + add %o3, 0x40, %o3 + ba,pt %xcc, __restore_fp + add %o2, %o3, %o0 +ENDPROC(U3_retl_o2_plus_o3_sll_6_plus_0x40_fp) +ENTRY(U3_retl_o2_plus_GS_plus_0x10) + add GLOBAL_SPARE, 0x10, GLOBAL_SPARE + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_plus_GS_plus_0x10) +ENTRY(U3_retl_o2_plus_GS_plus_0x08) + add GLOBAL_SPARE, 0x08, GLOBAL_SPARE + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_plus_GS_plus_0x08) +ENTRY(U3_retl_o2_and_7_plus_GS) + and %o2, 7, %o2 + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_and_7_plus_GS) +ENTRY(U3_retl_o2_and_7_plus_GS_plus_8) + add GLOBAL_SPARE, 8, GLOBAL_SPARE + and %o2, 7, %o2 + retl + add %o2, GLOBAL_SPARE, %o0 +ENDPROC(U3_retl_o2_and_7_plus_GS_plus_8) +#endif + + .align 64 + + /* The cheetah's flexible spine, oversized liver, enlarged heart, + * slender muscular body, and claws make it the swiftest hunter + * in Africa and the fastest animal on land. Can reach speeds + * of up to 2.4GB per second. + */ + + .globl FUNC_NAME + .type FUNC_NAME,#function +FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ + srlx %o2, 31, %g2 + cmp %g2, 0 + + /* software trap 5 "Range Check" if dst >= 0x80000000 */ + tne %xcc, 5 + PREAMBLE + mov %o0, %o4 + + /* if len == 0 */ + cmp %o2, 0 + be,pn %XCC, end_return + or %o0, %o1, %o3 + + /* if len < 16 */ + cmp %o2, 16 + blu,a,pn %XCC, less_than_16 + or %o3, %o2, %o3 + + /* if len < 192 */ + cmp %o2, (3 * 64) + blu,pt %XCC, less_than_192 + andcc %o3, 0x7, %g0 + + /* Clobbers o5/g1/g2/g3/g7/icc/xcc. We must preserve + * o5 from here until we hit VISExitHalf. + */ + VISEntryHalf + + /* Is 'dst' already aligned on an 64-byte boundary? */ + andcc %o0, 0x3f, %g2 + be,pt %XCC, 2f + + /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number + * of bytes to copy to make 'dst' 64-byte aligned. We pre- + * subtract this from 'len'. + */ + sub %o0, %o1, GLOBAL_SPARE + sub %g2, 0x40, %g2 + sub %g0, %g2, %g2 + sub %o2, %g2, %o2 + andcc %g2, 0x7, %g1 + be,pt %icc, 2f + and %g2, 0x38, %g2 + +1: subcc %g1, 0x1, %g1 + EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3), U3_retl_o2_plus_g2_plus_g1_plus_1) + EX_ST_FP(STORE(stb, %o3, %o1 + GLOBAL_SPARE), U3_retl_o2_plus_g2_plus_g1_plus_1) + bgu,pt %XCC, 1b + add %o1, 0x1, %o1 + + add %o1, GLOBAL_SPARE, %o0 + +2: cmp %g2, 0x0 + and %o1, 0x7, %g1 + be,pt %icc, 3f + alignaddr %o1, %g0, %o1 + + EX_LD_FP(LOAD(ldd, %o1, %f4), U3_retl_o2_plus_g2) +1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6), U3_retl_o2_plus_g2) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f4, %f6, %f0 + EX_ST_FP(STORE(std, %f0, %o0), U3_retl_o2_plus_g2_plus_8) + be,pn %icc, 3f + add %o0, 0x8, %o0 + + EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4), U3_retl_o2_plus_g2) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f6, %f4, %f2 + EX_ST_FP(STORE(std, %f2, %o0), U3_retl_o2_plus_g2_plus_8) + bne,pt %icc, 1b + add %o0, 0x8, %o0 + +3: LOAD(prefetch, %o1 + 0x000, #one_read) + LOAD(prefetch, %o1 + 0x040, #one_read) + andn %o2, (0x40 - 1), GLOBAL_SPARE + LOAD(prefetch, %o1 + 0x080, #one_read) + LOAD(prefetch, %o1 + 0x0c0, #one_read) + LOAD(prefetch, %o1 + 0x100, #one_read) + EX_LD_FP(LOAD(ldd, %o1 + 0x000, %f0), U3_retl_o2) + LOAD(prefetch, %o1 + 0x140, #one_read) + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2) + LOAD(prefetch, %o1 + 0x180, #one_read) + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2) + LOAD(prefetch, %o1 + 0x1c0, #one_read) + faligndata %f0, %f2, %f16 + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2) + faligndata %f2, %f4, %f18 + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2) + faligndata %f4, %f6, %f20 + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2) + faligndata %f6, %f8, %f22 + + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2) + faligndata %f8, %f10, %f24 + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2) + faligndata %f10, %f12, %f26 + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2) + + and %o2, 0x3f, %o2 + subcc GLOBAL_SPARE, 0x80, GLOBAL_SPARE + add %o1, 0x40, %o1 + bgu,pt %XCC, 1f + srl GLOBAL_SPARE, 6, %o3 + ba,pt %xcc, 2f + nop + + .align 64 +1: + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) + faligndata %f12, %f14, %f28 + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) + faligndata %f14, %f0, %f30 + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) + faligndata %f0, %f2, %f16 + add %o0, 0x40, %o0 + + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) + faligndata %f2, %f4, %f18 + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) + faligndata %f4, %f6, %f20 + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) + subcc %o3, 0x01, %o3 + faligndata %f6, %f8, %f22 + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x80) + + faligndata %f8, %f10, %f24 + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x80) + LOAD(prefetch, %o1 + 0x1c0, #one_read) + faligndata %f10, %f12, %f26 + bg,pt %XCC, 1b + add %o1, 0x40, %o1 + + /* Finally we copy the last full 64-byte block. */ +2: + EX_LD_FP(LOAD(ldd, %o1 + 0x008, %f2), U3_retl_o2_plus_o3_sll_6_plus_0x80) + faligndata %f12, %f14, %f28 + EX_LD_FP(LOAD(ldd, %o1 + 0x010, %f4), U3_retl_o2_plus_o3_sll_6_plus_0x80) + faligndata %f14, %f0, %f30 + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x80) + EX_LD_FP(LOAD(ldd, %o1 + 0x018, %f6), U3_retl_o2_plus_o3_sll_6_plus_0x40) + faligndata %f0, %f2, %f16 + EX_LD_FP(LOAD(ldd, %o1 + 0x020, %f8), U3_retl_o2_plus_o3_sll_6_plus_0x40) + faligndata %f2, %f4, %f18 + EX_LD_FP(LOAD(ldd, %o1 + 0x028, %f10), U3_retl_o2_plus_o3_sll_6_plus_0x40) + faligndata %f4, %f6, %f20 + EX_LD_FP(LOAD(ldd, %o1 + 0x030, %f12), U3_retl_o2_plus_o3_sll_6_plus_0x40) + faligndata %f6, %f8, %f22 + EX_LD_FP(LOAD(ldd, %o1 + 0x038, %f14), U3_retl_o2_plus_o3_sll_6_plus_0x40) + faligndata %f8, %f10, %f24 + cmp %g1, 0 + be,pt %XCC, 1f + add %o0, 0x40, %o0 + EX_LD_FP(LOAD(ldd, %o1 + 0x040, %f0), U3_retl_o2_plus_o3_sll_6_plus_0x40) +1: faligndata %f10, %f12, %f26 + faligndata %f12, %f14, %f28 + faligndata %f14, %f0, %f30 + EX_ST_FP(STORE_BLK(%f16, %o0), U3_retl_o2_plus_o3_sll_6_plus_0x40) + add %o0, 0x40, %o0 + add %o1, 0x40, %o1 + membar #Sync + + /* Now we copy the (len modulo 64) bytes at the end. + * Note how we borrow the %f0 loaded above. + * + * Also notice how this code is careful not to perform a + * load past the end of the src buffer. + */ + andcc %o2, 0x38, %g2 + be,pn %XCC, 2f + subcc %g2, 0x8, %g2 + be,pn %XCC, 2f + cmp %g1, 0 + + sub %o2, %g2, %o2 + be,a,pt %XCC, 1f + EX_LD_FP(LOAD(ldd, %o1 + 0x00, %f0), U3_retl_o2_plus_g2) + +1: EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f2), U3_retl_o2_plus_g2) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f0, %f2, %f8 + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) + be,pn %XCC, 2f + add %o0, 0x8, %o0 + EX_LD_FP(LOAD(ldd, %o1 + 0x08, %f0), U3_retl_o2_plus_g2) + add %o1, 0x8, %o1 + subcc %g2, 0x8, %g2 + faligndata %f2, %f0, %f8 + EX_ST_FP(STORE(std, %f8, %o0), U3_retl_o2_plus_g2_plus_8) + bne,pn %XCC, 1b + add %o0, 0x8, %o0 + + /* If anything is left, we copy it one byte at a time. + * Note that %g1 is (src & 0x3) saved above before the + * alignaddr was performed. + */ +2: + cmp %o2, 0 + add %o1, %g1, %o1 + VISExitHalf + be,pn %XCC, end_return + sub %o0, %o1, %o3 + + andcc %g1, 0x7, %g0 + bne,pn %icc, 90f + andcc %o2, 0x8, %g0 + be,pt %icc, 1f + nop + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2) + add %o1, 0x8, %o1 + sub %o2, 8, %o2 + +1: andcc %o2, 0x4, %g0 + be,pt %icc, 1f + nop + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2) + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2) + add %o1, 0x4, %o1 + sub %o2, 4, %o2 + +1: andcc %o2, 0x2, %g0 + be,pt %icc, 1f + nop + EX_LD(LOAD(lduh, %o1, %o5), U3_retl_o2) + EX_ST(STORE(sth, %o5, %o1 + %o3), U3_retl_o2) + add %o1, 0x2, %o1 + sub %o2, 2, %o2 + +1: andcc %o2, 0x1, %g0 + be,pt %icc, end_return + nop + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2) + ba,pt %xcc, end_return + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2) + + .align 64 + /* 16 <= len < 192 */ +less_than_192: + bne,pn %XCC, 75f + sub %o0, %o1, %o3 + +72: + andn %o2, 0xf, GLOBAL_SPARE + and %o2, 0xf, %o2 +1: subcc GLOBAL_SPARE, 0x10, GLOBAL_SPARE + EX_LD(LOAD(ldx, %o1 + 0x00, %o5), U3_retl_o2_plus_GS_plus_0x10) + EX_LD(LOAD(ldx, %o1 + 0x08, %g1), U3_retl_o2_plus_GS_plus_0x10) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x10) + add %o1, 0x8, %o1 + EX_ST(STORE(stx, %g1, %o1 + %o3), U3_retl_o2_plus_GS_plus_0x08) + bgu,pt %XCC, 1b + add %o1, 0x8, %o1 +73: andcc %o2, 0x8, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x8, %o2 + EX_LD(LOAD(ldx, %o1, %o5), U3_retl_o2_plus_8) + EX_ST(STORE(stx, %o5, %o1 + %o3), U3_retl_o2_plus_8) + add %o1, 0x8, %o1 +1: andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x4, %o2 + EX_LD(LOAD(lduw, %o1, %o5), U3_retl_o2_plus_4) + EX_ST(STORE(stw, %o5, %o1 + %o3), U3_retl_o2_plus_4) + add %o1, 0x4, %o1 +1: cmp %o2, 0 + be,pt %XCC, end_return + nop + ba,pt %xcc, 90f + nop + +75: + andcc %o0, 0x7, %g1 + sub %g1, 0x8, %g1 + be,pn %icc, 2f + sub %g0, %g1, %g1 + sub %o2, %g1, %o2 + +1: subcc %g1, 1, %g1 + EX_LD(LOAD(ldub, %o1, %o5), U3_retl_o2_plus_g1_plus_1) + EX_ST(STORE(stb, %o5, %o1 + %o3), U3_retl_o2_plus_g1_plus_1) + bgu,pt %icc, 1b + add %o1, 1, %o1 + +2: add %o1, %o3, %o0 + andcc %o1, 0x7, %g1 + bne,pt %icc, 8f + sll %g1, 3, %g1 + + cmp %o2, 16 + bgeu,pt %icc, 72b + nop + ba,a,pt %xcc, 73b + +8: mov 64, %o3 + andn %o1, 0x7, %o1 + EX_LD(LOAD(ldx, %o1, %g2), U3_retl_o2) + sub %o3, %g1, %o3 + andn %o2, 0x7, GLOBAL_SPARE + sllx %g2, %g1, %g2 +1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3), U3_retl_o2_and_7_plus_GS) + subcc GLOBAL_SPARE, 0x8, GLOBAL_SPARE + add %o1, 0x8, %o1 + srlx %g3, %o3, %o5 + or %o5, %g2, %o5 + EX_ST(STORE(stx, %o5, %o0), U3_retl_o2_and_7_plus_GS_plus_8) + add %o0, 0x8, %o0 + bgu,pt %icc, 1b + sllx %g3, %g1, %g2 + + srl %g1, 3, %g1 + andcc %o2, 0x7, %o2 + be,pn %icc, end_return + add %o1, %g1, %o1 + ba,pt %xcc, 90f + sub %o0, %o1, %o3 + + .align 64 + /* 0 < len < 16 */ +less_than_16: + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + sub %o0, %o1, %o3 + +1: + subcc %o2, 4, %o2 + EX_LD(LOAD(lduw, %o1, %g1), U3_retl_o2_plus_4) + EX_ST(STORE(stw, %g1, %o1 + %o3), U3_retl_o2_plus_4) + bgu,pt %XCC, 1b + add %o1, 4, %o1 + +end_return: + retl + mov EX_RETVAL(%o4), %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + EX_LD(LOAD(ldub, %o1, %g1), U3_retl_o2_plus_1) + EX_ST(STORE(stb, %g1, %o1 + %o3), U3_retl_o2_plus_1) + bgu,pt %XCC, 90b + add %o1, 1, %o1 + retl + mov EX_RETVAL(%o4), %o0 + + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/U3patch.S b/arch/sparc/lib/U3patch.S new file mode 100644 index 000000000..9a888088f --- /dev/null +++ b/arch/sparc/lib/U3patch.S @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* U3patch.S: Patch Ultra-I routines with Ultra-III variant. + * + * Copyright (C) 2004 David S. Miller + */ + +#define BRANCH_ALWAYS 0x10680000 +#define NOP 0x01000000 +#define ULTRA3_DO_PATCH(OLD, NEW) \ + sethi %hi(NEW), %g1; \ + or %g1, %lo(NEW), %g1; \ + sethi %hi(OLD), %g2; \ + or %g2, %lo(OLD), %g2; \ + sub %g1, %g2, %g1; \ + sethi %hi(BRANCH_ALWAYS), %g3; \ + sll %g1, 11, %g1; \ + srl %g1, 11 + 2, %g1; \ + or %g3, %lo(BRANCH_ALWAYS), %g3; \ + or %g3, %g1, %g3; \ + stw %g3, [%g2]; \ + sethi %hi(NOP), %g3; \ + or %g3, %lo(NOP), %g3; \ + stw %g3, [%g2 + 0x4]; \ + flush %g2; + + .globl cheetah_patch_copyops + .type cheetah_patch_copyops,#function +cheetah_patch_copyops: + ULTRA3_DO_PATCH(memcpy, U3memcpy) + ULTRA3_DO_PATCH(raw_copy_from_user, U3copy_from_user) + ULTRA3_DO_PATCH(raw_copy_to_user, U3copy_to_user) + retl + nop + .size cheetah_patch_copyops,.-cheetah_patch_copyops diff --git a/arch/sparc/lib/VISsave.S b/arch/sparc/lib/VISsave.S new file mode 100644 index 000000000..31a0c336c --- /dev/null +++ b/arch/sparc/lib/VISsave.S @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * VISsave.S: Code for saving FPU register state for + * VIS routines. One should not call this directly, + * but use macros provided in . + * + * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#include +#include + +#include +#include +#include +#include +#include + + /* On entry: %o5=current FPRS value, %g7 is callers address */ + /* May clobber %o5, %g1, %g2, %g3, %g7, %icc, %xcc */ + + /* Nothing special need be done here to handle pre-emption, this + * FPU save/restore mechanism is already preemption safe. + */ + .text + .align 32 +ENTRY(VISenter) + ldub [%g6 + TI_FPDEPTH], %g1 + brnz,a,pn %g1, 1f + cmp %g1, 1 + stb %g0, [%g6 + TI_FPSAVED] + stx %fsr, [%g6 + TI_XFSR] +9: jmpl %g7 + %g0, %g0 + nop +1: bne,pn %icc, 2f + + srl %g1, 1, %g1 +vis1: ldub [%g6 + TI_FPSAVED], %g3 + stx %fsr, [%g6 + TI_XFSR] + or %g3, %o5, %g3 + stb %g3, [%g6 + TI_FPSAVED] + rd %gsr, %g3 + clr %g1 + ba,pt %xcc, 3f + + stx %g3, [%g6 + TI_GSR] +2: add %g6, %g1, %g3 + mov FPRS_DU | FPRS_DL | FPRS_FEF, %o5 + sll %g1, 3, %g1 + stb %o5, [%g3 + TI_FPSAVED] + rd %gsr, %g2 + add %g6, %g1, %g3 + stx %g2, [%g3 + TI_GSR] + + add %g6, %g1, %g2 + stx %fsr, [%g2 + TI_XFSR] + sll %g1, 5, %g1 +3: andcc %o5, FPRS_DL|FPRS_DU, %g0 + be,pn %icc, 9b + add %g6, TI_FPREGS, %g2 + andcc %o5, FPRS_DL, %g0 + + be,pn %icc, 4f + add %g6, TI_FPREGS+0x40, %g3 + membar #Sync + stda %f0, [%g2 + %g1] ASI_BLK_P + stda %f16, [%g3 + %g1] ASI_BLK_P + membar #Sync + andcc %o5, FPRS_DU, %g0 + be,pn %icc, 5f +4: add %g1, 128, %g1 + membar #Sync + stda %f32, [%g2 + %g1] ASI_BLK_P + + stda %f48, [%g3 + %g1] ASI_BLK_P +5: membar #Sync + ba,pt %xcc, 80f + nop + + .align 32 +80: jmpl %g7 + %g0, %g0 + nop +ENDPROC(VISenter) +EXPORT_SYMBOL(VISenter) diff --git a/arch/sparc/lib/ashldi3.S b/arch/sparc/lib/ashldi3.S new file mode 100644 index 000000000..2a9e7c4fb --- /dev/null +++ b/arch/sparc/lib/ashldi3.S @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ashldi3.S: GCC emits these for certain drivers playing + * with long longs. + * + * Copyright (C) 1999 David S. Miller (davem@redhat.com) + */ + +#include +#include + + .text +ENTRY(__ashldi3) + cmp %o2, 0 + be 9f + mov 0x20, %g2 + + sub %g2, %o2, %g2 + cmp %g2, 0 + bg 7f + sll %o0, %o2, %g3 + + neg %g2 + clr %o5 + b 8f + sll %o1, %g2, %o4 +7: + srl %o1, %g2, %g2 + sll %o1, %o2, %o5 + or %g3, %g2, %o4 +8: + mov %o4, %o0 + mov %o5, %o1 +9: + retl + nop +ENDPROC(__ashldi3) +EXPORT_SYMBOL(__ashldi3) diff --git a/arch/sparc/lib/ashrdi3.S b/arch/sparc/lib/ashrdi3.S new file mode 100644 index 000000000..8fd0b3117 --- /dev/null +++ b/arch/sparc/lib/ashrdi3.S @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ashrdi3.S: The filesystem code creates all kinds of references to + * this little routine on the sparc with gcc. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include + + .text +ENTRY(__ashrdi3) + tst %o2 + be 3f + or %g0, 32, %g2 + + sub %g2, %o2, %g2 + + tst %g2 + bg 1f + sra %o0, %o2, %o4 + + sra %o0, 31, %o4 + sub %g0, %g2, %g2 + ba 2f + sra %o0, %g2, %o5 + +1: + sll %o0, %g2, %g3 + srl %o1, %o2, %g2 + or %g2, %g3, %o5 +2: + or %g0, %o4, %o0 + or %g0, %o5, %o1 +3: + jmpl %o7 + 8, %g0 + nop +ENDPROC(__ashrdi3) +EXPORT_SYMBOL(__ashrdi3) diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c new file mode 100644 index 000000000..8ae880ebf --- /dev/null +++ b/arch/sparc/lib/atomic32.c @@ -0,0 +1,197 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * atomic32.c: 32-bit atomic_t implementation + * + * Copyright (C) 2004 Keith M Wesolowski + * Copyright (C) 2007 Kyle McMartin + * + * Based on asm-parisc/atomic.h Copyright (C) 2000 Philipp Rumpf + */ + +#include +#include +#include + +#ifdef CONFIG_SMP +#define ATOMIC_HASH_SIZE 4 +#define ATOMIC_HASH(a) (&__atomic_hash[(((unsigned long)a)>>8) & (ATOMIC_HASH_SIZE-1)]) + +spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] = { + [0 ... (ATOMIC_HASH_SIZE-1)] = __SPIN_LOCK_UNLOCKED(__atomic_hash) +}; + +#else /* SMP */ + +static DEFINE_SPINLOCK(dummy); +#define ATOMIC_HASH_SIZE 1 +#define ATOMIC_HASH(a) (&dummy) + +#endif /* SMP */ + +#define ATOMIC_FETCH_OP(op, c_op) \ +int arch_atomic_fetch_##op(int i, atomic_t *v) \ +{ \ + int ret; \ + unsigned long flags; \ + spin_lock_irqsave(ATOMIC_HASH(v), flags); \ + \ + ret = v->counter; \ + v->counter c_op i; \ + \ + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ + return ret; \ +} \ +EXPORT_SYMBOL(arch_atomic_fetch_##op); + +#define ATOMIC_OP_RETURN(op, c_op) \ +int arch_atomic_##op##_return(int i, atomic_t *v) \ +{ \ + int ret; \ + unsigned long flags; \ + spin_lock_irqsave(ATOMIC_HASH(v), flags); \ + \ + ret = (v->counter c_op i); \ + \ + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ + return ret; \ +} \ +EXPORT_SYMBOL(arch_atomic_##op##_return); + +ATOMIC_OP_RETURN(add, +=) + +ATOMIC_FETCH_OP(add, +=) +ATOMIC_FETCH_OP(and, &=) +ATOMIC_FETCH_OP(or, |=) +ATOMIC_FETCH_OP(xor, ^=) + +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN + +int arch_atomic_xchg(atomic_t *v, int new) +{ + int ret; + unsigned long flags; + + spin_lock_irqsave(ATOMIC_HASH(v), flags); + ret = v->counter; + v->counter = new; + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); + return ret; +} +EXPORT_SYMBOL(arch_atomic_xchg); + +int arch_atomic_cmpxchg(atomic_t *v, int old, int new) +{ + int ret; + unsigned long flags; + + spin_lock_irqsave(ATOMIC_HASH(v), flags); + ret = v->counter; + if (likely(ret == old)) + v->counter = new; + + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); + return ret; +} +EXPORT_SYMBOL(arch_atomic_cmpxchg); + +int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + int ret; + unsigned long flags; + + spin_lock_irqsave(ATOMIC_HASH(v), flags); + ret = v->counter; + if (ret != u) + v->counter += a; + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); + return ret; +} +EXPORT_SYMBOL(arch_atomic_fetch_add_unless); + +/* Atomic operations are already serializing */ +void arch_atomic_set(atomic_t *v, int i) +{ + unsigned long flags; + + spin_lock_irqsave(ATOMIC_HASH(v), flags); + v->counter = i; + spin_unlock_irqrestore(ATOMIC_HASH(v), flags); +} +EXPORT_SYMBOL(arch_atomic_set); + +unsigned long sp32___set_bit(unsigned long *addr, unsigned long mask) +{ + unsigned long old, flags; + + spin_lock_irqsave(ATOMIC_HASH(addr), flags); + old = *addr; + *addr = old | mask; + spin_unlock_irqrestore(ATOMIC_HASH(addr), flags); + + return old & mask; +} +EXPORT_SYMBOL(sp32___set_bit); + +unsigned long sp32___clear_bit(unsigned long *addr, unsigned long mask) +{ + unsigned long old, flags; + + spin_lock_irqsave(ATOMIC_HASH(addr), flags); + old = *addr; + *addr = old & ~mask; + spin_unlock_irqrestore(ATOMIC_HASH(addr), flags); + + return old & mask; +} +EXPORT_SYMBOL(sp32___clear_bit); + +unsigned long sp32___change_bit(unsigned long *addr, unsigned long mask) +{ + unsigned long old, flags; + + spin_lock_irqsave(ATOMIC_HASH(addr), flags); + old = *addr; + *addr = old ^ mask; + spin_unlock_irqrestore(ATOMIC_HASH(addr), flags); + + return old & mask; +} +EXPORT_SYMBOL(sp32___change_bit); + +#define CMPXCHG(T) \ + T __cmpxchg_##T(volatile T *ptr, T old, T new) \ + { \ + unsigned long flags; \ + T prev; \ + \ + spin_lock_irqsave(ATOMIC_HASH(ptr), flags); \ + if ((prev = *ptr) == old) \ + *ptr = new; \ + spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags);\ + \ + return prev; \ + } + +CMPXCHG(u8) +CMPXCHG(u16) +CMPXCHG(u32) +CMPXCHG(u64) +EXPORT_SYMBOL(__cmpxchg_u8); +EXPORT_SYMBOL(__cmpxchg_u16); +EXPORT_SYMBOL(__cmpxchg_u32); +EXPORT_SYMBOL(__cmpxchg_u64); + +unsigned long __xchg_u32(volatile u32 *ptr, u32 new) +{ + unsigned long flags; + u32 prev; + + spin_lock_irqsave(ATOMIC_HASH(ptr), flags); + prev = *ptr; + *ptr = new; + spin_unlock_irqrestore(ATOMIC_HASH(ptr), flags); + + return (unsigned long)prev; +} +EXPORT_SYMBOL(__xchg_u32); diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S new file mode 100644 index 000000000..4f8cab2fb --- /dev/null +++ b/arch/sparc/lib/atomic_64.S @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* atomic.S: These things are too big to do inline. + * + * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include + + .text + + /* Three versions of the atomic routines, one that + * does not return a value and does not perform + * memory barriers, and a two which return + * a value, the new and old value resp. and does the + * barriers. + */ + +#define ATOMIC_OP(op) \ +ENTRY(arch_atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: lduw [%o1], %g1; \ + op %g1, %o0, %g7; \ + cas [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + nop; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic_##op); \ +EXPORT_SYMBOL(arch_atomic_##op); + +#define ATOMIC_OP_RETURN(op) \ +ENTRY(arch_atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */\ + BACKOFF_SETUP(%o2); \ +1: lduw [%o1], %g1; \ + op %g1, %o0, %g7; \ + cas [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ + op %g1, %o0, %g1; \ + retl; \ + sra %g1, 0, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic_##op##_return); \ +EXPORT_SYMBOL(arch_atomic_##op##_return); + +#define ATOMIC_FETCH_OP(op) \ +ENTRY(arch_atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: lduw [%o1], %g1; \ + op %g1, %o0, %g7; \ + cas [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + sra %g1, 0, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic_fetch_##op); \ +EXPORT_SYMBOL(arch_atomic_fetch_##op); + +ATOMIC_OP(add) +ATOMIC_OP_RETURN(add) +ATOMIC_FETCH_OP(add) + +ATOMIC_OP(sub) +ATOMIC_OP_RETURN(sub) +ATOMIC_FETCH_OP(sub) + +ATOMIC_OP(and) +ATOMIC_FETCH_OP(and) + +ATOMIC_OP(or) +ATOMIC_FETCH_OP(or) + +ATOMIC_OP(xor) +ATOMIC_FETCH_OP(xor) + +#undef ATOMIC_FETCH_OP +#undef ATOMIC_OP_RETURN +#undef ATOMIC_OP + +#define ATOMIC64_OP(op) \ +ENTRY(arch_atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: ldx [%o1], %g1; \ + op %g1, %o0, %g7; \ + casx [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + nop; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic64_##op); \ +EXPORT_SYMBOL(arch_atomic64_##op); + +#define ATOMIC64_OP_RETURN(op) \ +ENTRY(arch_atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: ldx [%o1], %g1; \ + op %g1, %o0, %g7; \ + casx [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + op %g1, %o0, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic64_##op##_return); \ +EXPORT_SYMBOL(arch_atomic64_##op##_return); + +#define ATOMIC64_FETCH_OP(op) \ +ENTRY(arch_atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: ldx [%o1], %g1; \ + op %g1, %o0, %g7; \ + casx [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + mov %g1, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(arch_atomic64_fetch_##op); \ +EXPORT_SYMBOL(arch_atomic64_fetch_##op); + +ATOMIC64_OP(add) +ATOMIC64_OP_RETURN(add) +ATOMIC64_FETCH_OP(add) + +ATOMIC64_OP(sub) +ATOMIC64_OP_RETURN(sub) +ATOMIC64_FETCH_OP(sub) + +ATOMIC64_OP(and) +ATOMIC64_FETCH_OP(and) + +ATOMIC64_OP(or) +ATOMIC64_FETCH_OP(or) + +ATOMIC64_OP(xor) +ATOMIC64_FETCH_OP(xor) + +#undef ATOMIC64_FETCH_OP +#undef ATOMIC64_OP_RETURN +#undef ATOMIC64_OP + +ENTRY(arch_atomic64_dec_if_positive) /* %o0 = atomic_ptr */ + BACKOFF_SETUP(%o2) +1: ldx [%o0], %g1 + brlez,pn %g1, 3f + sub %g1, 1, %g7 + casx [%o0], %g1, %g7 + cmp %g1, %g7 + bne,pn %xcc, BACKOFF_LABEL(2f, 1b) + nop +3: retl + sub %g1, 1, %o0 +2: BACKOFF_SPIN(%o2, %o3, 1b) +ENDPROC(arch_atomic64_dec_if_positive) +EXPORT_SYMBOL(arch_atomic64_dec_if_positive) diff --git a/arch/sparc/lib/bitext.c b/arch/sparc/lib/bitext.c new file mode 100644 index 000000000..32a5c1d94 --- /dev/null +++ b/arch/sparc/lib/bitext.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * bitext.c: kernel little helper (of bit shuffling variety). + * + * Copyright (C) 2002 Pete Zaitcev + * + * The algorithm to search a zero bit string is geared towards its application. + * We expect a couple of fixed sizes of requests, so a rotating counter, reset + * by align size, should provide fast enough search while maintaining low + * fragmentation. + */ + +#include +#include + +#include + +/** + * bit_map_string_get - find and set a bit string in bit map. + * @t: the bit map. + * @len: requested string length + * @align: requested alignment + * + * Returns offset in the map or -1 if out of space. + * + * Not safe to call from an interrupt (uses spin_lock). + */ +int bit_map_string_get(struct bit_map *t, int len, int align) +{ + int offset, count; /* siamese twins */ + int off_new; + int align1; + int i, color; + + if (t->num_colors) { + /* align is overloaded to be the page color */ + color = align; + align = t->num_colors; + } else { + color = 0; + if (align == 0) + align = 1; + } + align1 = align - 1; + if ((align & align1) != 0) + BUG(); + if (align < 0 || align >= t->size) + BUG(); + if (len <= 0 || len > t->size) + BUG(); + color &= align1; + + spin_lock(&t->lock); + if (len < t->last_size) + offset = t->first_free; + else + offset = t->last_off & ~align1; + count = 0; + for (;;) { + off_new = find_next_zero_bit(t->map, t->size, offset); + off_new = ((off_new + align1) & ~align1) + color; + count += off_new - offset; + offset = off_new; + if (offset >= t->size) + offset = 0; + if (count + len > t->size) { + spin_unlock(&t->lock); +/* P3 */ printk(KERN_ERR + "bitmap out: size %d used %d off %d len %d align %d count %d\n", + t->size, t->used, offset, len, align, count); + return -1; + } + + if (offset + len > t->size) { + count += t->size - offset; + offset = 0; + continue; + } + + i = 0; + while (test_bit(offset + i, t->map) == 0) { + i++; + if (i == len) { + bitmap_set(t->map, offset, len); + if (offset == t->first_free) + t->first_free = find_next_zero_bit + (t->map, t->size, + t->first_free + len); + if ((t->last_off = offset + len) >= t->size) + t->last_off = 0; + t->used += len; + t->last_size = len; + spin_unlock(&t->lock); + return offset; + } + } + count += i + 1; + if ((offset += i + 1) >= t->size) + offset = 0; + } +} + +void bit_map_clear(struct bit_map *t, int offset, int len) +{ + int i; + + if (t->used < len) + BUG(); /* Much too late to do any good, but alas... */ + spin_lock(&t->lock); + for (i = 0; i < len; i++) { + if (test_bit(offset + i, t->map) == 0) + BUG(); + __clear_bit(offset + i, t->map); + } + if (offset < t->first_free) + t->first_free = offset; + t->used -= len; + spin_unlock(&t->lock); +} + +void bit_map_init(struct bit_map *t, unsigned long *map, int size) +{ + bitmap_zero(map, size); + memset(t, 0, sizeof *t); + spin_lock_init(&t->lock); + t->map = map; + t->size = size; +} diff --git a/arch/sparc/lib/bitops.S b/arch/sparc/lib/bitops.S new file mode 100644 index 000000000..9c91cbb31 --- /dev/null +++ b/arch/sparc/lib/bitops.S @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* bitops.S: Sparc64 atomic bit operations. + * + * Copyright (C) 2000, 2007 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include + + .text + +ENTRY(test_and_set_bit) /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + or %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, BACKOFF_LABEL(2f, 1b) + and %g7, %o2, %g2 + clr %o0 + movrne %g2, 1, %o0 + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) +ENDPROC(test_and_set_bit) +EXPORT_SYMBOL(test_and_set_bit) + +ENTRY(test_and_clear_bit) /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + andn %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, BACKOFF_LABEL(2f, 1b) + and %g7, %o2, %g2 + clr %o0 + movrne %g2, 1, %o0 + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) +ENDPROC(test_and_clear_bit) +EXPORT_SYMBOL(test_and_clear_bit) + +ENTRY(test_and_change_bit) /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + xor %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, BACKOFF_LABEL(2f, 1b) + and %g7, %o2, %g2 + clr %o0 + movrne %g2, 1, %o0 + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) +ENDPROC(test_and_change_bit) +EXPORT_SYMBOL(test_and_change_bit) + +ENTRY(set_bit) /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + or %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, BACKOFF_LABEL(2f, 1b) + nop + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) +ENDPROC(set_bit) +EXPORT_SYMBOL(set_bit) + +ENTRY(clear_bit) /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + andn %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, BACKOFF_LABEL(2f, 1b) + nop + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) +ENDPROC(clear_bit) +EXPORT_SYMBOL(clear_bit) + +ENTRY(change_bit) /* %o0=nr, %o1=addr */ + BACKOFF_SETUP(%o3) + srlx %o0, 6, %g1 + mov 1, %o2 + sllx %g1, 3, %g3 + and %o0, 63, %g2 + sllx %o2, %g2, %o2 + add %o1, %g3, %o1 +1: ldx [%o1], %g7 + xor %g7, %o2, %g1 + casx [%o1], %g7, %g1 + cmp %g7, %g1 + bne,pn %xcc, BACKOFF_LABEL(2f, 1b) + nop + retl + nop +2: BACKOFF_SPIN(%o3, %o4, 1b) +ENDPROC(change_bit) +EXPORT_SYMBOL(change_bit) diff --git a/arch/sparc/lib/blockops.S b/arch/sparc/lib/blockops.S new file mode 100644 index 000000000..5b92959a4 --- /dev/null +++ b/arch/sparc/lib/blockops.S @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * blockops.S: Common block zero optimized routines. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include + + /* Zero out 64 bytes of memory at (buf + offset). + * Assumes %g1 contains zero. + */ +#define BLAST_BLOCK(buf, offset) \ + std %g0, [buf + offset + 0x38]; \ + std %g0, [buf + offset + 0x30]; \ + std %g0, [buf + offset + 0x28]; \ + std %g0, [buf + offset + 0x20]; \ + std %g0, [buf + offset + 0x18]; \ + std %g0, [buf + offset + 0x10]; \ + std %g0, [buf + offset + 0x08]; \ + std %g0, [buf + offset + 0x00]; + + /* Copy 32 bytes of memory at (src + offset) to + * (dst + offset). + */ +#define MIRROR_BLOCK(dst, src, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ + ldd [src + offset + 0x18], t0; \ + ldd [src + offset + 0x10], t2; \ + ldd [src + offset + 0x08], t4; \ + ldd [src + offset + 0x00], t6; \ + std t0, [dst + offset + 0x18]; \ + std t2, [dst + offset + 0x10]; \ + std t4, [dst + offset + 0x08]; \ + std t6, [dst + offset + 0x00]; + + /* Profiling evidence indicates that memset() is + * commonly called for blocks of size PAGE_SIZE, + * and (2 * PAGE_SIZE) (for kernel stacks) + * and with a second arg of zero. We assume in + * all of these cases that the buffer is aligned + * on at least an 8 byte boundary. + * + * Therefore we special case them to make them + * as fast as possible. + */ + + .text +ENTRY(bzero_1page) +/* NOTE: If you change the number of insns of this routine, please check + * arch/sparc/mm/hypersparc.S */ + /* %o0 = buf */ + or %g0, %g0, %g1 + or %o0, %g0, %o1 + or %g0, (PAGE_SIZE >> 8), %g2 +1: + BLAST_BLOCK(%o0, 0x00) + BLAST_BLOCK(%o0, 0x40) + BLAST_BLOCK(%o0, 0x80) + BLAST_BLOCK(%o0, 0xc0) + subcc %g2, 1, %g2 + bne 1b + add %o0, 0x100, %o0 + + retl + nop +ENDPROC(bzero_1page) +EXPORT_SYMBOL(bzero_1page) + +ENTRY(__copy_1page) +/* NOTE: If you change the number of insns of this routine, please check + * arch/sparc/mm/hypersparc.S */ + /* %o0 = dst, %o1 = src */ + or %g0, (PAGE_SIZE >> 8), %g1 +1: + MIRROR_BLOCK(%o0, %o1, 0x00, %o2, %o3, %o4, %o5, %g2, %g3, %g4, %g5) + MIRROR_BLOCK(%o0, %o1, 0x20, %o2, %o3, %o4, %o5, %g2, %g3, %g4, %g5) + MIRROR_BLOCK(%o0, %o1, 0x40, %o2, %o3, %o4, %o5, %g2, %g3, %g4, %g5) + MIRROR_BLOCK(%o0, %o1, 0x60, %o2, %o3, %o4, %o5, %g2, %g3, %g4, %g5) + MIRROR_BLOCK(%o0, %o1, 0x80, %o2, %o3, %o4, %o5, %g2, %g3, %g4, %g5) + MIRROR_BLOCK(%o0, %o1, 0xa0, %o2, %o3, %o4, %o5, %g2, %g3, %g4, %g5) + MIRROR_BLOCK(%o0, %o1, 0xc0, %o2, %o3, %o4, %o5, %g2, %g3, %g4, %g5) + MIRROR_BLOCK(%o0, %o1, 0xe0, %o2, %o3, %o4, %o5, %g2, %g3, %g4, %g5) + subcc %g1, 1, %g1 + add %o0, 0x100, %o0 + bne 1b + add %o1, 0x100, %o1 + + retl + nop +ENDPROC(__copy_1page) +EXPORT_SYMBOL(__copy_1page) diff --git a/arch/sparc/lib/bzero.S b/arch/sparc/lib/bzero.S new file mode 100644 index 000000000..2bfa44a6b --- /dev/null +++ b/arch/sparc/lib/bzero.S @@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* bzero.S: Simple prefetching memset, bzero, and clear_user + * implementations. + * + * Copyright (C) 2005 David S. Miller + */ + +#include +#include + + .text + +ENTRY(memset) /* %o0=buf, %o1=pat, %o2=len */ + and %o1, 0xff, %o3 + mov %o2, %o1 + sllx %o3, 8, %g1 + or %g1, %o3, %o2 + sllx %o2, 16, %g1 + or %g1, %o2, %o2 + sllx %o2, 32, %g1 + ba,pt %xcc, 1f + or %g1, %o2, %o2 + +ENTRY(__bzero) /* %o0=buf, %o1=len */ + clr %o2 +1: mov %o0, %o3 + brz,pn %o1, __bzero_done + cmp %o1, 16 + bl,pn %icc, __bzero_tiny + prefetch [%o0 + 0x000], #n_writes + andcc %o0, 0x3, %g0 + be,pt %icc, 2f +1: stb %o2, [%o0 + 0x00] + add %o0, 1, %o0 + andcc %o0, 0x3, %g0 + bne,pn %icc, 1b + sub %o1, 1, %o1 +2: andcc %o0, 0x7, %g0 + be,pt %icc, 3f + stw %o2, [%o0 + 0x00] + sub %o1, 4, %o1 + add %o0, 4, %o0 +3: and %o1, 0x38, %g1 + cmp %o1, 0x40 + andn %o1, 0x3f, %o4 + bl,pn %icc, 5f + and %o1, 0x7, %o1 + prefetch [%o0 + 0x040], #n_writes + prefetch [%o0 + 0x080], #n_writes + prefetch [%o0 + 0x0c0], #n_writes + prefetch [%o0 + 0x100], #n_writes + prefetch [%o0 + 0x140], #n_writes +4: prefetch [%o0 + 0x180], #n_writes + stx %o2, [%o0 + 0x00] + stx %o2, [%o0 + 0x08] + stx %o2, [%o0 + 0x10] + stx %o2, [%o0 + 0x18] + stx %o2, [%o0 + 0x20] + stx %o2, [%o0 + 0x28] + stx %o2, [%o0 + 0x30] + stx %o2, [%o0 + 0x38] + subcc %o4, 0x40, %o4 + bne,pt %icc, 4b + add %o0, 0x40, %o0 + brz,pn %g1, 6f + nop +5: stx %o2, [%o0 + 0x00] + subcc %g1, 8, %g1 + bne,pt %icc, 5b + add %o0, 0x8, %o0 +6: brz,pt %o1, __bzero_done + nop +__bzero_tiny: +1: stb %o2, [%o0 + 0x00] + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 1, %o0 +__bzero_done: + retl + mov %o3, %o0 +ENDPROC(__bzero) +ENDPROC(memset) +EXPORT_SYMBOL(__bzero) +EXPORT_SYMBOL(memset) + +#define EX_ST(x,y) \ +98: x,y; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, __retl_o1; \ + .text; \ + .align 4; + +ENTRY(__clear_user) /* %o0=buf, %o1=len */ + brz,pn %o1, __clear_user_done + cmp %o1, 16 + bl,pn %icc, __clear_user_tiny + EX_ST(prefetcha [%o0 + 0x00] %asi, #n_writes) + andcc %o0, 0x3, %g0 + be,pt %icc, 2f +1: EX_ST(stba %g0, [%o0 + 0x00] %asi) + add %o0, 1, %o0 + andcc %o0, 0x3, %g0 + bne,pn %icc, 1b + sub %o1, 1, %o1 +2: andcc %o0, 0x7, %g0 + be,pt %icc, 3f + EX_ST(stwa %g0, [%o0 + 0x00] %asi) + sub %o1, 4, %o1 + add %o0, 4, %o0 +3: and %o1, 0x38, %g1 + cmp %o1, 0x40 + andn %o1, 0x3f, %o4 + bl,pn %icc, 5f + and %o1, 0x7, %o1 + EX_ST(prefetcha [%o0 + 0x040] %asi, #n_writes) + EX_ST(prefetcha [%o0 + 0x080] %asi, #n_writes) + EX_ST(prefetcha [%o0 + 0x0c0] %asi, #n_writes) + EX_ST(prefetcha [%o0 + 0x100] %asi, #n_writes) + EX_ST(prefetcha [%o0 + 0x140] %asi, #n_writes) +4: EX_ST(prefetcha [%o0 + 0x180] %asi, #n_writes) + EX_ST(stxa %g0, [%o0 + 0x00] %asi) + EX_ST(stxa %g0, [%o0 + 0x08] %asi) + EX_ST(stxa %g0, [%o0 + 0x10] %asi) + EX_ST(stxa %g0, [%o0 + 0x18] %asi) + EX_ST(stxa %g0, [%o0 + 0x20] %asi) + EX_ST(stxa %g0, [%o0 + 0x28] %asi) + EX_ST(stxa %g0, [%o0 + 0x30] %asi) + EX_ST(stxa %g0, [%o0 + 0x38] %asi) + subcc %o4, 0x40, %o4 + bne,pt %icc, 4b + add %o0, 0x40, %o0 + brz,pn %g1, 6f + nop +5: EX_ST(stxa %g0, [%o0 + 0x00] %asi) + subcc %g1, 8, %g1 + bne,pt %icc, 5b + add %o0, 0x8, %o0 +6: brz,pt %o1, __clear_user_done + nop +__clear_user_tiny: +1: EX_ST(stba %g0, [%o0 + 0x00] %asi) + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 1, %o0 +__clear_user_done: + retl + clr %o0 +ENDPROC(__clear_user) +EXPORT_SYMBOL(__clear_user) diff --git a/arch/sparc/lib/checksum_32.S b/arch/sparc/lib/checksum_32.S new file mode 100644 index 000000000..66eda40fc --- /dev/null +++ b/arch/sparc/lib/checksum_32.S @@ -0,0 +1,457 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* checksum.S: Sparc optimized checksum code. + * + * Copyright(C) 1995 Linus Torvalds + * Copyright(C) 1995 Miguel de Icaza + * Copyright(C) 1996 David S. Miller + * Copyright(C) 1997 Jakub Jelinek + * + * derived from: + * Linux/Alpha checksum c-code + * Linux/ix86 inline checksum assembly + * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code) + * David Mosberger-Tang for optimized reference c-code + * BSD4.4 portable checksum routine + */ + +#include +#include + +#define CSUM_BIGCHUNK(buf, offset, sum, t0, t1, t2, t3, t4, t5) \ + ldd [buf + offset + 0x00], t0; \ + ldd [buf + offset + 0x08], t2; \ + addxcc t0, sum, sum; \ + addxcc t1, sum, sum; \ + ldd [buf + offset + 0x10], t4; \ + addxcc t2, sum, sum; \ + addxcc t3, sum, sum; \ + ldd [buf + offset + 0x18], t0; \ + addxcc t4, sum, sum; \ + addxcc t5, sum, sum; \ + addxcc t0, sum, sum; \ + addxcc t1, sum, sum; + +#define CSUM_LASTCHUNK(buf, offset, sum, t0, t1, t2, t3) \ + ldd [buf - offset - 0x08], t0; \ + ldd [buf - offset - 0x00], t2; \ + addxcc t0, sum, sum; \ + addxcc t1, sum, sum; \ + addxcc t2, sum, sum; \ + addxcc t3, sum, sum; + + /* Do end cruft out of band to get better cache patterns. */ +csum_partial_end_cruft: + be 1f ! caller asks %o1 & 0x8 + andcc %o1, 4, %g0 ! nope, check for word remaining + ldd [%o0], %g2 ! load two + addcc %g2, %o2, %o2 ! add first word to sum + addxcc %g3, %o2, %o2 ! add second word as well + add %o0, 8, %o0 ! advance buf ptr + addx %g0, %o2, %o2 ! add in final carry + andcc %o1, 4, %g0 ! check again for word remaining +1: be 1f ! nope, skip this code + andcc %o1, 3, %o1 ! check for trailing bytes + ld [%o0], %g2 ! load it + addcc %g2, %o2, %o2 ! add to sum + add %o0, 4, %o0 ! advance buf ptr + addx %g0, %o2, %o2 ! add in final carry + andcc %o1, 3, %g0 ! check again for trailing bytes +1: be 1f ! no trailing bytes, return + addcc %o1, -1, %g0 ! only one byte remains? + bne 2f ! at least two bytes more + subcc %o1, 2, %o1 ! only two bytes more? + b 4f ! only one byte remains + or %g0, %g0, %o4 ! clear fake hword value +2: lduh [%o0], %o4 ! get hword + be 6f ! jmp if only hword remains + add %o0, 2, %o0 ! advance buf ptr either way + sll %o4, 16, %o4 ! create upper hword +4: ldub [%o0], %o5 ! get final byte + sll %o5, 8, %o5 ! put into place + or %o5, %o4, %o4 ! coalese with hword (if any) +6: addcc %o4, %o2, %o2 ! add to sum +1: retl ! get outta here + addx %g0, %o2, %o0 ! add final carry into retval + + /* Also do alignment out of band to get better cache patterns. */ +csum_partial_fix_alignment: + cmp %o1, 6 + bl cpte - 0x4 + andcc %o0, 0x2, %g0 + be 1f + andcc %o0, 0x4, %g0 + lduh [%o0 + 0x00], %g2 + sub %o1, 2, %o1 + add %o0, 2, %o0 + sll %g2, 16, %g2 + addcc %g2, %o2, %o2 + srl %o2, 16, %g3 + addx %g0, %g3, %g2 + sll %o2, 16, %o2 + sll %g2, 16, %g3 + srl %o2, 16, %o2 + andcc %o0, 0x4, %g0 + or %g3, %o2, %o2 +1: be cpa + andcc %o1, 0xffffff80, %o3 + ld [%o0 + 0x00], %g2 + sub %o1, 4, %o1 + addcc %g2, %o2, %o2 + add %o0, 4, %o0 + addx %g0, %o2, %o2 + b cpa + andcc %o1, 0xffffff80, %o3 + + /* The common case is to get called with a nicely aligned + * buffer of size 0x20. Follow the code path for that case. + */ + .globl csum_partial + EXPORT_SYMBOL(csum_partial) +csum_partial: /* %o0=buf, %o1=len, %o2=sum */ + andcc %o0, 0x7, %g0 ! alignment problems? + bne csum_partial_fix_alignment ! yep, handle it + sethi %hi(cpte - 8), %g7 ! prepare table jmp ptr + andcc %o1, 0xffffff80, %o3 ! num loop iterations +cpa: be 3f ! none to do + andcc %o1, 0x70, %g1 ! clears carry flag too +5: CSUM_BIGCHUNK(%o0, 0x00, %o2, %o4, %o5, %g2, %g3, %g4, %g5) + CSUM_BIGCHUNK(%o0, 0x20, %o2, %o4, %o5, %g2, %g3, %g4, %g5) + CSUM_BIGCHUNK(%o0, 0x40, %o2, %o4, %o5, %g2, %g3, %g4, %g5) + CSUM_BIGCHUNK(%o0, 0x60, %o2, %o4, %o5, %g2, %g3, %g4, %g5) + addx %g0, %o2, %o2 ! sink in final carry + subcc %o3, 128, %o3 ! detract from loop iters + bne 5b ! more to do + add %o0, 128, %o0 ! advance buf ptr + andcc %o1, 0x70, %g1 ! clears carry flag too +3: be cpte ! nope + andcc %o1, 0xf, %g0 ! anything left at all? + srl %g1, 1, %o4 ! compute offset + sub %g7, %g1, %g7 ! adjust jmp ptr + sub %g7, %o4, %g7 ! final jmp ptr adjust + jmp %g7 + %lo(cpte - 8) ! enter the table + add %o0, %g1, %o0 ! advance buf ptr +cptbl: CSUM_LASTCHUNK(%o0, 0x68, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x58, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x48, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x38, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x28, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x18, %o2, %g2, %g3, %g4, %g5) + CSUM_LASTCHUNK(%o0, 0x08, %o2, %g2, %g3, %g4, %g5) + addx %g0, %o2, %o2 ! fetch final carry + andcc %o1, 0xf, %g0 ! anything left at all? +cpte: bne csum_partial_end_cruft ! yep, handle it + andcc %o1, 8, %g0 ! check how much +cpout: retl ! get outta here + mov %o2, %o0 ! return computed csum + +/* Work around cpp -rob */ +#define ALLOC #alloc +#define EXECINSTR #execinstr +#define EX(x,y) \ +98: x,y; \ + .section __ex_table,ALLOC; \ + .align 4; \ + .word 98b, cc_fault; \ + .text; \ + .align 4 + + /* This aligned version executes typically in 8.5 superscalar cycles, this + * is the best I can do. I say 8.5 because the final add will pair with + * the next ldd in the main unrolled loop. Thus the pipe is always full. + * If you change these macros (including order of instructions), + * please check the fixup code below as well. + */ +#define CSUMCOPY_BIGCHUNK_ALIGNED(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \ + EX(ldd [src + off + 0x00], t0); \ + EX(ldd [src + off + 0x08], t2); \ + addxcc t0, sum, sum; \ + EX(ldd [src + off + 0x10], t4); \ + addxcc t1, sum, sum; \ + EX(ldd [src + off + 0x18], t6); \ + addxcc t2, sum, sum; \ + EX(std t0, [dst + off + 0x00]); \ + addxcc t3, sum, sum; \ + EX(std t2, [dst + off + 0x08]); \ + addxcc t4, sum, sum; \ + EX(std t4, [dst + off + 0x10]); \ + addxcc t5, sum, sum; \ + EX(std t6, [dst + off + 0x18]); \ + addxcc t6, sum, sum; \ + addxcc t7, sum, sum; + + /* 12 superscalar cycles seems to be the limit for this case, + * because of this we thus do all the ldd's together to get + * Viking MXCC into streaming mode. Ho hum... + */ +#define CSUMCOPY_BIGCHUNK(src, dst, sum, off, t0, t1, t2, t3, t4, t5, t6, t7) \ + EX(ldd [src + off + 0x00], t0); \ + EX(ldd [src + off + 0x08], t2); \ + EX(ldd [src + off + 0x10], t4); \ + EX(ldd [src + off + 0x18], t6); \ + EX(st t0, [dst + off + 0x00]); \ + addxcc t0, sum, sum; \ + EX(st t1, [dst + off + 0x04]); \ + addxcc t1, sum, sum; \ + EX(st t2, [dst + off + 0x08]); \ + addxcc t2, sum, sum; \ + EX(st t3, [dst + off + 0x0c]); \ + addxcc t3, sum, sum; \ + EX(st t4, [dst + off + 0x10]); \ + addxcc t4, sum, sum; \ + EX(st t5, [dst + off + 0x14]); \ + addxcc t5, sum, sum; \ + EX(st t6, [dst + off + 0x18]); \ + addxcc t6, sum, sum; \ + EX(st t7, [dst + off + 0x1c]); \ + addxcc t7, sum, sum; + + /* Yuck, 6 superscalar cycles... */ +#define CSUMCOPY_LASTCHUNK(src, dst, sum, off, t0, t1, t2, t3) \ + EX(ldd [src - off - 0x08], t0); \ + EX(ldd [src - off - 0x00], t2); \ + addxcc t0, sum, sum; \ + EX(st t0, [dst - off - 0x08]); \ + addxcc t1, sum, sum; \ + EX(st t1, [dst - off - 0x04]); \ + addxcc t2, sum, sum; \ + EX(st t2, [dst - off - 0x00]); \ + addxcc t3, sum, sum; \ + EX(st t3, [dst - off + 0x04]); + + /* Handle the end cruft code out of band for better cache patterns. */ +cc_end_cruft: + be 1f + andcc %o3, 4, %g0 + EX(ldd [%o0 + 0x00], %g2) + add %o1, 8, %o1 + addcc %g2, %g7, %g7 + add %o0, 8, %o0 + addxcc %g3, %g7, %g7 + EX(st %g2, [%o1 - 0x08]) + addx %g0, %g7, %g7 + andcc %o3, 4, %g0 + EX(st %g3, [%o1 - 0x04]) +1: be 1f + andcc %o3, 3, %o3 + EX(ld [%o0 + 0x00], %g2) + add %o1, 4, %o1 + addcc %g2, %g7, %g7 + EX(st %g2, [%o1 - 0x04]) + addx %g0, %g7, %g7 + andcc %o3, 3, %g0 + add %o0, 4, %o0 +1: be 1f + addcc %o3, -1, %g0 + bne 2f + subcc %o3, 2, %o3 + b 4f + or %g0, %g0, %o4 +2: EX(lduh [%o0 + 0x00], %o4) + add %o0, 2, %o0 + EX(sth %o4, [%o1 + 0x00]) + be 6f + add %o1, 2, %o1 + sll %o4, 16, %o4 +4: EX(ldub [%o0 + 0x00], %o5) + EX(stb %o5, [%o1 + 0x00]) + sll %o5, 8, %o5 + or %o5, %o4, %o4 +6: addcc %o4, %g7, %g7 +1: retl + addx %g0, %g7, %o0 + + /* Also, handle the alignment code out of band. */ +cc_dword_align: + cmp %g1, 16 + bge 1f + srl %g1, 1, %o3 +2: cmp %o3, 0 + be,a ccte + andcc %g1, 0xf, %o3 + andcc %o3, %o0, %g0 ! Check %o0 only (%o1 has the same last 2 bits) + be,a 2b + srl %o3, 1, %o3 +1: andcc %o0, 0x1, %g0 + bne ccslow + andcc %o0, 0x2, %g0 + be 1f + andcc %o0, 0x4, %g0 + EX(lduh [%o0 + 0x00], %g4) + sub %g1, 2, %g1 + EX(sth %g4, [%o1 + 0x00]) + add %o0, 2, %o0 + sll %g4, 16, %g4 + addcc %g4, %g7, %g7 + add %o1, 2, %o1 + srl %g7, 16, %g3 + addx %g0, %g3, %g4 + sll %g7, 16, %g7 + sll %g4, 16, %g3 + srl %g7, 16, %g7 + andcc %o0, 0x4, %g0 + or %g3, %g7, %g7 +1: be 3f + andcc %g1, 0xffffff80, %g0 + EX(ld [%o0 + 0x00], %g4) + sub %g1, 4, %g1 + EX(st %g4, [%o1 + 0x00]) + add %o0, 4, %o0 + addcc %g4, %g7, %g7 + add %o1, 4, %o1 + addx %g0, %g7, %g7 + b 3f + andcc %g1, 0xffffff80, %g0 + + /* Sun, you just can't beat me, you just can't. Stop trying, + * give up. I'm serious, I am going to kick the living shit + * out of you, game over, lights out. + */ + .align 8 + .globl __csum_partial_copy_sparc_generic + EXPORT_SYMBOL(__csum_partial_copy_sparc_generic) +__csum_partial_copy_sparc_generic: + /* %o0=src, %o1=dest, %g1=len, %g7=sum */ + xor %o0, %o1, %o4 ! get changing bits + andcc %o4, 3, %g0 ! check for mismatched alignment + bne ccslow ! better this than unaligned/fixups + andcc %o0, 7, %g0 ! need to align things? + bne cc_dword_align ! yes, we check for short lengths there + andcc %g1, 0xffffff80, %g0 ! can we use unrolled loop? +3: be 3f ! nope, less than one loop remains + andcc %o1, 4, %g0 ! dest aligned on 4 or 8 byte boundary? + be ccdbl + 4 ! 8 byte aligned, kick ass +5: CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x00,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + sub %g1, 128, %g1 ! detract from length + addx %g0, %g7, %g7 ! add in last carry bit + andcc %g1, 0xffffff80, %g0 ! more to csum? + add %o0, 128, %o0 ! advance src ptr + bne 5b ! we did not go negative, continue looping + add %o1, 128, %o1 ! advance dest ptr +3: andcc %g1, 0x70, %o2 ! can use table? +ccmerge:be ccte ! nope, go and check for end cruft + andcc %g1, 0xf, %o3 ! get low bits of length (clears carry btw) + srl %o2, 1, %o4 ! begin negative offset computation + sethi %hi(12f), %o5 ! set up table ptr end + add %o0, %o2, %o0 ! advance src ptr + sub %o5, %o4, %o5 ! continue table calculation + sll %o2, 1, %g2 ! constant multiplies are fun... + sub %o5, %g2, %o5 ! some more adjustments + jmp %o5 + %lo(12f) ! jump into it, duff style, wheee... + add %o1, %o2, %o1 ! advance dest ptr (carry is clear btw) +cctbl: CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x68,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x58,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x48,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x38,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x28,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x18,%g2,%g3,%g4,%g5) + CSUMCOPY_LASTCHUNK(%o0,%o1,%g7,0x08,%g2,%g3,%g4,%g5) +12: addx %g0, %g7, %g7 + andcc %o3, 0xf, %g0 ! check for low bits set +ccte: bne cc_end_cruft ! something left, handle it out of band + andcc %o3, 8, %g0 ! begin checks for that code + retl ! return + mov %g7, %o0 ! give em the computed checksum +ccdbl: CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x00,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x20,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x40,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + CSUMCOPY_BIGCHUNK_ALIGNED(%o0,%o1,%g7,0x60,%o4,%o5,%g2,%g3,%g4,%g5,%o2,%o3) + sub %g1, 128, %g1 ! detract from length + addx %g0, %g7, %g7 ! add in last carry bit + andcc %g1, 0xffffff80, %g0 ! more to csum? + add %o0, 128, %o0 ! advance src ptr + bne ccdbl ! we did not go negative, continue looping + add %o1, 128, %o1 ! advance dest ptr + b ccmerge ! finish it off, above + andcc %g1, 0x70, %o2 ! can use table? (clears carry btw) + +ccslow: cmp %g1, 0 + mov 0, %g5 + bleu 4f + andcc %o0, 1, %o5 + be,a 1f + srl %g1, 1, %g4 + sub %g1, 1, %g1 + EX(ldub [%o0], %g5) + add %o0, 1, %o0 + EX(stb %g5, [%o1]) + srl %g1, 1, %g4 + add %o1, 1, %o1 +1: cmp %g4, 0 + be,a 3f + andcc %g1, 1, %g0 + andcc %o0, 2, %g0 + be,a 1f + srl %g4, 1, %g4 + EX(lduh [%o0], %o4) + sub %g1, 2, %g1 + srl %o4, 8, %g2 + sub %g4, 1, %g4 + EX(stb %g2, [%o1]) + add %o4, %g5, %g5 + EX(stb %o4, [%o1 + 1]) + add %o0, 2, %o0 + srl %g4, 1, %g4 + add %o1, 2, %o1 +1: cmp %g4, 0 + be,a 2f + andcc %g1, 2, %g0 + EX(ld [%o0], %o4) +5: srl %o4, 24, %g2 + srl %o4, 16, %g3 + EX(stb %g2, [%o1]) + srl %o4, 8, %g2 + EX(stb %g3, [%o1 + 1]) + add %o0, 4, %o0 + EX(stb %g2, [%o1 + 2]) + addcc %o4, %g5, %g5 + EX(stb %o4, [%o1 + 3]) + addx %g5, %g0, %g5 ! I am now to lazy to optimize this (question it + add %o1, 4, %o1 ! is worthy). Maybe some day - with the sll/srl + subcc %g4, 1, %g4 ! tricks + bne,a 5b + EX(ld [%o0], %o4) + sll %g5, 16, %g2 + srl %g5, 16, %g5 + srl %g2, 16, %g2 + andcc %g1, 2, %g0 + add %g2, %g5, %g5 +2: be,a 3f + andcc %g1, 1, %g0 + EX(lduh [%o0], %o4) + andcc %g1, 1, %g0 + srl %o4, 8, %g2 + add %o0, 2, %o0 + EX(stb %g2, [%o1]) + add %g5, %o4, %g5 + EX(stb %o4, [%o1 + 1]) + add %o1, 2, %o1 +3: be,a 1f + sll %g5, 16, %o4 + EX(ldub [%o0], %g2) + sll %g2, 8, %o4 + EX(stb %g2, [%o1]) + add %g5, %o4, %g5 + sll %g5, 16, %o4 +1: addcc %o4, %g5, %g5 + srl %g5, 16, %o4 + addx %g0, %o4, %g5 + orcc %o5, %g0, %g0 + be 4f + srl %g5, 8, %o4 + and %g5, 0xff, %g2 + and %o4, 0xff, %o4 + sll %g2, 8, %g2 + or %g2, %o4, %g5 +4: addcc %g7, %g5, %g7 + retl + addx %g0, %g7, %o0 + +/* We do these strange calculations for the csum_*_from_user case only, ie. + * we only bother with faults on loads... */ + +cc_fault: + retl + clr %o0 diff --git a/arch/sparc/lib/checksum_64.S b/arch/sparc/lib/checksum_64.S new file mode 100644 index 000000000..32b626f3f --- /dev/null +++ b/arch/sparc/lib/checksum_64.S @@ -0,0 +1,177 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* checksum.S: Sparc V9 optimized checksum code. + * + * Copyright(C) 1995 Linus Torvalds + * Copyright(C) 1995 Miguel de Icaza + * Copyright(C) 1996, 2000 David S. Miller + * Copyright(C) 1997 Jakub Jelinek + * + * derived from: + * Linux/Alpha checksum c-code + * Linux/ix86 inline checksum assembly + * RFC1071 Computing the Internet Checksum (esp. Jacobsons m68k code) + * David Mosberger-Tang for optimized reference c-code + * BSD4.4 portable checksum routine + */ + +#include + .text + +csum_partial_fix_alignment: + /* We checked for zero length already, so there must be + * at least one byte. + */ + be,pt %icc, 1f + nop + ldub [%o0 + 0x00], %o4 + add %o0, 1, %o0 + sub %o1, 1, %o1 +1: andcc %o0, 0x2, %g0 + be,pn %icc, csum_partial_post_align + cmp %o1, 2 + blu,pn %icc, csum_partial_end_cruft + nop + lduh [%o0 + 0x00], %o5 + add %o0, 2, %o0 + sub %o1, 2, %o1 + ba,pt %xcc, csum_partial_post_align + add %o5, %o4, %o4 + + .align 32 + .globl csum_partial + .type csum_partial,#function + EXPORT_SYMBOL(csum_partial) +csum_partial: /* %o0=buff, %o1=len, %o2=sum */ + prefetch [%o0 + 0x000], #n_reads + clr %o4 + prefetch [%o0 + 0x040], #n_reads + brz,pn %o1, csum_partial_finish + andcc %o0, 0x3, %g0 + + /* We "remember" whether the lowest bit in the address + * was set in %g7. Because if it is, we have to swap + * upper and lower 8 bit fields of the sum we calculate. + */ + bne,pn %icc, csum_partial_fix_alignment + andcc %o0, 0x1, %g7 + +csum_partial_post_align: + prefetch [%o0 + 0x080], #n_reads + andncc %o1, 0x3f, %o3 + + prefetch [%o0 + 0x0c0], #n_reads + sub %o1, %o3, %o1 + brz,pn %o3, 2f + prefetch [%o0 + 0x100], #n_reads + + /* So that we don't need to use the non-pairing + * add-with-carry instructions we accumulate 32-bit + * values into a 64-bit register. At the end of the + * loop we fold it down to 32-bits and so on. + */ + prefetch [%o0 + 0x140], #n_reads +1: lduw [%o0 + 0x00], %o5 + lduw [%o0 + 0x04], %g1 + lduw [%o0 + 0x08], %g2 + add %o4, %o5, %o4 + lduw [%o0 + 0x0c], %g3 + add %o4, %g1, %o4 + lduw [%o0 + 0x10], %o5 + add %o4, %g2, %o4 + lduw [%o0 + 0x14], %g1 + add %o4, %g3, %o4 + lduw [%o0 + 0x18], %g2 + add %o4, %o5, %o4 + lduw [%o0 + 0x1c], %g3 + add %o4, %g1, %o4 + lduw [%o0 + 0x20], %o5 + add %o4, %g2, %o4 + lduw [%o0 + 0x24], %g1 + add %o4, %g3, %o4 + lduw [%o0 + 0x28], %g2 + add %o4, %o5, %o4 + lduw [%o0 + 0x2c], %g3 + add %o4, %g1, %o4 + lduw [%o0 + 0x30], %o5 + add %o4, %g2, %o4 + lduw [%o0 + 0x34], %g1 + add %o4, %g3, %o4 + lduw [%o0 + 0x38], %g2 + add %o4, %o5, %o4 + lduw [%o0 + 0x3c], %g3 + add %o4, %g1, %o4 + prefetch [%o0 + 0x180], #n_reads + add %o4, %g2, %o4 + subcc %o3, 0x40, %o3 + add %o0, 0x40, %o0 + bne,pt %icc, 1b + add %o4, %g3, %o4 + +2: and %o1, 0x3c, %o3 + brz,pn %o3, 2f + sub %o1, %o3, %o1 +1: lduw [%o0 + 0x00], %o5 + subcc %o3, 0x4, %o3 + add %o0, 0x4, %o0 + bne,pt %icc, 1b + add %o4, %o5, %o4 + +2: + /* fold 64-->32 */ + srlx %o4, 32, %o5 + srl %o4, 0, %o4 + add %o4, %o5, %o4 + srlx %o4, 32, %o5 + srl %o4, 0, %o4 + add %o4, %o5, %o4 + + /* fold 32-->16 */ + sethi %hi(0xffff0000), %g1 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + +csum_partial_end_cruft: + /* %o4 has the 16-bit sum we have calculated so-far. */ + cmp %o1, 2 + blu,pt %icc, 1f + nop + lduh [%o0 + 0x00], %o5 + sub %o1, 2, %o1 + add %o0, 2, %o0 + add %o4, %o5, %o4 +1: brz,pt %o1, 1f + nop + ldub [%o0 + 0x00], %o5 + sub %o1, 1, %o1 + add %o0, 1, %o0 + sllx %o5, 8, %o5 + add %o4, %o5, %o4 +1: + /* fold 32-->16 */ + sethi %hi(0xffff0000), %g1 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + +1: brz,pt %g7, 1f + nop + + /* We started with an odd byte, byte-swap the result. */ + srl %o4, 8, %o5 + and %o4, 0xff, %g1 + sll %g1, 8, %g1 + or %o5, %g1, %o4 + +1: addcc %o2, %o4, %o2 + addc %g0, %o2, %o2 + +csum_partial_finish: + retl + srl %o2, 0, %o0 diff --git a/arch/sparc/lib/clear_page.S b/arch/sparc/lib/clear_page.S new file mode 100644 index 000000000..e63458194 --- /dev/null +++ b/arch/sparc/lib/clear_page.S @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* clear_page.S: UltraSparc optimized clear page. + * + * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com) + * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include + + /* What we used to do was lock a TLB entry into a specific + * TLB slot, clear the page with interrupts disabled, then + * restore the original TLB entry. This was great for + * disturbing the TLB as little as possible, but it meant + * we had to keep interrupts disabled for a long time. + * + * Now, we simply use the normal TLB loading mechanism, + * and this makes the cpu choose a slot all by itself. + * Then we do a normal TLB flush on exit. We need only + * disable preemption during the clear. + */ + + .text + + .globl _clear_page + EXPORT_SYMBOL(_clear_page) +_clear_page: /* %o0=dest */ + ba,pt %xcc, clear_page_common + clr %o4 + + /* This thing is pretty important, it shows up + * on the profiles via do_anonymous_page(). + */ + .align 32 + .globl clear_user_page + EXPORT_SYMBOL(clear_user_page) +clear_user_page: /* %o0=dest, %o1=vaddr */ + lduw [%g6 + TI_PRE_COUNT], %o2 + sethi %hi(PAGE_OFFSET), %g2 + sethi %hi(PAGE_SIZE), %o4 + + ldx [%g2 + %lo(PAGE_OFFSET)], %g2 + sethi %hi(PAGE_KERNEL_LOCKED), %g3 + + ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 + sub %o0, %g2, %g1 ! paddr + + and %o1, %o4, %o0 ! vaddr D-cache alias bit + + or %g1, %g3, %g1 ! TTE data + sethi %hi(TLBTEMP_BASE), %o3 + + add %o2, 1, %o4 + add %o0, %o3, %o0 ! TTE vaddr + + /* Disable preemption. */ + mov TLB_TAG_ACCESS, %g3 + stw %o4, [%g6 + TI_PRE_COUNT] + + /* Load TLB entry. */ + rdpr %pstate, %o4 + wrpr %o4, PSTATE_IE, %pstate + stxa %o0, [%g3] ASI_DMMU + stxa %g1, [%g0] ASI_DTLB_DATA_IN + sethi %hi(KERNBASE), %g1 + flush %g1 + wrpr %o4, 0x0, %pstate + + mov 1, %o4 + +clear_page_common: + VISEntryHalf + membar #StoreLoad | #StoreStore | #LoadStore + fzero %f0 + sethi %hi(PAGE_SIZE/64), %o1 + mov %o0, %g1 ! remember vaddr for tlbflush + fzero %f2 + or %o1, %lo(PAGE_SIZE/64), %o1 + faddd %f0, %f2, %f4 + fmuld %f0, %f2, %f6 + faddd %f0, %f2, %f8 + fmuld %f0, %f2, %f10 + + faddd %f0, %f2, %f12 + fmuld %f0, %f2, %f14 +1: stda %f0, [%o0 + %g0] ASI_BLK_P + subcc %o1, 1, %o1 + bne,pt %icc, 1b + add %o0, 0x40, %o0 + membar #Sync + VISExitHalf + + brz,pn %o4, out + nop + + stxa %g0, [%g1] ASI_DMMU_DEMAP + membar #Sync + stw %o2, [%g6 + TI_PRE_COUNT] + +out: retl + nop + diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S new file mode 100644 index 000000000..e23e6a69f --- /dev/null +++ b/arch/sparc/lib/copy_in_user.S @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* copy_in_user.S: Copy from userspace to userspace. + * + * Copyright (C) 1999, 2000, 2004 David S. Miller (davem@redhat.com) + */ + +#include +#include +#include + +#define XCC xcc + +#define EX(x,y,z) \ +98: x,y; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, z; \ + .text; \ + .align 4; + +#define EX_O4(x,y) EX(x,y,__retl_o4_plus_8) +#define EX_O2_4(x,y) EX(x,y,__retl_o2_plus_4) +#define EX_O2_1(x,y) EX(x,y,__retl_o2_plus_1) + + .register %g2,#scratch + .register %g3,#scratch + + .text +__retl_o4_plus_8: + add %o4, %o2, %o4 + retl + add %o4, 8, %o0 +__retl_o2_plus_4: + retl + add %o2, 4, %o0 +__retl_o2_plus_1: + retl + add %o2, 1, %o0 + + .align 32 + + /* Don't try to get too fancy here, just nice and + * simple. This is predominantly used for well aligned + * small copies in the compat layer. It is also used + * to copy register windows around during thread cloning. + */ + +ENTRY(raw_copy_in_user) /* %o0=dst, %o1=src, %o2=len */ + cmp %o2, 0 + be,pn %XCC, 85f + or %o0, %o1, %o3 + cmp %o2, 16 + bleu,a,pn %XCC, 80f + or %o3, %o2, %o3 + + /* 16 < len <= 64 */ + andcc %o3, 0x7, %g0 + bne,pn %XCC, 90f + nop + + andn %o2, 0x7, %o4 + and %o2, 0x7, %o2 +1: subcc %o4, 0x8, %o4 + EX_O4(ldxa [%o1] %asi, %o5) + EX_O4(stxa %o5, [%o0] %asi) + add %o1, 0x8, %o1 + bgu,pt %XCC, 1b + add %o0, 0x8, %o0 + andcc %o2, 0x4, %g0 + be,pt %XCC, 1f + nop + sub %o2, 0x4, %o2 + EX_O2_4(lduwa [%o1] %asi, %o5) + EX_O2_4(stwa %o5, [%o0] %asi) + add %o1, 0x4, %o1 + add %o0, 0x4, %o0 +1: cmp %o2, 0 + be,pt %XCC, 85f + nop + ba,pt %xcc, 90f + nop + +80: /* 0 < len <= 16 */ + andcc %o3, 0x3, %g0 + bne,pn %XCC, 90f + nop + +82: + subcc %o2, 4, %o2 + EX_O2_4(lduwa [%o1] %asi, %g1) + EX_O2_4(stwa %g1, [%o0] %asi) + add %o1, 4, %o1 + bgu,pt %XCC, 82b + add %o0, 4, %o0 + +85: retl + clr %o0 + + .align 32 +90: + subcc %o2, 1, %o2 + EX_O2_1(lduba [%o1] %asi, %g1) + EX_O2_1(stba %g1, [%o0] %asi) + add %o1, 1, %o1 + bgu,pt %XCC, 90b + add %o0, 1, %o0 + retl + clr %o0 +ENDPROC(raw_copy_in_user) +EXPORT_SYMBOL(raw_copy_in_user) diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S new file mode 100644 index 000000000..7a041f3eb --- /dev/null +++ b/arch/sparc/lib/copy_page.S @@ -0,0 +1,253 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* clear_page.S: UltraSparc optimized copy page. + * + * Copyright (C) 1996, 1998, 1999, 2000, 2004 David S. Miller (davem@redhat.com) + * Copyright (C) 1997 Jakub Jelinek (jakub@redhat.com) + */ + +#include +#include +#include +#include +#include +#include +#include + + /* What we used to do was lock a TLB entry into a specific + * TLB slot, clear the page with interrupts disabled, then + * restore the original TLB entry. This was great for + * disturbing the TLB as little as possible, but it meant + * we had to keep interrupts disabled for a long time. + * + * Now, we simply use the normal TLB loading mechanism, + * and this makes the cpu choose a slot all by itself. + * Then we do a normal TLB flush on exit. We need only + * disable preemption during the clear. + */ + +#define DCACHE_SIZE (PAGE_SIZE * 2) + +#if (PAGE_SHIFT == 13) +#define PAGE_SIZE_REM 0x80 +#elif (PAGE_SHIFT == 16) +#define PAGE_SIZE_REM 0x100 +#else +#error Wrong PAGE_SHIFT specified +#endif + +#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \ + fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \ + fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \ + fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \ + fsrc2 %reg6, %f60; fsrc2 %reg7, %f62; + + .text + + .align 32 + .globl copy_user_page + .type copy_user_page,#function + EXPORT_SYMBOL(copy_user_page) +copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ + lduw [%g6 + TI_PRE_COUNT], %o4 + sethi %hi(PAGE_OFFSET), %g2 + sethi %hi(PAGE_SIZE), %o3 + + ldx [%g2 + %lo(PAGE_OFFSET)], %g2 + sethi %hi(PAGE_KERNEL_LOCKED), %g3 + + ldx [%g3 + %lo(PAGE_KERNEL_LOCKED)], %g3 + sub %o0, %g2, %g1 ! dest paddr + + sub %o1, %g2, %g2 ! src paddr + + and %o2, %o3, %o0 ! vaddr D-cache alias bit + or %g1, %g3, %g1 ! dest TTE data + + or %g2, %g3, %g2 ! src TTE data + sethi %hi(TLBTEMP_BASE), %o3 + + sethi %hi(DCACHE_SIZE), %o1 + add %o0, %o3, %o0 ! dest TTE vaddr + + add %o4, 1, %o2 + add %o0, %o1, %o1 ! src TTE vaddr + + /* Disable preemption. */ + mov TLB_TAG_ACCESS, %g3 + stw %o2, [%g6 + TI_PRE_COUNT] + + /* Load TLB entries. */ + rdpr %pstate, %o2 + wrpr %o2, PSTATE_IE, %pstate + stxa %o0, [%g3] ASI_DMMU + stxa %g1, [%g0] ASI_DTLB_DATA_IN + membar #Sync + stxa %o1, [%g3] ASI_DMMU + stxa %g2, [%g0] ASI_DTLB_DATA_IN + membar #Sync + wrpr %o2, 0x0, %pstate + +cheetah_copy_page_insn: + ba,pt %xcc, 9f + nop + +1: + VISEntryHalf + membar #StoreLoad | #StoreStore | #LoadStore + sethi %hi((PAGE_SIZE/64)-2), %o2 + mov %o0, %g1 + prefetch [%o1 + 0x000], #one_read + or %o2, %lo((PAGE_SIZE/64)-2), %o2 + prefetch [%o1 + 0x040], #one_read + prefetch [%o1 + 0x080], #one_read + prefetch [%o1 + 0x0c0], #one_read + ldd [%o1 + 0x000], %f0 + prefetch [%o1 + 0x100], #one_read + ldd [%o1 + 0x008], %f2 + prefetch [%o1 + 0x140], #one_read + ldd [%o1 + 0x010], %f4 + prefetch [%o1 + 0x180], #one_read + fsrc2 %f0, %f16 + ldd [%o1 + 0x018], %f6 + fsrc2 %f2, %f18 + ldd [%o1 + 0x020], %f8 + fsrc2 %f4, %f20 + ldd [%o1 + 0x028], %f10 + fsrc2 %f6, %f22 + ldd [%o1 + 0x030], %f12 + fsrc2 %f8, %f24 + ldd [%o1 + 0x038], %f14 + fsrc2 %f10, %f26 + ldd [%o1 + 0x040], %f0 +1: ldd [%o1 + 0x048], %f2 + fsrc2 %f12, %f28 + ldd [%o1 + 0x050], %f4 + fsrc2 %f14, %f30 + stda %f16, [%o0] ASI_BLK_P + ldd [%o1 + 0x058], %f6 + fsrc2 %f0, %f16 + ldd [%o1 + 0x060], %f8 + fsrc2 %f2, %f18 + ldd [%o1 + 0x068], %f10 + fsrc2 %f4, %f20 + ldd [%o1 + 0x070], %f12 + fsrc2 %f6, %f22 + ldd [%o1 + 0x078], %f14 + fsrc2 %f8, %f24 + ldd [%o1 + 0x080], %f0 + prefetch [%o1 + 0x180], #one_read + fsrc2 %f10, %f26 + subcc %o2, 1, %o2 + add %o0, 0x40, %o0 + bne,pt %xcc, 1b + add %o1, 0x40, %o1 + + ldd [%o1 + 0x048], %f2 + fsrc2 %f12, %f28 + ldd [%o1 + 0x050], %f4 + fsrc2 %f14, %f30 + stda %f16, [%o0] ASI_BLK_P + ldd [%o1 + 0x058], %f6 + fsrc2 %f0, %f16 + ldd [%o1 + 0x060], %f8 + fsrc2 %f2, %f18 + ldd [%o1 + 0x068], %f10 + fsrc2 %f4, %f20 + ldd [%o1 + 0x070], %f12 + fsrc2 %f6, %f22 + add %o0, 0x40, %o0 + ldd [%o1 + 0x078], %f14 + fsrc2 %f8, %f24 + fsrc2 %f10, %f26 + fsrc2 %f12, %f28 + fsrc2 %f14, %f30 + stda %f16, [%o0] ASI_BLK_P + membar #Sync + VISExitHalf + ba,pt %xcc, 5f + nop + +9: + VISEntry + ldub [%g6 + TI_FAULT_CODE], %g3 + mov %o0, %g1 + cmp %g3, 0 + rd %asi, %g3 + be,a,pt %icc, 1f + wr %g0, ASI_BLK_P, %asi + wr %g0, ASI_BLK_COMMIT_P, %asi +1: ldda [%o1] ASI_BLK_P, %f0 + add %o1, 0x40, %o1 + ldda [%o1] ASI_BLK_P, %f16 + add %o1, 0x40, %o1 + sethi %hi(PAGE_SIZE), %o2 +1: TOUCH(f0, f2, f4, f6, f8, f10, f12, f14) + ldda [%o1] ASI_BLK_P, %f32 + stda %f48, [%o0] %asi + add %o1, 0x40, %o1 + sub %o2, 0x40, %o2 + add %o0, 0x40, %o0 + TOUCH(f16, f18, f20, f22, f24, f26, f28, f30) + ldda [%o1] ASI_BLK_P, %f0 + stda %f48, [%o0] %asi + add %o1, 0x40, %o1 + sub %o2, 0x40, %o2 + add %o0, 0x40, %o0 + TOUCH(f32, f34, f36, f38, f40, f42, f44, f46) + ldda [%o1] ASI_BLK_P, %f16 + stda %f48, [%o0] %asi + sub %o2, 0x40, %o2 + add %o1, 0x40, %o1 + cmp %o2, PAGE_SIZE_REM + bne,pt %xcc, 1b + add %o0, 0x40, %o0 +#if (PAGE_SHIFT == 16) + TOUCH(f0, f2, f4, f6, f8, f10, f12, f14) + ldda [%o1] ASI_BLK_P, %f32 + stda %f48, [%o0] %asi + add %o1, 0x40, %o1 + sub %o2, 0x40, %o2 + add %o0, 0x40, %o0 + TOUCH(f16, f18, f20, f22, f24, f26, f28, f30) + ldda [%o1] ASI_BLK_P, %f0 + stda %f48, [%o0] %asi + add %o1, 0x40, %o1 + sub %o2, 0x40, %o2 + add %o0, 0x40, %o0 + membar #Sync + stda %f32, [%o0] %asi + add %o0, 0x40, %o0 + stda %f0, [%o0] %asi +#else + membar #Sync + stda %f0, [%o0] %asi + add %o0, 0x40, %o0 + stda %f16, [%o0] %asi +#endif + membar #Sync + wr %g3, 0x0, %asi + VISExit + +5: + stxa %g0, [%g1] ASI_DMMU_DEMAP + membar #Sync + + sethi %hi(DCACHE_SIZE), %g2 + stxa %g0, [%g1 + %g2] ASI_DMMU_DEMAP + membar #Sync + + retl + stw %o4, [%g6 + TI_PRE_COUNT] + + .size copy_user_page, .-copy_user_page + + .globl cheetah_patch_copy_page +cheetah_patch_copy_page: + sethi %hi(0x01000000), %o1 ! NOP + sethi %hi(cheetah_copy_page_insn), %o0 + or %o0, %lo(cheetah_copy_page_insn), %o0 + stw %o1, [%o0] + membar #StoreStore + flush %o0 + retl + nop diff --git a/arch/sparc/lib/copy_user.S b/arch/sparc/lib/copy_user.S new file mode 100644 index 000000000..7bb2ef688 --- /dev/null +++ b/arch/sparc/lib/copy_user.S @@ -0,0 +1,394 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* copy_user.S: Sparc optimized copy_from_user and copy_to_user code. + * + * Copyright(C) 1995 Linus Torvalds + * Copyright(C) 1996 David S. Miller + * Copyright(C) 1996 Eddie C. Dost + * Copyright(C) 1996,1998 Jakub Jelinek + * + * derived from: + * e-mail between David and Eddie. + * + * Returns 0 if successful, otherwise count of bytes not copied yet + */ + +#include +#include +#include +#include +#include + +/* Work around cpp -rob */ +#define ALLOC #alloc +#define EXECINSTR #execinstr + +#define EX_ENTRY(l1, l2) \ + .section __ex_table,ALLOC; \ + .align 4; \ + .word l1, l2; \ + .text; + +#define EX(x,y,a,b) \ +98: x,y; \ + .section .fixup,ALLOC,EXECINSTR; \ + .align 4; \ +99: retl; \ + a, b, %o0; \ + EX_ENTRY(98b, 99b) + +#define EX2(x,y,c,d,e,a,b) \ +98: x,y; \ + .section .fixup,ALLOC,EXECINSTR; \ + .align 4; \ +99: c, d, e; \ + retl; \ + a, b, %o0; \ + EX_ENTRY(98b, 99b) + +#define EXO2(x,y) \ +98: x, y; \ + EX_ENTRY(98b, 97f) + +#define LD(insn, src, offset, reg, label) \ +98: insn [%src + (offset)], %reg; \ + .section .fixup,ALLOC,EXECINSTR; \ +99: ba label; \ + mov offset, %g5; \ + EX_ENTRY(98b, 99b) + +#define ST(insn, dst, offset, reg, label) \ +98: insn %reg, [%dst + (offset)]; \ + .section .fixup,ALLOC,EXECINSTR; \ +99: ba label; \ + mov offset, %g5; \ + EX_ENTRY(98b, 99b) + +/* Both these macros have to start with exactly the same insn */ +/* left: g7 + (g1 % 128) - offset */ +#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ + LD(ldd, src, offset + 0x00, t0, bigchunk_fault) \ + LD(ldd, src, offset + 0x08, t2, bigchunk_fault) \ + LD(ldd, src, offset + 0x10, t4, bigchunk_fault) \ + LD(ldd, src, offset + 0x18, t6, bigchunk_fault) \ + ST(st, dst, offset + 0x00, t0, bigchunk_fault) \ + ST(st, dst, offset + 0x04, t1, bigchunk_fault) \ + ST(st, dst, offset + 0x08, t2, bigchunk_fault) \ + ST(st, dst, offset + 0x0c, t3, bigchunk_fault) \ + ST(st, dst, offset + 0x10, t4, bigchunk_fault) \ + ST(st, dst, offset + 0x14, t5, bigchunk_fault) \ + ST(st, dst, offset + 0x18, t6, bigchunk_fault) \ + ST(st, dst, offset + 0x1c, t7, bigchunk_fault) + +/* left: g7 + (g1 % 128) - offset */ +#define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ + LD(ldd, src, offset + 0x00, t0, bigchunk_fault) \ + LD(ldd, src, offset + 0x08, t2, bigchunk_fault) \ + LD(ldd, src, offset + 0x10, t4, bigchunk_fault) \ + LD(ldd, src, offset + 0x18, t6, bigchunk_fault) \ + ST(std, dst, offset + 0x00, t0, bigchunk_fault) \ + ST(std, dst, offset + 0x08, t2, bigchunk_fault) \ + ST(std, dst, offset + 0x10, t4, bigchunk_fault) \ + ST(std, dst, offset + 0x18, t6, bigchunk_fault) + + .section .fixup,#alloc,#execinstr +bigchunk_fault: + sub %g7, %g5, %o0 + and %g1, 127, %g1 + retl + add %o0, %g1, %o0 + +/* left: offset + 16 + (g1 % 16) */ +#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \ + LD(ldd, src, -(offset + 0x10), t0, lastchunk_fault) \ + LD(ldd, src, -(offset + 0x08), t2, lastchunk_fault) \ + ST(st, dst, -(offset + 0x10), t0, lastchunk_fault) \ + ST(st, dst, -(offset + 0x0c), t1, lastchunk_fault) \ + ST(st, dst, -(offset + 0x08), t2, lastchunk_fault) \ + ST(st, dst, -(offset + 0x04), t3, lastchunk_fault) + + .section .fixup,#alloc,#execinstr +lastchunk_fault: + and %g1, 15, %g1 + retl + sub %g1, %g5, %o0 + +/* left: o3 + (o2 % 16) - offset */ +#define MOVE_HALFCHUNK(src, dst, offset, t0, t1, t2, t3) \ + LD(lduh, src, offset + 0x00, t0, halfchunk_fault) \ + LD(lduh, src, offset + 0x02, t1, halfchunk_fault) \ + LD(lduh, src, offset + 0x04, t2, halfchunk_fault) \ + LD(lduh, src, offset + 0x06, t3, halfchunk_fault) \ + ST(sth, dst, offset + 0x00, t0, halfchunk_fault) \ + ST(sth, dst, offset + 0x02, t1, halfchunk_fault) \ + ST(sth, dst, offset + 0x04, t2, halfchunk_fault) \ + ST(sth, dst, offset + 0x06, t3, halfchunk_fault) + +/* left: o3 + (o2 % 16) + offset + 2 */ +#define MOVE_SHORTCHUNK(src, dst, offset, t0, t1) \ + LD(ldub, src, -(offset + 0x02), t0, halfchunk_fault) \ + LD(ldub, src, -(offset + 0x01), t1, halfchunk_fault) \ + ST(stb, dst, -(offset + 0x02), t0, halfchunk_fault) \ + ST(stb, dst, -(offset + 0x01), t1, halfchunk_fault) + + .section .fixup,#alloc,#execinstr +halfchunk_fault: + and %o2, 15, %o2 + sub %o3, %g5, %o3 + retl + add %o2, %o3, %o0 + +/* left: offset + 2 + (o2 % 2) */ +#define MOVE_LAST_SHORTCHUNK(src, dst, offset, t0, t1) \ + LD(ldub, src, -(offset + 0x02), t0, last_shortchunk_fault) \ + LD(ldub, src, -(offset + 0x01), t1, last_shortchunk_fault) \ + ST(stb, dst, -(offset + 0x02), t0, last_shortchunk_fault) \ + ST(stb, dst, -(offset + 0x01), t1, last_shortchunk_fault) + + .section .fixup,#alloc,#execinstr +last_shortchunk_fault: + and %o2, 1, %o2 + retl + sub %o2, %g5, %o0 + + .text + .align 4 + + .globl __copy_user_begin +__copy_user_begin: + + .globl __copy_user + EXPORT_SYMBOL(__copy_user) +dword_align: + andcc %o1, 1, %g0 + be 4f + andcc %o1, 2, %g0 + + EXO2(ldub [%o1], %g2) + add %o1, 1, %o1 + EXO2(stb %g2, [%o0]) + sub %o2, 1, %o2 + bne 3f + add %o0, 1, %o0 + + EXO2(lduh [%o1], %g2) + add %o1, 2, %o1 + EXO2(sth %g2, [%o0]) + sub %o2, 2, %o2 + b 3f + add %o0, 2, %o0 +4: + EXO2(lduh [%o1], %g2) + add %o1, 2, %o1 + EXO2(sth %g2, [%o0]) + sub %o2, 2, %o2 + b 3f + add %o0, 2, %o0 + +__copy_user: /* %o0=dst %o1=src %o2=len */ + xor %o0, %o1, %o4 +1: + andcc %o4, 3, %o5 +2: + bne cannot_optimize + cmp %o2, 15 + + bleu short_aligned_end + andcc %o1, 3, %g0 + + bne dword_align +3: + andcc %o1, 4, %g0 + + be 2f + mov %o2, %g1 + + EXO2(ld [%o1], %o4) + sub %g1, 4, %g1 + EXO2(st %o4, [%o0]) + add %o1, 4, %o1 + add %o0, 4, %o0 +2: + andcc %g1, 0xffffff80, %g7 + be 3f + andcc %o0, 4, %g0 + + be ldd_std + 4 +5: + MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) + subcc %g7, 128, %g7 + add %o1, 128, %o1 + bne 5b + add %o0, 128, %o0 +3: + andcc %g1, 0x70, %g7 + be copy_user_table_end + andcc %g1, 8, %g0 + + sethi %hi(copy_user_table_end), %o5 + srl %g7, 1, %o4 + add %g7, %o4, %o4 + add %o1, %g7, %o1 + sub %o5, %o4, %o5 + jmpl %o5 + %lo(copy_user_table_end), %g0 + add %o0, %g7, %o0 + + MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5) +copy_user_table_end: + be copy_user_last7 + andcc %g1, 4, %g0 + + EX(ldd [%o1], %g2, and %g1, 0xf) + add %o0, 8, %o0 + add %o1, 8, %o1 + EX(st %g2, [%o0 - 0x08], and %g1, 0xf) + EX2(st %g3, [%o0 - 0x04], and %g1, 0xf, %g1, sub %g1, 4) +copy_user_last7: + be 1f + andcc %g1, 2, %g0 + + EX(ld [%o1], %g2, and %g1, 7) + add %o1, 4, %o1 + EX(st %g2, [%o0], and %g1, 7) + add %o0, 4, %o0 +1: + be 1f + andcc %g1, 1, %g0 + + EX(lduh [%o1], %g2, and %g1, 3) + add %o1, 2, %o1 + EX(sth %g2, [%o0], and %g1, 3) + add %o0, 2, %o0 +1: + be 1f + nop + + EX(ldub [%o1], %g2, add %g0, 1) + EX(stb %g2, [%o0], add %g0, 1) +1: + retl + clr %o0 + +ldd_std: + MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) + subcc %g7, 128, %g7 + add %o1, 128, %o1 + bne ldd_std + add %o0, 128, %o0 + + andcc %g1, 0x70, %g7 + be copy_user_table_end + andcc %g1, 8, %g0 + + sethi %hi(copy_user_table_end), %o5 + srl %g7, 1, %o4 + add %g7, %o4, %o4 + add %o1, %g7, %o1 + sub %o5, %o4, %o5 + jmpl %o5 + %lo(copy_user_table_end), %g0 + add %o0, %g7, %o0 + +cannot_optimize: + bleu short_end + cmp %o5, 2 + + bne byte_chunk + and %o2, 0xfffffff0, %o3 + + andcc %o1, 1, %g0 + be 10f + nop + + EXO2(ldub [%o1], %g2) + add %o1, 1, %o1 + EXO2(stb %g2, [%o0]) + sub %o2, 1, %o2 + andcc %o2, 0xfffffff0, %o3 + be short_end + add %o0, 1, %o0 +10: + MOVE_HALFCHUNK(o1, o0, 0x00, g2, g3, g4, g5) + MOVE_HALFCHUNK(o1, o0, 0x08, g2, g3, g4, g5) + subcc %o3, 0x10, %o3 + add %o1, 0x10, %o1 + bne 10b + add %o0, 0x10, %o0 + b 2f + and %o2, 0xe, %o3 + +byte_chunk: + MOVE_SHORTCHUNK(o1, o0, -0x02, g2, g3) + MOVE_SHORTCHUNK(o1, o0, -0x04, g2, g3) + MOVE_SHORTCHUNK(o1, o0, -0x06, g2, g3) + MOVE_SHORTCHUNK(o1, o0, -0x08, g2, g3) + MOVE_SHORTCHUNK(o1, o0, -0x0a, g2, g3) + MOVE_SHORTCHUNK(o1, o0, -0x0c, g2, g3) + MOVE_SHORTCHUNK(o1, o0, -0x0e, g2, g3) + MOVE_SHORTCHUNK(o1, o0, -0x10, g2, g3) + subcc %o3, 0x10, %o3 + add %o1, 0x10, %o1 + bne byte_chunk + add %o0, 0x10, %o0 + +short_end: + and %o2, 0xe, %o3 +2: + sethi %hi(short_table_end), %o5 + sll %o3, 3, %o4 + add %o0, %o3, %o0 + sub %o5, %o4, %o5 + add %o1, %o3, %o1 + jmpl %o5 + %lo(short_table_end), %g0 + andcc %o2, 1, %g0 + MOVE_LAST_SHORTCHUNK(o1, o0, 0x0c, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x0a, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x08, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x06, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x04, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x02, g2, g3) + MOVE_LAST_SHORTCHUNK(o1, o0, 0x00, g2, g3) +short_table_end: + be 1f + nop + EX(ldub [%o1], %g2, add %g0, 1) + EX(stb %g2, [%o0], add %g0, 1) +1: + retl + clr %o0 + +short_aligned_end: + bne short_end + andcc %o2, 8, %g0 + + be 1f + andcc %o2, 4, %g0 + + EXO2(ld [%o1 + 0x00], %g2) + EXO2(ld [%o1 + 0x04], %g3) + add %o1, 8, %o1 + EXO2(st %g2, [%o0 + 0x00]) + EX(st %g3, [%o0 + 0x04], sub %o2, 4) + add %o0, 8, %o0 +1: + b copy_user_last7 + mov %o2, %g1 + + .section .fixup,#alloc,#execinstr + .align 4 +97: + retl + mov %o2, %o0 + + .globl __copy_user_end +__copy_user_end: diff --git a/arch/sparc/lib/csum_copy.S b/arch/sparc/lib/csum_copy.S new file mode 100644 index 000000000..f968e83bc --- /dev/null +++ b/arch/sparc/lib/csum_copy.S @@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* csum_copy.S: Checksum+copy code for sparc64 + * + * Copyright (C) 2005 David S. Miller + */ + +#include + +#ifdef __KERNEL__ +#define GLOBAL_SPARE %g7 +#else +#define GLOBAL_SPARE %g5 +#endif + +#ifndef EX_LD +#define EX_LD(x) x +#endif + +#ifndef EX_ST +#define EX_ST(x) x +#endif + +#ifndef EX_RETVAL +#define EX_RETVAL(x) x +#endif + +#ifndef LOAD +#define LOAD(type,addr,dest) type [addr], dest +#endif + +#ifndef STORE +#define STORE(type,src,addr) type src, [addr] +#endif + +#ifndef FUNC_NAME +#define FUNC_NAME csum_partial_copy_nocheck +#endif + + .register %g2, #scratch + .register %g3, #scratch + + .text + +90: + /* We checked for zero length already, so there must be + * at least one byte. + */ + be,pt %icc, 1f + nop + EX_LD(LOAD(ldub, %o0 + 0x00, %o4)) + add %o0, 1, %o0 + sub %o2, 1, %o2 + EX_ST(STORE(stb, %o4, %o1 + 0x00)) + add %o1, 1, %o1 +1: andcc %o0, 0x2, %g0 + be,pn %icc, 80f + cmp %o2, 2 + blu,pn %icc, 60f + nop + EX_LD(LOAD(lduh, %o0 + 0x00, %o5)) + add %o0, 2, %o0 + sub %o2, 2, %o2 + EX_ST(STORE(sth, %o5, %o1 + 0x00)) + add %o1, 2, %o1 + ba,pt %xcc, 80f + add %o5, %o4, %o4 + + .globl FUNC_NAME + .type FUNC_NAME,#function + EXPORT_SYMBOL(FUNC_NAME) +FUNC_NAME: /* %o0=src, %o1=dst, %o2=len */ + LOAD(prefetch, %o0 + 0x000, #n_reads) + xor %o0, %o1, %g1 + mov -1, %o3 + clr %o4 + andcc %g1, 0x3, %g0 + bne,pn %icc, 95f + LOAD(prefetch, %o0 + 0x040, #n_reads) + + brz,pn %o2, 70f + andcc %o0, 0x3, %g0 + + /* We "remember" whether the lowest bit in the address + * was set in GLOBAL_SPARE. Because if it is, we have to swap + * upper and lower 8 bit fields of the sum we calculate. + */ + bne,pn %icc, 90b + andcc %o0, 0x1, GLOBAL_SPARE + +80: + LOAD(prefetch, %o0 + 0x080, #n_reads) + andncc %o2, 0x3f, %g3 + + LOAD(prefetch, %o0 + 0x0c0, #n_reads) + sub %o2, %g3, %o2 + brz,pn %g3, 2f + LOAD(prefetch, %o0 + 0x100, #n_reads) + + /* So that we don't need to use the non-pairing + * add-with-carry instructions we accumulate 32-bit + * values into a 64-bit register. At the end of the + * loop we fold it down to 32-bits and so on. + */ + ba,pt %xcc, 1f + LOAD(prefetch, %o0 + 0x140, #n_reads) + + .align 32 +1: EX_LD(LOAD(lduw, %o0 + 0x00, %o5)) + EX_LD(LOAD(lduw, %o0 + 0x04, %g1)) + EX_LD(LOAD(lduw, %o0 + 0x08, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x00)) + EX_LD(LOAD(lduw, %o0 + 0x0c, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x04)) + EX_LD(LOAD(lduw, %o0 + 0x10, %g1)) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x08)) + EX_LD(LOAD(lduw, %o0 + 0x14, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x0c)) + EX_LD(LOAD(lduw, %o0 + 0x18, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x10)) + EX_LD(LOAD(lduw, %o0 + 0x1c, %g1)) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x14)) + EX_LD(LOAD(lduw, %o0 + 0x20, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x18)) + EX_LD(LOAD(lduw, %o0 + 0x24, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x1c)) + EX_LD(LOAD(lduw, %o0 + 0x28, %g1)) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x20)) + EX_LD(LOAD(lduw, %o0 + 0x2c, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x24)) + EX_LD(LOAD(lduw, %o0 + 0x30, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x28)) + EX_LD(LOAD(lduw, %o0 + 0x34, %g1)) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x2c)) + EX_LD(LOAD(lduw, %o0 + 0x38, %g2)) + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x30)) + EX_LD(LOAD(lduw, %o0 + 0x3c, %o5)) + add %o4, %g1, %o4 + EX_ST(STORE(stw, %g1, %o1 + 0x34)) + LOAD(prefetch, %o0 + 0x180, #n_reads) + add %o4, %g2, %o4 + EX_ST(STORE(stw, %g2, %o1 + 0x38)) + subcc %g3, 0x40, %g3 + add %o0, 0x40, %o0 + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x3c)) + bne,pt %icc, 1b + add %o1, 0x40, %o1 + +2: and %o2, 0x3c, %g3 + brz,pn %g3, 2f + sub %o2, %g3, %o2 +1: EX_LD(LOAD(lduw, %o0 + 0x00, %o5)) + subcc %g3, 0x4, %g3 + add %o0, 0x4, %o0 + add %o4, %o5, %o4 + EX_ST(STORE(stw, %o5, %o1 + 0x00)) + bne,pt %icc, 1b + add %o1, 0x4, %o1 + +2: + /* fold 64-->32 */ + srlx %o4, 32, %o5 + srl %o4, 0, %o4 + add %o4, %o5, %o4 + srlx %o4, 32, %o5 + srl %o4, 0, %o4 + add %o4, %o5, %o4 + + /* fold 32-->16 */ + sethi %hi(0xffff0000), %g1 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + +60: + /* %o4 has the 16-bit sum we have calculated so-far. */ + cmp %o2, 2 + blu,pt %icc, 1f + nop + EX_LD(LOAD(lduh, %o0 + 0x00, %o5)) + sub %o2, 2, %o2 + add %o0, 2, %o0 + add %o4, %o5, %o4 + EX_ST(STORE(sth, %o5, %o1 + 0x00)) + add %o1, 0x2, %o1 +1: brz,pt %o2, 1f + nop + EX_LD(LOAD(ldub, %o0 + 0x00, %o5)) + sub %o2, 1, %o2 + add %o0, 1, %o0 + EX_ST(STORE(stb, %o5, %o1 + 0x00)) + sllx %o5, 8, %o5 + add %o1, 1, %o1 + add %o4, %o5, %o4 +1: + /* fold 32-->16 */ + sethi %hi(0xffff0000), %g1 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + srl %o4, 16, %o5 + andn %o4, %g1, %g2 + add %o5, %g2, %o4 + +1: brz,pt GLOBAL_SPARE, 1f + nop + + /* We started with an odd byte, byte-swap the result. */ + srl %o4, 8, %o5 + and %o4, 0xff, %g1 + sll %g1, 8, %g1 + or %o5, %g1, %o4 + +1: addcc %o3, %o4, %o3 + addc %g0, %o3, %o3 + +70: + retl + srl %o3, 0, %o0 + +95: mov 0, GLOBAL_SPARE + brlez,pn %o2, 4f + andcc %o0, 1, %o5 + be,a,pt %icc, 1f + srl %o2, 1, %g1 + sub %o2, 1, %o2 + EX_LD(LOAD(ldub, %o0, GLOBAL_SPARE)) + add %o0, 1, %o0 + EX_ST(STORE(stb, GLOBAL_SPARE, %o1)) + srl %o2, 1, %g1 + add %o1, 1, %o1 +1: brz,a,pn %g1, 3f + andcc %o2, 1, %g0 + andcc %o0, 2, %g0 + be,a,pt %icc, 1f + srl %g1, 1, %g1 + EX_LD(LOAD(lduh, %o0, %o4)) + sub %o2, 2, %o2 + srl %o4, 8, %g2 + sub %g1, 1, %g1 + EX_ST(STORE(stb, %g2, %o1)) + add %o4, GLOBAL_SPARE, GLOBAL_SPARE + EX_ST(STORE(stb, %o4, %o1 + 1)) + add %o0, 2, %o0 + srl %g1, 1, %g1 + add %o1, 2, %o1 +1: brz,a,pn %g1, 2f + andcc %o2, 2, %g0 + EX_LD(LOAD(lduw, %o0, %o4)) +5: srl %o4, 24, %g2 + srl %o4, 16, %g3 + EX_ST(STORE(stb, %g2, %o1)) + srl %o4, 8, %g2 + EX_ST(STORE(stb, %g3, %o1 + 1)) + add %o0, 4, %o0 + EX_ST(STORE(stb, %g2, %o1 + 2)) + addcc %o4, GLOBAL_SPARE, GLOBAL_SPARE + EX_ST(STORE(stb, %o4, %o1 + 3)) + addc GLOBAL_SPARE, %g0, GLOBAL_SPARE + add %o1, 4, %o1 + subcc %g1, 1, %g1 + bne,a,pt %icc, 5b + EX_LD(LOAD(lduw, %o0, %o4)) + sll GLOBAL_SPARE, 16, %g2 + srl GLOBAL_SPARE, 16, GLOBAL_SPARE + srl %g2, 16, %g2 + andcc %o2, 2, %g0 + add %g2, GLOBAL_SPARE, GLOBAL_SPARE +2: be,a,pt %icc, 3f + andcc %o2, 1, %g0 + EX_LD(LOAD(lduh, %o0, %o4)) + andcc %o2, 1, %g0 + srl %o4, 8, %g2 + add %o0, 2, %o0 + EX_ST(STORE(stb, %g2, %o1)) + add GLOBAL_SPARE, %o4, GLOBAL_SPARE + EX_ST(STORE(stb, %o4, %o1 + 1)) + add %o1, 2, %o1 +3: be,a,pt %icc, 1f + sll GLOBAL_SPARE, 16, %o4 + EX_LD(LOAD(ldub, %o0, %g2)) + sll %g2, 8, %o4 + EX_ST(STORE(stb, %g2, %o1)) + add GLOBAL_SPARE, %o4, GLOBAL_SPARE + sll GLOBAL_SPARE, 16, %o4 +1: addcc %o4, GLOBAL_SPARE, GLOBAL_SPARE + srl GLOBAL_SPARE, 16, %o4 + addc %g0, %o4, GLOBAL_SPARE + brz,pt %o5, 4f + srl GLOBAL_SPARE, 8, %o4 + and GLOBAL_SPARE, 0xff, %g2 + and %o4, 0xff, %o4 + sll %g2, 8, %g2 + or %g2, %o4, GLOBAL_SPARE +4: addcc %o3, GLOBAL_SPARE, %o3 + addc %g0, %o3, %o0 + retl + srl %o0, 0, %o0 + .size FUNC_NAME, .-FUNC_NAME diff --git a/arch/sparc/lib/csum_copy_from_user.S b/arch/sparc/lib/csum_copy_from_user.S new file mode 100644 index 000000000..b0ba8d4dd --- /dev/null +++ b/arch/sparc/lib/csum_copy_from_user.S @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* csum_copy_from_user.S: Checksum+copy from userspace. + * + * Copyright (C) 2005 David S. Miller (davem@davemloft.net) + */ + +#define EX_LD(x) \ +98: x; \ + .section .fixup, "ax"; \ + .align 4; \ +99: retl; \ + mov 0, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#define FUNC_NAME csum_and_copy_from_user +#define LOAD(type,addr,dest) type##a [addr] %asi, dest + +#include "csum_copy.S" diff --git a/arch/sparc/lib/csum_copy_to_user.S b/arch/sparc/lib/csum_copy_to_user.S new file mode 100644 index 000000000..91ba36dbf --- /dev/null +++ b/arch/sparc/lib/csum_copy_to_user.S @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* csum_copy_to_user.S: Checksum+copy to userspace. + * + * Copyright (C) 2005 David S. Miller (davem@davemloft.net) + */ + +#define EX_ST(x) \ +98: x; \ + .section .fixup,"ax"; \ + .align 4; \ +99: retl; \ + mov 0, %o0; \ + .section __ex_table,"a";\ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#define FUNC_NAME csum_and_copy_to_user +#define STORE(type,src,addr) type##a src, [addr] %asi + +#include "csum_copy.S" diff --git a/arch/sparc/lib/divdi3.S b/arch/sparc/lib/divdi3.S new file mode 100644 index 000000000..4ba901acd --- /dev/null +++ b/arch/sparc/lib/divdi3.S @@ -0,0 +1,271 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 1989, 1992, 1993, 1994, 1995 Free Software Foundation, Inc. + +This file is part of GNU CC. + + */ + +#include + .text + .align 4 + .globl __divdi3 +__divdi3: + save %sp,-104,%sp + cmp %i0,0 + bge .LL40 + mov 0,%l4 + mov -1,%l4 + sub %g0,%i1,%o0 + mov %o0,%o5 + subcc %g0,%o0,%g0 + sub %g0,%i0,%o0 + subx %o0,0,%o4 + mov %o4,%i0 + mov %o5,%i1 +.LL40: + cmp %i2,0 + bge .LL84 + mov %i3,%o4 + xnor %g0,%l4,%l4 + sub %g0,%i3,%o0 + mov %o0,%o3 + subcc %g0,%o0,%g0 + sub %g0,%i2,%o0 + subx %o0,0,%o2 + mov %o2,%i2 + mov %o3,%i3 + mov %i3,%o4 +.LL84: + cmp %i2,0 + bne .LL45 + mov %i1,%i3 + cmp %o4,%i0 + bleu .LL46 + mov %i3,%o1 + mov 32,%g1 + subcc %i0,%o4,%g0 +1: bcs 5f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + sub %i0,%o4,%i0 ! this kills msb of n + addx %i0,%i0,%i0 ! so this cannot give carry + subcc %g1,1,%g1 +2: bne 1b + subcc %i0,%o4,%g0 + bcs 3f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + b 3f + sub %i0,%o4,%i0 ! this kills msb of n +4: sub %i0,%o4,%i0 +5: addxcc %i0,%i0,%i0 + bcc 2b + subcc %g1,1,%g1 +! Got carry from n. Subtract next step to cancel this carry. + bne 4b + addcc %o1,%o1,%o1 ! shift n1n0 and a 0-bit in lsb + sub %i0,%o4,%i0 +3: xnor %o1,0,%o1 + b .LL50 + mov 0,%o2 +.LL46: + cmp %o4,0 + bne .LL85 + mov %i0,%o2 + mov 1,%o0 + mov 0,%o1 + wr %g0, 0, %y + udiv %o0, %o1, %o0 + mov %o0,%o4 + mov %i0,%o2 +.LL85: + mov 0,%g3 + mov 32,%g1 + subcc %g3,%o4,%g0 +1: bcs 5f + addxcc %o2,%o2,%o2 ! shift n1n0 and a q-bit in lsb + sub %g3,%o4,%g3 ! this kills msb of n + addx %g3,%g3,%g3 ! so this cannot give carry + subcc %g1,1,%g1 +2: bne 1b + subcc %g3,%o4,%g0 + bcs 3f + addxcc %o2,%o2,%o2 ! shift n1n0 and a q-bit in lsb + b 3f + sub %g3,%o4,%g3 ! this kills msb of n +4: sub %g3,%o4,%g3 +5: addxcc %g3,%g3,%g3 + bcc 2b + subcc %g1,1,%g1 +! Got carry from n. Subtract next step to cancel this carry. + bne 4b + addcc %o2,%o2,%o2 ! shift n1n0 and a 0-bit in lsb + sub %g3,%o4,%g3 +3: xnor %o2,0,%o2 + mov %g3,%i0 + mov %i3,%o1 + mov 32,%g1 + subcc %i0,%o4,%g0 +1: bcs 5f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + sub %i0,%o4,%i0 ! this kills msb of n + addx %i0,%i0,%i0 ! so this cannot give carry + subcc %g1,1,%g1 +2: bne 1b + subcc %i0,%o4,%g0 + bcs 3f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + b 3f + sub %i0,%o4,%i0 ! this kills msb of n +4: sub %i0,%o4,%i0 +5: addxcc %i0,%i0,%i0 + bcc 2b + subcc %g1,1,%g1 +! Got carry from n. Subtract next step to cancel this carry. + bne 4b + addcc %o1,%o1,%o1 ! shift n1n0 and a 0-bit in lsb + sub %i0,%o4,%i0 +3: xnor %o1,0,%o1 + b .LL86 + mov %o1,%l1 +.LL45: + cmp %i2,%i0 + bleu .LL51 + sethi %hi(65535),%o0 + b .LL78 + mov 0,%o1 +.LL51: + or %o0,%lo(65535),%o0 + cmp %i2,%o0 + bgu .LL58 + mov %i2,%o1 + cmp %i2,256 + addx %g0,-1,%o0 + b .LL64 + and %o0,8,%o2 +.LL58: + sethi %hi(16777215),%o0 + or %o0,%lo(16777215),%o0 + cmp %i2,%o0 + bgu .LL64 + mov 24,%o2 + mov 16,%o2 +.LL64: + srl %o1,%o2,%o0 + sethi %hi(__clz_tab),%o1 + or %o1,%lo(__clz_tab),%o1 + ldub [%o0+%o1],%o0 + add %o0,%o2,%o0 + mov 32,%o1 + subcc %o1,%o0,%o3 + bne,a .LL72 + sub %o1,%o3,%o1 + cmp %i0,%i2 + bgu .LL74 + cmp %i3,%o4 + blu .LL78 + mov 0,%o1 +.LL74: + b .LL78 + mov 1,%o1 +.LL72: + sll %i2,%o3,%o2 + srl %o4,%o1,%o0 + or %o2,%o0,%i2 + sll %o4,%o3,%o4 + srl %i0,%o1,%o2 + sll %i0,%o3,%o0 + srl %i3,%o1,%o1 + or %o0,%o1,%i0 + sll %i3,%o3,%i3 + mov %i0,%o1 + mov 32,%g1 + subcc %o2,%i2,%g0 +1: bcs 5f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + sub %o2,%i2,%o2 ! this kills msb of n + addx %o2,%o2,%o2 ! so this cannot give carry + subcc %g1,1,%g1 +2: bne 1b + subcc %o2,%i2,%g0 + bcs 3f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + b 3f + sub %o2,%i2,%o2 ! this kills msb of n +4: sub %o2,%i2,%o2 +5: addxcc %o2,%o2,%o2 + bcc 2b + subcc %g1,1,%g1 +! Got carry from n. Subtract next step to cancel this carry. + bne 4b + addcc %o1,%o1,%o1 ! shift n1n0 and a 0-bit in lsb + sub %o2,%i2,%o2 +3: xnor %o1,0,%o1 + mov %o2,%i0 + wr %g0,%o1,%y ! SPARC has 0-3 delay insn after a wr + sra %o4,31,%g2 ! Do not move this insn + and %o1,%g2,%g2 ! Do not move this insn + andcc %g0,0,%g1 ! Do not move this insn + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,%o4,%g1 + mulscc %g1,0,%g1 + add %g1,%g2,%o0 + rd %y,%o2 + cmp %o0,%i0 + bgu,a .LL78 + add %o1,-1,%o1 + bne,a .LL50 + mov 0,%o2 + cmp %o2,%i3 + bleu .LL50 + mov 0,%o2 + add %o1,-1,%o1 +.LL78: + mov 0,%o2 +.LL50: + mov %o1,%l1 +.LL86: + mov %o2,%l0 + mov %l0,%i0 + mov %l1,%i1 + cmp %l4,0 + be .LL81 + sub %g0,%i1,%o0 + mov %o0,%l3 + subcc %g0,%o0,%g0 + sub %g0,%i0,%o0 + subx %o0,0,%l2 + mov %l2,%i0 + mov %l3,%i1 +.LL81: + ret + restore +EXPORT_SYMBOL(__divdi3) diff --git a/arch/sparc/lib/ffs.S b/arch/sparc/lib/ffs.S new file mode 100644 index 000000000..3a9ad8ffd --- /dev/null +++ b/arch/sparc/lib/ffs.S @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include + + .register %g2,#scratch + + .text + .align 32 + +ENTRY(ffs) + brnz,pt %o0, 1f + mov 1, %o1 + retl + clr %o0 + nop + nop +ENTRY(__ffs) + sllx %o0, 32, %g1 /* 1 */ + srlx %o0, 32, %g2 + + clr %o1 /* 2 */ + movrz %g1, %g2, %o0 + + movrz %g1, 32, %o1 /* 3 */ +1: clr %o2 + + sllx %o0, (64 - 16), %g1 /* 4 */ + srlx %o0, 16, %g2 + + movrz %g1, %g2, %o0 /* 5 */ + clr %o3 + + movrz %g1, 16, %o2 /* 6 */ + clr %o4 + + and %o0, 0xff, %g1 /* 7 */ + srlx %o0, 8, %g2 + + movrz %g1, %g2, %o0 /* 8 */ + clr %o5 + + movrz %g1, 8, %o3 /* 9 */ + add %o2, %o1, %o2 + + and %o0, 0xf, %g1 /* 10 */ + srlx %o0, 4, %g2 + + movrz %g1, %g2, %o0 /* 11 */ + add %o2, %o3, %o2 + + movrz %g1, 4, %o4 /* 12 */ + + and %o0, 0x3, %g1 /* 13 */ + srlx %o0, 2, %g2 + + movrz %g1, %g2, %o0 /* 14 */ + add %o2, %o4, %o2 + + movrz %g1, 2, %o5 /* 15 */ + + and %o0, 0x1, %g1 /* 16 */ + + add %o2, %o5, %o2 /* 17 */ + xor %g1, 0x1, %g1 + + retl /* 18 */ + add %o2, %g1, %o0 +ENDPROC(ffs) +ENDPROC(__ffs) +EXPORT_SYMBOL(__ffs) +EXPORT_SYMBOL(ffs) + + .section .popc_6insn_patch, "ax" + .word ffs + brz,pn %o0, 98f + neg %o0, %g1 + xnor %o0, %g1, %o1 + popc %o1, %o0 +98: retl + nop + .word __ffs + neg %o0, %g1 + xnor %o0, %g1, %o1 + popc %o1, %o0 + retl + sub %o0, 1, %o0 + nop + .previous diff --git a/arch/sparc/lib/fls.S b/arch/sparc/lib/fls.S new file mode 100644 index 000000000..ccf97fb7d --- /dev/null +++ b/arch/sparc/lib/fls.S @@ -0,0 +1,67 @@ +/* fls.S: SPARC default fls definition. + * + * SPARC default fls definition, which follows the same algorithm as + * in generic fls(). This function will be boot time patched on T4 + * and onward. + */ + +#include +#include + + .text + .register %g2, #scratch + .register %g3, #scratch +ENTRY(fls) + brz,pn %o0, 6f + mov 0, %o1 + sethi %hi(0xffff0000), %g3 + mov %o0, %g2 + andcc %o0, %g3, %g0 + be,pt %icc, 8f + mov 32, %o1 + sethi %hi(0xff000000), %g3 + andcc %g2, %g3, %g0 + bne,pt %icc, 3f + sethi %hi(0xf0000000), %g3 + sll %o0, 8, %o0 +1: + add %o1, -8, %o1 + sra %o0, 0, %o0 + mov %o0, %g2 +2: + sethi %hi(0xf0000000), %g3 +3: + andcc %g2, %g3, %g0 + bne,pt %icc, 4f + sethi %hi(0xc0000000), %g3 + sll %o0, 4, %o0 + add %o1, -4, %o1 + sra %o0, 0, %o0 + mov %o0, %g2 +4: + andcc %g2, %g3, %g0 + be,a,pt %icc, 7f + sll %o0, 2, %o0 +5: + xnor %g0, %o0, %o0 + srl %o0, 31, %o0 + sub %o1, %o0, %o1 +6: + jmp %o7 + 8 + sra %o1, 0, %o0 +7: + add %o1, -2, %o1 + ba,pt %xcc, 5b + sra %o0, 0, %o0 +8: + sll %o0, 16, %o0 + sethi %hi(0xff000000), %g3 + sra %o0, 0, %o0 + mov %o0, %g2 + andcc %g2, %g3, %g0 + bne,pt %icc, 2b + mov 16, %o1 + ba,pt %xcc, 1b + sll %o0, 8, %o0 +ENDPROC(fls) +EXPORT_SYMBOL(fls) diff --git a/arch/sparc/lib/fls64.S b/arch/sparc/lib/fls64.S new file mode 100644 index 000000000..87005b67d --- /dev/null +++ b/arch/sparc/lib/fls64.S @@ -0,0 +1,61 @@ +/* fls64.S: SPARC default __fls definition. + * + * SPARC default __fls definition, which follows the same algorithm as + * in generic __fls(). This function will be boot time patched on T4 + * and onward. + */ + +#include +#include + + .text + .register %g2, #scratch + .register %g3, #scratch +ENTRY(__fls) + mov -1, %g2 + sllx %g2, 32, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 1f + mov 63, %g1 + sllx %o0, 32, %o0 + mov 31, %g1 +1: + mov -1, %g2 + sllx %g2, 48, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 2f + mov -1, %g2 + sllx %o0, 16, %o0 + add %g1, -16, %g1 +2: + mov -1, %g2 + sllx %g2, 56, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 3f + mov -1, %g2 + sllx %o0, 8, %o0 + add %g1, -8, %g1 +3: + sllx %g2, 60, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 4f + mov -1, %g2 + sllx %o0, 4, %o0 + add %g1, -4, %g1 +4: + sllx %g2, 62, %g2 + and %o0, %g2, %g2 + brnz,pt %g2, 5f + mov -1, %g3 + sllx %o0, 2, %o0 + add %g1, -2, %g1 +5: + mov 0, %g2 + sllx %g3, 63, %g3 + and %o0, %g3, %o0 + movre %o0, 1, %g2 + sub %g1, %g2, %g1 + jmp %o7+8 + sra %g1, 0, %o0 +ENDPROC(__fls) +EXPORT_SYMBOL(__fls) diff --git a/arch/sparc/lib/hweight.S b/arch/sparc/lib/hweight.S new file mode 100644 index 000000000..eebee59b0 --- /dev/null +++ b/arch/sparc/lib/hweight.S @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include + + .text + .align 32 +ENTRY(__arch_hweight8) + sethi %hi(__sw_hweight8), %g1 + jmpl %g1 + %lo(__sw_hweight8), %g0 + nop +ENDPROC(__arch_hweight8) +EXPORT_SYMBOL(__arch_hweight8) + .section .popc_3insn_patch, "ax" + .word __arch_hweight8 + sllx %o0, 64-8, %g1 + retl + popc %g1, %o0 + .previous + +ENTRY(__arch_hweight16) + sethi %hi(__sw_hweight16), %g1 + jmpl %g1 + %lo(__sw_hweight16), %g0 + nop +ENDPROC(__arch_hweight16) +EXPORT_SYMBOL(__arch_hweight16) + .section .popc_3insn_patch, "ax" + .word __arch_hweight16 + sllx %o0, 64-16, %g1 + retl + popc %g1, %o0 + .previous + +ENTRY(__arch_hweight32) + sethi %hi(__sw_hweight32), %g1 + jmpl %g1 + %lo(__sw_hweight32), %g0 + nop +ENDPROC(__arch_hweight32) +EXPORT_SYMBOL(__arch_hweight32) + .section .popc_3insn_patch, "ax" + .word __arch_hweight32 + sllx %o0, 64-32, %g1 + retl + popc %g1, %o0 + .previous + +ENTRY(__arch_hweight64) + sethi %hi(__sw_hweight64), %g1 + jmpl %g1 + %lo(__sw_hweight64), %g0 + nop +ENDPROC(__arch_hweight64) +EXPORT_SYMBOL(__arch_hweight64) + .section .popc_3insn_patch, "ax" + .word __arch_hweight64 + retl + popc %o0, %o0 + nop + .previous diff --git a/arch/sparc/lib/iomap.c b/arch/sparc/lib/iomap.c new file mode 100644 index 000000000..f3a8cd491 --- /dev/null +++ b/arch/sparc/lib/iomap.c @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Implement the sparc iomap interfaces + */ +#include +#include +#include + +/* Create a virtual mapping cookie for an IO port range */ +void __iomem *ioport_map(unsigned long port, unsigned int nr) +{ + return (void __iomem *) (unsigned long) port; +} + +void ioport_unmap(void __iomem *addr) +{ + /* Nothing to do */ +} +EXPORT_SYMBOL(ioport_map); +EXPORT_SYMBOL(ioport_unmap); + +#ifdef CONFIG_PCI +void pci_iounmap(struct pci_dev *dev, void __iomem * addr) +{ + /* nothing to do */ +} +EXPORT_SYMBOL(pci_iounmap); +#endif diff --git a/arch/sparc/lib/ipcsum.S b/arch/sparc/lib/ipcsum.S new file mode 100644 index 000000000..7fa8fd4b7 --- /dev/null +++ b/arch/sparc/lib/ipcsum.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include + + .text +ENTRY(ip_fast_csum) /* %o0 = iph, %o1 = ihl */ + sub %o1, 4, %g7 + lduw [%o0 + 0x00], %o2 + lduw [%o0 + 0x04], %g2 + lduw [%o0 + 0x08], %g3 + addcc %g2, %o2, %o2 + lduw [%o0 + 0x0c], %g2 + addccc %g3, %o2, %o2 + lduw [%o0 + 0x10], %g3 + + addccc %g2, %o2, %o2 + addc %o2, %g0, %o2 +1: addcc %g3, %o2, %o2 + add %o0, 4, %o0 + addccc %o2, %g0, %o2 + subcc %g7, 1, %g7 + be,a,pt %icc, 2f + sll %o2, 16, %g2 + + lduw [%o0 + 0x10], %g3 + ba,pt %xcc, 1b + nop +2: addcc %o2, %g2, %g2 + srl %g2, 16, %o2 + addc %o2, %g0, %o2 + xnor %g0, %o2, %o2 + set 0xffff, %o1 + retl + and %o2, %o1, %o0 +ENDPROC(ip_fast_csum) +EXPORT_SYMBOL(ip_fast_csum) diff --git a/arch/sparc/lib/libgcc.h b/arch/sparc/lib/libgcc.h new file mode 100644 index 000000000..79845c941 --- /dev/null +++ b/arch/sparc/lib/libgcc.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_LIBGCC_H +#define __ASM_LIBGCC_H + +#include + +typedef int word_type __attribute__ ((mode (__word__))); + +struct DWstruct { + int high, low; +}; + +typedef union +{ + struct DWstruct s; + long long ll; +} DWunion; + +#endif /* __ASM_LIBGCC_H */ diff --git a/arch/sparc/lib/locks.S b/arch/sparc/lib/locks.S new file mode 100644 index 000000000..47a39f438 --- /dev/null +++ b/arch/sparc/lib/locks.S @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * locks.S: SMP low-level lock primitives on Sparc. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1998 Anton Blanchard (anton@progsoc.uts.edu.au) + * Copyright (C) 1998 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#include +#include +#include +#include +#include + + .text + .align 4 + + /* Read/writer locks, as usual this is overly clever to make it + * as fast as possible. + */ + + /* caches... */ +___rw_read_enter_spin_on_wlock: + orcc %g2, 0x0, %g0 + be,a ___rw_read_enter + ldstub [%g1 + 3], %g2 + b ___rw_read_enter_spin_on_wlock + ldub [%g1 + 3], %g2 +___rw_read_try_spin_on_wlock: + andcc %g2, 0xff, %g0 + be,a ___rw_read_try + ldstub [%g1 + 3], %g2 + xnorcc %g2, 0x0, %o0 /* if g2 is ~0, set o0 to 0 and bugger off */ + bne,a ___rw_read_enter_spin_on_wlock + ld [%g1], %g2 + retl + mov %g4, %o7 +___rw_read_exit_spin_on_wlock: + orcc %g2, 0x0, %g0 + be,a ___rw_read_exit + ldstub [%g1 + 3], %g2 + b ___rw_read_exit_spin_on_wlock + ldub [%g1 + 3], %g2 +___rw_write_enter_spin_on_wlock: + orcc %g2, 0x0, %g0 + be,a ___rw_write_enter + ldstub [%g1 + 3], %g2 + b ___rw_write_enter_spin_on_wlock + ld [%g1], %g2 + + .globl ___rw_read_enter +EXPORT_SYMBOL(___rw_read_enter) +___rw_read_enter: + orcc %g2, 0x0, %g0 + bne,a ___rw_read_enter_spin_on_wlock + ldub [%g1 + 3], %g2 + ld [%g1], %g2 + add %g2, 1, %g2 + st %g2, [%g1] + retl + mov %g4, %o7 + + .globl ___rw_read_exit +EXPORT_SYMBOL(___rw_read_exit) +___rw_read_exit: + orcc %g2, 0x0, %g0 + bne,a ___rw_read_exit_spin_on_wlock + ldub [%g1 + 3], %g2 + ld [%g1], %g2 + sub %g2, 0x1ff, %g2 + st %g2, [%g1] + retl + mov %g4, %o7 + + .globl ___rw_read_try +EXPORT_SYMBOL(___rw_read_try) +___rw_read_try: + orcc %g2, 0x0, %g0 + bne ___rw_read_try_spin_on_wlock + ld [%g1], %g2 + add %g2, 1, %g2 + st %g2, [%g1] + set 1, %o1 + retl + mov %g4, %o7 + + .globl ___rw_write_enter +EXPORT_SYMBOL(___rw_write_enter) +___rw_write_enter: + orcc %g2, 0x0, %g0 + bne ___rw_write_enter_spin_on_wlock + ld [%g1], %g2 + andncc %g2, 0xff, %g0 + bne,a ___rw_write_enter_spin_on_wlock + stb %g0, [%g1 + 3] + retl + mov %g4, %o7 diff --git a/arch/sparc/lib/lshrdi3.S b/arch/sparc/lib/lshrdi3.S new file mode 100644 index 000000000..09bf581a0 --- /dev/null +++ b/arch/sparc/lib/lshrdi3.S @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include + +ENTRY(__lshrdi3) + cmp %o2, 0 + be 3f + mov 0x20, %g2 + + sub %g2, %o2, %g2 + cmp %g2, 0 + bg 1f + srl %o0, %o2, %o4 + + clr %o4 + neg %g2 + b 2f + srl %o0, %g2, %o5 +1: + sll %o0, %g2, %g3 + srl %o1, %o2, %g2 + or %g2, %g3, %o5 +2: + mov %o4, %o0 + mov %o5, %o1 +3: + retl + nop +ENDPROC(__lshrdi3) +EXPORT_SYMBOL(__lshrdi3) diff --git a/arch/sparc/lib/mcount.S b/arch/sparc/lib/mcount.S new file mode 100644 index 000000000..0309ba2c4 --- /dev/null +++ b/arch/sparc/lib/mcount.S @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2000 Anton Blanchard (anton@linuxcare.com) + * + * This file implements mcount(), which is used to collect profiling data. + * This can also be tweaked for kernel stack overflow detection. + */ + +#include +#include + +/* + * This is the main variant and is called by C code. GCC's -pg option + * automatically instruments every C function with a call to this. + */ + + .text + .align 32 + .globl _mcount + .type _mcount,#function + EXPORT_SYMBOL(_mcount) + .globl mcount + .type mcount,#function + EXPORT_SYMBOL(mcount) +_mcount: +mcount: +#ifdef CONFIG_FUNCTION_TRACER +#ifdef CONFIG_DYNAMIC_FTRACE + /* Do nothing, the retl/nop below is all we need. */ +#else + sethi %hi(ftrace_trace_function), %g1 + sethi %hi(ftrace_stub), %g2 + ldx [%g1 + %lo(ftrace_trace_function)], %g1 + or %g2, %lo(ftrace_stub), %g2 + cmp %g1, %g2 + be,pn %icc, 1f + mov %i7, %g3 + save %sp, -176, %sp + mov %g3, %o1 + jmpl %g1, %o7 + mov %i7, %o0 + ret + restore + /* not reached */ +1: +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + sethi %hi(ftrace_graph_return), %g1 + ldx [%g1 + %lo(ftrace_graph_return)], %g3 + cmp %g2, %g3 + bne,pn %xcc, 5f + sethi %hi(ftrace_graph_entry_stub), %g2 + sethi %hi(ftrace_graph_entry), %g1 + or %g2, %lo(ftrace_graph_entry_stub), %g2 + ldx [%g1 + %lo(ftrace_graph_entry)], %g1 + cmp %g1, %g2 + be,pt %xcc, 2f + nop +5: mov %i7, %g2 + mov %fp, %g3 + save %sp, -176, %sp + mov %g2, %l0 + ba,pt %xcc, ftrace_graph_caller + mov %g3, %l1 +#endif +2: +#endif +#endif + retl + nop + .size _mcount,.-_mcount + .size mcount,.-mcount + +#ifdef CONFIG_FUNCTION_TRACER + .globl ftrace_stub + .type ftrace_stub,#function +ftrace_stub: + retl + nop + .size ftrace_stub,.-ftrace_stub +#ifdef CONFIG_DYNAMIC_FTRACE + .globl ftrace_caller + .type ftrace_caller,#function +ftrace_caller: + mov %i7, %g2 + mov %fp, %g3 + save %sp, -176, %sp + mov %g2, %o1 + mov %g2, %l0 + mov %g3, %l1 + .globl ftrace_call +ftrace_call: + call ftrace_stub + mov %i7, %o0 +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + .globl ftrace_graph_call +ftrace_graph_call: + call ftrace_stub + nop +#endif + ret + restore +#ifdef CONFIG_FUNCTION_GRAPH_TRACER + .size ftrace_graph_call,.-ftrace_graph_call +#endif + .size ftrace_call,.-ftrace_call + .size ftrace_caller,.-ftrace_caller +#endif +#endif + +#ifdef CONFIG_FUNCTION_GRAPH_TRACER +ENTRY(ftrace_graph_caller) + mov %l0, %o0 + mov %i7, %o1 + call prepare_ftrace_return + mov %l1, %o2 + ret + restore %o0, -8, %i7 +END(ftrace_graph_caller) + +ENTRY(return_to_handler) + save %sp, -176, %sp + call ftrace_return_to_handler + mov %fp, %o0 + jmpl %o0 + 8, %g0 + restore +END(return_to_handler) +#endif diff --git a/arch/sparc/lib/memcmp.S b/arch/sparc/lib/memcmp.S new file mode 100644 index 000000000..c87e8000f --- /dev/null +++ b/arch/sparc/lib/memcmp.S @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Sparc optimized memcmp code. + * + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 2000, 2008 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include + + .text +ENTRY(memcmp) + cmp %o2, 0 +1: BRANCH32(be, pn, 2f) + nop + ldub [%o0], %g7 + ldub [%o1], %g3 + sub %o2, 1, %o2 + add %o0, 1, %o0 + add %o1, 1, %o1 + subcc %g7, %g3, %g3 + BRANCH32(be, pt, 1b) + cmp %o2, 0 + retl + mov %g3, %o0 +2: retl + mov 0, %o0 +ENDPROC(memcmp) +EXPORT_SYMBOL(memcmp) diff --git a/arch/sparc/lib/memcpy.S b/arch/sparc/lib/memcpy.S new file mode 100644 index 000000000..57b1ae0f5 --- /dev/null +++ b/arch/sparc/lib/memcpy.S @@ -0,0 +1,463 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* memcpy.S: Sparc optimized memcpy and memmove code + * Hand optimized from GNU libc's memcpy and memmove + * Copyright (C) 1991,1996 Free Software Foundation + * Copyright (C) 1995 Linus Torvalds (Linus.Torvalds@helsinki.fi) + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include + +#define FUNC(x) \ + .globl x; \ + .type x,@function; \ + .align 4; \ +x: + +/* Both these macros have to start with exactly the same insn */ +#define MOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ + ldd [%src + (offset) + 0x00], %t0; \ + ldd [%src + (offset) + 0x08], %t2; \ + ldd [%src + (offset) + 0x10], %t4; \ + ldd [%src + (offset) + 0x18], %t6; \ + st %t0, [%dst + (offset) + 0x00]; \ + st %t1, [%dst + (offset) + 0x04]; \ + st %t2, [%dst + (offset) + 0x08]; \ + st %t3, [%dst + (offset) + 0x0c]; \ + st %t4, [%dst + (offset) + 0x10]; \ + st %t5, [%dst + (offset) + 0x14]; \ + st %t6, [%dst + (offset) + 0x18]; \ + st %t7, [%dst + (offset) + 0x1c]; + +#define MOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ + ldd [%src + (offset) + 0x00], %t0; \ + ldd [%src + (offset) + 0x08], %t2; \ + ldd [%src + (offset) + 0x10], %t4; \ + ldd [%src + (offset) + 0x18], %t6; \ + std %t0, [%dst + (offset) + 0x00]; \ + std %t2, [%dst + (offset) + 0x08]; \ + std %t4, [%dst + (offset) + 0x10]; \ + std %t6, [%dst + (offset) + 0x18]; + +#define MOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \ + ldd [%src - (offset) - 0x10], %t0; \ + ldd [%src - (offset) - 0x08], %t2; \ + st %t0, [%dst - (offset) - 0x10]; \ + st %t1, [%dst - (offset) - 0x0c]; \ + st %t2, [%dst - (offset) - 0x08]; \ + st %t3, [%dst - (offset) - 0x04]; + +#define MOVE_LASTALIGNCHUNK(src, dst, offset, t0, t1, t2, t3) \ + ldd [%src - (offset) - 0x10], %t0; \ + ldd [%src - (offset) - 0x08], %t2; \ + std %t0, [%dst - (offset) - 0x10]; \ + std %t2, [%dst - (offset) - 0x08]; + +#define MOVE_SHORTCHUNK(src, dst, offset, t0, t1) \ + ldub [%src - (offset) - 0x02], %t0; \ + ldub [%src - (offset) - 0x01], %t1; \ + stb %t0, [%dst - (offset) - 0x02]; \ + stb %t1, [%dst - (offset) - 0x01]; + + .text + .align 4 + +FUNC(memmove) +EXPORT_SYMBOL(memmove) + cmp %o0, %o1 + mov %o0, %g7 + bleu 9f + sub %o0, %o1, %o4 + + add %o1, %o2, %o3 + cmp %o3, %o0 + bleu 0f + andcc %o4, 3, %o5 + + add %o1, %o2, %o1 + add %o0, %o2, %o0 + sub %o1, 1, %o1 + sub %o0, 1, %o0 + +1: /* reverse_bytes */ + + ldub [%o1], %o4 + subcc %o2, 1, %o2 + stb %o4, [%o0] + sub %o1, 1, %o1 + bne 1b + sub %o0, 1, %o0 + + retl + mov %g7, %o0 + +/* NOTE: This code is executed just for the cases, + where %src (=%o1) & 3 is != 0. + We need to align it to 4. So, for (%src & 3) + 1 we need to do ldub,lduh + 2 lduh + 3 just ldub + so even if it looks weird, the branches + are correct here. -jj + */ +78: /* dword_align */ + + andcc %o1, 1, %g0 + be 4f + andcc %o1, 2, %g0 + + ldub [%o1], %g2 + add %o1, 1, %o1 + stb %g2, [%o0] + sub %o2, 1, %o2 + bne 3f + add %o0, 1, %o0 +4: + lduh [%o1], %g2 + add %o1, 2, %o1 + sth %g2, [%o0] + sub %o2, 2, %o2 + b 3f + add %o0, 2, %o0 + +FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ +EXPORT_SYMBOL(memcpy) + + sub %o0, %o1, %o4 + mov %o0, %g7 +9: + andcc %o4, 3, %o5 +0: + bne 86f + cmp %o2, 15 + + bleu 90f + andcc %o1, 3, %g0 + + bne 78b +3: + andcc %o1, 4, %g0 + + be 2f + mov %o2, %g1 + + ld [%o1], %o4 + sub %g1, 4, %g1 + st %o4, [%o0] + add %o1, 4, %o1 + add %o0, 4, %o0 +2: + andcc %g1, 0xffffff80, %g0 + be 3f + andcc %o0, 4, %g0 + + be 82f + 4 +5: + MOVE_BIGCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) + sub %g1, 128, %g1 + add %o1, 128, %o1 + cmp %g1, 128 + bge 5b + add %o0, 128, %o0 +3: + andcc %g1, 0x70, %g4 + be 80f + andcc %g1, 8, %g0 + + sethi %hi(80f), %o5 + srl %g4, 1, %o4 + add %g4, %o4, %o4 + add %o1, %g4, %o1 + sub %o5, %o4, %o5 + jmpl %o5 + %lo(80f), %g0 + add %o0, %g4, %o0 + +79: /* memcpy_table */ + + MOVE_LASTCHUNK(o1, o0, 0x60, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x50, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x40, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x30, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x20, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x10, g2, g3, g4, g5) + MOVE_LASTCHUNK(o1, o0, 0x00, g2, g3, g4, g5) + +80: /* memcpy_table_end */ + be 81f + andcc %g1, 4, %g0 + + ldd [%o1], %g2 + add %o0, 8, %o0 + st %g2, [%o0 - 0x08] + add %o1, 8, %o1 + st %g3, [%o0 - 0x04] + +81: /* memcpy_last7 */ + + be 1f + andcc %g1, 2, %g0 + + ld [%o1], %g2 + add %o1, 4, %o1 + st %g2, [%o0] + add %o0, 4, %o0 +1: + be 1f + andcc %g1, 1, %g0 + + lduh [%o1], %g2 + add %o1, 2, %o1 + sth %g2, [%o0] + add %o0, 2, %o0 +1: + be 1f + nop + + ldub [%o1], %g2 + stb %g2, [%o0] +1: + retl + mov %g7, %o0 + +82: /* ldd_std */ + MOVE_BIGALIGNCHUNK(o1, o0, 0x00, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGALIGNCHUNK(o1, o0, 0x20, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGALIGNCHUNK(o1, o0, 0x40, o2, o3, o4, o5, g2, g3, g4, g5) + MOVE_BIGALIGNCHUNK(o1, o0, 0x60, o2, o3, o4, o5, g2, g3, g4, g5) + subcc %g1, 128, %g1 + add %o1, 128, %o1 + cmp %g1, 128 + bge 82b + add %o0, 128, %o0 + + andcc %g1, 0x70, %g4 + be 84f + andcc %g1, 8, %g0 + + sethi %hi(84f), %o5 + add %o1, %g4, %o1 + sub %o5, %g4, %o5 + jmpl %o5 + %lo(84f), %g0 + add %o0, %g4, %o0 + +83: /* amemcpy_table */ + + MOVE_LASTALIGNCHUNK(o1, o0, 0x60, g2, g3, g4, g5) + MOVE_LASTALIGNCHUNK(o1, o0, 0x50, g2, g3, g4, g5) + MOVE_LASTALIGNCHUNK(o1, o0, 0x40, g2, g3, g4, g5) + MOVE_LASTALIGNCHUNK(o1, o0, 0x30, g2, g3, g4, g5) + MOVE_LASTALIGNCHUNK(o1, o0, 0x20, g2, g3, g4, g5) + MOVE_LASTALIGNCHUNK(o1, o0, 0x10, g2, g3, g4, g5) + MOVE_LASTALIGNCHUNK(o1, o0, 0x00, g2, g3, g4, g5) + +84: /* amemcpy_table_end */ + be 85f + andcc %g1, 4, %g0 + + ldd [%o1], %g2 + add %o0, 8, %o0 + std %g2, [%o0 - 0x08] + add %o1, 8, %o1 +85: /* amemcpy_last7 */ + be 1f + andcc %g1, 2, %g0 + + ld [%o1], %g2 + add %o1, 4, %o1 + st %g2, [%o0] + add %o0, 4, %o0 +1: + be 1f + andcc %g1, 1, %g0 + + lduh [%o1], %g2 + add %o1, 2, %o1 + sth %g2, [%o0] + add %o0, 2, %o0 +1: + be 1f + nop + + ldub [%o1], %g2 + stb %g2, [%o0] +1: + retl + mov %g7, %o0 + +86: /* non_aligned */ + cmp %o2, 6 + bleu 88f + nop + + save %sp, -96, %sp + andcc %i0, 3, %g0 + be 61f + andcc %i0, 1, %g0 + be 60f + andcc %i0, 2, %g0 + + ldub [%i1], %g5 + add %i1, 1, %i1 + stb %g5, [%i0] + sub %i2, 1, %i2 + bne 61f + add %i0, 1, %i0 +60: + ldub [%i1], %g3 + add %i1, 2, %i1 + stb %g3, [%i0] + sub %i2, 2, %i2 + ldub [%i1 - 1], %g3 + add %i0, 2, %i0 + stb %g3, [%i0 - 1] +61: + and %i1, 3, %g2 + and %i2, 0xc, %g3 + and %i1, -4, %i1 + cmp %g3, 4 + sll %g2, 3, %g4 + mov 32, %g2 + be 4f + sub %g2, %g4, %l0 + + blu 3f + cmp %g3, 0x8 + + be 2f + srl %i2, 2, %g3 + + ld [%i1], %i3 + add %i0, -8, %i0 + ld [%i1 + 4], %i4 + b 8f + add %g3, 1, %g3 +2: + ld [%i1], %i4 + add %i0, -12, %i0 + ld [%i1 + 4], %i5 + add %g3, 2, %g3 + b 9f + add %i1, -4, %i1 +3: + ld [%i1], %g1 + add %i0, -4, %i0 + ld [%i1 + 4], %i3 + srl %i2, 2, %g3 + b 7f + add %i1, 4, %i1 +4: + ld [%i1], %i5 + cmp %i2, 7 + ld [%i1 + 4], %g1 + srl %i2, 2, %g3 + bleu 10f + add %i1, 8, %i1 + + ld [%i1], %i3 + add %g3, -1, %g3 +5: + sll %i5, %g4, %g2 + srl %g1, %l0, %g5 + or %g2, %g5, %g2 + st %g2, [%i0] +7: + ld [%i1 + 4], %i4 + sll %g1, %g4, %g2 + srl %i3, %l0, %g5 + or %g2, %g5, %g2 + st %g2, [%i0 + 4] +8: + ld [%i1 + 8], %i5 + sll %i3, %g4, %g2 + srl %i4, %l0, %g5 + or %g2, %g5, %g2 + st %g2, [%i0 + 8] +9: + ld [%i1 + 12], %g1 + sll %i4, %g4, %g2 + srl %i5, %l0, %g5 + addcc %g3, -4, %g3 + or %g2, %g5, %g2 + add %i1, 16, %i1 + st %g2, [%i0 + 12] + add %i0, 16, %i0 + bne,a 5b + ld [%i1], %i3 +10: + sll %i5, %g4, %g2 + srl %g1, %l0, %g5 + srl %l0, 3, %g3 + or %g2, %g5, %g2 + sub %i1, %g3, %i1 + andcc %i2, 2, %g0 + st %g2, [%i0] + be 1f + andcc %i2, 1, %g0 + + ldub [%i1], %g2 + add %i1, 2, %i1 + stb %g2, [%i0 + 4] + add %i0, 2, %i0 + ldub [%i1 - 1], %g2 + stb %g2, [%i0 + 3] +1: + be 1f + nop + ldub [%i1], %g2 + stb %g2, [%i0 + 4] +1: + ret + restore %g7, %g0, %o0 + +88: /* short_end */ + + and %o2, 0xe, %o3 +20: + sethi %hi(89f), %o5 + sll %o3, 3, %o4 + add %o0, %o3, %o0 + sub %o5, %o4, %o5 + add %o1, %o3, %o1 + jmpl %o5 + %lo(89f), %g0 + andcc %o2, 1, %g0 + + MOVE_SHORTCHUNK(o1, o0, 0x0c, g2, g3) + MOVE_SHORTCHUNK(o1, o0, 0x0a, g2, g3) + MOVE_SHORTCHUNK(o1, o0, 0x08, g2, g3) + MOVE_SHORTCHUNK(o1, o0, 0x06, g2, g3) + MOVE_SHORTCHUNK(o1, o0, 0x04, g2, g3) + MOVE_SHORTCHUNK(o1, o0, 0x02, g2, g3) + MOVE_SHORTCHUNK(o1, o0, 0x00, g2, g3) + +89: /* short_table_end */ + + be 1f + nop + + ldub [%o1], %g2 + stb %g2, [%o0] +1: + retl + mov %g7, %o0 + +90: /* short_aligned_end */ + bne 88b + andcc %o2, 8, %g0 + + be 1f + andcc %o2, 4, %g0 + + ld [%o1 + 0x00], %g2 + ld [%o1 + 0x04], %g3 + add %o1, 8, %o1 + st %g2, [%o0 + 0x00] + st %g3, [%o0 + 0x04] + add %o0, 8, %o0 +1: + b 81b + mov %o2, %g1 diff --git a/arch/sparc/lib/memmove.S b/arch/sparc/lib/memmove.S new file mode 100644 index 000000000..543dda7b9 --- /dev/null +++ b/arch/sparc/lib/memmove.S @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* memmove.S: Simple memmove implementation. + * + * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com) + * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#include +#include + + .text +ENTRY(memmove) /* o0=dst o1=src o2=len */ + brz,pn %o2, 99f + mov %o0, %g1 + + cmp %o0, %o1 + bleu,pt %xcc, 2f + add %o1, %o2, %g7 + cmp %g7, %o0 + bleu,pt %xcc, memcpy + add %o0, %o2, %o5 + sub %g7, 1, %o1 + + sub %o5, 1, %o0 +1: ldub [%o1], %g7 + subcc %o2, 1, %o2 + sub %o1, 1, %o1 + stb %g7, [%o0] + bne,pt %icc, 1b + sub %o0, 1, %o0 +99: + retl + mov %g1, %o0 + + /* We can't just call memcpy for these memmove cases. On some + * chips the memcpy uses cache initializing stores and when dst + * and src are close enough, those can clobber the source data + * before we've loaded it in. + */ +2: or %o0, %o1, %g7 + or %o2, %g7, %g7 + andcc %g7, 0x7, %g0 + bne,pn %xcc, 4f + nop + +3: ldx [%o1], %g7 + add %o1, 8, %o1 + subcc %o2, 8, %o2 + add %o0, 8, %o0 + bne,pt %icc, 3b + stx %g7, [%o0 - 0x8] + ba,a,pt %xcc, 99b + +4: ldub [%o1], %g7 + add %o1, 1, %o1 + subcc %o2, 1, %o2 + add %o0, 1, %o0 + bne,pt %icc, 4b + stb %g7, [%o0 - 0x1] + ba,a,pt %xcc, 99b +ENDPROC(memmove) +EXPORT_SYMBOL(memmove) diff --git a/arch/sparc/lib/memscan_32.S b/arch/sparc/lib/memscan_32.S new file mode 100644 index 000000000..5386a3a20 --- /dev/null +++ b/arch/sparc/lib/memscan_32.S @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * memscan.S: Optimized memscan for the Sparc. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + */ + +#include + +/* In essence, this is just a fancy strlen. */ + +#define LO_MAGIC 0x01010101 +#define HI_MAGIC 0x80808080 + + .text + .align 4 + .globl __memscan_zero, __memscan_generic + .globl memscan +EXPORT_SYMBOL(__memscan_zero) +EXPORT_SYMBOL(__memscan_generic) +__memscan_zero: + /* %o0 = addr, %o1 = size */ + cmp %o1, 0 + bne,a 1f + andcc %o0, 3, %g0 + + retl + nop + +1: + be mzero_scan_word + sethi %hi(HI_MAGIC), %g2 + + ldsb [%o0], %g3 +mzero_still_not_word_aligned: + cmp %g3, 0 + bne 1f + add %o0, 1, %o0 + + retl + sub %o0, 1, %o0 + +1: + subcc %o1, 1, %o1 + bne,a 1f + andcc %o0, 3, %g0 + + retl + nop + +1: + bne,a mzero_still_not_word_aligned + ldsb [%o0], %g3 + + sethi %hi(HI_MAGIC), %g2 +mzero_scan_word: + or %g2, %lo(HI_MAGIC), %o3 + sethi %hi(LO_MAGIC), %g3 + or %g3, %lo(LO_MAGIC), %o2 +mzero_next_word: + ld [%o0], %g2 +mzero_next_word_preloaded: + sub %g2, %o2, %g2 +mzero_next_word_preloaded_next: + andcc %g2, %o3, %g0 + bne mzero_byte_zero + add %o0, 4, %o0 + +mzero_check_out_of_fuel: + subcc %o1, 4, %o1 + bg,a 1f + ld [%o0], %g2 + + retl + nop + +1: + b mzero_next_word_preloaded_next + sub %g2, %o2, %g2 + + /* Check every byte. */ +mzero_byte_zero: + ldsb [%o0 - 4], %g2 + cmp %g2, 0 + bne mzero_byte_one + sub %o0, 4, %g3 + + retl + mov %g3, %o0 + +mzero_byte_one: + ldsb [%o0 - 3], %g2 + cmp %g2, 0 + bne,a mzero_byte_two_and_three + ldsb [%o0 - 2], %g2 + + retl + sub %o0, 3, %o0 + +mzero_byte_two_and_three: + cmp %g2, 0 + bne,a 1f + ldsb [%o0 - 1], %g2 + + retl + sub %o0, 2, %o0 + +1: + cmp %g2, 0 + bne,a mzero_next_word_preloaded + ld [%o0], %g2 + + retl + sub %o0, 1, %o0 + +mzero_found_it: + retl + sub %o0, 2, %o0 + +memscan: +__memscan_generic: + /* %o0 = addr, %o1 = c, %o2 = size */ + cmp %o2, 0 + bne,a 0f + ldub [%o0], %g2 + + b,a 2f +1: + ldub [%o0], %g2 +0: + cmp %g2, %o1 + be 2f + addcc %o2, -1, %o2 + bne 1b + add %o0, 1, %o0 +2: + retl + nop diff --git a/arch/sparc/lib/memscan_64.S b/arch/sparc/lib/memscan_64.S new file mode 100644 index 000000000..70a4f2105 --- /dev/null +++ b/arch/sparc/lib/memscan_64.S @@ -0,0 +1,136 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * memscan.S: Optimized memscan for Sparc64. + * + * Copyright (C) 1997,1998 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 1998 David S. Miller (davem@redhat.com) + */ + +#include + +#define HI_MAGIC 0x8080808080808080 +#define LO_MAGIC 0x0101010101010101 +#define ASI_PL 0x88 + + .text + .align 32 + .globl __memscan_zero, __memscan_generic + .type __memscan_zero,#function + .type __memscan_generic,#function + .globl memscan + EXPORT_SYMBOL(__memscan_zero) + EXPORT_SYMBOL(__memscan_generic) + +__memscan_zero: + /* %o0 = bufp, %o1 = size */ + brlez,pn %o1, szzero + andcc %o0, 7, %g0 + be,pt %icc, we_are_aligned + sethi %hi(HI_MAGIC), %o4 + ldub [%o0], %o5 +1: subcc %o1, 1, %o1 + brz,pn %o5, 10f + add %o0, 1, %o0 + + be,pn %xcc, szzero + andcc %o0, 7, %g0 + bne,a,pn %icc, 1b + ldub [%o0], %o5 +we_are_aligned: + ldxa [%o0] ASI_PL, %o5 + or %o4, %lo(HI_MAGIC), %o3 + sllx %o3, 32, %o4 + or %o4, %o3, %o3 + + srlx %o3, 7, %o2 +msloop: + sub %o1, 8, %o1 + add %o0, 8, %o0 + sub %o5, %o2, %o4 + xor %o4, %o5, %o4 + andcc %o4, %o3, %g3 + bne,pn %xcc, check_bytes + srlx %o4, 32, %g3 + + brgz,a,pt %o1, msloop + ldxa [%o0] ASI_PL, %o5 +check_bytes: + bne,a,pn %icc, 2f + andcc %o5, 0xff, %g0 + add %o0, -5, %g2 + ba,pt %xcc, 3f + srlx %o5, 32, %g7 + +2: srlx %o5, 8, %g7 + be,pn %icc, 1f + add %o0, -8, %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g3, %o3, %g0 + + be,a,pn %icc, 2f + mov %o0, %g2 +3: andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + be,pn %icc, 1f + inc %g2 + andcc %g7, 0xff, %g0 + srlx %g7, 8, %g7 + + be,pn %icc, 1f + inc %g2 +2: brgz,a,pt %o1, msloop + ldxa [%o0] ASI_PL, %o5 + inc %g2 +1: add %o0, %o1, %o0 + cmp %g2, %o0 + retl + + movle %xcc, %g2, %o0 +10: retl + sub %o0, 1, %o0 +szzero: retl + nop + +memscan: +__memscan_generic: + /* %o0 = addr, %o1 = c, %o2 = size */ + brz,pn %o2, 3f + add %o0, %o2, %o3 + ldub [%o0], %o5 + sub %g0, %o2, %o4 +1: + cmp %o5, %o1 + be,pn %icc, 2f + addcc %o4, 1, %o4 + bne,a,pt %xcc, 1b + ldub [%o3 + %o4], %o5 + retl + /* The delay slot is the same as the next insn, this is just to make it look more awful */ +2: + add %o3, %o4, %o0 + retl + sub %o0, 1, %o0 +3: + retl + nop diff --git a/arch/sparc/lib/memset.S b/arch/sparc/lib/memset.S new file mode 100644 index 000000000..a33419dbb --- /dev/null +++ b/arch/sparc/lib/memset.S @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* linux/arch/sparc/lib/memset.S: Sparc optimized memset, bzero and clear_user code + * Copyright (C) 1991,1996 Free Software Foundation + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * + * Calls to memset returns initial %o0. Calls to bzero returns 0, if ok, and + * number of bytes not yet set if exception occurs and we were called as + * clear_user. + */ + +#include +#include + +/* Work around cpp -rob */ +#define ALLOC #alloc +#define EXECINSTR #execinstr +#define EX(x,y,a,b) \ +98: x,y; \ + .section .fixup,ALLOC,EXECINSTR; \ + .align 4; \ +99: retl; \ + a, b, %o0; \ + .section __ex_table,ALLOC; \ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4 + +#define STORE(source, base, offset, n) \ +98: std source, [base + offset + n]; \ + .section .fixup,ALLOC,EXECINSTR; \ + .align 4; \ +99: ba 30f; \ + sub %o3, n - offset, %o3; \ + .section __ex_table,ALLOC; \ + .align 4; \ + .word 98b, 99b; \ + .text; \ + .align 4; + +#define STORE_LAST(source, base, offset, n) \ + EX(std source, [base - offset - n], \ + add %o1, offset + n); + +/* Please don't change these macros, unless you change the logic + * in the .fixup section below as well. + * Store 64 bytes at (BASE + OFFSET) using value SOURCE. */ +#define ZERO_BIG_BLOCK(base, offset, source) \ + STORE(source, base, offset, 0x00); \ + STORE(source, base, offset, 0x08); \ + STORE(source, base, offset, 0x10); \ + STORE(source, base, offset, 0x18); \ + STORE(source, base, offset, 0x20); \ + STORE(source, base, offset, 0x28); \ + STORE(source, base, offset, 0x30); \ + STORE(source, base, offset, 0x38); + +#define ZERO_LAST_BLOCKS(base, offset, source) \ + STORE_LAST(source, base, offset, 0x38); \ + STORE_LAST(source, base, offset, 0x30); \ + STORE_LAST(source, base, offset, 0x28); \ + STORE_LAST(source, base, offset, 0x20); \ + STORE_LAST(source, base, offset, 0x18); \ + STORE_LAST(source, base, offset, 0x10); \ + STORE_LAST(source, base, offset, 0x08); \ + STORE_LAST(source, base, offset, 0x00); + + .text + .align 4 + + .globl __bzero_begin +__bzero_begin: + + .globl __bzero + .type __bzero,#function + .globl memset + EXPORT_SYMBOL(__bzero) + EXPORT_SYMBOL(memset) +memset: + mov %o0, %g1 + mov 1, %g4 + and %o1, 0xff, %g3 + sll %g3, 8, %g2 + or %g3, %g2, %g3 + sll %g3, 16, %g2 + or %g3, %g2, %g3 + b 1f + mov %o2, %o1 +3: + cmp %o2, 3 + be 2f + EX(stb %g3, [%o0], sub %o1, 0) + + cmp %o2, 2 + be 2f + EX(stb %g3, [%o0 + 0x01], sub %o1, 1) + + EX(stb %g3, [%o0 + 0x02], sub %o1, 2) +2: + sub %o2, 4, %o2 + add %o1, %o2, %o1 + b 4f + sub %o0, %o2, %o0 + +__bzero: + clr %g4 + mov %g0, %g3 +1: + cmp %o1, 7 + bleu 7f + andcc %o0, 3, %o2 + + bne 3b +4: + andcc %o0, 4, %g0 + + be 2f + mov %g3, %g2 + + EX(st %g3, [%o0], sub %o1, 0) + sub %o1, 4, %o1 + add %o0, 4, %o0 +2: + andcc %o1, 0xffffff80, %o3 ! Now everything is 8 aligned and o1 is len to run + be 9f + andcc %o1, 0x78, %o2 +10: + ZERO_BIG_BLOCK(%o0, 0x00, %g2) + subcc %o3, 128, %o3 + ZERO_BIG_BLOCK(%o0, 0x40, %g2) + bne 10b + add %o0, 128, %o0 + + orcc %o2, %g0, %g0 +9: + be 13f + andcc %o1, 7, %o1 + + srl %o2, 1, %o3 + set 13f, %o4 + sub %o4, %o3, %o4 + jmp %o4 + add %o0, %o2, %o0 + + ZERO_LAST_BLOCKS(%o0, 0x48, %g2) + ZERO_LAST_BLOCKS(%o0, 0x08, %g2) +13: + be 8f + andcc %o1, 4, %g0 + + be 1f + andcc %o1, 2, %g0 + + EX(st %g3, [%o0], and %o1, 7) + add %o0, 4, %o0 +1: + be 1f + andcc %o1, 1, %g0 + + EX(sth %g3, [%o0], and %o1, 3) + add %o0, 2, %o0 +1: + bne,a 8f + EX(stb %g3, [%o0], and %o1, 1) +8: + b 0f + nop +7: + be 13b + orcc %o1, 0, %g0 + + be 0f +8: + add %o0, 1, %o0 + subcc %o1, 1, %o1 + bne 8b + EX(stb %g3, [%o0 - 1], add %o1, 1) +0: + andcc %g4, 1, %g0 + be 5f + nop + retl + mov %g1, %o0 +5: + retl + clr %o0 + + .section .fixup,#alloc,#execinstr + .align 4 +30: + and %o1, 0x7f, %o1 + retl + add %o3, %o1, %o0 + + .globl __bzero_end +__bzero_end: diff --git a/arch/sparc/lib/muldi3.S b/arch/sparc/lib/muldi3.S new file mode 100644 index 000000000..7e1e8cd30 --- /dev/null +++ b/arch/sparc/lib/muldi3.S @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 1989, 1992, 1993, 1994, 1995 Free Software Foundation, Inc. + +This file is part of GNU CC. + + */ + +#include + .text + .align 4 + .globl __muldi3 +__muldi3: + save %sp, -104, %sp + wr %g0, %i1, %y + sra %i3, 0x1f, %g2 + and %i1, %g2, %g2 + andcc %g0, 0, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, %i3, %g1 + mulscc %g1, 0, %g1 + add %g1, %g2, %l2 + rd %y, %o1 + mov %o1, %l3 + mov %i1, %o0 + mov %i2, %o1 + umul %o0, %o1, %o0 + mov %o0, %l0 + mov %i0, %o0 + mov %i3, %o1 + umul %o0, %o1, %o0 + add %l0, %o0, %l0 + mov %l2, %i0 + add %l2, %l0, %i0 + ret + restore %g0, %l3, %o1 +EXPORT_SYMBOL(__muldi3) diff --git a/arch/sparc/lib/multi3.S b/arch/sparc/lib/multi3.S new file mode 100644 index 000000000..5bb4c122a --- /dev/null +++ b/arch/sparc/lib/multi3.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include + + .text + .align 4 +ENTRY(__multi3) /* %o0 = u, %o1 = v */ + mov %o1, %g1 + srl %o3, 0, %o4 + mulx %o4, %g1, %o1 + srlx %g1, 0x20, %g3 + mulx %g3, %o4, %g7 + sllx %g7, 0x20, %o5 + srl %g1, 0, %o4 + sub %o1, %o5, %o5 + srlx %o5, 0x20, %o5 + addcc %g7, %o5, %g7 + srlx %o3, 0x20, %o5 + mulx %o4, %o5, %o4 + mulx %g3, %o5, %o5 + sethi %hi(0x80000000), %g3 + addcc %g7, %o4, %g7 + srlx %g7, 0x20, %g7 + add %g3, %g3, %g3 + movcc %xcc, %g0, %g3 + addcc %o5, %g7, %o5 + sllx %o4, 0x20, %o4 + add %o1, %o4, %o1 + add %o5, %g3, %g2 + mulx %g1, %o2, %g1 + add %g1, %g2, %g1 + mulx %o0, %o3, %o0 + retl + add %g1, %o0, %o0 +ENDPROC(__multi3) +EXPORT_SYMBOL(__multi3) diff --git a/arch/sparc/lib/strlen.S b/arch/sparc/lib/strlen.S new file mode 100644 index 000000000..27478b3f1 --- /dev/null +++ b/arch/sparc/lib/strlen.S @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* strlen.S: Sparc optimized strlen code + * Hand optimized from GNU libc's strlen + * Copyright (C) 1991,1996 Free Software Foundation + * Copyright (C) 1996,2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996, 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include + +#define LO_MAGIC 0x01010101 +#define HI_MAGIC 0x80808080 + + .text +ENTRY(strlen) + mov %o0, %o1 + andcc %o0, 3, %g0 + BRANCH32(be, pt, 9f) + sethi %hi(HI_MAGIC), %o4 + ldub [%o0], %o5 + BRANCH_REG_ZERO(pn, %o5, 11f) + add %o0, 1, %o0 + andcc %o0, 3, %g0 + BRANCH32(be, pn, 4f) + or %o4, %lo(HI_MAGIC), %o3 + ldub [%o0], %o5 + BRANCH_REG_ZERO(pn, %o5, 12f) + add %o0, 1, %o0 + andcc %o0, 3, %g0 + BRANCH32(be, pt, 5f) + sethi %hi(LO_MAGIC), %o4 + ldub [%o0], %o5 + BRANCH_REG_ZERO(pn, %o5, 13f) + add %o0, 1, %o0 + BRANCH32(ba, pt, 8f) + or %o4, %lo(LO_MAGIC), %o2 +9: + or %o4, %lo(HI_MAGIC), %o3 +4: + sethi %hi(LO_MAGIC), %o4 +5: + or %o4, %lo(LO_MAGIC), %o2 +8: + ld [%o0], %o5 +2: + sub %o5, %o2, %o4 + andcc %o4, %o3, %g0 + BRANCH32(be, pt, 8b) + add %o0, 4, %o0 + + /* Check every byte. */ + srl %o5, 24, %g7 + andcc %g7, 0xff, %g0 + BRANCH32(be, pn, 1f) + add %o0, -4, %o4 + srl %o5, 16, %g7 + andcc %g7, 0xff, %g0 + BRANCH32(be, pn, 1f) + add %o4, 1, %o4 + srl %o5, 8, %g7 + andcc %g7, 0xff, %g0 + BRANCH32(be, pn, 1f) + add %o4, 1, %o4 + andcc %o5, 0xff, %g0 + BRANCH32_ANNUL(bne, pt, 2b) + ld [%o0], %o5 + add %o4, 1, %o4 +1: + retl + sub %o4, %o1, %o0 +11: + retl + mov 0, %o0 +12: + retl + mov 1, %o0 +13: + retl + mov 2, %o0 +ENDPROC(strlen) +EXPORT_SYMBOL(strlen) diff --git a/arch/sparc/lib/strncmp_32.S b/arch/sparc/lib/strncmp_32.S new file mode 100644 index 000000000..387bbf621 --- /dev/null +++ b/arch/sparc/lib/strncmp_32.S @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * strncmp.S: Hand optimized Sparc assembly of GCC output from GNU libc + * generic strncmp routine. + */ + +#include +#include + + .text +ENTRY(strncmp) + mov %o0, %g3 + mov 0, %o3 + + cmp %o2, 3 + ble 7f + mov 0, %g2 + + sra %o2, 2, %o4 + ldub [%g3], %o3 + +0: + ldub [%o1], %g2 + add %g3, 1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be,a 1f + ldub [%g3], %o3 + + retl + sub %o0, %g2, %o0 + +1: + ldub [%o1], %g2 + add %g3,1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be,a 1f + ldub [%g3], %o3 + + retl + sub %o0, %g2, %o0 + +1: + ldub [%o1], %g2 + add %g3, 1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be,a 1f + ldub [%g3], %o3 + + retl + sub %o0, %g2, %o0 + +1: + ldub [%o1], %g2 + add %g3, 1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be 1f + add %o4, -1, %o4 + + retl + sub %o0, %g2, %o0 + +1: + + cmp %o4, 0 + bg,a 0b + ldub [%g3], %o3 + + b 7f + and %o2, 3, %o2 + +9: + ldub [%o1], %g2 + add %g3, 1, %g3 + and %o3, 0xff, %o0 + + cmp %o0, 0 + be 8f + add %o1, 1, %o1 + + cmp %o0, %g2 + be 7f + add %o2, -1, %o2 + +8: + retl + sub %o0, %g2, %o0 + +7: + cmp %o2, 0 + bg,a 9b + ldub [%g3], %o3 + + and %g2, 0xff, %o0 + retl + sub %o3, %o0, %o0 +ENDPROC(strncmp) +EXPORT_SYMBOL(strncmp) diff --git a/arch/sparc/lib/strncmp_64.S b/arch/sparc/lib/strncmp_64.S new file mode 100644 index 000000000..76c1207ec --- /dev/null +++ b/arch/sparc/lib/strncmp_64.S @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Sparc64 optimized strncmp code. + * + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include + + .text +ENTRY(strncmp) + brlez,pn %o2, 3f + lduba [%o0] (ASI_PNF), %o3 +1: + add %o0, 1, %o0 + ldub [%o1], %o4 + brz,pn %o3, 2f + add %o1, 1, %o1 + cmp %o3, %o4 + bne,pn %icc, 2f + subcc %o2, 1, %o2 + bne,a,pt %xcc, 1b + ldub [%o0], %o3 +2: + retl + sub %o3, %o4, %o0 +3: + retl + clr %o0 +ENDPROC(strncmp) +EXPORT_SYMBOL(strncmp) diff --git a/arch/sparc/lib/udivdi3.S b/arch/sparc/lib/udivdi3.S new file mode 100644 index 000000000..7a1117ec7 --- /dev/null +++ b/arch/sparc/lib/udivdi3.S @@ -0,0 +1,247 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* Copyright (C) 1989, 1992, 1993, 1994, 1995 Free Software Foundation, Inc. + +This file is part of GNU CC. + + */ + + .text + .align 4 + .globl __udivdi3 +__udivdi3: + save %sp,-104,%sp + mov %i3,%o3 + cmp %i2,0 + bne .LL40 + mov %i1,%i3 + cmp %o3,%i0 + bleu .LL41 + mov %i3,%o1 + ! Inlined udiv_qrnnd + mov 32,%g1 + subcc %i0,%o3,%g0 +1: bcs 5f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + sub %i0,%o3,%i0 ! this kills msb of n + addx %i0,%i0,%i0 ! so this cannot give carry + subcc %g1,1,%g1 +2: bne 1b + subcc %i0,%o3,%g0 + bcs 3f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + b 3f + sub %i0,%o3,%i0 ! this kills msb of n +4: sub %i0,%o3,%i0 +5: addxcc %i0,%i0,%i0 + bcc 2b + subcc %g1,1,%g1 +! Got carry from n. Subtract next step to cancel this carry. + bne 4b + addcc %o1,%o1,%o1 ! shift n1n0 and a 0-bit in lsb + sub %i0,%o3,%i0 +3: xnor %o1,0,%o1 + ! End of inline udiv_qrnnd + b .LL45 + mov 0,%o2 +.LL41: + cmp %o3,0 + bne .LL77 + mov %i0,%o2 + mov 1,%o0 + mov 0,%o1 + wr %g0, 0, %y + udiv %o0, %o1, %o0 + mov %o0,%o3 + mov %i0,%o2 +.LL77: + mov 0,%o4 + ! Inlined udiv_qrnnd + mov 32,%g1 + subcc %o4,%o3,%g0 +1: bcs 5f + addxcc %o2,%o2,%o2 ! shift n1n0 and a q-bit in lsb + sub %o4,%o3,%o4 ! this kills msb of n + addx %o4,%o4,%o4 ! so this cannot give carry + subcc %g1,1,%g1 +2: bne 1b + subcc %o4,%o3,%g0 + bcs 3f + addxcc %o2,%o2,%o2 ! shift n1n0 and a q-bit in lsb + b 3f + sub %o4,%o3,%o4 ! this kills msb of n +4: sub %o4,%o3,%o4 +5: addxcc %o4,%o4,%o4 + bcc 2b + subcc %g1,1,%g1 +! Got carry from n. Subtract next step to cancel this carry. + bne 4b + addcc %o2,%o2,%o2 ! shift n1n0 and a 0-bit in lsb + sub %o4,%o3,%o4 +3: xnor %o2,0,%o2 + ! End of inline udiv_qrnnd + mov %o4,%i0 + mov %i3,%o1 + ! Inlined udiv_qrnnd + mov 32,%g1 + subcc %i0,%o3,%g0 +1: bcs 5f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + sub %i0,%o3,%i0 ! this kills msb of n + addx %i0,%i0,%i0 ! so this cannot give carry + subcc %g1,1,%g1 +2: bne 1b + subcc %i0,%o3,%g0 + bcs 3f + addxcc %o1,%o1,%o1 ! shift n1n0 and a q-bit in lsb + b 3f + sub %i0,%o3,%i0 ! this kills msb of n +4: sub %i0,%o3,%i0 +5: addxcc %i0,%i0,%i0 + bcc 2b + subcc %g1,1,%g1 +! Got carry from n. Subtract next step to cancel this carry. + bne 4b + addcc %o1,%o1,%o1 ! shift n1n0 and a 0-bit in lsb + sub %i0,%o3,%i0 +3: xnor %o1,0,%o1 + ! End of inline udiv_qrnnd + b .LL78 + mov %o1,%l1 +.LL40: + cmp %i2,%i0 + bleu .LL46 + sethi %hi(65535),%o0 + b .LL73 + mov 0,%o1 +.LL46: + or %o0,%lo(65535),%o0 + cmp %i2,%o0 + bgu .LL53 + mov %i2,%o1 + cmp %i2,256 + addx %g0,-1,%o0 + b .LL59 + and %o0,8,%o2 +.LL53: + sethi %hi(16777215),%o0 + or %o0,%lo(16777215),%o0 + cmp %o1,%o0 + bgu .LL59 + mov 24,%o2 + mov 16,%o2 +.LL59: + srl %o1,%o2,%o1 + sethi %hi(__clz_tab),%o0 + or %o0,%lo(__clz_tab),%o0 + ldub [%o1+%o0],%o0 + add %o0,%o2,%o0 + mov 32,%o1 + subcc %o1,%o0,%o2 + bne,a .LL67 + mov 32,%o0 + cmp %i0,%i2 + bgu .LL69 + cmp %i3,%o3 + blu .LL73 + mov 0,%o1 +.LL69: + b .LL73 + mov 1,%o1 +.LL67: + sub %o0,%o2,%o0 + sll %i2,%o2,%i2 + srl %o3,%o0,%o1 + or %i2,%o1,%i2 + sll %o3,%o2,%o3 + srl %i0,%o0,%o1 + sll %i0,%o2,%i0 + srl %i3,%o0,%o0 + or %i0,%o0,%i0 + sll %i3,%o2,%i3 + mov %i0,%o5 + mov %o1,%o4 + ! Inlined udiv_qrnnd + mov 32,%g1 + subcc %o4,%i2,%g0 +1: bcs 5f + addxcc %o5,%o5,%o5 ! shift n1n0 and a q-bit in lsb + sub %o4,%i2,%o4 ! this kills msb of n + addx %o4,%o4,%o4 ! so this cannot give carry + subcc %g1,1,%g1 +2: bne 1b + subcc %o4,%i2,%g0 + bcs 3f + addxcc %o5,%o5,%o5 ! shift n1n0 and a q-bit in lsb + b 3f + sub %o4,%i2,%o4 ! this kills msb of n +4: sub %o4,%i2,%o4 +5: addxcc %o4,%o4,%o4 + bcc 2b + subcc %g1,1,%g1 +! Got carry from n. Subtract next step to cancel this carry. + bne 4b + addcc %o5,%o5,%o5 ! shift n1n0 and a 0-bit in lsb + sub %o4,%i2,%o4 +3: xnor %o5,0,%o5 + ! End of inline udiv_qrnnd + mov %o4,%i0 + mov %o5,%o1 + ! Inlined umul_ppmm + wr %g0,%o1,%y ! SPARC has 0-3 delay insn after a wr + sra %o3,31,%g2 ! Do not move this insn + and %o1,%g2,%g2 ! Do not move this insn + andcc %g0,0,%g1 ! Do not move this insn + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,%o3,%g1 + mulscc %g1,0,%g1 + add %g1,%g2,%o0 + rd %y,%o2 + cmp %o0,%i0 + bgu,a .LL73 + add %o1,-1,%o1 + bne,a .LL45 + mov 0,%o2 + cmp %o2,%i3 + bleu .LL45 + mov 0,%o2 + add %o1,-1,%o1 +.LL73: + mov 0,%o2 +.LL45: + mov %o1,%l1 +.LL78: + mov %o2,%l0 + mov %l0,%i0 + mov %l1,%i1 + ret + restore diff --git a/arch/sparc/math-emu/Makefile b/arch/sparc/math-emu/Makefile new file mode 100644 index 000000000..aea805979 --- /dev/null +++ b/arch/sparc/math-emu/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the FPU instruction emulation. +# + +# suppress all warnings - as math.c produces a lot! +ccflags-y := -w + +obj-y := math_$(BITS).o diff --git a/arch/sparc/math-emu/math_32.c b/arch/sparc/math-emu/math_32.c new file mode 100644 index 000000000..d5beec856 --- /dev/null +++ b/arch/sparc/math-emu/math_32.c @@ -0,0 +1,515 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/sparc/math-emu/math.c + * + * Copyright (C) 1998 Peter Maydell (pmaydell@chiark.greenend.org.uk) + * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 1999 David S. Miller (davem@redhat.com) + * + * This is a good place to start if you're trying to understand the + * emulation code, because it's pretty simple. What we do is + * essentially analyse the instruction to work out what the operation + * is and which registers are involved. We then execute the appropriate + * FXXXX function. [The floating point queue introduces a minor wrinkle; + * see below...] + * The fxxxxx.c files each emulate a single insn. They look relatively + * simple because the complexity is hidden away in an unholy tangle + * of preprocessor macros. + * + * The first layer of macros is single.h, double.h, quad.h. Generally + * these files define macros for working with floating point numbers + * of the three IEEE formats. FP_ADD_D(R,A,B) is for adding doubles, + * for instance. These macros are usually defined as calls to more + * generic macros (in this case _FP_ADD(D,2,R,X,Y) where the number + * of machine words required to store the given IEEE format is passed + * as a parameter. [double.h and co check the number of bits in a word + * and define FP_ADD_D & co appropriately]. + * The generic macros are defined in op-common.h. This is where all + * the grotty stuff like handling NaNs is coded. To handle the possible + * word sizes macros in op-common.h use macros like _FP_FRAC_SLL_##wc() + * where wc is the 'number of machine words' parameter (here 2). + * These are defined in the third layer of macros: op-1.h, op-2.h + * and op-4.h. These handle operations on floating point numbers composed + * of 1,2 and 4 machine words respectively. [For example, on sparc64 + * doubles are one machine word so macros in double.h eventually use + * constructs in op-1.h, but on sparc32 they use op-2.h definitions.] + * soft-fp.h is on the same level as op-common.h, and defines some + * macros which are independent of both word size and FP format. + * Finally, sfp-machine.h is the machine dependent part of the + * code: it defines the word size and what type a word is. It also + * defines how _FP_MUL_MEAT_t() maps to _FP_MUL_MEAT_n_* : op-n.h + * provide several possible flavours of multiply algorithm, most + * of which require that you supply some form of asm or C primitive to + * do the actual multiply. (such asm primitives should be defined + * in sfp-machine.h too). udivmodti4.c is the same sort of thing. + * + * There may be some errors here because I'm working from a + * SPARC architecture manual V9, and what I really want is V8... + * Also, the insns which can generate exceptions seem to be a + * greater subset of the FPops than for V9 (for example, FCMPED + * has to be emulated on V8). So I think I'm going to have + * to emulate them all just to be on the safe side... + * + * Emulation routines originate from soft-fp package, which is + * part of glibc and has appropriate copyrights in it (allegedly). + * + * NB: on sparc int == long == 4 bytes, long long == 8 bytes. + * Most bits of the kernel seem to go for long rather than int, + * so we follow that practice... + */ + +/* TODO: + * fpsave() saves the FP queue but fpload() doesn't reload it. + * Therefore when we context switch or change FPU ownership + * we have to check to see if the queue had anything in it and + * emulate it if it did. This is going to be a pain. + */ + +#include +#include +#include +#include +#include + +#include "sfp-util_32.h" +#include +#include +#include +#include + +#define FLOATFUNC(x) extern int x(void *,void *,void *) + +/* The Vn labels indicate what version of the SPARC architecture gas thinks + * each insn is. This is from the binutils source :-> + */ +/* quadword instructions */ +#define FSQRTQ 0x02b /* v8 */ +#define FADDQ 0x043 /* v8 */ +#define FSUBQ 0x047 /* v8 */ +#define FMULQ 0x04b /* v8 */ +#define FDIVQ 0x04f /* v8 */ +#define FDMULQ 0x06e /* v8 */ +#define FQTOS 0x0c7 /* v8 */ +#define FQTOD 0x0cb /* v8 */ +#define FITOQ 0x0cc /* v8 */ +#define FSTOQ 0x0cd /* v8 */ +#define FDTOQ 0x0ce /* v8 */ +#define FQTOI 0x0d3 /* v8 */ +#define FCMPQ 0x053 /* v8 */ +#define FCMPEQ 0x057 /* v8 */ +/* single/double instructions (subnormal): should all work */ +#define FSQRTS 0x029 /* v7 */ +#define FSQRTD 0x02a /* v7 */ +#define FADDS 0x041 /* v6 */ +#define FADDD 0x042 /* v6 */ +#define FSUBS 0x045 /* v6 */ +#define FSUBD 0x046 /* v6 */ +#define FMULS 0x049 /* v6 */ +#define FMULD 0x04a /* v6 */ +#define FDIVS 0x04d /* v6 */ +#define FDIVD 0x04e /* v6 */ +#define FSMULD 0x069 /* v6 */ +#define FDTOS 0x0c6 /* v6 */ +#define FSTOD 0x0c9 /* v6 */ +#define FSTOI 0x0d1 /* v6 */ +#define FDTOI 0x0d2 /* v6 */ +#define FABSS 0x009 /* v6 */ +#define FCMPS 0x051 /* v6 */ +#define FCMPES 0x055 /* v6 */ +#define FCMPD 0x052 /* v6 */ +#define FCMPED 0x056 /* v6 */ +#define FMOVS 0x001 /* v6 */ +#define FNEGS 0x005 /* v6 */ +#define FITOS 0x0c4 /* v6 */ +#define FITOD 0x0c8 /* v6 */ + +#define FSR_TEM_SHIFT 23UL +#define FSR_TEM_MASK (0x1fUL << FSR_TEM_SHIFT) +#define FSR_AEXC_SHIFT 5UL +#define FSR_AEXC_MASK (0x1fUL << FSR_AEXC_SHIFT) +#define FSR_CEXC_SHIFT 0UL +#define FSR_CEXC_MASK (0x1fUL << FSR_CEXC_SHIFT) + +static int do_one_mathemu(u32 insn, unsigned long *fsr, unsigned long *fregs); + +/* Unlike the Sparc64 version (which has a struct fpustate), we + * pass the taskstruct corresponding to the task which currently owns the + * FPU. This is partly because we don't have the fpustate struct and + * partly because the task owning the FPU isn't always current (as is + * the case for the Sparc64 port). This is probably SMP-related... + * This function returns 1 if all queued insns were emulated successfully. + * The test for unimplemented FPop in kernel mode has been moved into + * kernel/traps.c for simplicity. + */ +int do_mathemu(struct pt_regs *regs, struct task_struct *fpt) +{ + /* regs->pc isn't necessarily the PC at which the offending insn is sitting. + * The FPU maintains a queue of FPops which cause traps. + * When it hits an instruction that requires that the trapped op succeeded + * (usually because it reads a reg. that the trapped op wrote) then it + * causes this exception. We need to emulate all the insns on the queue + * and then allow the op to proceed. + * This code should also handle the case where the trap was precise, + * in which case the queue length is zero and regs->pc points at the + * single FPop to be emulated. (this case is untested, though :->) + * You'll need this case if you want to be able to emulate all FPops + * because the FPU either doesn't exist or has been software-disabled. + * [The UltraSPARC makes FP a precise trap; this isn't as stupid as it + * might sound because the Ultra does funky things with a superscalar + * architecture.] + */ + + /* You wouldn't believe how often I typed 'ftp' when I meant 'fpt' :-> */ + + int i; + int retcode = 0; /* assume all succeed */ + unsigned long insn; + + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + +#ifdef DEBUG_MATHEMU + printk("In do_mathemu()... pc is %08lx\n", regs->pc); + printk("fpqdepth is %ld\n", fpt->thread.fpqdepth); + for (i = 0; i < fpt->thread.fpqdepth; i++) + printk("%d: %08lx at %08lx\n", i, fpt->thread.fpqueue[i].insn, + (unsigned long)fpt->thread.fpqueue[i].insn_addr); +#endif + + if (fpt->thread.fpqdepth == 0) { /* no queue, guilty insn is at regs->pc */ +#ifdef DEBUG_MATHEMU + printk("precise trap at %08lx\n", regs->pc); +#endif + if (!get_user(insn, (u32 __user *) regs->pc)) { + retcode = do_one_mathemu(insn, &fpt->thread.fsr, fpt->thread.float_regs); + if (retcode) { + /* in this case we need to fix up PC & nPC */ + regs->pc = regs->npc; + regs->npc += 4; + } + } + return retcode; + } + + /* Normal case: need to empty the queue... */ + for (i = 0; i < fpt->thread.fpqdepth; i++) { + retcode = do_one_mathemu(fpt->thread.fpqueue[i].insn, &(fpt->thread.fsr), fpt->thread.float_regs); + if (!retcode) /* insn failed, no point doing any more */ + break; + } + /* Now empty the queue and clear the queue_not_empty flag */ + if (retcode) + fpt->thread.fsr &= ~(0x3000 | FSR_CEXC_MASK); + else + fpt->thread.fsr &= ~0x3000; + fpt->thread.fpqdepth = 0; + + return retcode; +} + +/* All routines returning an exception to raise should detect + * such exceptions _before_ rounding to be consistent with + * the behavior of the hardware in the implemented cases + * (and thus with the recommendations in the V9 architecture + * manual). + * + * We return 0 if a SIGFPE should be sent, 1 otherwise. + */ +static inline int record_exception(unsigned long *pfsr, int eflag) +{ + unsigned long fsr = *pfsr; + int would_trap; + + /* Determine if this exception would have generated a trap. */ + would_trap = (fsr & ((long)eflag << FSR_TEM_SHIFT)) != 0UL; + + /* If trapping, we only want to signal one bit. */ + if (would_trap != 0) { + eflag &= ((fsr & FSR_TEM_MASK) >> FSR_TEM_SHIFT); + if ((eflag & (eflag - 1)) != 0) { + if (eflag & FP_EX_INVALID) + eflag = FP_EX_INVALID; + else if (eflag & FP_EX_OVERFLOW) + eflag = FP_EX_OVERFLOW; + else if (eflag & FP_EX_UNDERFLOW) + eflag = FP_EX_UNDERFLOW; + else if (eflag & FP_EX_DIVZERO) + eflag = FP_EX_DIVZERO; + else if (eflag & FP_EX_INEXACT) + eflag = FP_EX_INEXACT; + } + } + + /* Set CEXC, here is the rule: + * + * In general all FPU ops will set one and only one + * bit in the CEXC field, this is always the case + * when the IEEE exception trap is enabled in TEM. + */ + fsr &= ~(FSR_CEXC_MASK); + fsr |= ((long)eflag << FSR_CEXC_SHIFT); + + /* Set the AEXC field, rule is: + * + * If a trap would not be generated, the + * CEXC just generated is OR'd into the + * existing value of AEXC. + */ + if (would_trap == 0) + fsr |= ((long)eflag << FSR_AEXC_SHIFT); + + /* If trapping, indicate fault trap type IEEE. */ + if (would_trap != 0) + fsr |= (1UL << 14); + + *pfsr = fsr; + + return (would_trap ? 0 : 1); +} + +typedef union { + u32 s; + u64 d; + u64 q[2]; +} *argp; + +static int do_one_mathemu(u32 insn, unsigned long *pfsr, unsigned long *fregs) +{ + /* Emulate the given insn, updating fsr and fregs appropriately. */ + int type = 0; + /* r is rd, b is rs2 and a is rs1. The *u arg tells + whether the argument should be packed/unpacked (0 - do not unpack/pack, 1 - unpack/pack) + non-u args tells the size of the argument (0 - no argument, 1 - single, 2 - double, 3 - quad */ +#define TYPE(dummy, r, ru, b, bu, a, au) type = (au << 2) | (a << 0) | (bu << 5) | (b << 3) | (ru << 8) | (r << 6) + int freg; + argp rs1 = NULL, rs2 = NULL, rd = NULL; + FP_DECL_EX; + FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); + FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); + FP_DECL_Q(QA); FP_DECL_Q(QB); FP_DECL_Q(QR); + int IR; + long fsr; + +#ifdef DEBUG_MATHEMU + printk("In do_mathemu(), emulating %08lx\n", insn); +#endif + + if ((insn & 0xc1f80000) == 0x81a00000) /* FPOP1 */ { + switch ((insn >> 5) & 0x1ff) { + case FSQRTQ: TYPE(3,3,1,3,1,0,0); break; + case FADDQ: + case FSUBQ: + case FMULQ: + case FDIVQ: TYPE(3,3,1,3,1,3,1); break; + case FDMULQ: TYPE(3,3,1,2,1,2,1); break; + case FQTOS: TYPE(3,1,1,3,1,0,0); break; + case FQTOD: TYPE(3,2,1,3,1,0,0); break; + case FITOQ: TYPE(3,3,1,1,0,0,0); break; + case FSTOQ: TYPE(3,3,1,1,1,0,0); break; + case FDTOQ: TYPE(3,3,1,2,1,0,0); break; + case FQTOI: TYPE(3,1,0,3,1,0,0); break; + case FSQRTS: TYPE(2,1,1,1,1,0,0); break; + case FSQRTD: TYPE(2,2,1,2,1,0,0); break; + case FADDD: + case FSUBD: + case FMULD: + case FDIVD: TYPE(2,2,1,2,1,2,1); break; + case FADDS: + case FSUBS: + case FMULS: + case FDIVS: TYPE(2,1,1,1,1,1,1); break; + case FSMULD: TYPE(2,2,1,1,1,1,1); break; + case FDTOS: TYPE(2,1,1,2,1,0,0); break; + case FSTOD: TYPE(2,2,1,1,1,0,0); break; + case FSTOI: TYPE(2,1,0,1,1,0,0); break; + case FDTOI: TYPE(2,1,0,2,1,0,0); break; + case FITOS: TYPE(2,1,1,1,0,0,0); break; + case FITOD: TYPE(2,2,1,1,0,0,0); break; + case FMOVS: + case FABSS: + case FNEGS: TYPE(2,1,0,1,0,0,0); break; + } + } else if ((insn & 0xc1f80000) == 0x81a80000) /* FPOP2 */ { + switch ((insn >> 5) & 0x1ff) { + case FCMPS: TYPE(3,0,0,1,1,1,1); break; + case FCMPES: TYPE(3,0,0,1,1,1,1); break; + case FCMPD: TYPE(3,0,0,2,1,2,1); break; + case FCMPED: TYPE(3,0,0,2,1,2,1); break; + case FCMPQ: TYPE(3,0,0,3,1,3,1); break; + case FCMPEQ: TYPE(3,0,0,3,1,3,1); break; + } + } + + if (!type) { /* oops, didn't recognise that FPop */ +#ifdef DEBUG_MATHEMU + printk("attempt to emulate unrecognised FPop!\n"); +#endif + return 0; + } + + /* Decode the registers to be used */ + freg = (*pfsr >> 14) & 0xf; + + *pfsr &= ~0x1c000; /* clear the traptype bits */ + + freg = ((insn >> 14) & 0x1f); + switch (type & 0x3) { /* is rs1 single, double or quad? */ + case 3: + if (freg & 3) { /* quadwords must have bits 4&5 of the */ + /* encoded reg. number set to zero. */ + *pfsr |= (6 << 14); + return 0; /* simulate invalid_fp_register exception */ + } + fallthrough; + case 2: + if (freg & 1) { /* doublewords must have bit 5 zeroed */ + *pfsr |= (6 << 14); + return 0; + } + } + rs1 = (argp)&fregs[freg]; + switch (type & 0x7) { + case 7: FP_UNPACK_QP (QA, rs1); break; + case 6: FP_UNPACK_DP (DA, rs1); break; + case 5: FP_UNPACK_SP (SA, rs1); break; + } + freg = (insn & 0x1f); + switch ((type >> 3) & 0x3) { /* same again for rs2 */ + case 3: + if (freg & 3) { /* quadwords must have bits 4&5 of the */ + /* encoded reg. number set to zero. */ + *pfsr |= (6 << 14); + return 0; /* simulate invalid_fp_register exception */ + } + fallthrough; + case 2: + if (freg & 1) { /* doublewords must have bit 5 zeroed */ + *pfsr |= (6 << 14); + return 0; + } + } + rs2 = (argp)&fregs[freg]; + switch ((type >> 3) & 0x7) { + case 7: FP_UNPACK_QP (QB, rs2); break; + case 6: FP_UNPACK_DP (DB, rs2); break; + case 5: FP_UNPACK_SP (SB, rs2); break; + } + freg = ((insn >> 25) & 0x1f); + switch ((type >> 6) & 0x3) { /* and finally rd. This one's a bit different */ + case 0: /* dest is fcc. (this must be FCMPQ or FCMPEQ) */ + if (freg) { /* V8 has only one set of condition codes, so */ + /* anything but 0 in the rd field is an error */ + *pfsr |= (6 << 14); /* (should probably flag as invalid opcode */ + return 0; /* but SIGFPE will do :-> ) */ + } + break; + case 3: + if (freg & 3) { /* quadwords must have bits 4&5 of the */ + /* encoded reg. number set to zero. */ + *pfsr |= (6 << 14); + return 0; /* simulate invalid_fp_register exception */ + } + fallthrough; + case 2: + if (freg & 1) { /* doublewords must have bit 5 zeroed */ + *pfsr |= (6 << 14); + return 0; + } + fallthrough; + case 1: + rd = (void *)&fregs[freg]; + break; + } +#ifdef DEBUG_MATHEMU + printk("executing insn...\n"); +#endif + /* do the Right Thing */ + switch ((insn >> 5) & 0x1ff) { + /* + */ + case FADDS: FP_ADD_S (SR, SA, SB); break; + case FADDD: FP_ADD_D (DR, DA, DB); break; + case FADDQ: FP_ADD_Q (QR, QA, QB); break; + /* - */ + case FSUBS: FP_SUB_S (SR, SA, SB); break; + case FSUBD: FP_SUB_D (DR, DA, DB); break; + case FSUBQ: FP_SUB_Q (QR, QA, QB); break; + /* * */ + case FMULS: FP_MUL_S (SR, SA, SB); break; + case FSMULD: FP_CONV (D, S, 2, 1, DA, SA); + FP_CONV (D, S, 2, 1, DB, SB); + case FMULD: FP_MUL_D (DR, DA, DB); break; + case FDMULQ: FP_CONV (Q, D, 4, 2, QA, DA); + FP_CONV (Q, D, 4, 2, QB, DB); + case FMULQ: FP_MUL_Q (QR, QA, QB); break; + /* / */ + case FDIVS: FP_DIV_S (SR, SA, SB); break; + case FDIVD: FP_DIV_D (DR, DA, DB); break; + case FDIVQ: FP_DIV_Q (QR, QA, QB); break; + /* sqrt */ + case FSQRTS: FP_SQRT_S (SR, SB); break; + case FSQRTD: FP_SQRT_D (DR, DB); break; + case FSQRTQ: FP_SQRT_Q (QR, QB); break; + /* mov */ + case FMOVS: rd->s = rs2->s; break; + case FABSS: rd->s = rs2->s & 0x7fffffff; break; + case FNEGS: rd->s = rs2->s ^ 0x80000000; break; + /* float to int */ + case FSTOI: FP_TO_INT_S (IR, SB, 32, 1); break; + case FDTOI: FP_TO_INT_D (IR, DB, 32, 1); break; + case FQTOI: FP_TO_INT_Q (IR, QB, 32, 1); break; + /* int to float */ + case FITOS: IR = rs2->s; FP_FROM_INT_S (SR, IR, 32, int); break; + case FITOD: IR = rs2->s; FP_FROM_INT_D (DR, IR, 32, int); break; + case FITOQ: IR = rs2->s; FP_FROM_INT_Q (QR, IR, 32, int); break; + /* float to float */ + case FSTOD: FP_CONV (D, S, 2, 1, DR, SB); break; + case FSTOQ: FP_CONV (Q, S, 4, 1, QR, SB); break; + case FDTOQ: FP_CONV (Q, D, 4, 2, QR, DB); break; + case FDTOS: FP_CONV (S, D, 1, 2, SR, DB); break; + case FQTOS: FP_CONV (S, Q, 1, 4, SR, QB); break; + case FQTOD: FP_CONV (D, Q, 2, 4, DR, QB); break; + /* comparison */ + case FCMPS: + case FCMPES: + FP_CMP_S(IR, SB, SA, 3); + if (IR == 3 && + (((insn >> 5) & 0x1ff) == FCMPES || + FP_ISSIGNAN_S(SA) || + FP_ISSIGNAN_S(SB))) + FP_SET_EXCEPTION (FP_EX_INVALID); + break; + case FCMPD: + case FCMPED: + FP_CMP_D(IR, DB, DA, 3); + if (IR == 3 && + (((insn >> 5) & 0x1ff) == FCMPED || + FP_ISSIGNAN_D(DA) || + FP_ISSIGNAN_D(DB))) + FP_SET_EXCEPTION (FP_EX_INVALID); + break; + case FCMPQ: + case FCMPEQ: + FP_CMP_Q(IR, QB, QA, 3); + if (IR == 3 && + (((insn >> 5) & 0x1ff) == FCMPEQ || + FP_ISSIGNAN_Q(QA) || + FP_ISSIGNAN_Q(QB))) + FP_SET_EXCEPTION (FP_EX_INVALID); + } + if (!FP_INHIBIT_RESULTS) { + switch ((type >> 6) & 0x7) { + case 0: fsr = *pfsr; + if (IR == -1) IR = 2; + /* fcc is always fcc0 */ + fsr &= ~0xc00; fsr |= (IR << 10); + *pfsr = fsr; + break; + case 1: rd->s = IR; break; + case 5: FP_PACK_SP (rd, SR); break; + case 6: FP_PACK_DP (rd, DR); break; + case 7: FP_PACK_QP (rd, QR); break; + } + } + if (_fex == 0) + return 1; /* success! */ + return record_exception(pfsr, _fex); +} diff --git a/arch/sparc/math-emu/math_64.c b/arch/sparc/math-emu/math_64.c new file mode 100644 index 000000000..1379dee26 --- /dev/null +++ b/arch/sparc/math-emu/math_64.c @@ -0,0 +1,525 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/sparc64/math-emu/math.c + * + * Copyright (C) 1997,1999 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 1999 David S. Miller (davem@redhat.com) + * + * Emulation routines originate from soft-fp package, which is part + * of glibc and has appropriate copyrights in it. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "sfp-util_64.h" +#include +#include +#include +#include + +/* QUAD - ftt == 3 */ +#define FMOVQ 0x003 +#define FNEGQ 0x007 +#define FABSQ 0x00b +#define FSQRTQ 0x02b +#define FADDQ 0x043 +#define FSUBQ 0x047 +#define FMULQ 0x04b +#define FDIVQ 0x04f +#define FDMULQ 0x06e +#define FQTOX 0x083 +#define FXTOQ 0x08c +#define FQTOS 0x0c7 +#define FQTOD 0x0cb +#define FITOQ 0x0cc +#define FSTOQ 0x0cd +#define FDTOQ 0x0ce +#define FQTOI 0x0d3 +/* SUBNORMAL - ftt == 2 */ +#define FSQRTS 0x029 +#define FSQRTD 0x02a +#define FADDS 0x041 +#define FADDD 0x042 +#define FSUBS 0x045 +#define FSUBD 0x046 +#define FMULS 0x049 +#define FMULD 0x04a +#define FDIVS 0x04d +#define FDIVD 0x04e +#define FSMULD 0x069 +#define FSTOX 0x081 +#define FDTOX 0x082 +#define FDTOS 0x0c6 +#define FSTOD 0x0c9 +#define FSTOI 0x0d1 +#define FDTOI 0x0d2 +#define FXTOS 0x084 /* Only Ultra-III generates this. */ +#define FXTOD 0x088 /* Only Ultra-III generates this. */ +#if 0 /* Optimized inline in sparc64/kernel/entry.S */ +#define FITOS 0x0c4 /* Only Ultra-III generates this. */ +#endif +#define FITOD 0x0c8 /* Only Ultra-III generates this. */ +/* FPOP2 */ +#define FCMPQ 0x053 +#define FCMPEQ 0x057 +#define FMOVQ0 0x003 +#define FMOVQ1 0x043 +#define FMOVQ2 0x083 +#define FMOVQ3 0x0c3 +#define FMOVQI 0x103 +#define FMOVQX 0x183 +#define FMOVQZ 0x027 +#define FMOVQLE 0x047 +#define FMOVQLZ 0x067 +#define FMOVQNZ 0x0a7 +#define FMOVQGZ 0x0c7 +#define FMOVQGE 0x0e7 + +#define FSR_TEM_SHIFT 23UL +#define FSR_TEM_MASK (0x1fUL << FSR_TEM_SHIFT) +#define FSR_AEXC_SHIFT 5UL +#define FSR_AEXC_MASK (0x1fUL << FSR_AEXC_SHIFT) +#define FSR_CEXC_SHIFT 0UL +#define FSR_CEXC_MASK (0x1fUL << FSR_CEXC_SHIFT) + +/* All routines returning an exception to raise should detect + * such exceptions _before_ rounding to be consistent with + * the behavior of the hardware in the implemented cases + * (and thus with the recommendations in the V9 architecture + * manual). + * + * We return 0 if a SIGFPE should be sent, 1 otherwise. + */ +static inline int record_exception(struct pt_regs *regs, int eflag) +{ + u64 fsr = current_thread_info()->xfsr[0]; + int would_trap; + + /* Determine if this exception would have generated a trap. */ + would_trap = (fsr & ((long)eflag << FSR_TEM_SHIFT)) != 0UL; + + /* If trapping, we only want to signal one bit. */ + if(would_trap != 0) { + eflag &= ((fsr & FSR_TEM_MASK) >> FSR_TEM_SHIFT); + if((eflag & (eflag - 1)) != 0) { + if(eflag & FP_EX_INVALID) + eflag = FP_EX_INVALID; + else if(eflag & FP_EX_OVERFLOW) + eflag = FP_EX_OVERFLOW; + else if(eflag & FP_EX_UNDERFLOW) + eflag = FP_EX_UNDERFLOW; + else if(eflag & FP_EX_DIVZERO) + eflag = FP_EX_DIVZERO; + else if(eflag & FP_EX_INEXACT) + eflag = FP_EX_INEXACT; + } + } + + /* Set CEXC, here is the rule: + * + * In general all FPU ops will set one and only one + * bit in the CEXC field, this is always the case + * when the IEEE exception trap is enabled in TEM. + */ + fsr &= ~(FSR_CEXC_MASK); + fsr |= ((long)eflag << FSR_CEXC_SHIFT); + + /* Set the AEXC field, rule is: + * + * If a trap would not be generated, the + * CEXC just generated is OR'd into the + * existing value of AEXC. + */ + if(would_trap == 0) + fsr |= ((long)eflag << FSR_AEXC_SHIFT); + + /* If trapping, indicate fault trap type IEEE. */ + if(would_trap != 0) + fsr |= (1UL << 14); + + current_thread_info()->xfsr[0] = fsr; + + /* If we will not trap, advance the program counter over + * the instruction being handled. + */ + if(would_trap == 0) { + regs->tpc = regs->tnpc; + regs->tnpc += 4; + } + + return (would_trap ? 0 : 1); +} + +typedef union { + u32 s; + u64 d; + u64 q[2]; +} *argp; + +int do_mathemu(struct pt_regs *regs, struct fpustate *f, bool illegal_insn_trap) +{ + unsigned long pc = regs->tpc; + unsigned long tstate = regs->tstate; + u32 insn = 0; + int type = 0; + /* ftt tells which ftt it may happen in, r is rd, b is rs2 and a is rs1. The *u arg tells + whether the argument should be packed/unpacked (0 - do not unpack/pack, 1 - unpack/pack) + non-u args tells the size of the argument (0 - no argument, 1 - single, 2 - double, 3 - quad */ +#define TYPE(ftt, r, ru, b, bu, a, au) type = (au << 2) | (a << 0) | (bu << 5) | (b << 3) | (ru << 8) | (r << 6) | (ftt << 9) + int freg; + static u64 zero[2] = { 0L, 0L }; + int flags; + FP_DECL_EX; + FP_DECL_S(SA); FP_DECL_S(SB); FP_DECL_S(SR); + FP_DECL_D(DA); FP_DECL_D(DB); FP_DECL_D(DR); + FP_DECL_Q(QA); FP_DECL_Q(QB); FP_DECL_Q(QR); + int IR; + long XR, xfsr; + + if (tstate & TSTATE_PRIV) + die_if_kernel("unfinished/unimplemented FPop from kernel", regs); + perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, 0); + if (test_thread_flag(TIF_32BIT)) + pc = (u32)pc; + if (get_user(insn, (u32 __user *) pc) != -EFAULT) { + if ((insn & 0xc1f80000) == 0x81a00000) /* FPOP1 */ { + switch ((insn >> 5) & 0x1ff) { + /* QUAD - ftt == 3 */ + case FMOVQ: + case FNEGQ: + case FABSQ: TYPE(3,3,0,3,0,0,0); break; + case FSQRTQ: TYPE(3,3,1,3,1,0,0); break; + case FADDQ: + case FSUBQ: + case FMULQ: + case FDIVQ: TYPE(3,3,1,3,1,3,1); break; + case FDMULQ: TYPE(3,3,1,2,1,2,1); break; + case FQTOX: TYPE(3,2,0,3,1,0,0); break; + case FXTOQ: TYPE(3,3,1,2,0,0,0); break; + case FQTOS: TYPE(3,1,1,3,1,0,0); break; + case FQTOD: TYPE(3,2,1,3,1,0,0); break; + case FITOQ: TYPE(3,3,1,1,0,0,0); break; + case FSTOQ: TYPE(3,3,1,1,1,0,0); break; + case FDTOQ: TYPE(3,3,1,2,1,0,0); break; + case FQTOI: TYPE(3,1,0,3,1,0,0); break; + + /* We can get either unimplemented or unfinished + * for these cases. Pre-Niagara systems generate + * unfinished fpop for SUBNORMAL cases, and Niagara + * always gives unimplemented fpop for fsqrt{s,d}. + */ + case FSQRTS: { + unsigned long x = current_thread_info()->xfsr[0]; + + x = (x >> 14) & 0x7; + TYPE(x,1,1,1,1,0,0); + break; + } + + case FSQRTD: { + unsigned long x = current_thread_info()->xfsr[0]; + + x = (x >> 14) & 0x7; + TYPE(x,2,1,2,1,0,0); + break; + } + + /* SUBNORMAL - ftt == 2 */ + case FADDD: + case FSUBD: + case FMULD: + case FDIVD: TYPE(2,2,1,2,1,2,1); break; + case FADDS: + case FSUBS: + case FMULS: + case FDIVS: TYPE(2,1,1,1,1,1,1); break; + case FSMULD: TYPE(2,2,1,1,1,1,1); break; + case FSTOX: TYPE(2,2,0,1,1,0,0); break; + case FDTOX: TYPE(2,2,0,2,1,0,0); break; + case FDTOS: TYPE(2,1,1,2,1,0,0); break; + case FSTOD: TYPE(2,2,1,1,1,0,0); break; + case FSTOI: TYPE(2,1,0,1,1,0,0); break; + case FDTOI: TYPE(2,1,0,2,1,0,0); break; + + /* Only Ultra-III generates these */ + case FXTOS: TYPE(2,1,1,2,0,0,0); break; + case FXTOD: TYPE(2,2,1,2,0,0,0); break; +#if 0 /* Optimized inline in sparc64/kernel/entry.S */ + case FITOS: TYPE(2,1,1,1,0,0,0); break; +#endif + case FITOD: TYPE(2,2,1,1,0,0,0); break; + } + } + else if ((insn & 0xc1f80000) == 0x81a80000) /* FPOP2 */ { + IR = 2; + switch ((insn >> 5) & 0x1ff) { + case FCMPQ: TYPE(3,0,0,3,1,3,1); break; + case FCMPEQ: TYPE(3,0,0,3,1,3,1); break; + /* Now the conditional fmovq support */ + case FMOVQ0: + case FMOVQ1: + case FMOVQ2: + case FMOVQ3: + /* fmovq %fccX, %fY, %fZ */ + if (!((insn >> 11) & 3)) + XR = current_thread_info()->xfsr[0] >> 10; + else + XR = current_thread_info()->xfsr[0] >> (30 + ((insn >> 10) & 0x6)); + XR &= 3; + IR = 0; + switch ((insn >> 14) & 0x7) { + /* case 0: IR = 0; break; */ /* Never */ + case 1: if (XR) IR = 1; break; /* Not Equal */ + case 2: if (XR == 1 || XR == 2) IR = 1; break; /* Less or Greater */ + case 3: if (XR & 1) IR = 1; break; /* Unordered or Less */ + case 4: if (XR == 1) IR = 1; break; /* Less */ + case 5: if (XR & 2) IR = 1; break; /* Unordered or Greater */ + case 6: if (XR == 2) IR = 1; break; /* Greater */ + case 7: if (XR == 3) IR = 1; break; /* Unordered */ + } + if ((insn >> 14) & 8) + IR ^= 1; + break; + case FMOVQI: + case FMOVQX: + /* fmovq %[ix]cc, %fY, %fZ */ + XR = regs->tstate >> 32; + if ((insn >> 5) & 0x80) + XR >>= 4; + XR &= 0xf; + IR = 0; + freg = ((XR >> 2) ^ XR) & 2; + switch ((insn >> 14) & 0x7) { + /* case 0: IR = 0; break; */ /* Never */ + case 1: if (XR & 4) IR = 1; break; /* Equal */ + case 2: if ((XR & 4) || freg) IR = 1; break; /* Less or Equal */ + case 3: if (freg) IR = 1; break; /* Less */ + case 4: if (XR & 5) IR = 1; break; /* Less or Equal Unsigned */ + case 5: if (XR & 1) IR = 1; break; /* Carry Set */ + case 6: if (XR & 8) IR = 1; break; /* Negative */ + case 7: if (XR & 2) IR = 1; break; /* Overflow Set */ + } + if ((insn >> 14) & 8) + IR ^= 1; + break; + case FMOVQZ: + case FMOVQLE: + case FMOVQLZ: + case FMOVQNZ: + case FMOVQGZ: + case FMOVQGE: + freg = (insn >> 14) & 0x1f; + if (!freg) + XR = 0; + else if (freg < 16) + XR = regs->u_regs[freg]; + else if (!test_thread_64bit_stack(regs->u_regs[UREG_FP])) { + struct reg_window32 __user *win32; + flushw_user (); + win32 = (struct reg_window32 __user *)((unsigned long)((u32)regs->u_regs[UREG_FP])); + get_user(XR, &win32->locals[freg - 16]); + } else { + struct reg_window __user *win; + flushw_user (); + win = (struct reg_window __user *)(regs->u_regs[UREG_FP] + STACK_BIAS); + get_user(XR, &win->locals[freg - 16]); + } + IR = 0; + switch ((insn >> 10) & 3) { + case 1: if (!XR) IR = 1; break; /* Register Zero */ + case 2: if (XR <= 0) IR = 1; break; /* Register Less Than or Equal to Zero */ + case 3: if (XR < 0) IR = 1; break; /* Register Less Than Zero */ + } + if ((insn >> 10) & 4) + IR ^= 1; + break; + } + if (IR == 0) { + /* The fmov test was false. Do a nop instead */ + current_thread_info()->xfsr[0] &= ~(FSR_CEXC_MASK); + regs->tpc = regs->tnpc; + regs->tnpc += 4; + return 1; + } else if (IR == 1) { + /* Change the instruction into plain fmovq */ + insn = (insn & 0x3e00001f) | 0x81a00060; + TYPE(3,3,0,3,0,0,0); + } + } + } + if (type) { + argp rs1 = NULL, rs2 = NULL, rd = NULL; + + /* Starting with UltraSPARC-T2, the cpu does not set the FP Trap + * Type field in the %fsr to unimplemented_FPop. Nor does it + * use the fp_exception_other trap. Instead it signals an + * illegal instruction and leaves the FP trap type field of + * the %fsr unchanged. + */ + if (!illegal_insn_trap) { + int ftt = (current_thread_info()->xfsr[0] >> 14) & 0x7; + if (ftt != (type >> 9)) + goto err; + } + current_thread_info()->xfsr[0] &= ~0x1c000; + freg = ((insn >> 14) & 0x1f); + switch (type & 0x3) { + case 3: if (freg & 2) { + current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; + goto err; + } + case 2: freg = ((freg & 1) << 5) | (freg & 0x1e); + case 1: rs1 = (argp)&f->regs[freg]; + flags = (freg < 32) ? FPRS_DL : FPRS_DU; + if (!(current_thread_info()->fpsaved[0] & flags)) + rs1 = (argp)&zero; + break; + } + switch (type & 0x7) { + case 7: FP_UNPACK_QP (QA, rs1); break; + case 6: FP_UNPACK_DP (DA, rs1); break; + case 5: FP_UNPACK_SP (SA, rs1); break; + } + freg = (insn & 0x1f); + switch ((type >> 3) & 0x3) { + case 3: if (freg & 2) { + current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; + goto err; + } + case 2: freg = ((freg & 1) << 5) | (freg & 0x1e); + case 1: rs2 = (argp)&f->regs[freg]; + flags = (freg < 32) ? FPRS_DL : FPRS_DU; + if (!(current_thread_info()->fpsaved[0] & flags)) + rs2 = (argp)&zero; + break; + } + switch ((type >> 3) & 0x7) { + case 7: FP_UNPACK_QP (QB, rs2); break; + case 6: FP_UNPACK_DP (DB, rs2); break; + case 5: FP_UNPACK_SP (SB, rs2); break; + } + freg = ((insn >> 25) & 0x1f); + switch ((type >> 6) & 0x3) { + case 3: if (freg & 2) { + current_thread_info()->xfsr[0] |= (6 << 14) /* invalid_fp_register */; + goto err; + } + case 2: freg = ((freg & 1) << 5) | (freg & 0x1e); + case 1: rd = (argp)&f->regs[freg]; + flags = (freg < 32) ? FPRS_DL : FPRS_DU; + if (!(current_thread_info()->fpsaved[0] & FPRS_FEF)) { + current_thread_info()->fpsaved[0] = FPRS_FEF; + current_thread_info()->gsr[0] = 0; + } + if (!(current_thread_info()->fpsaved[0] & flags)) { + if (freg < 32) + memset(f->regs, 0, 32*sizeof(u32)); + else + memset(f->regs+32, 0, 32*sizeof(u32)); + } + current_thread_info()->fpsaved[0] |= flags; + break; + } + switch ((insn >> 5) & 0x1ff) { + /* + */ + case FADDS: FP_ADD_S (SR, SA, SB); break; + case FADDD: FP_ADD_D (DR, DA, DB); break; + case FADDQ: FP_ADD_Q (QR, QA, QB); break; + /* - */ + case FSUBS: FP_SUB_S (SR, SA, SB); break; + case FSUBD: FP_SUB_D (DR, DA, DB); break; + case FSUBQ: FP_SUB_Q (QR, QA, QB); break; + /* * */ + case FMULS: FP_MUL_S (SR, SA, SB); break; + case FSMULD: FP_CONV (D, S, 1, 1, DA, SA); + FP_CONV (D, S, 1, 1, DB, SB); + case FMULD: FP_MUL_D (DR, DA, DB); break; + case FDMULQ: FP_CONV (Q, D, 2, 1, QA, DA); + FP_CONV (Q, D, 2, 1, QB, DB); + case FMULQ: FP_MUL_Q (QR, QA, QB); break; + /* / */ + case FDIVS: FP_DIV_S (SR, SA, SB); break; + case FDIVD: FP_DIV_D (DR, DA, DB); break; + case FDIVQ: FP_DIV_Q (QR, QA, QB); break; + /* sqrt */ + case FSQRTS: FP_SQRT_S (SR, SB); break; + case FSQRTD: FP_SQRT_D (DR, DB); break; + case FSQRTQ: FP_SQRT_Q (QR, QB); break; + /* mov */ + case FMOVQ: rd->q[0] = rs2->q[0]; rd->q[1] = rs2->q[1]; break; + case FABSQ: rd->q[0] = rs2->q[0] & 0x7fffffffffffffffUL; rd->q[1] = rs2->q[1]; break; + case FNEGQ: rd->q[0] = rs2->q[0] ^ 0x8000000000000000UL; rd->q[1] = rs2->q[1]; break; + /* float to int */ + case FSTOI: FP_TO_INT_S (IR, SB, 32, 1); break; + case FDTOI: FP_TO_INT_D (IR, DB, 32, 1); break; + case FQTOI: FP_TO_INT_Q (IR, QB, 32, 1); break; + case FSTOX: FP_TO_INT_S (XR, SB, 64, 1); break; + case FDTOX: FP_TO_INT_D (XR, DB, 64, 1); break; + case FQTOX: FP_TO_INT_Q (XR, QB, 64, 1); break; + /* int to float */ + case FITOQ: IR = rs2->s; FP_FROM_INT_Q (QR, IR, 32, int); break; + case FXTOQ: XR = rs2->d; FP_FROM_INT_Q (QR, XR, 64, long); break; + /* Only Ultra-III generates these */ + case FXTOS: XR = rs2->d; FP_FROM_INT_S (SR, XR, 64, long); break; + case FXTOD: XR = rs2->d; FP_FROM_INT_D (DR, XR, 64, long); break; +#if 0 /* Optimized inline in sparc64/kernel/entry.S */ + case FITOS: IR = rs2->s; FP_FROM_INT_S (SR, IR, 32, int); break; +#endif + case FITOD: IR = rs2->s; FP_FROM_INT_D (DR, IR, 32, int); break; + /* float to float */ + case FSTOD: FP_CONV (D, S, 1, 1, DR, SB); break; + case FSTOQ: FP_CONV (Q, S, 2, 1, QR, SB); break; + case FDTOQ: FP_CONV (Q, D, 2, 1, QR, DB); break; + case FDTOS: FP_CONV (S, D, 1, 1, SR, DB); break; + case FQTOS: FP_CONV (S, Q, 1, 2, SR, QB); break; + case FQTOD: FP_CONV (D, Q, 1, 2, DR, QB); break; + /* comparison */ + case FCMPQ: + case FCMPEQ: + FP_CMP_Q(XR, QB, QA, 3); + if (XR == 3 && + (((insn >> 5) & 0x1ff) == FCMPEQ || + FP_ISSIGNAN_Q(QA) || + FP_ISSIGNAN_Q(QB))) + FP_SET_EXCEPTION (FP_EX_INVALID); + } + if (!FP_INHIBIT_RESULTS) { + switch ((type >> 6) & 0x7) { + case 0: xfsr = current_thread_info()->xfsr[0]; + if (XR == -1) XR = 2; + switch (freg & 3) { + /* fcc0, 1, 2, 3 */ + case 0: xfsr &= ~0xc00; xfsr |= (XR << 10); break; + case 1: xfsr &= ~0x300000000UL; xfsr |= (XR << 32); break; + case 2: xfsr &= ~0xc00000000UL; xfsr |= (XR << 34); break; + case 3: xfsr &= ~0x3000000000UL; xfsr |= (XR << 36); break; + } + current_thread_info()->xfsr[0] = xfsr; + break; + case 1: rd->s = IR; break; + case 2: rd->d = XR; break; + case 5: FP_PACK_SP (rd, SR); break; + case 6: FP_PACK_DP (rd, DR); break; + case 7: FP_PACK_QP (rd, QR); break; + } + } + + if(_fex != 0) + return record_exception(regs, _fex); + + /* Success and no exceptions detected. */ + current_thread_info()->xfsr[0] &= ~(FSR_CEXC_MASK); + regs->tpc = regs->tnpc; + regs->tnpc += 4; + return 1; + } +err: return 0; +} diff --git a/arch/sparc/math-emu/sfp-util_32.h b/arch/sparc/math-emu/sfp-util_32.h new file mode 100644 index 000000000..b57375ff2 --- /dev/null +++ b/arch/sparc/math-emu/sfp-util_32.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include + +#define add_ssaaaa(sh, sl, ah, al, bh, bl) \ + __asm__ ("addcc %r4,%5,%1\n\t" \ + "addx %r2,%3,%0\n" \ + : "=r" (sh), \ + "=&r" (sl) \ + : "%rJ" ((USItype)(ah)), \ + "rI" ((USItype)(bh)), \ + "%rJ" ((USItype)(al)), \ + "rI" ((USItype)(bl)) \ + : "cc") +#define sub_ddmmss(sh, sl, ah, al, bh, bl) \ + __asm__ ("subcc %r4,%5,%1\n\t" \ + "subx %r2,%3,%0\n" \ + : "=r" (sh), \ + "=&r" (sl) \ + : "rJ" ((USItype)(ah)), \ + "rI" ((USItype)(bh)), \ + "rJ" ((USItype)(al)), \ + "rI" ((USItype)(bl)) \ + : "cc") + +#define umul_ppmm(w1, w0, u, v) \ + __asm__ ("! Inlined umul_ppmm\n\t" \ + "wr %%g0,%2,%%y ! SPARC has 0-3 delay insn after a wr\n\t" \ + "sra %3,31,%%g2 ! Don't move this insn\n\t" \ + "and %2,%%g2,%%g2 ! Don't move this insn\n\t" \ + "andcc %%g0,0,%%g1 ! Don't move this insn\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,%3,%%g1\n\t" \ + "mulscc %%g1,0,%%g1\n\t" \ + "add %%g1,%%g2,%0\n\t" \ + "rd %%y,%1\n" \ + : "=r" (w1), \ + "=r" (w0) \ + : "%rI" ((USItype)(u)), \ + "r" ((USItype)(v)) \ + : "%g1", "%g2", "cc") + +/* It's quite necessary to add this much assembler for the sparc. + The default udiv_qrnnd (in C) is more than 10 times slower! */ +#define udiv_qrnnd(q, r, n1, n0, d) \ + __asm__ ("! Inlined udiv_qrnnd\n\t" \ + "mov 32,%%g1\n\t" \ + "subcc %1,%2,%%g0\n\t" \ + "1: bcs 5f\n\t" \ + "addxcc %0,%0,%0 ! shift n1n0 and a q-bit in lsb\n\t" \ + "sub %1,%2,%1 ! this kills msb of n\n\t" \ + "addx %1,%1,%1 ! so this can't give carry\n\t" \ + "subcc %%g1,1,%%g1\n\t" \ + "2: bne 1b\n\t" \ + "subcc %1,%2,%%g0\n\t" \ + "bcs 3f\n\t" \ + "addxcc %0,%0,%0 ! shift n1n0 and a q-bit in lsb\n\t" \ + "b 3f\n\t" \ + "sub %1,%2,%1 ! this kills msb of n\n\t" \ + "4: sub %1,%2,%1\n\t" \ + "5: addxcc %1,%1,%1\n\t" \ + "bcc 2b\n\t" \ + "subcc %%g1,1,%%g1\n\t" \ + "! Got carry from n. Subtract next step to cancel this carry.\n\t" \ + "bne 4b\n\t" \ + "addcc %0,%0,%0 ! shift n1n0 and a 0-bit in lsb\n\t" \ + "sub %1,%2,%1\n\t" \ + "3: xnor %0,0,%0\n\t" \ + "! End of inline udiv_qrnnd\n" \ + : "=&r" (q), \ + "=&r" (r) \ + : "r" ((USItype)(d)), \ + "1" ((USItype)(n1)), \ + "0" ((USItype)(n0)) : "%g1", "cc") +#define UDIV_NEEDS_NORMALIZATION 0 + +#define abort() \ + return 0 + +#ifdef __BIG_ENDIAN +#define __BYTE_ORDER __BIG_ENDIAN +#else +#define __BYTE_ORDER __LITTLE_ENDIAN +#endif diff --git a/arch/sparc/math-emu/sfp-util_64.h b/arch/sparc/math-emu/sfp-util_64.h new file mode 100644 index 000000000..8fdb55aae --- /dev/null +++ b/arch/sparc/math-emu/sfp-util_64.h @@ -0,0 +1,121 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arch/sparc64/math-emu/sfp-util.h + * + * Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 1999 David S. Miller (davem@redhat.com) + * + */ + +#include +#include +#include +#include + +#define add_ssaaaa(sh, sl, ah, al, bh, bl) \ + __asm__ ("addcc %4,%5,%1\n\t" \ + "add %2,%3,%0\n\t" \ + "bcs,a,pn %%xcc, 1f\n\t" \ + "add %0, 1, %0\n" \ + "1:" \ + : "=r" (sh), \ + "=&r" (sl) \ + : "r" ((UDItype)(ah)), \ + "r" ((UDItype)(bh)), \ + "r" ((UDItype)(al)), \ + "r" ((UDItype)(bl)) \ + : "cc") + +#define sub_ddmmss(sh, sl, ah, al, bh, bl) \ + __asm__ ("subcc %4,%5,%1\n\t" \ + "sub %2,%3,%0\n\t" \ + "bcs,a,pn %%xcc, 1f\n\t" \ + "sub %0, 1, %0\n" \ + "1:" \ + : "=r" (sh), \ + "=&r" (sl) \ + : "r" ((UDItype)(ah)), \ + "r" ((UDItype)(bh)), \ + "r" ((UDItype)(al)), \ + "r" ((UDItype)(bl)) \ + : "cc") + +#define umul_ppmm(wh, wl, u, v) \ + do { \ + UDItype tmp1, tmp2, tmp3, tmp4; \ + __asm__ __volatile__ ( \ + "srl %7,0,%3\n\t" \ + "mulx %3,%6,%1\n\t" \ + "srlx %6,32,%2\n\t" \ + "mulx %2,%3,%4\n\t" \ + "sllx %4,32,%5\n\t" \ + "srl %6,0,%3\n\t" \ + "sub %1,%5,%5\n\t" \ + "srlx %5,32,%5\n\t" \ + "addcc %4,%5,%4\n\t" \ + "srlx %7,32,%5\n\t" \ + "mulx %3,%5,%3\n\t" \ + "mulx %2,%5,%5\n\t" \ + "sethi %%hi(0x80000000),%2\n\t" \ + "addcc %4,%3,%4\n\t" \ + "srlx %4,32,%4\n\t" \ + "add %2,%2,%2\n\t" \ + "movcc %%xcc,%%g0,%2\n\t" \ + "addcc %5,%4,%5\n\t" \ + "sllx %3,32,%3\n\t" \ + "add %1,%3,%1\n\t" \ + "add %5,%2,%0" \ + : "=r" (wh), \ + "=&r" (wl), \ + "=&r" (tmp1), "=&r" (tmp2), "=&r" (tmp3), "=&r" (tmp4) \ + : "r" ((UDItype)(u)), \ + "r" ((UDItype)(v)) \ + : "cc"); \ + } while (0) + +#define udiv_qrnnd(q, r, n1, n0, d) \ + do { \ + UWtype __d1, __d0, __q1, __q0, __r1, __r0, __m; \ + __d1 = (d >> 32); \ + __d0 = (USItype)d; \ + \ + __r1 = (n1) % __d1; \ + __q1 = (n1) / __d1; \ + __m = (UWtype) __q1 * __d0; \ + __r1 = (__r1 << 32) | (n0 >> 32); \ + if (__r1 < __m) \ + { \ + __q1--, __r1 += (d); \ + if (__r1 >= (d)) /* i.e. we didn't get carry when adding to __r1 */ \ + if (__r1 < __m) \ + __q1--, __r1 += (d); \ + } \ + __r1 -= __m; \ + \ + __r0 = __r1 % __d1; \ + __q0 = __r1 / __d1; \ + __m = (UWtype) __q0 * __d0; \ + __r0 = (__r0 << 32) | ((USItype)n0); \ + if (__r0 < __m) \ + { \ + __q0--, __r0 += (d); \ + if (__r0 >= (d)) \ + if (__r0 < __m) \ + __q0--, __r0 += (d); \ + } \ + __r0 -= __m; \ + \ + (q) = (UWtype) (__q1 << 32) | __q0; \ + (r) = __r0; \ + } while (0) + +#define UDIV_NEEDS_NORMALIZATION 1 + +#define abort() \ + return 0 + +#ifdef __BIG_ENDIAN +#define __BYTE_ORDER __BIG_ENDIAN +#else +#define __BYTE_ORDER __LITTLE_ENDIAN +#endif diff --git a/arch/sparc/mm/Makefile b/arch/sparc/mm/Makefile new file mode 100644 index 000000000..e9d232561 --- /dev/null +++ b/arch/sparc/mm/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for the linux Sparc-specific parts of the memory manager. +# + +obj-$(CONFIG_SPARC64) += ultra.o tlb.o tsb.o +obj-y += fault_$(BITS).o +obj-y += init_$(BITS).o +obj-$(CONFIG_SPARC32) += srmmu.o iommu.o io-unit.o +obj-$(CONFIG_SPARC32) += srmmu_access.o +obj-$(CONFIG_SPARC32) += hypersparc.o viking.o tsunami.o swift.o +obj-$(CONFIG_SPARC32) += leon_mm.o + +# Only used by sparc64 +obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o + +obj-$(CONFIG_EXECMEM) += execmem.o diff --git a/arch/sparc/mm/execmem.c b/arch/sparc/mm/execmem.c new file mode 100644 index 000000000..0fac97dd5 --- /dev/null +++ b/arch/sparc/mm/execmem.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include + +static struct execmem_info execmem_info __ro_after_init; + +struct execmem_info __init *execmem_arch_setup(void) +{ + execmem_info = (struct execmem_info){ + .ranges = { + [EXECMEM_DEFAULT] = { + .start = MODULES_VADDR, + .end = MODULES_END, + .pgprot = PAGE_KERNEL, + .alignment = 1, + }, + }, + }; + + return &execmem_info; +} diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c new file mode 100644 index 000000000..86a831ebd --- /dev/null +++ b/arch/sparc/mm/fault_32.c @@ -0,0 +1,382 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * fault.c: Page fault handlers for the Sparc. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "mm_32.h" + +int show_unhandled_signals = 1; + +static void __noreturn unhandled_fault(unsigned long address, + struct task_struct *tsk, + struct pt_regs *regs) +{ + if ((unsigned long) address < PAGE_SIZE) { + printk(KERN_ALERT + "Unable to handle kernel NULL pointer dereference\n"); + } else { + printk(KERN_ALERT "Unable to handle kernel paging request at virtual address %08lx\n", + address); + } + printk(KERN_ALERT "tsk->{mm,active_mm}->context = %08lx\n", + (tsk->mm ? tsk->mm->context : tsk->active_mm->context)); + printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %08lx\n", + (tsk->mm ? (unsigned long) tsk->mm->pgd : + (unsigned long) tsk->active_mm->pgd)); + die_if_kernel("Oops", regs); +} + +static inline void +show_signal_msg(struct pt_regs *regs, int sig, int code, + unsigned long address, struct task_struct *tsk) +{ + if (!unhandled_signal(tsk, sig)) + return; + + if (!printk_ratelimit()) + return; + + printk("%s%s[%d]: segfault at %lx ip %px (rpc %px) sp %px error %x", + task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG, + tsk->comm, task_pid_nr(tsk), address, + (void *)regs->pc, (void *)regs->u_regs[UREG_I7], + (void *)regs->u_regs[UREG_FP], code); + + print_vma_addr(KERN_CONT " in ", regs->pc); + + printk(KERN_CONT "\n"); +} + +static void __do_fault_siginfo(int code, int sig, struct pt_regs *regs, + unsigned long addr) +{ + if (unlikely(show_unhandled_signals)) + show_signal_msg(regs, sig, code, + addr, current); + + force_sig_fault(sig, code, (void __user *) addr); +} + +static unsigned long compute_si_addr(struct pt_regs *regs, int text_fault) +{ + unsigned int insn; + + if (text_fault) + return regs->pc; + + if (regs->psr & PSR_PS) + insn = *(unsigned int *) regs->pc; + else + __get_user(insn, (unsigned int *) regs->pc); + + return safe_compute_effective_address(regs, insn); +} + +static noinline void do_fault_siginfo(int code, int sig, struct pt_regs *regs, + int text_fault) +{ + unsigned long addr = compute_si_addr(regs, text_fault); + + __do_fault_siginfo(code, sig, regs, addr); +} + +asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, + unsigned long address) +{ + struct vm_area_struct *vma; + struct task_struct *tsk = current; + struct mm_struct *mm = tsk->mm; + int from_user = !(regs->psr & PSR_PS); + int code; + vm_fault_t fault; + unsigned int flags = FAULT_FLAG_DEFAULT; + + if (text_fault) + address = regs->pc; + + /* + * We fault-in kernel-space virtual memory on-demand. The + * 'reference' page table is init_mm.pgd. + * + * NOTE! We MUST NOT take any locks for this case. We may + * be in an interrupt or a critical region, and should + * only copy the information from the master page table, + * nothing more. + */ + code = SEGV_MAPERR; + if (address >= TASK_SIZE) + goto vmalloc_fault; + + /* + * If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (pagefault_disabled() || !mm) + goto no_context; + + if (!from_user && address >= PAGE_OFFSET) + goto no_context; + + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); + +retry: + vma = lock_mm_and_find_vma(mm, address, regs); + if (!vma) + goto bad_area_nosemaphore; + /* + * Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ + code = SEGV_ACCERR; + if (write) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + } else { + /* Allow reads even for write-only mappings */ + if (!(vma->vm_flags & (VM_READ | VM_EXEC))) + goto bad_area; + } + + if (from_user) + flags |= FAULT_FLAG_USER; + if (write) + flags |= FAULT_FLAG_WRITE; + + /* + * If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ + fault = handle_mm_fault(vma, address, flags, regs); + + if (fault_signal_pending(fault, regs)) { + if (!from_user) + goto no_context; + return; + } + + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + return; + + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) + goto out_of_memory; + else if (fault & VM_FAULT_SIGSEGV) + goto bad_area; + else if (fault & VM_FAULT_SIGBUS) + goto do_sigbus; + BUG(); + } + + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; + + /* No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; + } + + mmap_read_unlock(mm); + return; + + /* + * Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: + mmap_read_unlock(mm); + +bad_area_nosemaphore: + /* User mode accesses just cause a SIGSEGV */ + if (from_user) { + do_fault_siginfo(code, SIGSEGV, regs, text_fault); + return; + } + + /* Is this in ex_table? */ +no_context: + if (!from_user) { + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->pc); +#ifdef DEBUG_EXCEPTIONS + printk("Exception: PC<%08lx> faddr<%08lx>\n", + regs->pc, address); + printk("EX_TABLE: insn<%08lx> fixup<%08x>\n", + regs->pc, entry->fixup); +#endif + regs->pc = entry->fixup; + regs->npc = regs->pc + 4; + return; + } + + unhandled_fault(address, tsk, regs); + +/* + * We ran out of memory, or some other thing happened to us that made + * us unable to handle the page fault gracefully. + */ +out_of_memory: + mmap_read_unlock(mm); + if (from_user) { + pagefault_out_of_memory(); + return; + } + goto no_context; + +do_sigbus: + mmap_read_unlock(mm); + do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, text_fault); + if (!from_user) + goto no_context; + +vmalloc_fault: + { + /* + * Synchronize this task's top level page-table + * with the 'reference' page table. + */ + int offset = pgd_index(address); + pgd_t *pgd, *pgd_k; + p4d_t *p4d, *p4d_k; + pud_t *pud, *pud_k; + pmd_t *pmd, *pmd_k; + + pgd = tsk->active_mm->pgd + offset; + pgd_k = init_mm.pgd + offset; + + if (!pgd_present(*pgd)) { + if (!pgd_present(*pgd_k)) + goto bad_area_nosemaphore; + pgd_val(*pgd) = pgd_val(*pgd_k); + return; + } + + p4d = p4d_offset(pgd, address); + pud = pud_offset(p4d, address); + pmd = pmd_offset(pud, address); + + p4d_k = p4d_offset(pgd_k, address); + pud_k = pud_offset(p4d_k, address); + pmd_k = pmd_offset(pud_k, address); + + if (pmd_present(*pmd) || !pmd_present(*pmd_k)) + goto bad_area_nosemaphore; + + *pmd = *pmd_k; + return; + } +} + +/* This always deals with user addresses. */ +static void force_user_fault(unsigned long address, int write) +{ + struct vm_area_struct *vma; + struct task_struct *tsk = current; + struct mm_struct *mm = tsk->mm; + unsigned int flags = FAULT_FLAG_USER; + int code; + + code = SEGV_MAPERR; + + vma = lock_mm_and_find_vma(mm, address, NULL); + if (!vma) + goto bad_area_nosemaphore; + code = SEGV_ACCERR; + if (write) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + flags |= FAULT_FLAG_WRITE; + } else { + if (!(vma->vm_flags & (VM_READ | VM_EXEC))) + goto bad_area; + } + switch (handle_mm_fault(vma, address, flags, NULL)) { + case VM_FAULT_SIGBUS: + case VM_FAULT_OOM: + goto do_sigbus; + } + mmap_read_unlock(mm); + return; +bad_area: + mmap_read_unlock(mm); +bad_area_nosemaphore: + __do_fault_siginfo(code, SIGSEGV, tsk->thread.kregs, address); + return; + +do_sigbus: + mmap_read_unlock(mm); + __do_fault_siginfo(BUS_ADRERR, SIGBUS, tsk->thread.kregs, address); +} + +static void check_stack_aligned(unsigned long sp) +{ + if (sp & 0x7UL) + force_sig(SIGILL); +} + +void window_overflow_fault(void) +{ + unsigned long sp; + + sp = current_thread_info()->rwbuf_stkptrs[0]; + if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) + force_user_fault(sp + 0x38, 1); + force_user_fault(sp, 1); + + check_stack_aligned(sp); +} + +void window_underflow_fault(unsigned long sp) +{ + if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) + force_user_fault(sp + 0x38, 0); + force_user_fault(sp, 0); + + check_stack_aligned(sp); +} + +void window_ret_fault(struct pt_regs *regs) +{ + unsigned long sp; + + sp = regs->u_regs[UREG_FP]; + if (((sp + 0x38) & PAGE_MASK) != (sp & PAGE_MASK)) + force_user_fault(sp + 0x38, 0); + force_user_fault(sp, 0); + + check_stack_aligned(sp); +} diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c new file mode 100644 index 000000000..e326caf70 --- /dev/null +++ b/arch/sparc/mm/fault_64.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/sparc64/mm/fault.c: Page fault handlers for the 64-bit Sparc. + * + * Copyright (C) 1996, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz) + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +int show_unhandled_signals = 1; + +static void __kprobes unhandled_fault(unsigned long address, + struct task_struct *tsk, + struct pt_regs *regs) +{ + if ((unsigned long) address < PAGE_SIZE) { + printk(KERN_ALERT "Unable to handle kernel NULL " + "pointer dereference\n"); + } else { + printk(KERN_ALERT "Unable to handle kernel paging request " + "at virtual address %016lx\n", (unsigned long)address); + } + printk(KERN_ALERT "tsk->{mm,active_mm}->context = %016lx\n", + (tsk->mm ? + CTX_HWBITS(tsk->mm->context) : + CTX_HWBITS(tsk->active_mm->context))); + printk(KERN_ALERT "tsk->{mm,active_mm}->pgd = %016lx\n", + (tsk->mm ? (unsigned long) tsk->mm->pgd : + (unsigned long) tsk->active_mm->pgd)); + die_if_kernel("Oops", regs); +} + +static void __kprobes bad_kernel_pc(struct pt_regs *regs, unsigned long vaddr) +{ + printk(KERN_CRIT "OOPS: Bogus kernel PC [%016lx] in fault handler\n", + regs->tpc); + printk(KERN_CRIT "OOPS: RPC [%016lx]\n", regs->u_regs[15]); + printk("OOPS: RPC <%pS>\n", (void *) regs->u_regs[15]); + printk(KERN_CRIT "OOPS: Fault was to vaddr[%lx]\n", vaddr); + dump_stack(); + unhandled_fault(regs->tpc, current, regs); +} + +/* + * We now make sure that mmap_lock is held in all paths that call + * this. Additionally, to prevent kswapd from ripping ptes from + * under us, raise interrupts around the time that we look at the + * pte, kswapd will have to wait to get his smp ipi response from + * us. vmtruncate likewise. This saves us having to get pte lock. + */ +static unsigned int get_user_insn(unsigned long tpc) +{ + pgd_t *pgdp = pgd_offset(current->mm, tpc); + p4d_t *p4dp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep, pte; + unsigned long pa; + u32 insn = 0; + + if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) + goto out; + p4dp = p4d_offset(pgdp, tpc); + if (p4d_none(*p4dp) || unlikely(p4d_bad(*p4dp))) + goto out; + pudp = pud_offset(p4dp, tpc); + if (pud_none(*pudp) || unlikely(pud_bad(*pudp))) + goto out; + + /* This disables preemption for us as well. */ + local_irq_disable(); + + pmdp = pmd_offset(pudp, tpc); +again: + if (pmd_none(*pmdp) || unlikely(pmd_bad(*pmdp))) + goto out_irq_enable; + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (is_hugetlb_pmd(*pmdp)) { + pa = pmd_pfn(*pmdp) << PAGE_SHIFT; + pa += tpc & ~HPAGE_MASK; + + /* Use phys bypass so we don't pollute dtlb/dcache. */ + __asm__ __volatile__("lduwa [%1] %2, %0" + : "=r" (insn) + : "r" (pa), "i" (ASI_PHYS_USE_EC)); + } else +#endif + { + ptep = pte_offset_map(pmdp, tpc); + if (!ptep) + goto again; + pte = *ptep; + if (pte_present(pte)) { + pa = (pte_pfn(pte) << PAGE_SHIFT); + pa += (tpc & ~PAGE_MASK); + + /* Use phys bypass so we don't pollute dtlb/dcache. */ + __asm__ __volatile__("lduwa [%1] %2, %0" + : "=r" (insn) + : "r" (pa), "i" (ASI_PHYS_USE_EC)); + } + pte_unmap(ptep); + } +out_irq_enable: + local_irq_enable(); +out: + return insn; +} + +static inline void +show_signal_msg(struct pt_regs *regs, int sig, int code, + unsigned long address, struct task_struct *tsk) +{ + if (!unhandled_signal(tsk, sig)) + return; + + if (!printk_ratelimit()) + return; + + printk("%s%s[%d]: segfault at %lx ip %px (rpc %px) sp %px error %x", + task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG, + tsk->comm, task_pid_nr(tsk), address, + (void *)regs->tpc, (void *)regs->u_regs[UREG_I7], + (void *)regs->u_regs[UREG_FP], code); + + print_vma_addr(KERN_CONT " in ", regs->tpc); + + printk(KERN_CONT "\n"); +} + +static void do_fault_siginfo(int code, int sig, struct pt_regs *regs, + unsigned long fault_addr, unsigned int insn, + int fault_code) +{ + unsigned long addr; + + if (fault_code & FAULT_CODE_ITLB) { + addr = regs->tpc; + } else { + /* If we were able to probe the faulting instruction, use it + * to compute a precise fault address. Otherwise use the fault + * time provided address which may only have page granularity. + */ + if (insn) + addr = compute_effective_address(regs, insn, 0); + else + addr = fault_addr; + } + + if (unlikely(show_unhandled_signals)) + show_signal_msg(regs, sig, code, addr, current); + + force_sig_fault(sig, code, (void __user *) addr); +} + +static unsigned int get_fault_insn(struct pt_regs *regs, unsigned int insn) +{ + if (!insn) { + if (!regs->tpc || (regs->tpc & 0x3)) + return 0; + if (regs->tstate & TSTATE_PRIV) { + insn = *(unsigned int *) regs->tpc; + } else { + insn = get_user_insn(regs->tpc); + } + } + return insn; +} + +static void __kprobes do_kernel_fault(struct pt_regs *regs, int si_code, + int fault_code, unsigned int insn, + unsigned long address) +{ + unsigned char asi = ASI_P; + + if ((!insn) && (regs->tstate & TSTATE_PRIV)) + goto cannot_handle; + + /* If user insn could be read (thus insn is zero), that + * is fine. We will just gun down the process with a signal + * in that case. + */ + + if (!(fault_code & (FAULT_CODE_WRITE|FAULT_CODE_ITLB)) && + (insn & 0xc0800000) == 0xc0800000) { + if (insn & 0x2000) + asi = (regs->tstate >> 24); + else + asi = (insn >> 5); + if ((asi & 0xf2) == 0x82) { + if (insn & 0x1000000) { + handle_ldf_stq(insn, regs); + } else { + /* This was a non-faulting load. Just clear the + * destination register(s) and continue with the next + * instruction. -jj + */ + handle_ld_nf(insn, regs); + } + return; + } + } + + /* Is this in ex_table? */ + if (regs->tstate & TSTATE_PRIV) { + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return; + } + } else { + /* The si_code was set to make clear whether + * this was a SEGV_MAPERR or SEGV_ACCERR fault. + */ + do_fault_siginfo(si_code, SIGSEGV, regs, address, insn, fault_code); + return; + } + +cannot_handle: + unhandled_fault (address, current, regs); +} + +static void noinline __kprobes bogus_32bit_fault_tpc(struct pt_regs *regs) +{ + static int times; + + if (times++ < 10) + printk(KERN_ERR "FAULT[%s:%d]: 32-bit process reports " + "64-bit TPC [%lx]\n", + current->comm, current->pid, + regs->tpc); + show_regs(regs); +} + +asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) +{ + enum ctx_state prev_state = exception_enter(); + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + unsigned int insn = 0; + int si_code, fault_code; + vm_fault_t fault; + unsigned long address, mm_rss; + unsigned int flags = FAULT_FLAG_DEFAULT; + + fault_code = get_thread_fault_code(); + + if (kprobe_page_fault(regs, 0)) + goto exit_exception; + + si_code = SEGV_MAPERR; + address = current_thread_info()->fault_address; + + if ((fault_code & FAULT_CODE_ITLB) && + (fault_code & FAULT_CODE_DTLB)) + BUG(); + + if (test_thread_flag(TIF_32BIT)) { + if (!(regs->tstate & TSTATE_PRIV)) { + if (unlikely((regs->tpc >> 32) != 0)) { + bogus_32bit_fault_tpc(regs); + goto intr_or_no_mm; + } + } + if (unlikely((address >> 32) != 0)) + goto intr_or_no_mm; + } + + if (regs->tstate & TSTATE_PRIV) { + unsigned long tpc = regs->tpc; + + /* Sanity check the PC. */ + if ((tpc >= KERNBASE && tpc < (unsigned long) __init_end) || + (tpc >= MODULES_VADDR && tpc < MODULES_END)) { + /* Valid, no problems... */ + } else { + bad_kernel_pc(regs, address); + goto exit_exception; + } + } else + flags |= FAULT_FLAG_USER; + + /* + * If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (faulthandler_disabled() || !mm) + goto intr_or_no_mm; + + perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); + + if (!mmap_read_trylock(mm)) { + if ((regs->tstate & TSTATE_PRIV) && + !search_exception_tables(regs->tpc)) { + insn = get_fault_insn(regs, insn); + goto handle_kernel_fault; + } + +retry: + mmap_read_lock(mm); + } + + if (fault_code & FAULT_CODE_BAD_RA) + goto do_sigbus; + + vma = find_vma(mm, address); + if (!vma) + goto bad_area; + + /* Pure DTLB misses do not tell us whether the fault causing + * load/store/atomic was a write or not, it only says that there + * was no match. So in such a case we (carefully) read the + * instruction to try and figure this out. It's an optimization + * so it's ok if we can't do this. + * + * Special hack, window spill/fill knows the exact fault type. + */ + if (((fault_code & + (FAULT_CODE_DTLB | FAULT_CODE_WRITE | FAULT_CODE_WINFIXUP)) == FAULT_CODE_DTLB) && + (vma->vm_flags & VM_WRITE) != 0) { + insn = get_fault_insn(regs, 0); + if (!insn) + goto continue_fault; + /* All loads, stores and atomics have bits 30 and 31 both set + * in the instruction. Bit 21 is set in all stores, but we + * have to avoid prefetches which also have bit 21 set. + */ + if ((insn & 0xc0200000) == 0xc0200000 && + (insn & 0x01780000) != 0x01680000) { + /* Don't bother updating thread struct value, + * because update_mmu_cache only cares which tlb + * the access came from. + */ + fault_code |= FAULT_CODE_WRITE; + } + } +continue_fault: + + if (vma->vm_start <= address) + goto good_area; + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto bad_area; + if (!(fault_code & FAULT_CODE_WRITE)) { + /* Non-faulting loads shouldn't expand stack. */ + insn = get_fault_insn(regs, insn); + if ((insn & 0xc0800000) == 0xc0800000) { + unsigned char asi; + + if (insn & 0x2000) + asi = (regs->tstate >> 24); + else + asi = (insn >> 5); + if ((asi & 0xf2) == 0x82) + goto bad_area; + } + } + vma = expand_stack(mm, address); + if (!vma) + goto bad_area_nosemaphore; + /* + * Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ +good_area: + si_code = SEGV_ACCERR; + + /* If we took a ITLB miss on a non-executable page, catch + * that here. + */ + if ((fault_code & FAULT_CODE_ITLB) && !(vma->vm_flags & VM_EXEC)) { + WARN(address != regs->tpc, + "address (%lx) != regs->tpc (%lx)\n", address, regs->tpc); + WARN_ON(regs->tstate & TSTATE_PRIV); + goto bad_area; + } + + if (fault_code & FAULT_CODE_WRITE) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + + /* Spitfire has an icache which does not snoop + * processor stores. Later processors do... + */ + if (tlb_type == spitfire && + (vma->vm_flags & VM_EXEC) != 0 && + vma->vm_file != NULL) + set_thread_fault_code(fault_code | + FAULT_CODE_BLKCOMMIT); + + flags |= FAULT_FLAG_WRITE; + } else { + /* Allow reads even for write-only mappings */ + if (!(vma->vm_flags & (VM_READ | VM_EXEC))) + goto bad_area; + } + + fault = handle_mm_fault(vma, address, flags, regs); + + if (fault_signal_pending(fault, regs)) { + if (regs->tstate & TSTATE_PRIV) { + insn = get_fault_insn(regs, insn); + goto handle_kernel_fault; + } + goto exit_exception; + } + + /* The fault is fully completed (including releasing mmap lock) */ + if (fault & VM_FAULT_COMPLETED) + goto lock_released; + + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) + goto out_of_memory; + else if (fault & VM_FAULT_SIGSEGV) + goto bad_area; + else if (fault & VM_FAULT_SIGBUS) + goto do_sigbus; + BUG(); + } + + if (fault & VM_FAULT_RETRY) { + flags |= FAULT_FLAG_TRIED; + + /* No need to mmap_read_unlock(mm) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; + } + mmap_read_unlock(mm); + +lock_released: + mm_rss = get_mm_rss(mm); +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) + mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE)); +#endif + if (unlikely(mm_rss > + mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit)) + tsb_grow(mm, MM_TSB_BASE, mm_rss); +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count; + mm_rss *= REAL_HPAGE_PER_HPAGE; + if (unlikely(mm_rss > + mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) { + if (mm->context.tsb_block[MM_TSB_HUGE].tsb) + tsb_grow(mm, MM_TSB_HUGE, mm_rss); + else + hugetlb_setup(regs); + + } +#endif +exit_exception: + exception_exit(prev_state); + return; + + /* + * Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: + mmap_read_unlock(mm); +bad_area_nosemaphore: + insn = get_fault_insn(regs, insn); + +handle_kernel_fault: + do_kernel_fault(regs, si_code, fault_code, insn, address); + goto exit_exception; + +/* + * We ran out of memory, or some other thing happened to us that made + * us unable to handle the page fault gracefully. + */ +out_of_memory: + insn = get_fault_insn(regs, insn); + mmap_read_unlock(mm); + if (!(regs->tstate & TSTATE_PRIV)) { + pagefault_out_of_memory(); + goto exit_exception; + } + goto handle_kernel_fault; + +intr_or_no_mm: + insn = get_fault_insn(regs, 0); + goto handle_kernel_fault; + +do_sigbus: + insn = get_fault_insn(regs, insn); + mmap_read_unlock(mm); + + /* + * Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + do_fault_siginfo(BUS_ADRERR, SIGBUS, regs, address, insn, fault_code); + + /* Kernel mode? Handle exceptions or die */ + if (regs->tstate & TSTATE_PRIV) + goto handle_kernel_fault; +} diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c new file mode 100644 index 000000000..4652e8686 --- /dev/null +++ b/arch/sparc/mm/hugetlbpage.c @@ -0,0 +1,317 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SPARC64 Huge TLB page support. + * + * Copyright (C) 2002, 2003, 2006 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +static pte_t sun4u_hugepage_shift_to_tte(pte_t entry, unsigned int shift) +{ + unsigned long hugepage_size = _PAGE_SZ4MB_4U; + + pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4U; + + switch (shift) { + case HPAGE_256MB_SHIFT: + hugepage_size = _PAGE_SZ256MB_4U; + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_SHIFT: + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_64K_SHIFT: + hugepage_size = _PAGE_SZ64K_4U; + break; + default: + WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift); + } + + pte_val(entry) = pte_val(entry) | hugepage_size; + return entry; +} + +static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, unsigned int shift) +{ + unsigned long hugepage_size = _PAGE_SZ4MB_4V; + + pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V; + + switch (shift) { + case HPAGE_16GB_SHIFT: + hugepage_size = _PAGE_SZ16GB_4V; + pte_val(entry) |= _PAGE_PUD_HUGE; + break; + case HPAGE_2GB_SHIFT: + hugepage_size = _PAGE_SZ2GB_4V; + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_256MB_SHIFT: + hugepage_size = _PAGE_SZ256MB_4V; + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_SHIFT: + pte_val(entry) |= _PAGE_PMD_HUGE; + break; + case HPAGE_64K_SHIFT: + hugepage_size = _PAGE_SZ64K_4V; + break; + default: + WARN_ONCE(1, "unsupported hugepage shift=%u\n", shift); + } + + pte_val(entry) = pte_val(entry) | hugepage_size; + return entry; +} + +static pte_t hugepage_shift_to_tte(pte_t entry, unsigned int shift) +{ + if (tlb_type == hypervisor) + return sun4v_hugepage_shift_to_tte(entry, shift); + else + return sun4u_hugepage_shift_to_tte(entry, shift); +} + +pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) +{ + pte_t pte; + + entry = pte_mkhuge(entry); + pte = hugepage_shift_to_tte(entry, shift); + +#ifdef CONFIG_SPARC64 + /* If this vma has ADI enabled on it, turn on TTE.mcd + */ + if (flags & VM_SPARC_ADI) + return pte_mkmcd(pte); + else + return pte_mknotmcd(pte); +#else + return pte; +#endif +} + +static unsigned int sun4v_huge_tte_to_shift(pte_t entry) +{ + unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4V; + unsigned int shift; + + switch (tte_szbits) { + case _PAGE_SZ16GB_4V: + shift = HPAGE_16GB_SHIFT; + break; + case _PAGE_SZ2GB_4V: + shift = HPAGE_2GB_SHIFT; + break; + case _PAGE_SZ256MB_4V: + shift = HPAGE_256MB_SHIFT; + break; + case _PAGE_SZ4MB_4V: + shift = REAL_HPAGE_SHIFT; + break; + case _PAGE_SZ64K_4V: + shift = HPAGE_64K_SHIFT; + break; + default: + shift = PAGE_SHIFT; + break; + } + return shift; +} + +static unsigned int sun4u_huge_tte_to_shift(pte_t entry) +{ + unsigned long tte_szbits = pte_val(entry) & _PAGE_SZALL_4U; + unsigned int shift; + + switch (tte_szbits) { + case _PAGE_SZ256MB_4U: + shift = HPAGE_256MB_SHIFT; + break; + case _PAGE_SZ4MB_4U: + shift = REAL_HPAGE_SHIFT; + break; + case _PAGE_SZ64K_4U: + shift = HPAGE_64K_SHIFT; + break; + default: + shift = PAGE_SHIFT; + break; + } + return shift; +} + +static unsigned long tte_to_shift(pte_t entry) +{ + if (tlb_type == hypervisor) + return sun4v_huge_tte_to_shift(entry); + + return sun4u_huge_tte_to_shift(entry); +} + +static unsigned int huge_tte_to_shift(pte_t entry) +{ + unsigned long shift = tte_to_shift(entry); + + if (shift == PAGE_SHIFT) + WARN_ONCE(1, "tto_to_shift: invalid hugepage tte=0x%lx\n", + pte_val(entry)); + + return shift; +} + +static unsigned long huge_tte_to_size(pte_t pte) +{ + unsigned long size = 1UL << huge_tte_to_shift(pte); + + if (size == REAL_HPAGE_SIZE) + size = HPAGE_SIZE; + return size; +} + +unsigned long pud_leaf_size(pud_t pud) { return 1UL << tte_to_shift(*(pte_t *)&pud); } +unsigned long pmd_leaf_size(pmd_t pmd) { return 1UL << tte_to_shift(*(pte_t *)&pmd); } +unsigned long pte_leaf_size(pte_t pte) { return 1UL << tte_to_shift(pte); } + +pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, + unsigned long addr, unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + pgd = pgd_offset(mm, addr); + p4d = p4d_offset(pgd, addr); + pud = pud_alloc(mm, p4d, addr); + if (!pud) + return NULL; + if (sz >= PUD_SIZE) + return (pte_t *)pud; + pmd = pmd_alloc(mm, pud, addr); + if (!pmd) + return NULL; + if (sz >= PMD_SIZE) + return (pte_t *)pmd; + return pte_alloc_huge(mm, pmd, addr); +} + +pte_t *huge_pte_offset(struct mm_struct *mm, + unsigned long addr, unsigned long sz) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + + pgd = pgd_offset(mm, addr); + if (pgd_none(*pgd)) + return NULL; + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) + return NULL; + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) + return NULL; + if (is_hugetlb_pud(*pud)) + return (pte_t *)pud; + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) + return NULL; + if (is_hugetlb_pmd(*pmd)) + return (pte_t *)pmd; + return pte_offset_huge(pmd, addr); +} + +void __set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t entry) +{ + unsigned int nptes, orig_shift, shift; + unsigned long i, size; + pte_t orig; + + size = huge_tte_to_size(entry); + + shift = PAGE_SHIFT; + if (size >= PUD_SIZE) + shift = PUD_SHIFT; + else if (size >= PMD_SIZE) + shift = PMD_SHIFT; + else + shift = PAGE_SHIFT; + + nptes = size >> shift; + + if (!pte_present(*ptep) && pte_present(entry)) + mm->context.hugetlb_pte_count += nptes; + + addr &= ~(size - 1); + orig = *ptep; + orig_shift = pte_none(orig) ? PAGE_SHIFT : huge_tte_to_shift(orig); + + for (i = 0; i < nptes; i++) + ptep[i] = __pte(pte_val(entry) + (i << shift)); + + maybe_tlb_batch_add(mm, addr, ptep, orig, 0, orig_shift); + /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ + if (size == HPAGE_SIZE) + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0, + orig_shift); +} + +void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t entry, unsigned long sz) +{ + __set_huge_pte_at(mm, addr, ptep, entry); +} + +pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, unsigned long sz) +{ + unsigned int i, nptes, orig_shift, shift; + unsigned long size; + pte_t entry; + + entry = *ptep; + size = huge_tte_to_size(entry); + + shift = PAGE_SHIFT; + if (size >= PUD_SIZE) + shift = PUD_SHIFT; + else if (size >= PMD_SIZE) + shift = PMD_SHIFT; + else + shift = PAGE_SHIFT; + + nptes = size >> shift; + orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry); + + if (pte_present(entry)) + mm->context.hugetlb_pte_count -= nptes; + + addr &= ~(size - 1); + for (i = 0; i < nptes; i++) + ptep[i] = __pte(0UL); + + maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift); + /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ + if (size == HPAGE_SIZE) + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0, + orig_shift); + + return entry; +} diff --git a/arch/sparc/mm/hypersparc.S b/arch/sparc/mm/hypersparc.S new file mode 100644 index 000000000..6c2521e85 --- /dev/null +++ b/arch/sparc/mm/hypersparc.S @@ -0,0 +1,414 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * hypersparc.S: High speed Hypersparc mmu/cache operations. + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include +#include +#include + + .text + .align 4 + + .globl hypersparc_flush_cache_all, hypersparc_flush_cache_mm + .globl hypersparc_flush_cache_range, hypersparc_flush_cache_page + .globl hypersparc_flush_page_to_ram + .globl hypersparc_flush_page_for_dma, hypersparc_flush_sig_insns + .globl hypersparc_flush_tlb_all, hypersparc_flush_tlb_mm + .globl hypersparc_flush_tlb_range, hypersparc_flush_tlb_page + +hypersparc_flush_cache_all: + WINDOW_FLUSH(%g4, %g5) + sethi %hi(vac_cache_size), %g4 + ld [%g4 + %lo(vac_cache_size)], %g5 + sethi %hi(vac_line_size), %g1 + ld [%g1 + %lo(vac_line_size)], %g2 +1: + subcc %g5, %g2, %g5 ! hyper_flush_unconditional_combined + bne 1b + sta %g0, [%g5] ASI_M_FLUSH_CTX + retl + sta %g0, [%g0] ASI_M_FLUSH_IWHOLE ! hyper_flush_whole_icache + + /* We expand the window flush to get maximum performance. */ +hypersparc_flush_cache_mm: +#ifndef CONFIG_SMP + ld [%o0 + AOFF_mm_context], %g1 + cmp %g1, -1 + be hypersparc_flush_cache_mm_out +#endif + WINDOW_FLUSH(%g4, %g5) + + sethi %hi(vac_line_size), %g1 + ld [%g1 + %lo(vac_line_size)], %o1 + sethi %hi(vac_cache_size), %g2 + ld [%g2 + %lo(vac_cache_size)], %o0 + add %o1, %o1, %g1 + add %o1, %g1, %g2 + add %o1, %g2, %g3 + add %o1, %g3, %g4 + add %o1, %g4, %g5 + add %o1, %g5, %o4 + add %o1, %o4, %o5 + + /* BLAMMO! */ +1: + subcc %o0, %o5, %o0 ! hyper_flush_cache_user + sta %g0, [%o0 + %g0] ASI_M_FLUSH_USER + sta %g0, [%o0 + %o1] ASI_M_FLUSH_USER + sta %g0, [%o0 + %g1] ASI_M_FLUSH_USER + sta %g0, [%o0 + %g2] ASI_M_FLUSH_USER + sta %g0, [%o0 + %g3] ASI_M_FLUSH_USER + sta %g0, [%o0 + %g4] ASI_M_FLUSH_USER + sta %g0, [%o0 + %g5] ASI_M_FLUSH_USER + bne 1b + sta %g0, [%o0 + %o4] ASI_M_FLUSH_USER +hypersparc_flush_cache_mm_out: + retl + nop + + /* The things we do for performance... */ +hypersparc_flush_cache_range: + ld [%o0 + VMA_VM_MM], %o0 +#ifndef CONFIG_SMP + ld [%o0 + AOFF_mm_context], %g1 + cmp %g1, -1 + be hypersparc_flush_cache_range_out +#endif + WINDOW_FLUSH(%g4, %g5) + + sethi %hi(vac_line_size), %g1 + ld [%g1 + %lo(vac_line_size)], %o4 + sethi %hi(vac_cache_size), %g2 + ld [%g2 + %lo(vac_cache_size)], %o3 + + /* Here comes the fun part... */ + add %o2, (PAGE_SIZE - 1), %o2 + andn %o1, (PAGE_SIZE - 1), %o1 + add %o4, %o4, %o5 + andn %o2, (PAGE_SIZE - 1), %o2 + add %o4, %o5, %g1 + sub %o2, %o1, %g4 + add %o4, %g1, %g2 + sll %o3, 2, %g5 + add %o4, %g2, %g3 + cmp %g4, %g5 + add %o4, %g3, %g4 + blu 0f + add %o4, %g4, %g5 + add %o4, %g5, %g7 + + /* Flush entire user space, believe it or not this is quicker + * than page at a time flushings for range > (cache_size<<2). + */ +1: + subcc %o3, %g7, %o3 + sta %g0, [%o3 + %g0] ASI_M_FLUSH_USER + sta %g0, [%o3 + %o4] ASI_M_FLUSH_USER + sta %g0, [%o3 + %o5] ASI_M_FLUSH_USER + sta %g0, [%o3 + %g1] ASI_M_FLUSH_USER + sta %g0, [%o3 + %g2] ASI_M_FLUSH_USER + sta %g0, [%o3 + %g3] ASI_M_FLUSH_USER + sta %g0, [%o3 + %g4] ASI_M_FLUSH_USER + bne 1b + sta %g0, [%o3 + %g5] ASI_M_FLUSH_USER + retl + nop + + /* Below our threshold, flush one page at a time. */ +0: + ld [%o0 + AOFF_mm_context], %o0 + mov SRMMU_CTX_REG, %g7 + lda [%g7] ASI_M_MMUREGS, %o3 + sta %o0, [%g7] ASI_M_MMUREGS + add %o2, -PAGE_SIZE, %o0 +1: + or %o0, 0x400, %g7 + lda [%g7] ASI_M_FLUSH_PROBE, %g7 + orcc %g7, 0, %g0 + be,a 3f + mov %o0, %o2 + add %o4, %g5, %g7 +2: + sub %o2, %g7, %o2 + sta %g0, [%o2 + %g0] ASI_M_FLUSH_PAGE + sta %g0, [%o2 + %o4] ASI_M_FLUSH_PAGE + sta %g0, [%o2 + %o5] ASI_M_FLUSH_PAGE + sta %g0, [%o2 + %g1] ASI_M_FLUSH_PAGE + sta %g0, [%o2 + %g2] ASI_M_FLUSH_PAGE + sta %g0, [%o2 + %g3] ASI_M_FLUSH_PAGE + andcc %o2, 0xffc, %g0 + sta %g0, [%o2 + %g4] ASI_M_FLUSH_PAGE + bne 2b + sta %g0, [%o2 + %g5] ASI_M_FLUSH_PAGE +3: + cmp %o2, %o1 + bne 1b + add %o2, -PAGE_SIZE, %o0 + mov SRMMU_FAULT_STATUS, %g5 + lda [%g5] ASI_M_MMUREGS, %g0 + mov SRMMU_CTX_REG, %g7 + sta %o3, [%g7] ASI_M_MMUREGS +hypersparc_flush_cache_range_out: + retl + nop + + /* HyperSparc requires a valid mapping where we are about to flush + * in order to check for a physical tag match during the flush. + */ + /* Verified, my ass... */ +hypersparc_flush_cache_page: + ld [%o0 + VMA_VM_MM], %o0 + ld [%o0 + AOFF_mm_context], %g2 +#ifndef CONFIG_SMP + cmp %g2, -1 + be hypersparc_flush_cache_page_out +#endif + WINDOW_FLUSH(%g4, %g5) + + sethi %hi(vac_line_size), %g1 + ld [%g1 + %lo(vac_line_size)], %o4 + mov SRMMU_CTX_REG, %o3 + andn %o1, (PAGE_SIZE - 1), %o1 + lda [%o3] ASI_M_MMUREGS, %o2 + sta %g2, [%o3] ASI_M_MMUREGS + or %o1, 0x400, %o5 + lda [%o5] ASI_M_FLUSH_PROBE, %g1 + orcc %g0, %g1, %g0 + be 2f + add %o4, %o4, %o5 + sub %o1, -PAGE_SIZE, %o1 + add %o4, %o5, %g1 + add %o4, %g1, %g2 + add %o4, %g2, %g3 + add %o4, %g3, %g4 + add %o4, %g4, %g5 + add %o4, %g5, %g7 + + /* BLAMMO! */ +1: + sub %o1, %g7, %o1 + sta %g0, [%o1 + %g0] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o4] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o5] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %g1] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %g2] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %g3] ASI_M_FLUSH_PAGE + andcc %o1, 0xffc, %g0 + sta %g0, [%o1 + %g4] ASI_M_FLUSH_PAGE + bne 1b + sta %g0, [%o1 + %g5] ASI_M_FLUSH_PAGE +2: + mov SRMMU_FAULT_STATUS, %g7 + mov SRMMU_CTX_REG, %g4 + lda [%g7] ASI_M_MMUREGS, %g0 + sta %o2, [%g4] ASI_M_MMUREGS +hypersparc_flush_cache_page_out: + retl + nop + +hypersparc_flush_sig_insns: + flush %o1 + retl + flush %o1 + 4 + + /* HyperSparc is copy-back. */ +hypersparc_flush_page_to_ram: + sethi %hi(vac_line_size), %g1 + ld [%g1 + %lo(vac_line_size)], %o4 + andn %o0, (PAGE_SIZE - 1), %o0 + add %o4, %o4, %o5 + or %o0, 0x400, %g7 + lda [%g7] ASI_M_FLUSH_PROBE, %g5 + add %o4, %o5, %g1 + orcc %g5, 0, %g0 + be 2f + add %o4, %g1, %g2 + add %o4, %g2, %g3 + sub %o0, -PAGE_SIZE, %o0 + add %o4, %g3, %g4 + add %o4, %g4, %g5 + add %o4, %g5, %g7 + + /* BLAMMO! */ +1: + sub %o0, %g7, %o0 + sta %g0, [%o0 + %g0] ASI_M_FLUSH_PAGE + sta %g0, [%o0 + %o4] ASI_M_FLUSH_PAGE + sta %g0, [%o0 + %o5] ASI_M_FLUSH_PAGE + sta %g0, [%o0 + %g1] ASI_M_FLUSH_PAGE + sta %g0, [%o0 + %g2] ASI_M_FLUSH_PAGE + sta %g0, [%o0 + %g3] ASI_M_FLUSH_PAGE + andcc %o0, 0xffc, %g0 + sta %g0, [%o0 + %g4] ASI_M_FLUSH_PAGE + bne 1b + sta %g0, [%o0 + %g5] ASI_M_FLUSH_PAGE +2: + mov SRMMU_FAULT_STATUS, %g1 + retl + lda [%g1] ASI_M_MMUREGS, %g0 + + /* HyperSparc is IO cache coherent. */ +hypersparc_flush_page_for_dma: + retl + nop + + /* It was noted that at boot time a TLB flush all in a delay slot + * can deliver an illegal instruction to the processor if the timing + * is just right... + */ +hypersparc_flush_tlb_all: + mov 0x400, %g1 + sta %g0, [%g1] ASI_M_FLUSH_PROBE + retl + nop + +hypersparc_flush_tlb_mm: + mov SRMMU_CTX_REG, %g1 + ld [%o0 + AOFF_mm_context], %o1 + lda [%g1] ASI_M_MMUREGS, %g5 +#ifndef CONFIG_SMP + cmp %o1, -1 + be hypersparc_flush_tlb_mm_out +#endif + mov 0x300, %g2 + sta %o1, [%g1] ASI_M_MMUREGS + sta %g0, [%g2] ASI_M_FLUSH_PROBE +hypersparc_flush_tlb_mm_out: + retl + sta %g5, [%g1] ASI_M_MMUREGS + +hypersparc_flush_tlb_range: + ld [%o0 + VMA_VM_MM], %o0 + mov SRMMU_CTX_REG, %g1 + ld [%o0 + AOFF_mm_context], %o3 + lda [%g1] ASI_M_MMUREGS, %g5 +#ifndef CONFIG_SMP + cmp %o3, -1 + be hypersparc_flush_tlb_range_out +#endif + sethi %hi(~((1 << PGDIR_SHIFT) - 1)), %o4 + sta %o3, [%g1] ASI_M_MMUREGS + and %o1, %o4, %o1 + add %o1, 0x200, %o1 + sta %g0, [%o1] ASI_M_FLUSH_PROBE +1: + sub %o1, %o4, %o1 + cmp %o1, %o2 + blu,a 1b + sta %g0, [%o1] ASI_M_FLUSH_PROBE +hypersparc_flush_tlb_range_out: + retl + sta %g5, [%g1] ASI_M_MMUREGS + +hypersparc_flush_tlb_page: + ld [%o0 + VMA_VM_MM], %o0 + mov SRMMU_CTX_REG, %g1 + ld [%o0 + AOFF_mm_context], %o3 + andn %o1, (PAGE_SIZE - 1), %o1 +#ifndef CONFIG_SMP + cmp %o3, -1 + be hypersparc_flush_tlb_page_out +#endif + lda [%g1] ASI_M_MMUREGS, %g5 + sta %o3, [%g1] ASI_M_MMUREGS + sta %g0, [%o1] ASI_M_FLUSH_PROBE +hypersparc_flush_tlb_page_out: + retl + sta %g5, [%g1] ASI_M_MMUREGS + + __INIT + + /* High speed page clear/copy. */ +hypersparc_bzero_1page: +/* NOTE: This routine has to be shorter than 40insns --jj */ + clr %g1 + mov 32, %g2 + mov 64, %g3 + mov 96, %g4 + mov 128, %g5 + mov 160, %g7 + mov 192, %o2 + mov 224, %o3 + mov 16, %o1 +1: + stda %g0, [%o0 + %g0] ASI_M_BFILL + stda %g0, [%o0 + %g2] ASI_M_BFILL + stda %g0, [%o0 + %g3] ASI_M_BFILL + stda %g0, [%o0 + %g4] ASI_M_BFILL + stda %g0, [%o0 + %g5] ASI_M_BFILL + stda %g0, [%o0 + %g7] ASI_M_BFILL + stda %g0, [%o0 + %o2] ASI_M_BFILL + stda %g0, [%o0 + %o3] ASI_M_BFILL + subcc %o1, 1, %o1 + bne 1b + add %o0, 256, %o0 + + retl + nop + +hypersparc_copy_1page: +/* NOTE: This routine has to be shorter than 70insns --jj */ + sub %o1, %o0, %o2 ! difference + mov 16, %g1 +1: + sta %o0, [%o0 + %o2] ASI_M_BCOPY + add %o0, 32, %o0 + sta %o0, [%o0 + %o2] ASI_M_BCOPY + add %o0, 32, %o0 + sta %o0, [%o0 + %o2] ASI_M_BCOPY + add %o0, 32, %o0 + sta %o0, [%o0 + %o2] ASI_M_BCOPY + add %o0, 32, %o0 + sta %o0, [%o0 + %o2] ASI_M_BCOPY + add %o0, 32, %o0 + sta %o0, [%o0 + %o2] ASI_M_BCOPY + add %o0, 32, %o0 + sta %o0, [%o0 + %o2] ASI_M_BCOPY + add %o0, 32, %o0 + sta %o0, [%o0 + %o2] ASI_M_BCOPY + subcc %g1, 1, %g1 + bne 1b + add %o0, 32, %o0 + + retl + nop + + .globl hypersparc_setup_blockops +hypersparc_setup_blockops: + sethi %hi(bzero_1page), %o0 + or %o0, %lo(bzero_1page), %o0 + sethi %hi(hypersparc_bzero_1page), %o1 + or %o1, %lo(hypersparc_bzero_1page), %o1 + sethi %hi(hypersparc_copy_1page), %o2 + or %o2, %lo(hypersparc_copy_1page), %o2 + ld [%o1], %o4 +1: + add %o1, 4, %o1 + st %o4, [%o0] + add %o0, 4, %o0 + cmp %o1, %o2 + bne 1b + ld [%o1], %o4 + sethi %hi(__copy_1page), %o0 + or %o0, %lo(__copy_1page), %o0 + sethi %hi(hypersparc_setup_blockops), %o2 + or %o2, %lo(hypersparc_setup_blockops), %o2 + ld [%o1], %o4 +1: + add %o1, 4, %o1 + st %o4, [%o0] + add %o0, 4, %o0 + cmp %o1, %o2 + bne 1b + ld [%o1], %o4 + sta %g0, [%g0] ASI_M_FLUSH_IWHOLE + retl + nop diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c new file mode 100644 index 000000000..e0e66f91c --- /dev/null +++ b/arch/sparc/mm/init_32.c @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/sparc/mm/init.c + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1995 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 2000 Anton Blanchard (anton@samba.org) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "mm_32.h" + +static unsigned long *sparc_valid_addr_bitmap; + +unsigned long phys_base; +EXPORT_SYMBOL(phys_base); + +unsigned long pfn_base; +EXPORT_SYMBOL(pfn_base); + +struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1]; + +/* Initial ramdisk setup */ +extern unsigned int sparc_ramdisk_image; +extern unsigned int sparc_ramdisk_size; + +unsigned long highstart_pfn, highend_pfn; + +unsigned long last_valid_pfn; + +unsigned long calc_highpages(void) +{ + int i; + int nr = 0; + + for (i = 0; sp_banks[i].num_bytes != 0; i++) { + unsigned long start_pfn = sp_banks[i].base_addr >> PAGE_SHIFT; + unsigned long end_pfn = (sp_banks[i].base_addr + sp_banks[i].num_bytes) >> PAGE_SHIFT; + + if (end_pfn <= max_low_pfn) + continue; + + if (start_pfn < max_low_pfn) + start_pfn = max_low_pfn; + + nr += end_pfn - start_pfn; + } + + return nr; +} + +static unsigned long calc_max_low_pfn(void) +{ + int i; + unsigned long tmp = pfn_base + (SRMMU_MAXMEM >> PAGE_SHIFT); + unsigned long curr_pfn, last_pfn; + + last_pfn = (sp_banks[0].base_addr + sp_banks[0].num_bytes) >> PAGE_SHIFT; + for (i = 1; sp_banks[i].num_bytes != 0; i++) { + curr_pfn = sp_banks[i].base_addr >> PAGE_SHIFT; + + if (curr_pfn >= tmp) { + if (last_pfn < tmp) + tmp = last_pfn; + break; + } + + last_pfn = (sp_banks[i].base_addr + sp_banks[i].num_bytes) >> PAGE_SHIFT; + } + + return tmp; +} + +static void __init find_ramdisk(unsigned long end_of_phys_memory) +{ +#ifdef CONFIG_BLK_DEV_INITRD + unsigned long size; + + /* Now have to check initial ramdisk, so that it won't pass + * the end of memory + */ + if (sparc_ramdisk_image) { + if (sparc_ramdisk_image >= (unsigned long)&_end - 2 * PAGE_SIZE) + sparc_ramdisk_image -= KERNBASE; + initrd_start = sparc_ramdisk_image + phys_base; + initrd_end = initrd_start + sparc_ramdisk_size; + if (initrd_end > end_of_phys_memory) { + printk(KERN_CRIT "initrd extends beyond end of memory " + "(0x%016lx > 0x%016lx)\ndisabling initrd\n", + initrd_end, end_of_phys_memory); + initrd_start = 0; + } else { + /* Reserve the initrd image area. */ + size = initrd_end - initrd_start; + memblock_reserve(initrd_start, size); + + initrd_start = (initrd_start - phys_base) + PAGE_OFFSET; + initrd_end = (initrd_end - phys_base) + PAGE_OFFSET; + } + } +#endif +} + +unsigned long __init bootmem_init(unsigned long *pages_avail) +{ + unsigned long start_pfn, bytes_avail, size; + unsigned long end_of_phys_memory = 0; + unsigned long high_pages = 0; + int i; + + memblock_set_bottom_up(true); + memblock_allow_resize(); + + bytes_avail = 0UL; + for (i = 0; sp_banks[i].num_bytes != 0; i++) { + end_of_phys_memory = sp_banks[i].base_addr + + sp_banks[i].num_bytes; + bytes_avail += sp_banks[i].num_bytes; + if (cmdline_memory_size) { + if (bytes_avail > cmdline_memory_size) { + unsigned long slack = bytes_avail - cmdline_memory_size; + + bytes_avail -= slack; + end_of_phys_memory -= slack; + + sp_banks[i].num_bytes -= slack; + if (sp_banks[i].num_bytes == 0) { + sp_banks[i].base_addr = 0xdeadbeef; + } else { + memblock_add(sp_banks[i].base_addr, + sp_banks[i].num_bytes); + sp_banks[i+1].num_bytes = 0; + sp_banks[i+1].base_addr = 0xdeadbeef; + } + break; + } + } + memblock_add(sp_banks[i].base_addr, sp_banks[i].num_bytes); + } + + /* Start with page aligned address of last symbol in kernel + * image. + */ + start_pfn = (unsigned long)__pa(PAGE_ALIGN((unsigned long) &_end)); + + /* Now shift down to get the real physical page frame number. */ + start_pfn >>= PAGE_SHIFT; + + max_pfn = end_of_phys_memory >> PAGE_SHIFT; + + max_low_pfn = max_pfn; + highstart_pfn = highend_pfn = max_pfn; + + if (max_low_pfn > pfn_base + (SRMMU_MAXMEM >> PAGE_SHIFT)) { + highstart_pfn = pfn_base + (SRMMU_MAXMEM >> PAGE_SHIFT); + max_low_pfn = calc_max_low_pfn(); + high_pages = calc_highpages(); + printk(KERN_NOTICE "%ldMB HIGHMEM available.\n", + high_pages >> (20 - PAGE_SHIFT)); + } + + find_ramdisk(end_of_phys_memory); + + /* Reserve the kernel text/data/bss. */ + size = (start_pfn << PAGE_SHIFT) - phys_base; + memblock_reserve(phys_base, size); + memblock_add(phys_base, size); + + size = memblock_phys_mem_size() - memblock_reserved_size(); + *pages_avail = (size >> PAGE_SHIFT) - high_pages; + + /* Only allow low memory to be allocated via memblock allocation */ + memblock_set_current_limit(max_low_pfn << PAGE_SHIFT); + + return max_pfn; +} + +/* + * paging_init() sets up the page tables: We call the MMU specific + * init routine based upon the Sun model type on the Sparc. + * + */ +void __init paging_init(void) +{ + srmmu_paging_init(); + prom_build_devicetree(); + of_fill_in_cpu_data(); + device_scan(); +} + +static void __init taint_real_pages(void) +{ + int i; + + for (i = 0; sp_banks[i].num_bytes; i++) { + unsigned long start, end; + + start = sp_banks[i].base_addr; + end = start + sp_banks[i].num_bytes; + + while (start < end) { + set_bit(start >> 20, sparc_valid_addr_bitmap); + start += PAGE_SIZE; + } + } +} + +void __init arch_mm_preinit(void) +{ + int i; + + if (PKMAP_BASE+LAST_PKMAP*PAGE_SIZE >= FIXADDR_START) { + prom_printf("BUG: fixmap and pkmap areas overlap\n"); + prom_printf("pkbase: 0x%lx pkend: 0x%lx fixstart 0x%lx\n", + PKMAP_BASE, + (unsigned long)PKMAP_BASE+LAST_PKMAP*PAGE_SIZE, + FIXADDR_START); + prom_printf("Please mail sparclinux@vger.kernel.org.\n"); + prom_halt(); + } + + i = last_valid_pfn >> ((20 - PAGE_SHIFT) + 5); + i += 1; + sparc_valid_addr_bitmap = (unsigned long *) + memblock_alloc(i << 2, SMP_CACHE_BYTES); + + if (sparc_valid_addr_bitmap == NULL) { + prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n"); + prom_halt(); + } + memset(sparc_valid_addr_bitmap, 0, i << 2); + + taint_real_pages(); +} + +void sparc_flush_page_to_ram(struct page *page) +{ + unsigned long vaddr = (unsigned long)page_address(page); + + __flush_page_to_ram(vaddr); +} +EXPORT_SYMBOL(sparc_flush_page_to_ram); + +void sparc_flush_folio_to_ram(struct folio *folio) +{ + unsigned long vaddr = (unsigned long)folio_address(folio); + unsigned int i, nr = folio_nr_pages(folio); + + for (i = 0; i < nr; i++) + __flush_page_to_ram(vaddr + i * PAGE_SIZE); +} +EXPORT_SYMBOL(sparc_flush_folio_to_ram); + +static const pgprot_t protection_map[16] = { + [VM_NONE] = PAGE_NONE, + [VM_READ] = PAGE_READONLY, + [VM_WRITE] = PAGE_COPY, + [VM_WRITE | VM_READ] = PAGE_COPY, + [VM_EXEC] = PAGE_READONLY, + [VM_EXEC | VM_READ] = PAGE_READONLY, + [VM_EXEC | VM_WRITE] = PAGE_COPY, + [VM_EXEC | VM_WRITE | VM_READ] = PAGE_COPY, + [VM_SHARED] = PAGE_NONE, + [VM_SHARED | VM_READ] = PAGE_READONLY, + [VM_SHARED | VM_WRITE] = PAGE_SHARED, + [VM_SHARED | VM_WRITE | VM_READ] = PAGE_SHARED, + [VM_SHARED | VM_EXEC] = PAGE_READONLY, + [VM_SHARED | VM_EXEC | VM_READ] = PAGE_READONLY, + [VM_SHARED | VM_EXEC | VM_WRITE] = PAGE_SHARED, + [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = PAGE_SHARED +}; +DECLARE_VM_GET_PAGE_PROT diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c new file mode 100644 index 000000000..103db4683 --- /dev/null +++ b/arch/sparc/mm/init_64.c @@ -0,0 +1,3149 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arch/sparc64/mm/init.c + * + * Copyright (C) 1996-1999 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "init_64.h" + +unsigned long kern_linear_pte_xor[4] __read_mostly; +static unsigned long page_cache4v_flag; + +/* A bitmap, two bits for every 256MB of physical memory. These two + * bits determine what page size we use for kernel linear + * translations. They form an index into kern_linear_pte_xor[]. The + * value in the indexed slot is XOR'd with the TLB miss virtual + * address to form the resulting TTE. The mapping is: + * + * 0 ==> 4MB + * 1 ==> 256MB + * 2 ==> 2GB + * 3 ==> 16GB + * + * All sun4v chips support 256MB pages. Only SPARC-T4 and later + * support 2GB pages, and hopefully future cpus will support the 16GB + * pages as well. For slots 2 and 3, we encode a 256MB TTE xor there + * if these larger page sizes are not supported by the cpu. + * + * It would be nice to determine this from the machine description + * 'cpu' properties, but we need to have this table setup before the + * MDESC is initialized. + */ + +#ifndef CONFIG_DEBUG_PAGEALLOC +/* A special kernel TSB for 4MB, 256MB, 2GB and 16GB linear mappings. + * Space is allocated for this right after the trap table in + * arch/sparc64/kernel/head.S + */ +extern struct tsb swapper_4m_tsb[KERNEL_TSB4M_NENTRIES]; +#endif +extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; + +static unsigned long cpu_pgsz_mask; + +#define MAX_BANKS 1024 + +static struct linux_prom64_registers pavail[MAX_BANKS]; +static int pavail_ents; + +u64 numa_latency[MAX_NUMNODES][MAX_NUMNODES]; + +static int cmp_p64(const void *a, const void *b) +{ + const struct linux_prom64_registers *x = a, *y = b; + + if (x->phys_addr > y->phys_addr) + return 1; + if (x->phys_addr < y->phys_addr) + return -1; + return 0; +} + +static void __init read_obp_memory(const char *property, + struct linux_prom64_registers *regs, + int *num_ents) +{ + phandle node = prom_finddevice("/memory"); + int prop_size = prom_getproplen(node, property); + int ents, ret, i; + + ents = prop_size / sizeof(struct linux_prom64_registers); + if (ents > MAX_BANKS) { + prom_printf("The machine has more %s property entries than " + "this kernel can support (%d).\n", + property, MAX_BANKS); + prom_halt(); + } + + ret = prom_getproperty(node, property, (char *) regs, prop_size); + if (ret == -1) { + prom_printf("Couldn't get %s property from /memory.\n", + property); + prom_halt(); + } + + /* Sanitize what we got from the firmware, by page aligning + * everything. + */ + for (i = 0; i < ents; i++) { + unsigned long base, size; + + base = regs[i].phys_addr; + size = regs[i].reg_size; + + size &= PAGE_MASK; + if (base & ~PAGE_MASK) { + unsigned long new_base = PAGE_ALIGN(base); + + size -= new_base - base; + if ((long) size < 0L) + size = 0UL; + base = new_base; + } + if (size == 0UL) { + /* If it is empty, simply get rid of it. + * This simplifies the logic of the other + * functions that process these arrays. + */ + memmove(®s[i], ®s[i + 1], + (ents - i - 1) * sizeof(regs[0])); + i--; + ents--; + continue; + } + regs[i].phys_addr = base; + regs[i].reg_size = size; + } + + *num_ents = ents; + + sort(regs, ents, sizeof(struct linux_prom64_registers), + cmp_p64, NULL); +} + +/* Kernel physical address base and size in bytes. */ +unsigned long kern_base __read_mostly; +unsigned long kern_size __read_mostly; + +/* Initial ramdisk setup */ +extern unsigned long sparc_ramdisk_image64; +extern unsigned int sparc_ramdisk_image; +extern unsigned int sparc_ramdisk_size; + +unsigned int sparc64_highest_unlocked_tlb_ent __read_mostly; + +unsigned long sparc64_kern_pri_context __read_mostly; +unsigned long sparc64_kern_pri_nuc_bits __read_mostly; +unsigned long sparc64_kern_sec_context __read_mostly; + +int num_kernel_image_mappings; + +#ifdef CONFIG_DEBUG_DCFLUSH +atomic_t dcpage_flushes = ATOMIC_INIT(0); +#ifdef CONFIG_SMP +atomic_t dcpage_flushes_xcall = ATOMIC_INIT(0); +#endif +#endif + +inline void flush_dcache_folio_impl(struct folio *folio) +{ + unsigned int i, nr = folio_nr_pages(folio); + + BUG_ON(tlb_type == hypervisor); +#ifdef CONFIG_DEBUG_DCFLUSH + atomic_inc(&dcpage_flushes); +#endif + +#ifdef DCACHE_ALIASING_POSSIBLE + for (i = 0; i < nr; i++) + __flush_dcache_page(folio_address(folio) + i * PAGE_SIZE, + ((tlb_type == spitfire) && + folio_flush_mapping(folio) != NULL)); +#else + if (folio_flush_mapping(folio) != NULL && + tlb_type == spitfire) { + for (i = 0; i < nr; i++) + __flush_icache_page((pfn + i) * PAGE_SIZE); + } +#endif +} + +#define PG_dcache_dirty PG_arch_1 +#define PG_dcache_cpu_shift 32UL +#define PG_dcache_cpu_mask \ + ((1UL<flags.f >> PG_dcache_cpu_shift) & PG_dcache_cpu_mask) + +static inline void set_dcache_dirty(struct folio *folio, int this_cpu) +{ + unsigned long mask = this_cpu; + unsigned long non_cpu_bits; + + non_cpu_bits = ~(PG_dcache_cpu_mask << PG_dcache_cpu_shift); + mask = (mask << PG_dcache_cpu_shift) | (1UL << PG_dcache_dirty); + + __asm__ __volatile__("1:\n\t" + "ldx [%2], %%g7\n\t" + "and %%g7, %1, %%g1\n\t" + "or %%g1, %0, %%g1\n\t" + "casx [%2], %%g7, %%g1\n\t" + "cmp %%g7, %%g1\n\t" + "bne,pn %%xcc, 1b\n\t" + " nop" + : /* no outputs */ + : "r" (mask), "r" (non_cpu_bits), "r" (&folio->flags.f) + : "g1", "g7"); +} + +static inline void clear_dcache_dirty_cpu(struct folio *folio, unsigned long cpu) +{ + unsigned long mask = (1UL << PG_dcache_dirty); + + __asm__ __volatile__("! test_and_clear_dcache_dirty\n" + "1:\n\t" + "ldx [%2], %%g7\n\t" + "srlx %%g7, %4, %%g1\n\t" + "and %%g1, %3, %%g1\n\t" + "cmp %%g1, %0\n\t" + "bne,pn %%icc, 2f\n\t" + " andn %%g7, %1, %%g1\n\t" + "casx [%2], %%g7, %%g1\n\t" + "cmp %%g7, %%g1\n\t" + "bne,pn %%xcc, 1b\n\t" + " nop\n" + "2:" + : /* no outputs */ + : "r" (cpu), "r" (mask), "r" (&folio->flags.f), + "i" (PG_dcache_cpu_mask), + "i" (PG_dcache_cpu_shift) + : "g1", "g7"); +} + +static inline void tsb_insert(struct tsb *ent, unsigned long tag, unsigned long pte) +{ + unsigned long tsb_addr = (unsigned long) ent; + + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + tsb_addr = __pa(tsb_addr); + + __tsb_insert(tsb_addr, tag, pte); +} + +unsigned long _PAGE_ALL_SZ_BITS __read_mostly; + +static void flush_dcache(unsigned long pfn) +{ + struct page *page; + + page = pfn_to_page(pfn); + if (page) { + struct folio *folio = page_folio(page); + unsigned long pg_flags; + + pg_flags = folio->flags.f; + if (pg_flags & (1UL << PG_dcache_dirty)) { + int cpu = ((pg_flags >> PG_dcache_cpu_shift) & + PG_dcache_cpu_mask); + int this_cpu = get_cpu(); + + /* This is just to optimize away some function calls + * in the SMP case. + */ + if (cpu == this_cpu) + flush_dcache_folio_impl(folio); + else + smp_flush_dcache_folio_impl(folio, cpu); + + clear_dcache_dirty_cpu(folio, cpu); + + put_cpu(); + } + } +} + +/* mm->context.lock must be held */ +static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index, + unsigned long tsb_hash_shift, unsigned long address, + unsigned long tte) +{ + struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb; + unsigned long tag; + + if (unlikely(!tsb)) + return; + + tsb += ((address >> tsb_hash_shift) & + (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL)); + tag = (address >> 22UL); + tsb_insert(tsb, tag, tte); +} + +#ifdef CONFIG_HUGETLB_PAGE +static int __init hugetlbpage_init(void) +{ + hugetlb_add_hstate(HPAGE_64K_SHIFT - PAGE_SHIFT); + hugetlb_add_hstate(HPAGE_SHIFT - PAGE_SHIFT); + hugetlb_add_hstate(HPAGE_256MB_SHIFT - PAGE_SHIFT); + hugetlb_add_hstate(HPAGE_2GB_SHIFT - PAGE_SHIFT); + + return 0; +} + +arch_initcall(hugetlbpage_init); + +static void __init pud_huge_patch(void) +{ + struct pud_huge_patch_entry *p; + unsigned long addr; + + p = &__pud_huge_patch; + addr = p->addr; + *(unsigned int *)addr = p->insn; + + __asm__ __volatile__("flush %0" : : "r" (addr)); +} + +bool __init arch_hugetlb_valid_size(unsigned long size) +{ + unsigned int hugepage_shift = ilog2(size); + unsigned int hv_pgsz_mask; + + switch (hugepage_shift) { + case HPAGE_16GB_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_16GB; + pud_huge_patch(); + break; + case HPAGE_2GB_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_2GB; + break; + case HPAGE_256MB_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_256MB; + break; + case HPAGE_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_4MB; + break; + case HPAGE_64K_SHIFT: + hv_pgsz_mask = HV_PGSZ_MASK_64K; + break; + default: + hv_pgsz_mask = 0; + } + + if ((hv_pgsz_mask & cpu_pgsz_mask) == 0U) + return false; + + return true; +} +#endif /* CONFIG_HUGETLB_PAGE */ + +void update_mmu_cache_range(struct vm_fault *vmf, struct vm_area_struct *vma, + unsigned long address, pte_t *ptep, unsigned int nr) +{ + struct mm_struct *mm; + unsigned long flags; + bool is_huge_tsb; + pte_t pte = *ptep; + unsigned int i; + + if (tlb_type != hypervisor) { + unsigned long pfn = pte_pfn(pte); + + if (pfn_valid(pfn)) + flush_dcache(pfn); + } + + mm = vma->vm_mm; + + /* Don't insert a non-valid PTE into the TSB, we'll deadlock. */ + if (!pte_accessible(mm, pte)) + return; + + spin_lock_irqsave(&mm->context.lock, flags); + + is_huge_tsb = false; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (mm->context.hugetlb_pte_count || mm->context.thp_pte_count) { + unsigned long hugepage_size = PAGE_SIZE; + + if (is_vm_hugetlb_page(vma)) + hugepage_size = huge_page_size(hstate_vma(vma)); + + if (hugepage_size >= PUD_SIZE) { + unsigned long mask = 0x1ffc00000UL; + + /* Transfer bits [32:22] from address to resolve + * at 4M granularity. + */ + pte_val(pte) &= ~mask; + pte_val(pte) |= (address & mask); + } else if (hugepage_size >= PMD_SIZE) { + /* We are fabricating 8MB pages using 4MB + * real hw pages. + */ + pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT)); + } + + if (hugepage_size >= PMD_SIZE) { + __update_mmu_tsb_insert(mm, MM_TSB_HUGE, + REAL_HPAGE_SHIFT, address, pte_val(pte)); + is_huge_tsb = true; + } + } +#endif + if (!is_huge_tsb) { + for (i = 0; i < nr; i++) { + __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, + address, pte_val(pte)); + address += PAGE_SIZE; + pte_val(pte) += PAGE_SIZE; + } + } + + spin_unlock_irqrestore(&mm->context.lock, flags); +} + +void flush_dcache_folio(struct folio *folio) +{ + unsigned long pfn = folio_pfn(folio); + struct address_space *mapping; + int this_cpu; + + if (tlb_type == hypervisor) + return; + + /* Do not bother with the expensive D-cache flush if it + * is merely the zero page. The 'bigcore' testcase in GDB + * causes this case to run millions of times. + */ + if (is_zero_pfn(pfn)) + return; + + this_cpu = get_cpu(); + + mapping = folio_flush_mapping(folio); + if (mapping && !mapping_mapped(mapping)) { + bool dirty = test_bit(PG_dcache_dirty, &folio->flags.f); + if (dirty) { + int dirty_cpu = dcache_dirty_cpu(folio); + + if (dirty_cpu == this_cpu) + goto out; + smp_flush_dcache_folio_impl(folio, dirty_cpu); + } + set_dcache_dirty(folio, this_cpu); + } else { + /* We could delay the flush for the !folio_mapping + * case too. But that case is for exec env/arg + * pages and those are %99 certainly going to get + * faulted into the tlb (and thus flushed) anyways. + */ + flush_dcache_folio_impl(folio); + } + +out: + put_cpu(); +} +EXPORT_SYMBOL(flush_dcache_folio); + +void __kprobes flush_icache_range(unsigned long start, unsigned long end) +{ + /* Cheetah and Hypervisor platform cpus have coherent I-cache. */ + if (tlb_type == spitfire) { + unsigned long kaddr; + + /* This code only runs on Spitfire cpus so this is + * why we can assume _PAGE_PADDR_4U. + */ + for (kaddr = start; kaddr < end; kaddr += PAGE_SIZE) { + unsigned long paddr, mask = _PAGE_PADDR_4U; + + if (kaddr >= PAGE_OFFSET) + paddr = kaddr & mask; + else { + pte_t *ptep = virt_to_kpte(kaddr); + + paddr = pte_val(*ptep) & mask; + } + __flush_icache_page(paddr); + } + } +} +EXPORT_SYMBOL(flush_icache_range); + +void mmu_info(struct seq_file *m) +{ + static const char *pgsz_strings[] = { + "8K", "64K", "512K", "4MB", "32MB", + "256MB", "2GB", "16GB", + }; + int i, printed; + + if (tlb_type == cheetah) + seq_printf(m, "MMU Type\t: Cheetah\n"); + else if (tlb_type == cheetah_plus) + seq_printf(m, "MMU Type\t: Cheetah+\n"); + else if (tlb_type == spitfire) + seq_printf(m, "MMU Type\t: Spitfire\n"); + else if (tlb_type == hypervisor) + seq_printf(m, "MMU Type\t: Hypervisor (sun4v)\n"); + else + seq_printf(m, "MMU Type\t: ???\n"); + + seq_printf(m, "MMU PGSZs\t: "); + printed = 0; + for (i = 0; i < ARRAY_SIZE(pgsz_strings); i++) { + if (cpu_pgsz_mask & (1UL << i)) { + seq_printf(m, "%s%s", + printed ? "," : "", pgsz_strings[i]); + printed++; + } + } + seq_putc(m, '\n'); + +#ifdef CONFIG_DEBUG_DCFLUSH + seq_printf(m, "DCPageFlushes\t: %d\n", + atomic_read(&dcpage_flushes)); +#ifdef CONFIG_SMP + seq_printf(m, "DCPageFlushesXC\t: %d\n", + atomic_read(&dcpage_flushes_xcall)); +#endif /* CONFIG_SMP */ +#endif /* CONFIG_DEBUG_DCFLUSH */ +} + +struct linux_prom_translation prom_trans[512] __read_mostly; +unsigned int prom_trans_ents __read_mostly; + +unsigned long kern_locked_tte_data; + +/* The obp translations are saved based on 8k pagesize, since obp can + * use a mixture of pagesizes. Misses to the LOW_OBP_ADDRESS -> + * HI_OBP_ADDRESS range are handled in ktlb.S. + */ +static inline int in_obp_range(unsigned long vaddr) +{ + return (vaddr >= LOW_OBP_ADDRESS && + vaddr < HI_OBP_ADDRESS); +} + +static int cmp_ptrans(const void *a, const void *b) +{ + const struct linux_prom_translation *x = a, *y = b; + + if (x->virt > y->virt) + return 1; + if (x->virt < y->virt) + return -1; + return 0; +} + +/* Read OBP translations property into 'prom_trans[]'. */ +static void __init read_obp_translations(void) +{ + int n, node, ents, first, last, i; + + node = prom_finddevice("/virtual-memory"); + n = prom_getproplen(node, "translations"); + if (unlikely(n == 0 || n == -1)) { + prom_printf("prom_mappings: Couldn't get size.\n"); + prom_halt(); + } + if (unlikely(n > sizeof(prom_trans))) { + prom_printf("prom_mappings: Size %d is too big.\n", n); + prom_halt(); + } + + if ((n = prom_getproperty(node, "translations", + (char *)&prom_trans[0], + sizeof(prom_trans))) == -1) { + prom_printf("prom_mappings: Couldn't get property.\n"); + prom_halt(); + } + + n = n / sizeof(struct linux_prom_translation); + + ents = n; + + sort(prom_trans, ents, sizeof(struct linux_prom_translation), + cmp_ptrans, NULL); + + /* Now kick out all the non-OBP entries. */ + for (i = 0; i < ents; i++) { + if (in_obp_range(prom_trans[i].virt)) + break; + } + first = i; + for (; i < ents; i++) { + if (!in_obp_range(prom_trans[i].virt)) + break; + } + last = i; + + for (i = 0; i < (last - first); i++) { + struct linux_prom_translation *src = &prom_trans[i + first]; + struct linux_prom_translation *dest = &prom_trans[i]; + + *dest = *src; + } + for (; i < ents; i++) { + struct linux_prom_translation *dest = &prom_trans[i]; + dest->virt = dest->size = dest->data = 0x0UL; + } + + prom_trans_ents = last - first; + + if (tlb_type == spitfire) { + /* Clear diag TTE bits. */ + for (i = 0; i < prom_trans_ents; i++) + prom_trans[i].data &= ~0x0003fe0000000000UL; + } + + /* Force execute bit on. */ + for (i = 0; i < prom_trans_ents; i++) + prom_trans[i].data |= (tlb_type == hypervisor ? + _PAGE_EXEC_4V : _PAGE_EXEC_4U); +} + +static void __init hypervisor_tlb_lock(unsigned long vaddr, + unsigned long pte, + unsigned long mmu) +{ + unsigned long ret = sun4v_mmu_map_perm_addr(vaddr, 0, pte, mmu); + + if (ret != 0) { + prom_printf("hypervisor_tlb_lock[%lx:%x:%lx:%lx]: " + "errors with %lx\n", vaddr, 0, pte, mmu, ret); + prom_halt(); + } +} + +static unsigned long kern_large_tte(unsigned long paddr); + +static void __init remap_kernel(void) +{ + unsigned long phys_page, tte_vaddr, tte_data; + int i, tlb_ent = sparc64_highest_locked_tlbent(); + + tte_vaddr = (unsigned long) KERNBASE; + phys_page = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB; + tte_data = kern_large_tte(phys_page); + + kern_locked_tte_data = tte_data; + + /* Now lock us into the TLBs via Hypervisor or OBP. */ + if (tlb_type == hypervisor) { + for (i = 0; i < num_kernel_image_mappings; i++) { + hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_DMMU); + hypervisor_tlb_lock(tte_vaddr, tte_data, HV_MMU_IMMU); + tte_vaddr += 0x400000; + tte_data += 0x400000; + } + } else { + for (i = 0; i < num_kernel_image_mappings; i++) { + prom_dtlb_load(tlb_ent - i, tte_data, tte_vaddr); + prom_itlb_load(tlb_ent - i, tte_data, tte_vaddr); + tte_vaddr += 0x400000; + tte_data += 0x400000; + } + sparc64_highest_unlocked_tlb_ent = tlb_ent - i; + } + if (tlb_type == cheetah_plus) { + sparc64_kern_pri_context = (CTX_CHEETAH_PLUS_CTX0 | + CTX_CHEETAH_PLUS_NUC); + sparc64_kern_pri_nuc_bits = CTX_CHEETAH_PLUS_NUC; + sparc64_kern_sec_context = CTX_CHEETAH_PLUS_CTX0; + } +} + + +static void __init inherit_prom_mappings(void) +{ + /* Now fixup OBP's idea about where we really are mapped. */ + printk("Remapping the kernel... "); + remap_kernel(); + printk("done.\n"); +} + +void prom_world(int enter) +{ + /* + * No need to change the address space any more, just flush + * the register windows + */ + __asm__ __volatile__("flushw"); +} + +void __flush_dcache_range(unsigned long start, unsigned long end) +{ + unsigned long va; + + if (tlb_type == spitfire) { + int n = 0; + + for (va = start; va < end; va += 32) { + spitfire_put_dcache_tag(va & 0x3fe0, 0x0); + if (++n >= 512) + break; + } + } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { + start = __pa(start); + end = __pa(end); + for (va = start; va < end; va += 32) + __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (va), + "i" (ASI_DCACHE_INVALIDATE)); + } +} +EXPORT_SYMBOL(__flush_dcache_range); + +/* get_new_mmu_context() uses "cache + 1". */ +DEFINE_SPINLOCK(ctx_alloc_lock); +unsigned long tlb_context_cache = CTX_FIRST_VERSION; +#define MAX_CTX_NR (1UL << CTX_NR_BITS) +#define CTX_BMAP_SLOTS BITS_TO_LONGS(MAX_CTX_NR) +DECLARE_BITMAP(mmu_context_bmap, MAX_CTX_NR); +DEFINE_PER_CPU(struct mm_struct *, per_cpu_secondary_mm) = {0}; + +static void mmu_context_wrap(void) +{ + unsigned long old_ver = tlb_context_cache & CTX_VERSION_MASK; + unsigned long new_ver, new_ctx, old_ctx; + struct mm_struct *mm; + int cpu; + + bitmap_zero(mmu_context_bmap, 1 << CTX_NR_BITS); + + /* Reserve kernel context */ + set_bit(0, mmu_context_bmap); + + new_ver = (tlb_context_cache & CTX_VERSION_MASK) + CTX_FIRST_VERSION; + if (unlikely(new_ver == 0)) + new_ver = CTX_FIRST_VERSION; + tlb_context_cache = new_ver; + + /* + * Make sure that any new mm that are added into per_cpu_secondary_mm, + * are going to go through get_new_mmu_context() path. + */ + mb(); + + /* + * Updated versions to current on those CPUs that had valid secondary + * contexts + */ + for_each_online_cpu(cpu) { + /* + * If a new mm is stored after we took this mm from the array, + * it will go into get_new_mmu_context() path, because we + * already bumped the version in tlb_context_cache. + */ + mm = per_cpu(per_cpu_secondary_mm, cpu); + + if (unlikely(!mm || mm == &init_mm)) + continue; + + old_ctx = mm->context.sparc64_ctx_val; + if (likely((old_ctx & CTX_VERSION_MASK) == old_ver)) { + new_ctx = (old_ctx & ~CTX_VERSION_MASK) | new_ver; + set_bit(new_ctx & CTX_NR_MASK, mmu_context_bmap); + mm->context.sparc64_ctx_val = new_ctx; + } + } +} + +/* Caller does TLB context flushing on local CPU if necessary. + * The caller also ensures that CTX_VALID(mm->context) is false. + * + * We must be careful about boundary cases so that we never + * let the user have CTX 0 (nucleus) or we ever use a CTX + * version of zero (and thus NO_CONTEXT would not be caught + * by version mis-match tests in mmu_context.h). + * + * Always invoked with interrupts disabled. + */ +void get_new_mmu_context(struct mm_struct *mm) +{ + unsigned long ctx, new_ctx; + unsigned long orig_pgsz_bits; + + spin_lock(&ctx_alloc_lock); +retry: + /* wrap might have happened, test again if our context became valid */ + if (unlikely(CTX_VALID(mm->context))) + goto out; + orig_pgsz_bits = (mm->context.sparc64_ctx_val & CTX_PGSZ_MASK); + ctx = (tlb_context_cache + 1) & CTX_NR_MASK; + new_ctx = find_next_zero_bit(mmu_context_bmap, 1 << CTX_NR_BITS, ctx); + if (new_ctx >= (1 << CTX_NR_BITS)) { + new_ctx = find_next_zero_bit(mmu_context_bmap, ctx, 1); + if (new_ctx >= ctx) { + mmu_context_wrap(); + goto retry; + } + } + if (mm->context.sparc64_ctx_val) + cpumask_clear(mm_cpumask(mm)); + mmu_context_bmap[new_ctx>>6] |= (1UL << (new_ctx & 63)); + new_ctx |= (tlb_context_cache & CTX_VERSION_MASK); + tlb_context_cache = new_ctx; + mm->context.sparc64_ctx_val = new_ctx | orig_pgsz_bits; +out: + spin_unlock(&ctx_alloc_lock); +} + +static int numa_enabled = 1; +static int numa_debug; + +static int __init early_numa(char *p) +{ + if (!p) + return 0; + + if (strstr(p, "off")) + numa_enabled = 0; + + if (strstr(p, "debug")) + numa_debug = 1; + + return 0; +} +early_param("numa", early_numa); + +#define numadbg(f, a...) \ +do { if (numa_debug) \ + printk(KERN_INFO f, ## a); \ +} while (0) + +static void __init find_ramdisk(unsigned long phys_base) +{ +#ifdef CONFIG_BLK_DEV_INITRD + if (sparc_ramdisk_image || sparc_ramdisk_image64) { + unsigned long ramdisk_image; + + /* Older versions of the bootloader only supported a + * 32-bit physical address for the ramdisk image + * location, stored at sparc_ramdisk_image. Newer + * SILO versions set sparc_ramdisk_image to zero and + * provide a full 64-bit physical address at + * sparc_ramdisk_image64. + */ + ramdisk_image = sparc_ramdisk_image; + if (!ramdisk_image) + ramdisk_image = sparc_ramdisk_image64; + + /* Another bootloader quirk. The bootloader normalizes + * the physical address to KERNBASE, so we have to + * factor that back out and add in the lowest valid + * physical page address to get the true physical address. + */ + ramdisk_image -= KERNBASE; + ramdisk_image += phys_base; + + numadbg("Found ramdisk at physical address 0x%lx, size %u\n", + ramdisk_image, sparc_ramdisk_size); + + initrd_start = ramdisk_image; + initrd_end = ramdisk_image + sparc_ramdisk_size; + + memblock_reserve(initrd_start, sparc_ramdisk_size); + + initrd_start += PAGE_OFFSET; + initrd_end += PAGE_OFFSET; + } +#endif +} + +struct node_mem_mask { + unsigned long mask; + unsigned long match; +}; +static struct node_mem_mask node_masks[MAX_NUMNODES]; +static int num_node_masks; + +#ifdef CONFIG_NUMA + +struct mdesc_mlgroup { + u64 node; + u64 latency; + u64 match; + u64 mask; +}; + +static struct mdesc_mlgroup *mlgroups; +static int num_mlgroups; + +int numa_cpu_lookup_table[NR_CPUS]; +cpumask_t numa_cpumask_lookup_table[MAX_NUMNODES]; + +struct mdesc_mblock { + u64 base; + u64 size; + u64 offset; /* RA-to-PA */ +}; +static struct mdesc_mblock *mblocks; +static int num_mblocks; + +static struct mdesc_mblock * __init addr_to_mblock(unsigned long addr) +{ + struct mdesc_mblock *m = NULL; + int i; + + for (i = 0; i < num_mblocks; i++) { + m = &mblocks[i]; + + if (addr >= m->base && + addr < (m->base + m->size)) { + break; + } + } + + return m; +} + +static u64 __init memblock_nid_range_sun4u(u64 start, u64 end, int *nid) +{ + int prev_nid, new_nid; + + prev_nid = NUMA_NO_NODE; + for ( ; start < end; start += PAGE_SIZE) { + for (new_nid = 0; new_nid < num_node_masks; new_nid++) { + struct node_mem_mask *p = &node_masks[new_nid]; + + if ((start & p->mask) == p->match) { + if (prev_nid == NUMA_NO_NODE) + prev_nid = new_nid; + break; + } + } + + if (new_nid == num_node_masks) { + prev_nid = 0; + WARN_ONCE(1, "addr[%Lx] doesn't match a NUMA node rule. Some memory will be owned by node 0.", + start); + break; + } + + if (prev_nid != new_nid) + break; + } + *nid = prev_nid; + + return start > end ? end : start; +} + +static u64 __init memblock_nid_range(u64 start, u64 end, int *nid) +{ + u64 ret_end, pa_start, m_mask, m_match, m_end; + struct mdesc_mblock *mblock; + int _nid, i; + + if (tlb_type != hypervisor) + return memblock_nid_range_sun4u(start, end, nid); + + mblock = addr_to_mblock(start); + if (!mblock) { + WARN_ONCE(1, "memblock_nid_range: Can't find mblock addr[%Lx]", + start); + + _nid = 0; + ret_end = end; + goto done; + } + + pa_start = start + mblock->offset; + m_match = 0; + m_mask = 0; + + for (_nid = 0; _nid < num_node_masks; _nid++) { + struct node_mem_mask *const m = &node_masks[_nid]; + + if ((pa_start & m->mask) == m->match) { + m_match = m->match; + m_mask = m->mask; + break; + } + } + + if (num_node_masks == _nid) { + /* We could not find NUMA group, so default to 0, but lets + * search for latency group, so we could calculate the correct + * end address that we return + */ + _nid = 0; + + for (i = 0; i < num_mlgroups; i++) { + struct mdesc_mlgroup *const m = &mlgroups[i]; + + if ((pa_start & m->mask) == m->match) { + m_match = m->match; + m_mask = m->mask; + break; + } + } + + if (i == num_mlgroups) { + WARN_ONCE(1, "memblock_nid_range: Can't find latency group addr[%Lx]", + start); + + ret_end = end; + goto done; + } + } + + /* + * Each latency group has match and mask, and each memory block has an + * offset. An address belongs to a latency group if its address matches + * the following formula: ((addr + offset) & mask) == match + * It is, however, slow to check every single page if it matches a + * particular latency group. As optimization we calculate end value by + * using bit arithmetics. + */ + m_end = m_match + (1ul << __ffs(m_mask)) - mblock->offset; + m_end += pa_start & ~((1ul << fls64(m_mask)) - 1); + ret_end = m_end > end ? end : m_end; + +done: + *nid = _nid; + return ret_end; +} +#endif + +/* This must be invoked after performing all of the necessary + * memblock_set_node() calls for 'nid'. We need to be able to get + * correct data from get_pfn_range_for_nid(). + */ +static void __init allocate_node_data(int nid) +{ + struct pglist_data *p; + unsigned long start_pfn, end_pfn; + +#ifdef CONFIG_NUMA + alloc_node_data(nid); + + NODE_DATA(nid)->node_id = nid; +#endif + + p = NODE_DATA(nid); + + get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); + p->node_start_pfn = start_pfn; + p->node_spanned_pages = end_pfn - start_pfn; +} + +static void init_node_masks_nonnuma(void) +{ +#ifdef CONFIG_NUMA + int i; +#endif + + numadbg("Initializing tables for non-numa.\n"); + + node_masks[0].mask = 0; + node_masks[0].match = 0; + num_node_masks = 1; + +#ifdef CONFIG_NUMA + for (i = 0; i < NR_CPUS; i++) + numa_cpu_lookup_table[i] = 0; + + cpumask_setall(&numa_cpumask_lookup_table[0]); +#endif +} + +#ifdef CONFIG_NUMA + +EXPORT_SYMBOL(numa_cpu_lookup_table); +EXPORT_SYMBOL(numa_cpumask_lookup_table); + +static int scan_pio_for_cfg_handle(struct mdesc_handle *md, u64 pio, + u32 cfg_handle) +{ + u64 arc; + + mdesc_for_each_arc(arc, md, pio, MDESC_ARC_TYPE_FWD) { + u64 target = mdesc_arc_target(md, arc); + const u64 *val; + + val = mdesc_get_property(md, target, + "cfg-handle", NULL); + if (val && *val == cfg_handle) + return 0; + } + return -ENODEV; +} + +static int scan_arcs_for_cfg_handle(struct mdesc_handle *md, u64 grp, + u32 cfg_handle) +{ + u64 arc, candidate, best_latency = ~(u64)0; + + candidate = MDESC_NODE_NULL; + mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) { + u64 target = mdesc_arc_target(md, arc); + const char *name = mdesc_node_name(md, target); + const u64 *val; + + if (strcmp(name, "pio-latency-group")) + continue; + + val = mdesc_get_property(md, target, "latency", NULL); + if (!val) + continue; + + if (*val < best_latency) { + candidate = target; + best_latency = *val; + } + } + + if (candidate == MDESC_NODE_NULL) + return -ENODEV; + + return scan_pio_for_cfg_handle(md, candidate, cfg_handle); +} + +int of_node_to_nid(struct device_node *dp) +{ + const struct linux_prom64_registers *regs; + struct mdesc_handle *md; + u32 cfg_handle; + int count, nid; + u64 grp; + + /* This is the right thing to do on currently supported + * SUN4U NUMA platforms as well, as the PCI controller does + * not sit behind any particular memory controller. + */ + if (!mlgroups) + return -1; + + regs = of_get_property(dp, "reg", NULL); + if (!regs) + return -1; + + cfg_handle = (regs->phys_addr >> 32UL) & 0x0fffffff; + + md = mdesc_grab(); + + count = 0; + nid = NUMA_NO_NODE; + mdesc_for_each_node_by_name(md, grp, "group") { + if (!scan_arcs_for_cfg_handle(md, grp, cfg_handle)) { + nid = count; + break; + } + count++; + } + + mdesc_release(md); + + return nid; +} + +static void __init add_node_ranges(void) +{ + phys_addr_t start, end; + unsigned long prev_max; + u64 i; + +memblock_resized: + prev_max = memblock.memory.max; + + for_each_mem_range(i, &start, &end) { + while (start < end) { + unsigned long this_end; + int nid; + + this_end = memblock_nid_range(start, end, &nid); + + numadbg("Setting memblock NUMA node nid[%d] " + "start[%llx] end[%lx]\n", + nid, start, this_end); + + memblock_set_node(start, this_end - start, + &memblock.memory, nid); + if (memblock.memory.max != prev_max) + goto memblock_resized; + start = this_end; + } + } +} + +static int __init grab_mlgroups(struct mdesc_handle *md) +{ + unsigned long paddr; + int count = 0; + u64 node; + + mdesc_for_each_node_by_name(md, node, "memory-latency-group") + count++; + if (!count) + return -ENOENT; + + paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mlgroup), + SMP_CACHE_BYTES); + if (!paddr) + return -ENOMEM; + + mlgroups = __va(paddr); + num_mlgroups = count; + + count = 0; + mdesc_for_each_node_by_name(md, node, "memory-latency-group") { + struct mdesc_mlgroup *m = &mlgroups[count++]; + const u64 *val; + + m->node = node; + + val = mdesc_get_property(md, node, "latency", NULL); + m->latency = *val; + val = mdesc_get_property(md, node, "address-match", NULL); + m->match = *val; + val = mdesc_get_property(md, node, "address-mask", NULL); + m->mask = *val; + + numadbg("MLGROUP[%d]: node[%llx] latency[%llx] " + "match[%llx] mask[%llx]\n", + count - 1, m->node, m->latency, m->match, m->mask); + } + + return 0; +} + +static int __init grab_mblocks(struct mdesc_handle *md) +{ + unsigned long paddr; + int count = 0; + u64 node; + + mdesc_for_each_node_by_name(md, node, "mblock") + count++; + if (!count) + return -ENOENT; + + paddr = memblock_phys_alloc(count * sizeof(struct mdesc_mblock), + SMP_CACHE_BYTES); + if (!paddr) + return -ENOMEM; + + mblocks = __va(paddr); + num_mblocks = count; + + count = 0; + mdesc_for_each_node_by_name(md, node, "mblock") { + struct mdesc_mblock *m = &mblocks[count++]; + const u64 *val; + + val = mdesc_get_property(md, node, "base", NULL); + m->base = *val; + val = mdesc_get_property(md, node, "size", NULL); + m->size = *val; + val = mdesc_get_property(md, node, + "address-congruence-offset", NULL); + + /* The address-congruence-offset property is optional. + * Explicity zero it be identifty this. + */ + if (val) + m->offset = *val; + else + m->offset = 0UL; + + numadbg("MBLOCK[%d]: base[%llx] size[%llx] offset[%llx]\n", + count - 1, m->base, m->size, m->offset); + } + + return 0; +} + +static void __init numa_parse_mdesc_group_cpus(struct mdesc_handle *md, + u64 grp, cpumask_t *mask) +{ + u64 arc; + + cpumask_clear(mask); + + mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_BACK) { + u64 target = mdesc_arc_target(md, arc); + const char *name = mdesc_node_name(md, target); + const u64 *id; + + if (strcmp(name, "cpu")) + continue; + id = mdesc_get_property(md, target, "id", NULL); + if (*id < nr_cpu_ids) + cpumask_set_cpu(*id, mask); + } +} + +static struct mdesc_mlgroup * __init find_mlgroup(u64 node) +{ + int i; + + for (i = 0; i < num_mlgroups; i++) { + struct mdesc_mlgroup *m = &mlgroups[i]; + if (m->node == node) + return m; + } + return NULL; +} + +int __node_distance(int from, int to) +{ + if ((from >= MAX_NUMNODES) || (to >= MAX_NUMNODES)) { + pr_warn("Returning default NUMA distance value for %d->%d\n", + from, to); + return (from == to) ? LOCAL_DISTANCE : REMOTE_DISTANCE; + } + return numa_latency[from][to]; +} +EXPORT_SYMBOL(__node_distance); + +static int __init find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp) +{ + int i; + + for (i = 0; i < MAX_NUMNODES; i++) { + struct node_mem_mask *n = &node_masks[i]; + + if ((grp->mask == n->mask) && (grp->match == n->match)) + break; + } + return i; +} + +static void __init find_numa_latencies_for_group(struct mdesc_handle *md, + u64 grp, int index) +{ + u64 arc; + + mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) { + int tnode; + u64 target = mdesc_arc_target(md, arc); + struct mdesc_mlgroup *m = find_mlgroup(target); + + if (!m) + continue; + tnode = find_best_numa_node_for_mlgroup(m); + if (tnode == MAX_NUMNODES) + continue; + numa_latency[index][tnode] = m->latency; + } +} + +static int __init numa_attach_mlgroup(struct mdesc_handle *md, u64 grp, + int index) +{ + struct mdesc_mlgroup *candidate = NULL; + u64 arc, best_latency = ~(u64)0; + struct node_mem_mask *n; + + mdesc_for_each_arc(arc, md, grp, MDESC_ARC_TYPE_FWD) { + u64 target = mdesc_arc_target(md, arc); + struct mdesc_mlgroup *m = find_mlgroup(target); + if (!m) + continue; + if (m->latency < best_latency) { + candidate = m; + best_latency = m->latency; + } + } + if (!candidate) + return -ENOENT; + + if (num_node_masks != index) { + printk(KERN_ERR "Inconsistent NUMA state, " + "index[%d] != num_node_masks[%d]\n", + index, num_node_masks); + return -EINVAL; + } + + n = &node_masks[num_node_masks++]; + + n->mask = candidate->mask; + n->match = candidate->match; + + numadbg("NUMA NODE[%d]: mask[%lx] match[%lx] (latency[%llx])\n", + index, n->mask, n->match, candidate->latency); + + return 0; +} + +static int __init numa_parse_mdesc_group(struct mdesc_handle *md, u64 grp, + int index) +{ + cpumask_t mask; + int cpu; + + numa_parse_mdesc_group_cpus(md, grp, &mask); + + for_each_cpu(cpu, &mask) + numa_cpu_lookup_table[cpu] = index; + cpumask_copy(&numa_cpumask_lookup_table[index], &mask); + + if (numa_debug) { + printk(KERN_INFO "NUMA GROUP[%d]: cpus [ ", index); + for_each_cpu(cpu, &mask) + printk("%d ", cpu); + printk("]\n"); + } + + return numa_attach_mlgroup(md, grp, index); +} + +static int __init numa_parse_mdesc(void) +{ + struct mdesc_handle *md = mdesc_grab(); + int i, j, err, count; + u64 node; + + node = mdesc_node_by_name(md, MDESC_NODE_NULL, "latency-groups"); + if (node == MDESC_NODE_NULL) { + mdesc_release(md); + return -ENOENT; + } + + err = grab_mblocks(md); + if (err < 0) + goto out; + + err = grab_mlgroups(md); + if (err < 0) + goto out; + + count = 0; + mdesc_for_each_node_by_name(md, node, "group") { + err = numa_parse_mdesc_group(md, node, count); + if (err < 0) + break; + count++; + } + + count = 0; + mdesc_for_each_node_by_name(md, node, "group") { + find_numa_latencies_for_group(md, node, count); + count++; + } + + /* Normalize numa latency matrix according to ACPI SLIT spec. */ + for (i = 0; i < MAX_NUMNODES; i++) { + u64 self_latency = numa_latency[i][i]; + + for (j = 0; j < MAX_NUMNODES; j++) { + numa_latency[i][j] = + (numa_latency[i][j] * LOCAL_DISTANCE) / + self_latency; + } + } + + add_node_ranges(); + + for (i = 0; i < num_node_masks; i++) { + allocate_node_data(i); + node_set_online(i); + } + + err = 0; +out: + mdesc_release(md); + return err; +} + +static int __init numa_parse_jbus(void) +{ + unsigned long cpu, index; + + /* NUMA node id is encoded in bits 36 and higher, and there is + * a 1-to-1 mapping from CPU ID to NUMA node ID. + */ + index = 0; + for_each_present_cpu(cpu) { + numa_cpu_lookup_table[cpu] = index; + cpumask_copy(&numa_cpumask_lookup_table[index], cpumask_of(cpu)); + node_masks[index].mask = ~((1UL << 36UL) - 1UL); + node_masks[index].match = cpu << 36UL; + + index++; + } + num_node_masks = index; + + add_node_ranges(); + + for (index = 0; index < num_node_masks; index++) { + allocate_node_data(index); + node_set_online(index); + } + + return 0; +} + +static int __init numa_parse_sun4u(void) +{ + if (tlb_type == cheetah || tlb_type == cheetah_plus) { + unsigned long ver; + + __asm__ ("rdpr %%ver, %0" : "=r" (ver)); + if ((ver >> 32UL) == __JALAPENO_ID || + (ver >> 32UL) == __SERRANO_ID) + return numa_parse_jbus(); + } + return -1; +} + +static int __init bootmem_init_numa(void) +{ + int i, j; + int err = -1; + + numadbg("bootmem_init_numa()\n"); + + /* Some sane defaults for numa latency values */ + for (i = 0; i < MAX_NUMNODES; i++) { + for (j = 0; j < MAX_NUMNODES; j++) + numa_latency[i][j] = (i == j) ? + LOCAL_DISTANCE : REMOTE_DISTANCE; + } + + if (numa_enabled) { + if (tlb_type == hypervisor) + err = numa_parse_mdesc(); + else + err = numa_parse_sun4u(); + } + return err; +} + +#else + +static int bootmem_init_numa(void) +{ + return -1; +} + +#endif + +static void __init bootmem_init_nonnuma(void) +{ + unsigned long top_of_ram = memblock_end_of_DRAM(); + unsigned long total_ram = memblock_phys_mem_size(); + + numadbg("bootmem_init_nonnuma()\n"); + + printk(KERN_INFO "Top of RAM: 0x%lx, Total RAM: 0x%lx\n", + top_of_ram, total_ram); + printk(KERN_INFO "Memory hole size: %ldMB\n", + (top_of_ram - total_ram) >> 20); + + init_node_masks_nonnuma(); + memblock_set_node(0, PHYS_ADDR_MAX, &memblock.memory, 0); + allocate_node_data(0); + node_set_online(0); +} + +static unsigned long __init bootmem_init(unsigned long phys_base) +{ + unsigned long end_pfn; + + end_pfn = memblock_end_of_DRAM() >> PAGE_SHIFT; + max_pfn = max_low_pfn = end_pfn; + min_low_pfn = (phys_base >> PAGE_SHIFT); + + if (bootmem_init_numa() < 0) + bootmem_init_nonnuma(); + + /* Dump memblock with node info. */ + memblock_dump_all(); + + /* XXX cpu notifier XXX */ + + return end_pfn; +} + +static struct linux_prom64_registers pall[MAX_BANKS] __initdata; +static int pall_ents __initdata; + +static unsigned long max_phys_bits = 40; + +bool kern_addr_valid(unsigned long addr) +{ + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + if ((long)addr < 0L) { + unsigned long pa = __pa(addr); + + if ((pa >> max_phys_bits) != 0UL) + return false; + + return pfn_valid(pa >> PAGE_SHIFT); + } + + if (addr >= (unsigned long) KERNBASE && + addr < (unsigned long)&_end) + return true; + + pgd = pgd_offset_k(addr); + if (pgd_none(*pgd)) + return false; + + p4d = p4d_offset(pgd, addr); + if (p4d_none(*p4d)) + return false; + + pud = pud_offset(p4d, addr); + if (pud_none(*pud)) + return false; + + if (pud_leaf(*pud)) + return pfn_valid(pud_pfn(*pud)); + + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd)) + return false; + + if (pmd_leaf(*pmd)) + return pfn_valid(pmd_pfn(*pmd)); + + pte = pte_offset_kernel(pmd, addr); + if (pte_none(*pte)) + return false; + + return pfn_valid(pte_pfn(*pte)); +} + +static unsigned long __ref kernel_map_hugepud(unsigned long vstart, + unsigned long vend, + pud_t *pud) +{ + const unsigned long mask16gb = (1UL << 34) - 1UL; + u64 pte_val = vstart; + + /* Each PUD is 8GB */ + if ((vstart & mask16gb) || + (vend - vstart <= mask16gb)) { + pte_val ^= kern_linear_pte_xor[2]; + pud_val(*pud) = pte_val | _PAGE_PUD_HUGE; + + return vstart + PUD_SIZE; + } + + pte_val ^= kern_linear_pte_xor[3]; + pte_val |= _PAGE_PUD_HUGE; + + vend = vstart + mask16gb + 1UL; + while (vstart < vend) { + pud_val(*pud) = pte_val; + + pte_val += PUD_SIZE; + vstart += PUD_SIZE; + pud++; + } + return vstart; +} + +static bool kernel_can_map_hugepud(unsigned long vstart, unsigned long vend, + bool guard) +{ + if (guard && !(vstart & ~PUD_MASK) && (vend - vstart) >= PUD_SIZE) + return true; + + return false; +} + +static unsigned long __ref kernel_map_hugepmd(unsigned long vstart, + unsigned long vend, + pmd_t *pmd) +{ + const unsigned long mask256mb = (1UL << 28) - 1UL; + const unsigned long mask2gb = (1UL << 31) - 1UL; + u64 pte_val = vstart; + + /* Each PMD is 8MB */ + if ((vstart & mask256mb) || + (vend - vstart <= mask256mb)) { + pte_val ^= kern_linear_pte_xor[0]; + pmd_val(*pmd) = pte_val | _PAGE_PMD_HUGE; + + return vstart + PMD_SIZE; + } + + if ((vstart & mask2gb) || + (vend - vstart <= mask2gb)) { + pte_val ^= kern_linear_pte_xor[1]; + pte_val |= _PAGE_PMD_HUGE; + vend = vstart + mask256mb + 1UL; + } else { + pte_val ^= kern_linear_pte_xor[2]; + pte_val |= _PAGE_PMD_HUGE; + vend = vstart + mask2gb + 1UL; + } + + while (vstart < vend) { + pmd_val(*pmd) = pte_val; + + pte_val += PMD_SIZE; + vstart += PMD_SIZE; + pmd++; + } + + return vstart; +} + +static bool kernel_can_map_hugepmd(unsigned long vstart, unsigned long vend, + bool guard) +{ + if (guard && !(vstart & ~PMD_MASK) && (vend - vstart) >= PMD_SIZE) + return true; + + return false; +} + +static unsigned long __ref kernel_map_range(unsigned long pstart, + unsigned long pend, pgprot_t prot, + bool use_huge) +{ + unsigned long vstart = PAGE_OFFSET + pstart; + unsigned long vend = PAGE_OFFSET + pend; + unsigned long alloc_bytes = 0UL; + + if ((vstart & ~PAGE_MASK) || (vend & ~PAGE_MASK)) { + prom_printf("kernel_map: Unaligned physmem[%lx:%lx]\n", + vstart, vend); + prom_halt(); + } + + while (vstart < vend) { + unsigned long this_end, paddr = __pa(vstart); + pgd_t *pgd = pgd_offset_k(vstart); + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + + if (pgd_none(*pgd)) { + pud_t *new; + + new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, + PAGE_SIZE); + if (!new) + goto err_alloc; + alloc_bytes += PAGE_SIZE; + pgd_populate(&init_mm, pgd, new); + } + + p4d = p4d_offset(pgd, vstart); + if (p4d_none(*p4d)) { + pud_t *new; + + new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, + PAGE_SIZE); + if (!new) + goto err_alloc; + alloc_bytes += PAGE_SIZE; + p4d_populate(&init_mm, p4d, new); + } + + pud = pud_offset(p4d, vstart); + if (pud_none(*pud)) { + pmd_t *new; + + if (kernel_can_map_hugepud(vstart, vend, use_huge)) { + vstart = kernel_map_hugepud(vstart, vend, pud); + continue; + } + new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, + PAGE_SIZE); + if (!new) + goto err_alloc; + alloc_bytes += PAGE_SIZE; + pud_populate(&init_mm, pud, new); + } + + pmd = pmd_offset(pud, vstart); + if (pmd_none(*pmd)) { + pte_t *new; + + if (kernel_can_map_hugepmd(vstart, vend, use_huge)) { + vstart = kernel_map_hugepmd(vstart, vend, pmd); + continue; + } + new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, + PAGE_SIZE); + if (!new) + goto err_alloc; + alloc_bytes += PAGE_SIZE; + pmd_populate_kernel(&init_mm, pmd, new); + } + + pte = pte_offset_kernel(pmd, vstart); + this_end = (vstart + PMD_SIZE) & PMD_MASK; + if (this_end > vend) + this_end = vend; + + while (vstart < this_end) { + pte_val(*pte) = (paddr | pgprot_val(prot)); + + vstart += PAGE_SIZE; + paddr += PAGE_SIZE; + pte++; + } + } + + return alloc_bytes; + +err_alloc: + panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n", + __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); + return -ENOMEM; +} + +static void __init flush_all_kernel_tsbs(void) +{ + int i; + + for (i = 0; i < KERNEL_TSB_NENTRIES; i++) { + struct tsb *ent = &swapper_tsb[i]; + + ent->tag = (1UL << TSB_TAG_INVALID_BIT); + } +#ifndef CONFIG_DEBUG_PAGEALLOC + for (i = 0; i < KERNEL_TSB4M_NENTRIES; i++) { + struct tsb *ent = &swapper_4m_tsb[i]; + + ent->tag = (1UL << TSB_TAG_INVALID_BIT); + } +#endif +} + +extern unsigned int kvmap_linear_patch[1]; + +static void __init kernel_physical_mapping_init(void) +{ + unsigned long i, mem_alloced = 0UL; + bool use_huge = true; + +#ifdef CONFIG_DEBUG_PAGEALLOC + use_huge = false; +#endif + for (i = 0; i < pall_ents; i++) { + unsigned long phys_start, phys_end; + + phys_start = pall[i].phys_addr; + phys_end = phys_start + pall[i].reg_size; + + mem_alloced += kernel_map_range(phys_start, phys_end, + PAGE_KERNEL, use_huge); + } + + printk("Allocated %ld bytes for kernel page tables.\n", + mem_alloced); + + kvmap_linear_patch[0] = 0x01000000; /* nop */ + flushi(&kvmap_linear_patch[0]); + + flush_all_kernel_tsbs(); + + __flush_tlb_all(); +} + +#ifdef CONFIG_DEBUG_PAGEALLOC +void __kernel_map_pages(struct page *page, int numpages, int enable) +{ + unsigned long phys_start = page_to_pfn(page) << PAGE_SHIFT; + unsigned long phys_end = phys_start + (numpages * PAGE_SIZE); + + kernel_map_range(phys_start, phys_end, + (enable ? PAGE_KERNEL : __pgprot(0)), false); + + flush_tsb_kernel_range(PAGE_OFFSET + phys_start, + PAGE_OFFSET + phys_end); + + /* we should perform an IPI and flush all tlbs, + * but that can deadlock->flush only current cpu. + */ + __flush_tlb_kernel_range(PAGE_OFFSET + phys_start, + PAGE_OFFSET + phys_end); +} +#endif + +unsigned long __init find_ecache_flush_span(unsigned long size) +{ + int i; + + for (i = 0; i < pavail_ents; i++) { + if (pavail[i].reg_size >= size) + return pavail[i].phys_addr; + } + + return ~0UL; +} + +unsigned long PAGE_OFFSET; +EXPORT_SYMBOL(PAGE_OFFSET); + +unsigned long VMALLOC_END = 0x0000010000000000UL; +EXPORT_SYMBOL(VMALLOC_END); + +unsigned long sparc64_va_hole_top = 0xfffff80000000000UL; +unsigned long sparc64_va_hole_bottom = 0x0000080000000000UL; + +static void __init setup_page_offset(void) +{ + if (tlb_type == cheetah || tlb_type == cheetah_plus) { + /* Cheetah/Panther support a full 64-bit virtual + * address, so we can use all that our page tables + * support. + */ + sparc64_va_hole_top = 0xfff0000000000000UL; + sparc64_va_hole_bottom = 0x0010000000000000UL; + + max_phys_bits = 42; + } else if (tlb_type == hypervisor) { + switch (sun4v_chip_type) { + case SUN4V_CHIP_NIAGARA1: + case SUN4V_CHIP_NIAGARA2: + /* T1 and T2 support 48-bit virtual addresses. */ + sparc64_va_hole_top = 0xffff800000000000UL; + sparc64_va_hole_bottom = 0x0000800000000000UL; + + max_phys_bits = 39; + break; + case SUN4V_CHIP_NIAGARA3: + /* T3 supports 48-bit virtual addresses. */ + sparc64_va_hole_top = 0xffff800000000000UL; + sparc64_va_hole_bottom = 0x0000800000000000UL; + + max_phys_bits = 43; + break; + case SUN4V_CHIP_NIAGARA4: + case SUN4V_CHIP_NIAGARA5: + case SUN4V_CHIP_SPARC64X: + case SUN4V_CHIP_SPARC_M6: + /* T4 and later support 52-bit virtual addresses. */ + sparc64_va_hole_top = 0xfff8000000000000UL; + sparc64_va_hole_bottom = 0x0008000000000000UL; + max_phys_bits = 47; + break; + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_SN: + /* M7 and later support 52-bit virtual addresses. */ + sparc64_va_hole_top = 0xfff8000000000000UL; + sparc64_va_hole_bottom = 0x0008000000000000UL; + max_phys_bits = 49; + break; + case SUN4V_CHIP_SPARC_M8: + default: + /* M8 and later support 54-bit virtual addresses. + * However, restricting M8 and above VA bits to 53 + * as 4-level page table cannot support more than + * 53 VA bits. + */ + sparc64_va_hole_top = 0xfff0000000000000UL; + sparc64_va_hole_bottom = 0x0010000000000000UL; + max_phys_bits = 51; + break; + } + } + + if (max_phys_bits > MAX_PHYS_ADDRESS_BITS) { + prom_printf("MAX_PHYS_ADDRESS_BITS is too small, need %lu\n", + max_phys_bits); + prom_halt(); + } + + PAGE_OFFSET = sparc64_va_hole_top; + VMALLOC_END = ((sparc64_va_hole_bottom >> 1) + + (sparc64_va_hole_bottom >> 2)); + + pr_info("MM: PAGE_OFFSET is 0x%016lx (max_phys_bits == %lu)\n", + PAGE_OFFSET, max_phys_bits); + pr_info("MM: VMALLOC [0x%016lx --> 0x%016lx]\n", + VMALLOC_START, VMALLOC_END); + pr_info("MM: VMEMMAP [0x%016lx --> 0x%016lx]\n", + VMEMMAP_BASE, VMEMMAP_BASE << 1); +} + +static void __init tsb_phys_patch(void) +{ + struct tsb_ldquad_phys_patch_entry *pquad; + struct tsb_phys_patch_entry *p; + + pquad = &__tsb_ldquad_phys_patch; + while (pquad < &__tsb_ldquad_phys_patch_end) { + unsigned long addr = pquad->addr; + + if (tlb_type == hypervisor) + *(unsigned int *) addr = pquad->sun4v_insn; + else + *(unsigned int *) addr = pquad->sun4u_insn; + wmb(); + __asm__ __volatile__("flush %0" + : /* no outputs */ + : "r" (addr)); + + pquad++; + } + + p = &__tsb_phys_patch; + while (p < &__tsb_phys_patch_end) { + unsigned long addr = p->addr; + + *(unsigned int *) addr = p->insn; + wmb(); + __asm__ __volatile__("flush %0" + : /* no outputs */ + : "r" (addr)); + + p++; + } +} + +/* Don't mark as init, we give this to the Hypervisor. */ +#ifndef CONFIG_DEBUG_PAGEALLOC +#define NUM_KTSB_DESCR 2 +#else +#define NUM_KTSB_DESCR 1 +#endif +static struct hv_tsb_descr ktsb_descr[NUM_KTSB_DESCR]; + +/* The swapper TSBs are loaded with a base sequence of: + * + * sethi %uhi(SYMBOL), REG1 + * sethi %hi(SYMBOL), REG2 + * or REG1, %ulo(SYMBOL), REG1 + * or REG2, %lo(SYMBOL), REG2 + * sllx REG1, 32, REG1 + * or REG1, REG2, REG1 + * + * When we use physical addressing for the TSB accesses, we patch the + * first four instructions in the above sequence. + */ + +static void patch_one_ktsb_phys(unsigned int *start, unsigned int *end, unsigned long pa) +{ + unsigned long high_bits, low_bits; + + high_bits = (pa >> 32) & 0xffffffff; + low_bits = (pa >> 0) & 0xffffffff; + + while (start < end) { + unsigned int *ia = (unsigned int *)(unsigned long)*start; + + ia[0] = (ia[0] & ~0x3fffff) | (high_bits >> 10); + __asm__ __volatile__("flush %0" : : "r" (ia)); + + ia[1] = (ia[1] & ~0x3fffff) | (low_bits >> 10); + __asm__ __volatile__("flush %0" : : "r" (ia + 1)); + + ia[2] = (ia[2] & ~0x1fff) | (high_bits & 0x3ff); + __asm__ __volatile__("flush %0" : : "r" (ia + 2)); + + ia[3] = (ia[3] & ~0x1fff) | (low_bits & 0x3ff); + __asm__ __volatile__("flush %0" : : "r" (ia + 3)); + + start++; + } +} + +static void ktsb_phys_patch(void) +{ + extern unsigned int __swapper_tsb_phys_patch; + extern unsigned int __swapper_tsb_phys_patch_end; + unsigned long ktsb_pa; + + ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE); + patch_one_ktsb_phys(&__swapper_tsb_phys_patch, + &__swapper_tsb_phys_patch_end, ktsb_pa); +#ifndef CONFIG_DEBUG_PAGEALLOC + { + extern unsigned int __swapper_4m_tsb_phys_patch; + extern unsigned int __swapper_4m_tsb_phys_patch_end; + ktsb_pa = (kern_base + + ((unsigned long)&swapper_4m_tsb[0] - KERNBASE)); + patch_one_ktsb_phys(&__swapper_4m_tsb_phys_patch, + &__swapper_4m_tsb_phys_patch_end, ktsb_pa); + } +#endif +} + +static void __init sun4v_ktsb_init(void) +{ + unsigned long ktsb_pa; + + /* First KTSB for PAGE_SIZE mappings. */ + ktsb_pa = kern_base + ((unsigned long)&swapper_tsb[0] - KERNBASE); + + switch (PAGE_SIZE) { + case 8 * 1024: + default: + ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_8K; + ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_8K; + break; + + case 64 * 1024: + ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_64K; + ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_64K; + break; + + case 512 * 1024: + ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_512K; + ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_512K; + break; + + case 4 * 1024 * 1024: + ktsb_descr[0].pgsz_idx = HV_PGSZ_IDX_4MB; + ktsb_descr[0].pgsz_mask = HV_PGSZ_MASK_4MB; + break; + } + + ktsb_descr[0].assoc = 1; + ktsb_descr[0].num_ttes = KERNEL_TSB_NENTRIES; + ktsb_descr[0].ctx_idx = 0; + ktsb_descr[0].tsb_base = ktsb_pa; + ktsb_descr[0].resv = 0; + +#ifndef CONFIG_DEBUG_PAGEALLOC + /* Second KTSB for 4MB/256MB/2GB/16GB mappings. */ + ktsb_pa = (kern_base + + ((unsigned long)&swapper_4m_tsb[0] - KERNBASE)); + + ktsb_descr[1].pgsz_idx = HV_PGSZ_IDX_4MB; + ktsb_descr[1].pgsz_mask = ((HV_PGSZ_MASK_4MB | + HV_PGSZ_MASK_256MB | + HV_PGSZ_MASK_2GB | + HV_PGSZ_MASK_16GB) & + cpu_pgsz_mask); + ktsb_descr[1].assoc = 1; + ktsb_descr[1].num_ttes = KERNEL_TSB4M_NENTRIES; + ktsb_descr[1].ctx_idx = 0; + ktsb_descr[1].tsb_base = ktsb_pa; + ktsb_descr[1].resv = 0; +#endif +} + +void sun4v_ktsb_register(void) +{ + unsigned long pa, ret; + + pa = kern_base + ((unsigned long)&ktsb_descr[0] - KERNBASE); + + ret = sun4v_mmu_tsb_ctx0(NUM_KTSB_DESCR, pa); + if (ret != 0) { + prom_printf("hypervisor_mmu_tsb_ctx0[%lx]: " + "errors with %lx\n", pa, ret); + prom_halt(); + } +} + +static void __init sun4u_linear_pte_xor_finalize(void) +{ +#ifndef CONFIG_DEBUG_PAGEALLOC + /* This is where we would add Panther support for + * 32MB and 256MB pages. + */ +#endif +} + +static void __init sun4v_linear_pte_xor_finalize(void) +{ + unsigned long pagecv_flag; + + /* Bit 9 of TTE is no longer CV bit on M7 processor and it instead + * enables MCD error. Do not set bit 9 on M7 processor. + */ + switch (sun4v_chip_type) { + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_M8: + case SUN4V_CHIP_SPARC_SN: + pagecv_flag = 0x00; + break; + default: + pagecv_flag = _PAGE_CV_4V; + break; + } +#ifndef CONFIG_DEBUG_PAGEALLOC + if (cpu_pgsz_mask & HV_PGSZ_MASK_256MB) { + kern_linear_pte_xor[1] = (_PAGE_VALID | _PAGE_SZ256MB_4V) ^ + PAGE_OFFSET; + kern_linear_pte_xor[1] |= (_PAGE_CP_4V | pagecv_flag | + _PAGE_P_4V | _PAGE_W_4V); + } else { + kern_linear_pte_xor[1] = kern_linear_pte_xor[0]; + } + + if (cpu_pgsz_mask & HV_PGSZ_MASK_2GB) { + kern_linear_pte_xor[2] = (_PAGE_VALID | _PAGE_SZ2GB_4V) ^ + PAGE_OFFSET; + kern_linear_pte_xor[2] |= (_PAGE_CP_4V | pagecv_flag | + _PAGE_P_4V | _PAGE_W_4V); + } else { + kern_linear_pte_xor[2] = kern_linear_pte_xor[1]; + } + + if (cpu_pgsz_mask & HV_PGSZ_MASK_16GB) { + kern_linear_pte_xor[3] = (_PAGE_VALID | _PAGE_SZ16GB_4V) ^ + PAGE_OFFSET; + kern_linear_pte_xor[3] |= (_PAGE_CP_4V | pagecv_flag | + _PAGE_P_4V | _PAGE_W_4V); + } else { + kern_linear_pte_xor[3] = kern_linear_pte_xor[2]; + } +#endif +} + +/* paging_init() sets up the page tables */ + +static unsigned long last_valid_pfn; + +static void sun4u_pgprot_init(void); +static void sun4v_pgprot_init(void); + +#define _PAGE_CACHE_4U (_PAGE_CP_4U | _PAGE_CV_4U) +#define _PAGE_CACHE_4V (_PAGE_CP_4V | _PAGE_CV_4V) +#define __DIRTY_BITS_4U (_PAGE_MODIFIED_4U | _PAGE_WRITE_4U | _PAGE_W_4U) +#define __DIRTY_BITS_4V (_PAGE_MODIFIED_4V | _PAGE_WRITE_4V | _PAGE_W_4V) +#define __ACCESS_BITS_4U (_PAGE_ACCESSED_4U | _PAGE_READ_4U | _PAGE_R) +#define __ACCESS_BITS_4V (_PAGE_ACCESSED_4V | _PAGE_READ_4V | _PAGE_R) + +/* We need to exclude reserved regions. This exclusion will include + * vmlinux and initrd. To be more precise the initrd size could be used to + * compute a new lower limit because it is freed later during initialization. + */ +static void __init reduce_memory(phys_addr_t limit_ram) +{ + limit_ram += memblock_reserved_size(); + memblock_enforce_memory_limit(limit_ram); +} + +void __init arch_zone_limits_init(unsigned long *max_zone_pfns) +{ + max_zone_pfns[ZONE_NORMAL] = last_valid_pfn; +} + +void __init paging_init(void) +{ + unsigned long end_pfn, shift, phys_base; + unsigned long real_end, i; + + setup_page_offset(); + + /* These build time checkes make sure that the dcache_dirty_cpu() + * folio->flags usage will work. + * + * When a page gets marked as dcache-dirty, we store the + * cpu number starting at bit 32 in the folio->flags. Also, + * functions like clear_dcache_dirty_cpu use the cpu mask + * in 13-bit signed-immediate instruction fields. + */ + + /* + * Page flags must not reach into upper 32 bits that are used + * for the cpu number + */ + BUILD_BUG_ON(NR_PAGEFLAGS > 32); + + /* + * The bit fields placed in the high range must not reach below + * the 32 bit boundary. Otherwise we cannot place the cpu field + * at the 32 bit boundary. + */ + BUILD_BUG_ON(SECTIONS_WIDTH + NODES_WIDTH + ZONES_WIDTH + + ilog2(roundup_pow_of_two(NR_CPUS)) > 32); + + BUILD_BUG_ON(NR_CPUS > 4096); + + kern_base = (prom_boot_mapping_phys_low >> ILOG2_4MB) << ILOG2_4MB; + kern_size = (unsigned long)&_end - (unsigned long)KERNBASE; + + /* Invalidate both kernel TSBs. */ + memset(swapper_tsb, 0x40, sizeof(swapper_tsb)); +#ifndef CONFIG_DEBUG_PAGEALLOC + memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb)); +#endif + + /* TTE.cv bit on sparc v9 occupies the same position as TTE.mcde + * bit on M7 processor. This is a conflicting usage of the same + * bit. Enabling TTE.cv on M7 would turn on Memory Corruption + * Detection error on all pages and this will lead to problems + * later. Kernel does not run with MCD enabled and hence rest + * of the required steps to fully configure memory corruption + * detection are not taken. We need to ensure TTE.mcde is not + * set on M7 processor. Compute the value of cacheability + * flag for use later taking this into consideration. + */ + switch (sun4v_chip_type) { + case SUN4V_CHIP_SPARC_M7: + case SUN4V_CHIP_SPARC_M8: + case SUN4V_CHIP_SPARC_SN: + page_cache4v_flag = _PAGE_CP_4V; + break; + default: + page_cache4v_flag = _PAGE_CACHE_4V; + break; + } + + if (tlb_type == hypervisor) + sun4v_pgprot_init(); + else + sun4u_pgprot_init(); + + if (tlb_type == cheetah_plus || + tlb_type == hypervisor) { + tsb_phys_patch(); + ktsb_phys_patch(); + } + + if (tlb_type == hypervisor) + sun4v_patch_tlb_handlers(); + + /* Find available physical memory... + * + * Read it twice in order to work around a bug in openfirmware. + * The call to grab this table itself can cause openfirmware to + * allocate memory, which in turn can take away some space from + * the list of available memory. Reading it twice makes sure + * we really do get the final value. + */ + read_obp_translations(); + read_obp_memory("reg", &pall[0], &pall_ents); + read_obp_memory("available", &pavail[0], &pavail_ents); + read_obp_memory("available", &pavail[0], &pavail_ents); + + phys_base = 0xffffffffffffffffUL; + for (i = 0; i < pavail_ents; i++) { + phys_base = min(phys_base, pavail[i].phys_addr); + memblock_add(pavail[i].phys_addr, pavail[i].reg_size); + } + + memblock_reserve(kern_base, kern_size); + + find_ramdisk(phys_base); + + if (cmdline_memory_size) + reduce_memory(cmdline_memory_size); + + memblock_allow_resize(); + memblock_dump_all(); + + set_bit(0, mmu_context_bmap); + + shift = kern_base + PAGE_OFFSET - ((unsigned long)KERNBASE); + + real_end = (unsigned long)_end; + num_kernel_image_mappings = DIV_ROUND_UP(real_end - KERNBASE, 1 << ILOG2_4MB); + printk("Kernel: Using %d locked TLB entries for main kernel image.\n", + num_kernel_image_mappings); + + /* Set kernel pgd to upper alias so physical page computations + * work. + */ + init_mm.pgd += ((shift) / (sizeof(pgd_t))); + + memset(swapper_pg_dir, 0, sizeof(swapper_pg_dir)); + + inherit_prom_mappings(); + + /* Ok, we can use our TLB miss and window trap handlers safely. */ + setup_tba(); + + __flush_tlb_all(); + + prom_build_devicetree(); + of_populate_present_mask(); +#ifndef CONFIG_SMP + of_fill_in_cpu_data(); +#endif + + if (tlb_type == hypervisor) { + sun4v_mdesc_init(); + mdesc_populate_present_mask(cpu_all_mask); +#ifndef CONFIG_SMP + mdesc_fill_in_cpu_data(cpu_all_mask); +#endif + mdesc_get_page_sizes(cpu_all_mask, &cpu_pgsz_mask); + + sun4v_linear_pte_xor_finalize(); + + sun4v_ktsb_init(); + sun4v_ktsb_register(); + } else { + unsigned long impl, ver; + + cpu_pgsz_mask = (HV_PGSZ_MASK_8K | HV_PGSZ_MASK_64K | + HV_PGSZ_MASK_512K | HV_PGSZ_MASK_4MB); + + __asm__ __volatile__("rdpr %%ver, %0" : "=r" (ver)); + impl = ((ver >> 32) & 0xffff); + if (impl == PANTHER_IMPL) + cpu_pgsz_mask |= (HV_PGSZ_MASK_32MB | + HV_PGSZ_MASK_256MB); + + sun4u_linear_pte_xor_finalize(); + } + + /* Flush the TLBs and the 4M TSB so that the updated linear + * pte XOR settings are realized for all mappings. + */ + __flush_tlb_all(); +#ifndef CONFIG_DEBUG_PAGEALLOC + memset(swapper_4m_tsb, 0x40, sizeof(swapper_4m_tsb)); +#endif + __flush_tlb_all(); + + /* Setup bootmem... */ + last_valid_pfn = end_pfn = bootmem_init(phys_base); + + kernel_physical_mapping_init(); + + printk("Booting Linux...\n"); +} + +int page_in_phys_avail(unsigned long paddr) +{ + int i; + + paddr &= PAGE_MASK; + + for (i = 0; i < pavail_ents; i++) { + unsigned long start, end; + + start = pavail[i].phys_addr; + end = start + pavail[i].reg_size; + + if (paddr >= start && paddr < end) + return 1; + } + if (paddr >= kern_base && paddr < (kern_base + kern_size)) + return 1; +#ifdef CONFIG_BLK_DEV_INITRD + if (paddr >= __pa(initrd_start) && + paddr < __pa(PAGE_ALIGN(initrd_end))) + return 1; +#endif + + return 0; +} + +void __init arch_setup_zero_pages(void) +{ + phys_addr_t zero_page_pa = kern_base + + ((unsigned long)&empty_zero_page[0] - KERNBASE); + + __zero_page = phys_to_page(zero_page_pa); +} + +void __init mem_init(void) +{ + if (tlb_type == cheetah || tlb_type == cheetah_plus) + cheetah_ecache_flush_init(); +} + +void free_initmem(void) +{ + unsigned long addr, initend; + int do_free = 1; + + /* If the physical memory maps were trimmed by kernel command + * line options, don't even try freeing this initmem stuff up. + * The kernel image could have been in the trimmed out region + * and if so the freeing below will free invalid page structs. + */ + if (cmdline_memory_size) + do_free = 0; + + /* + * The init section is aligned to 8k in vmlinux.lds. Page align for >8k pagesizes. + */ + addr = PAGE_ALIGN((unsigned long)(__init_begin)); + initend = (unsigned long)(__init_end) & PAGE_MASK; + for (; addr < initend; addr += PAGE_SIZE) { + unsigned long page; + + page = (addr + + ((unsigned long) __va(kern_base)) - + ((unsigned long) KERNBASE)); + memset((void *)addr, POISON_FREE_INITMEM, PAGE_SIZE); + + if (do_free) + free_reserved_page(virt_to_page(page)); + } +} + +pgprot_t PAGE_KERNEL __read_mostly; +EXPORT_SYMBOL(PAGE_KERNEL); + +pgprot_t PAGE_KERNEL_LOCKED __read_mostly; +pgprot_t PAGE_COPY __read_mostly; + +pgprot_t PAGE_SHARED __read_mostly; +EXPORT_SYMBOL(PAGE_SHARED); + +unsigned long pg_iobits __read_mostly; + +unsigned long _PAGE_IE __read_mostly; +EXPORT_SYMBOL(_PAGE_IE); + +unsigned long _PAGE_E __read_mostly; +EXPORT_SYMBOL(_PAGE_E); + +unsigned long _PAGE_CACHE __read_mostly; +EXPORT_SYMBOL(_PAGE_CACHE); + +#ifdef CONFIG_SPARSEMEM_VMEMMAP +void __meminit vmemmap_set_pmd(pmd_t *pmd, void *p, int node, + unsigned long addr, unsigned long next) +{ + unsigned long pte_base; + + pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4U | + _PAGE_CP_4U | _PAGE_CV_4U | + _PAGE_P_4U | _PAGE_W_4U); + if (tlb_type == hypervisor) + pte_base = (_PAGE_VALID | _PAGE_SZ4MB_4V | + page_cache4v_flag | _PAGE_P_4V | _PAGE_W_4V); + + pte_base |= _PAGE_PMD_HUGE; + + pmd_val(*pmd) = pte_base | __pa(p); +} + +int __meminit vmemmap_populate(unsigned long vstart, unsigned long vend, + int node, struct vmem_altmap *altmap) +{ + return vmemmap_populate_hugepages(vstart, vend, node, NULL); +} +#endif /* CONFIG_SPARSEMEM_VMEMMAP */ + +/* These are actually filled in at boot time by sun4{u,v}_pgprot_init() */ +static pgprot_t protection_map[16] __ro_after_init; + +static void prot_init_common(unsigned long page_none, + unsigned long page_shared, + unsigned long page_copy, + unsigned long page_readonly, + unsigned long page_exec_bit) +{ + PAGE_COPY = __pgprot(page_copy); + PAGE_SHARED = __pgprot(page_shared); + + protection_map[0x0] = __pgprot(page_none); + protection_map[0x1] = __pgprot(page_readonly & ~page_exec_bit); + protection_map[0x2] = __pgprot(page_copy & ~page_exec_bit); + protection_map[0x3] = __pgprot(page_copy & ~page_exec_bit); + protection_map[0x4] = __pgprot(page_readonly); + protection_map[0x5] = __pgprot(page_readonly); + protection_map[0x6] = __pgprot(page_copy); + protection_map[0x7] = __pgprot(page_copy); + protection_map[0x8] = __pgprot(page_none); + protection_map[0x9] = __pgprot(page_readonly & ~page_exec_bit); + protection_map[0xa] = __pgprot(page_shared & ~page_exec_bit); + protection_map[0xb] = __pgprot(page_shared & ~page_exec_bit); + protection_map[0xc] = __pgprot(page_readonly); + protection_map[0xd] = __pgprot(page_readonly); + protection_map[0xe] = __pgprot(page_shared); + protection_map[0xf] = __pgprot(page_shared); +} + +static void __init sun4u_pgprot_init(void) +{ + unsigned long page_none, page_shared, page_copy, page_readonly; + unsigned long page_exec_bit; + int i; + + PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID | + _PAGE_CACHE_4U | _PAGE_P_4U | + __ACCESS_BITS_4U | __DIRTY_BITS_4U | + _PAGE_EXEC_4U); + PAGE_KERNEL_LOCKED = __pgprot (_PAGE_PRESENT_4U | _PAGE_VALID | + _PAGE_CACHE_4U | _PAGE_P_4U | + __ACCESS_BITS_4U | __DIRTY_BITS_4U | + _PAGE_EXEC_4U | _PAGE_L_4U); + + _PAGE_IE = _PAGE_IE_4U; + _PAGE_E = _PAGE_E_4U; + _PAGE_CACHE = _PAGE_CACHE_4U; + + pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4U | __DIRTY_BITS_4U | + __ACCESS_BITS_4U | _PAGE_E_4U); + +#ifdef CONFIG_DEBUG_PAGEALLOC + kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; +#else + kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4U) ^ + PAGE_OFFSET; +#endif + kern_linear_pte_xor[0] |= (_PAGE_CP_4U | _PAGE_CV_4U | + _PAGE_P_4U | _PAGE_W_4U); + + for (i = 1; i < 4; i++) + kern_linear_pte_xor[i] = kern_linear_pte_xor[0]; + + _PAGE_ALL_SZ_BITS = (_PAGE_SZ4MB_4U | _PAGE_SZ512K_4U | + _PAGE_SZ64K_4U | _PAGE_SZ8K_4U | + _PAGE_SZ32MB_4U | _PAGE_SZ256MB_4U); + + + page_none = _PAGE_PRESENT_4U | _PAGE_ACCESSED_4U | _PAGE_CACHE_4U; + page_shared = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U | + __ACCESS_BITS_4U | _PAGE_WRITE_4U | _PAGE_EXEC_4U); + page_copy = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U | + __ACCESS_BITS_4U | _PAGE_EXEC_4U); + page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4U | _PAGE_CACHE_4U | + __ACCESS_BITS_4U | _PAGE_EXEC_4U); + + page_exec_bit = _PAGE_EXEC_4U; + + prot_init_common(page_none, page_shared, page_copy, page_readonly, + page_exec_bit); +} + +static void __init sun4v_pgprot_init(void) +{ + unsigned long page_none, page_shared, page_copy, page_readonly; + unsigned long page_exec_bit; + int i; + + PAGE_KERNEL = __pgprot (_PAGE_PRESENT_4V | _PAGE_VALID | + page_cache4v_flag | _PAGE_P_4V | + __ACCESS_BITS_4V | __DIRTY_BITS_4V | + _PAGE_EXEC_4V); + PAGE_KERNEL_LOCKED = PAGE_KERNEL; + + _PAGE_IE = _PAGE_IE_4V; + _PAGE_E = _PAGE_E_4V; + _PAGE_CACHE = page_cache4v_flag; + +#ifdef CONFIG_DEBUG_PAGEALLOC + kern_linear_pte_xor[0] = _PAGE_VALID ^ PAGE_OFFSET; +#else + kern_linear_pte_xor[0] = (_PAGE_VALID | _PAGE_SZ4MB_4V) ^ + PAGE_OFFSET; +#endif + kern_linear_pte_xor[0] |= (page_cache4v_flag | _PAGE_P_4V | + _PAGE_W_4V); + + for (i = 1; i < 4; i++) + kern_linear_pte_xor[i] = kern_linear_pte_xor[0]; + + pg_iobits = (_PAGE_VALID | _PAGE_PRESENT_4V | __DIRTY_BITS_4V | + __ACCESS_BITS_4V | _PAGE_E_4V); + + _PAGE_ALL_SZ_BITS = (_PAGE_SZ16GB_4V | _PAGE_SZ2GB_4V | + _PAGE_SZ256MB_4V | _PAGE_SZ32MB_4V | + _PAGE_SZ4MB_4V | _PAGE_SZ512K_4V | + _PAGE_SZ64K_4V | _PAGE_SZ8K_4V); + + page_none = _PAGE_PRESENT_4V | _PAGE_ACCESSED_4V | page_cache4v_flag; + page_shared = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag | + __ACCESS_BITS_4V | _PAGE_WRITE_4V | _PAGE_EXEC_4V); + page_copy = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag | + __ACCESS_BITS_4V | _PAGE_EXEC_4V); + page_readonly = (_PAGE_VALID | _PAGE_PRESENT_4V | page_cache4v_flag | + __ACCESS_BITS_4V | _PAGE_EXEC_4V); + + page_exec_bit = _PAGE_EXEC_4V; + + prot_init_common(page_none, page_shared, page_copy, page_readonly, + page_exec_bit); +} + +unsigned long pte_sz_bits(unsigned long sz) +{ + if (tlb_type == hypervisor) { + switch (sz) { + case 8 * 1024: + default: + return _PAGE_SZ8K_4V; + case 64 * 1024: + return _PAGE_SZ64K_4V; + case 512 * 1024: + return _PAGE_SZ512K_4V; + case 4 * 1024 * 1024: + return _PAGE_SZ4MB_4V; + } + } else { + switch (sz) { + case 8 * 1024: + default: + return _PAGE_SZ8K_4U; + case 64 * 1024: + return _PAGE_SZ64K_4U; + case 512 * 1024: + return _PAGE_SZ512K_4U; + case 4 * 1024 * 1024: + return _PAGE_SZ4MB_4U; + } + } +} + +pte_t mk_pte_io(unsigned long page, pgprot_t prot, int space, unsigned long page_size) +{ + pte_t pte; + + pte_val(pte) = page | pgprot_val(pgprot_noncached(prot)); + pte_val(pte) |= (((unsigned long)space) << 32); + pte_val(pte) |= pte_sz_bits(page_size); + + return pte; +} + +static unsigned long kern_large_tte(unsigned long paddr) +{ + unsigned long val; + + val = (_PAGE_VALID | _PAGE_SZ4MB_4U | + _PAGE_CP_4U | _PAGE_CV_4U | _PAGE_P_4U | + _PAGE_EXEC_4U | _PAGE_L_4U | _PAGE_W_4U); + if (tlb_type == hypervisor) + val = (_PAGE_VALID | _PAGE_SZ4MB_4V | + page_cache4v_flag | _PAGE_P_4V | + _PAGE_EXEC_4V | _PAGE_W_4V); + + return val | paddr; +} + +/* If not locked, zap it. */ +void __flush_tlb_all(void) +{ + unsigned long pstate; + int i; + + __asm__ __volatile__("flushw\n\t" + "rdpr %%pstate, %0\n\t" + "wrpr %0, %1, %%pstate" + : "=r" (pstate) + : "i" (PSTATE_IE)); + if (tlb_type == hypervisor) { + sun4v_mmu_demap_all(); + } else if (tlb_type == spitfire) { + for (i = 0; i < 64; i++) { + /* Spitfire Errata #32 workaround */ + /* NOTE: Always runs on spitfire, so no + * cheetah+ page size encodings. + */ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "flush %%g6" + : /* No outputs */ + : "r" (0), + "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); + + if (!(spitfire_get_dtlb_data(i) & _PAGE_L_4U)) { + __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (TLB_TAG_ACCESS), "i" (ASI_DMMU)); + spitfire_put_dtlb_data(i, 0x0UL); + } + + /* Spitfire Errata #32 workaround */ + /* NOTE: Always runs on spitfire, so no + * cheetah+ page size encodings. + */ + __asm__ __volatile__("stxa %0, [%1] %2\n\t" + "flush %%g6" + : /* No outputs */ + : "r" (0), + "r" (PRIMARY_CONTEXT), "i" (ASI_DMMU)); + + if (!(spitfire_get_itlb_data(i) & _PAGE_L_4U)) { + __asm__ __volatile__("stxa %%g0, [%0] %1\n\t" + "membar #Sync" + : /* no outputs */ + : "r" (TLB_TAG_ACCESS), "i" (ASI_IMMU)); + spitfire_put_itlb_data(i, 0x0UL); + } + } + } else if (tlb_type == cheetah || tlb_type == cheetah_plus) { + cheetah_flush_dtlb_all(); + cheetah_flush_itlb_all(); + } + __asm__ __volatile__("wrpr %0, 0, %%pstate" + : : "r" (pstate)); +} + +static pte_t *__pte_alloc_one(struct mm_struct *mm) +{ + struct ptdesc *ptdesc = pagetable_alloc(GFP_KERNEL | __GFP_ZERO, 0); + + if (!ptdesc) + return NULL; + if (!pagetable_pte_ctor(mm, ptdesc)) { + pagetable_free(ptdesc); + return NULL; + } + return ptdesc_address(ptdesc); +} + +pte_t *pte_alloc_one_kernel(struct mm_struct *mm) +{ + return __pte_alloc_one(mm); +} + +pgtable_t pte_alloc_one(struct mm_struct *mm) +{ + return __pte_alloc_one(mm); +} + +static void __pte_free(pgtable_t pte) +{ + struct ptdesc *ptdesc = virt_to_ptdesc(pte); + + pagetable_dtor(ptdesc); + pagetable_free(ptdesc); +} + +void pte_free_kernel(struct mm_struct *mm, pte_t *pte) +{ + __pte_free(pte); +} + +void pte_free(struct mm_struct *mm, pgtable_t pte) +{ + __pte_free(pte); +} + +void pgtable_free(void *table, bool is_page) +{ + if (is_page) + __pte_free(table); + else + kmem_cache_free(pgtable_cache, table); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void pte_free_now(struct rcu_head *head) +{ + struct page *page; + + page = container_of(head, struct page, rcu_head); + __pte_free((pgtable_t)page_address(page)); +} + +void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) +{ + struct page *page; + + page = virt_to_page(pgtable); + call_rcu(&page->rcu_head, pte_free_now); +} + +void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr, + pmd_t *pmd) +{ + unsigned long pte, flags; + struct mm_struct *mm; + pmd_t entry = *pmd; + + if (!pmd_leaf(entry) || !pmd_young(entry)) + return; + + pte = pmd_val(entry); + + /* Don't insert a non-valid PMD into the TSB, we'll deadlock. */ + if (!(pte & _PAGE_VALID)) + return; + + /* We are fabricating 8MB pages using 4MB real hw pages. */ + pte |= (addr & (1UL << REAL_HPAGE_SHIFT)); + + mm = vma->vm_mm; + + spin_lock_irqsave(&mm->context.lock, flags); + + if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) + __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, + addr, pte); + + spin_unlock_irqrestore(&mm->context.lock, flags); +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +static void context_reload(void *__data) +{ + struct mm_struct *mm = __data; + + if (mm == current->mm) + load_secondary_context(mm); +} + +void hugetlb_setup(struct pt_regs *regs) +{ + struct mm_struct *mm = current->mm; + struct tsb_config *tp; + + if (faulthandler_disabled() || !mm) { + const struct exception_table_entry *entry; + + entry = search_exception_tables(regs->tpc); + if (entry) { + regs->tpc = entry->fixup; + regs->tnpc = regs->tpc + 4; + return; + } + pr_alert("Unexpected HugeTLB setup in atomic context.\n"); + die_if_kernel("HugeTSB in atomic", regs); + } + + tp = &mm->context.tsb_block[MM_TSB_HUGE]; + if (likely(tp->tsb == NULL)) + tsb_grow(mm, MM_TSB_HUGE, 0); + + tsb_context_switch(mm); + smp_tsb_sync(mm); + + /* On UltraSPARC-III+ and later, configure the second half of + * the Data-TLB for huge pages. + */ + if (tlb_type == cheetah_plus) { + bool need_context_reload = false; + unsigned long ctx; + + spin_lock_irq(&ctx_alloc_lock); + ctx = mm->context.sparc64_ctx_val; + ctx &= ~CTX_PGSZ_MASK; + ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT; + ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT; + + if (ctx != mm->context.sparc64_ctx_val) { + /* When changing the page size fields, we + * must perform a context flush so that no + * stale entries match. This flush must + * occur with the original context register + * settings. + */ + do_flush_tlb_mm(mm); + + /* Reload the context register of all processors + * also executing in this address space. + */ + mm->context.sparc64_ctx_val = ctx; + need_context_reload = true; + } + spin_unlock_irq(&ctx_alloc_lock); + + if (need_context_reload) + on_each_cpu(context_reload, mm, 0); + } +} +#endif + +static struct resource code_resource = { + .name = "Kernel code", + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM +}; + +static struct resource data_resource = { + .name = "Kernel data", + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM +}; + +static struct resource bss_resource = { + .name = "Kernel bss", + .flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM +}; + +static inline resource_size_t compute_kern_paddr(void *addr) +{ + return (resource_size_t) (addr - KERNBASE + kern_base); +} + +static void __init kernel_lds_init(void) +{ + code_resource.start = compute_kern_paddr(_text); + code_resource.end = compute_kern_paddr(_etext - 1); + data_resource.start = compute_kern_paddr(_etext); + data_resource.end = compute_kern_paddr(_edata - 1); + bss_resource.start = compute_kern_paddr(__bss_start); + bss_resource.end = compute_kern_paddr(_end - 1); +} + +static int __init report_memory(void) +{ + int i; + struct resource *res; + + kernel_lds_init(); + + for (i = 0; i < pavail_ents; i++) { + res = kzalloc_obj(struct resource); + + if (!res) { + pr_warn("Failed to allocate source.\n"); + break; + } + + res->name = "System RAM"; + res->start = pavail[i].phys_addr; + res->end = pavail[i].phys_addr + pavail[i].reg_size - 1; + res->flags = IORESOURCE_BUSY | IORESOURCE_SYSTEM_RAM; + + if (insert_resource(&iomem_resource, res) < 0) { + pr_warn("Resource insertion failed.\n"); + break; + } + + insert_resource(res, &code_resource); + insert_resource(res, &data_resource); + insert_resource(res, &bss_resource); + } + + return 0; +} +arch_initcall(report_memory); + +#ifdef CONFIG_SMP +#define do_flush_tlb_kernel_range smp_flush_tlb_kernel_range +#else +#define do_flush_tlb_kernel_range __flush_tlb_kernel_range +#endif + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + if (start < HI_OBP_ADDRESS && end > LOW_OBP_ADDRESS) { + if (start < LOW_OBP_ADDRESS) { + flush_tsb_kernel_range(start, LOW_OBP_ADDRESS); + do_flush_tlb_kernel_range(start, LOW_OBP_ADDRESS); + } + if (end > HI_OBP_ADDRESS) { + flush_tsb_kernel_range(HI_OBP_ADDRESS, end); + do_flush_tlb_kernel_range(HI_OBP_ADDRESS, end); + } + } else { + flush_tsb_kernel_range(start, end); + do_flush_tlb_kernel_range(start, end); + } +} + +void copy_user_highpage(struct page *to, struct page *from, + unsigned long vaddr, struct vm_area_struct *vma) +{ + char *vfrom, *vto; + + vfrom = kmap_atomic(from); + vto = kmap_atomic(to); + copy_user_page(vto, vfrom, vaddr, to); + kunmap_atomic(vto); + kunmap_atomic(vfrom); + + /* If this page has ADI enabled, copy over any ADI tags + * as well + */ + if (vma->vm_flags & VM_SPARC_ADI) { + unsigned long pfrom, pto, i, adi_tag; + + pfrom = page_to_phys(from); + pto = page_to_phys(to); + + for (i = pfrom; i < (pfrom + PAGE_SIZE); i += adi_blksize()) { + asm volatile("ldxa [%1] %2, %0\n\t" + : "=r" (adi_tag) + : "r" (i), "i" (ASI_MCD_REAL)); + asm volatile("stxa %0, [%1] %2\n\t" + : + : "r" (adi_tag), "r" (pto), + "i" (ASI_MCD_REAL)); + pto += adi_blksize(); + } + asm volatile("membar #Sync\n\t"); + } +} +EXPORT_SYMBOL(copy_user_highpage); + +void copy_highpage(struct page *to, struct page *from) +{ + char *vfrom, *vto; + + vfrom = kmap_atomic(from); + vto = kmap_atomic(to); + copy_page(vto, vfrom); + kunmap_atomic(vto); + kunmap_atomic(vfrom); + + /* If this platform is ADI enabled, copy any ADI tags + * as well + */ + if (adi_capable()) { + unsigned long pfrom, pto, i, adi_tag; + + pfrom = page_to_phys(from); + pto = page_to_phys(to); + + for (i = pfrom; i < (pfrom + PAGE_SIZE); i += adi_blksize()) { + asm volatile("ldxa [%1] %2, %0\n\t" + : "=r" (adi_tag) + : "r" (i), "i" (ASI_MCD_REAL)); + asm volatile("stxa %0, [%1] %2\n\t" + : + : "r" (adi_tag), "r" (pto), + "i" (ASI_MCD_REAL)); + pto += adi_blksize(); + } + asm volatile("membar #Sync\n\t"); + } +} +EXPORT_SYMBOL(copy_highpage); + +pgprot_t vm_get_page_prot(vm_flags_t vm_flags) +{ + unsigned long prot = pgprot_val(protection_map[vm_flags & + (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]); + + if (vm_flags & VM_SPARC_ADI) + prot |= _PAGE_MCD_4V; + + return __pgprot(prot); +} +EXPORT_SYMBOL(vm_get_page_prot); diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h new file mode 100644 index 000000000..d920a75b5 --- /dev/null +++ b/arch/sparc/mm/init_64.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _SPARC64_MM_INIT_H +#define _SPARC64_MM_INIT_H + +#include + +/* Most of the symbols in this file are defined in init.c and + * marked non-static so that assembler code can get at them. + */ + +#define MAX_PHYS_ADDRESS (1UL << MAX_PHYS_ADDRESS_BITS) + +extern unsigned long kern_linear_pte_xor[4]; +extern unsigned int sparc64_highest_unlocked_tlb_ent; +extern unsigned long sparc64_kern_pri_context; +extern unsigned long sparc64_kern_pri_nuc_bits; +extern unsigned long sparc64_kern_sec_context; +void mmu_info(struct seq_file *m); + +struct linux_prom_translation { + unsigned long virt; + unsigned long size; + unsigned long data; +}; + +/* Exported for kernel TLB miss handling in ktlb.S */ +extern struct linux_prom_translation prom_trans[512]; +extern unsigned int prom_trans_ents; + +/* Exported for SMP bootup purposes. */ +extern unsigned long kern_locked_tte_data; + +void prom_world(int enter); + +#endif /* _SPARC64_MM_INIT_H */ diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c new file mode 100644 index 000000000..2b5e40cc9 --- /dev/null +++ b/arch/sparc/mm/io-unit.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * io-unit.c: IO-UNIT specific routines for memory management. + * + * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "mm_32.h" + +/* #define IOUNIT_DEBUG */ +#ifdef IOUNIT_DEBUG +#define IOD(x) printk(x) +#else +#define IOD(x) do { } while (0) +#endif + +#define IOPERM (IOUPTE_CACHE | IOUPTE_WRITE | IOUPTE_VALID) +#define MKIOPTE(phys) __iopte((((phys)>>4) & IOUPTE_PAGE) | IOPERM) + +static const struct dma_map_ops iounit_dma_ops; + +static void __init iounit_iommu_init(struct platform_device *op) +{ + struct iounit_struct *iounit; + iopte_t __iomem *xpt; + iopte_t __iomem *xptend; + + iounit = kzalloc_obj(struct iounit_struct, GFP_ATOMIC); + if (!iounit) { + prom_printf("SUN4D: Cannot alloc iounit, halting.\n"); + prom_halt(); + } + + iounit->limit[0] = IOUNIT_BMAP1_START; + iounit->limit[1] = IOUNIT_BMAP2_START; + iounit->limit[2] = IOUNIT_BMAPM_START; + iounit->limit[3] = IOUNIT_BMAPM_END; + iounit->rotor[1] = IOUNIT_BMAP2_START; + iounit->rotor[2] = IOUNIT_BMAPM_START; + + xpt = of_ioremap(&op->resource[2], 0, PAGE_SIZE * 16, "XPT"); + if (!xpt) { + prom_printf("SUN4D: Cannot map External Page Table."); + prom_halt(); + } + + op->dev.archdata.iommu = iounit; + iounit->page_table = xpt; + spin_lock_init(&iounit->lock); + + xptend = iounit->page_table + (16 * PAGE_SIZE) / sizeof(iopte_t); + for (; xpt < xptend; xpt++) + sbus_writel(0, xpt); + + op->dev.dma_ops = &iounit_dma_ops; +} + +static int __init iounit_init(void) +{ + extern void sun4d_init_sbi_irq(void); + struct device_node *dp; + + for_each_node_by_name(dp, "sbi") { + struct platform_device *op = of_find_device_by_node(dp); + + iounit_iommu_init(op); + of_propagate_archdata(op); + } + + sun4d_init_sbi_irq(); + + return 0; +} + +subsys_initcall(iounit_init); + +/* One has to hold iounit->lock to call this */ +static dma_addr_t iounit_get_area(struct iounit_struct *iounit, + phys_addr_t phys, int size) +{ + int i, j, k, npages; + unsigned long rotor, scan, limit; + iopte_t iopte; + + npages = (offset_in_page(phys) + size + (PAGE_SIZE - 1)) >> PAGE_SHIFT; + + /* A tiny bit of magic ingredience :) */ + switch (npages) { + case 1: i = 0x0231; break; + case 2: i = 0x0132; break; + default: i = 0x0213; break; + } + + IOD(("%s(%pa,%d[%d])=", __func__, &phys, size, npages)); + +next: j = (i & 15); + rotor = iounit->rotor[j - 1]; + limit = iounit->limit[j]; + scan = rotor; +nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan); + if (scan + npages > limit) { + if (limit != rotor) { + limit = rotor; + scan = iounit->limit[j - 1]; + goto nexti; + } + i >>= 4; + if (!(i & 15)) + panic("iounit_get_area: Couldn't find free iopte slots for (%pa,%d)\n", + &phys, size); + goto next; + } + for (k = 1, scan++; k < npages; k++) + if (test_bit(scan++, iounit->bmap)) + goto nexti; + iounit->rotor[j - 1] = (scan < limit) ? scan : iounit->limit[j - 1]; + scan -= npages; + iopte = MKIOPTE(phys & PAGE_MASK); + phys = IOUNIT_DMA_BASE + (scan << PAGE_SHIFT) + offset_in_page(phys); + for (k = 0; k < npages; k++, iopte = __iopte(iopte_val(iopte) + 0x100), scan++) { + set_bit(scan, iounit->bmap); + sbus_writel(iopte_val(iopte), &iounit->page_table[scan]); + } + IOD(("%pa\n", &phys)); + return phys; +} + +static dma_addr_t iounit_map_phys(struct device *dev, phys_addr_t phys, + size_t len, enum dma_data_direction dir, unsigned long attrs) +{ + struct iounit_struct *iounit = dev->archdata.iommu; + unsigned long flags; + dma_addr_t ret; + + /* XXX So what is maxphys for us and how do drivers know it? */ + if (!len || len > 256 * 1024) + return DMA_MAPPING_ERROR; + + spin_lock_irqsave(&iounit->lock, flags); + ret = iounit_get_area(iounit, phys, len); + spin_unlock_irqrestore(&iounit->lock, flags); + return ret; +} + +static int iounit_map_sg(struct device *dev, struct scatterlist *sgl, int nents, + enum dma_data_direction dir, unsigned long attrs) +{ + struct iounit_struct *iounit = dev->archdata.iommu; + struct scatterlist *sg; + unsigned long flags; + int i; + + /* FIXME: Cache some resolved pages - often several sg entries are to the same page */ + spin_lock_irqsave(&iounit->lock, flags); + for_each_sg(sgl, sg, nents, i) { + sg->dma_address = + iounit_get_area(iounit, sg_phys(sg), sg->length); + sg->dma_length = sg->length; + } + spin_unlock_irqrestore(&iounit->lock, flags); + return nents; +} + +static void iounit_unmap_phys(struct device *dev, dma_addr_t vaddr, size_t len, + enum dma_data_direction dir, unsigned long attrs) +{ + struct iounit_struct *iounit = dev->archdata.iommu; + unsigned long flags; + + spin_lock_irqsave(&iounit->lock, flags); + len = ((vaddr & ~PAGE_MASK) + len + (PAGE_SIZE-1)) >> PAGE_SHIFT; + vaddr = (vaddr - IOUNIT_DMA_BASE) >> PAGE_SHIFT; + IOD(("iounit_release %08lx-%08lx\n", (long)vaddr, (long)len+vaddr)); + for (len += vaddr; vaddr < len; vaddr++) + clear_bit(vaddr, iounit->bmap); + spin_unlock_irqrestore(&iounit->lock, flags); +} + +static void iounit_unmap_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + struct iounit_struct *iounit = dev->archdata.iommu; + unsigned long flags, vaddr, len; + struct scatterlist *sg; + int i; + + spin_lock_irqsave(&iounit->lock, flags); + for_each_sg(sgl, sg, nents, i) { + len = ((sg->dma_address & ~PAGE_MASK) + sg->length + (PAGE_SIZE-1)) >> PAGE_SHIFT; + vaddr = (sg->dma_address - IOUNIT_DMA_BASE) >> PAGE_SHIFT; + IOD(("iounit_release %08lx-%08lx\n", (long)vaddr, (long)len+vaddr)); + for (len += vaddr; vaddr < len; vaddr++) + clear_bit(vaddr, iounit->bmap); + } + spin_unlock_irqrestore(&iounit->lock, flags); +} + +#ifdef CONFIG_SBUS +static void *iounit_alloc(struct device *dev, size_t len, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +{ + struct iounit_struct *iounit = dev->archdata.iommu; + unsigned long va, addr, page, end, ret; + pgprot_t dvma_prot; + iopte_t __iomem *iopte; + + /* XXX So what is maxphys for us and how do drivers know it? */ + if (!len || len > 256 * 1024) + return NULL; + + len = PAGE_ALIGN(len); + va = __get_free_pages(gfp | __GFP_ZERO, get_order(len)); + if (!va) + return NULL; + + addr = ret = sparc_dma_alloc_resource(dev, len); + if (!addr) + goto out_free_pages; + *dma_handle = addr; + + dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); + end = PAGE_ALIGN((addr + len)); + while(addr < end) { + page = va; + { + pmd_t *pmdp; + pte_t *ptep; + long i; + + pmdp = pmd_off_k(addr); + ptep = pte_offset_kernel(pmdp, addr); + + set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); + + i = ((addr - IOUNIT_DMA_BASE) >> PAGE_SHIFT); + + iopte = iounit->page_table + i; + sbus_writel(iopte_val(MKIOPTE(__pa(page))), iopte); + } + addr += PAGE_SIZE; + va += PAGE_SIZE; + } + flush_cache_all(); + flush_tlb_all(); + + return (void *)ret; + +out_free_pages: + free_pages(va, get_order(len)); + return NULL; +} + +static void iounit_free(struct device *dev, size_t size, void *cpu_addr, + dma_addr_t dma_addr, unsigned long attrs) +{ + /* XXX Somebody please fill this in */ +} +#endif + +static const struct dma_map_ops iounit_dma_ops = { +#ifdef CONFIG_SBUS + .alloc = iounit_alloc, + .free = iounit_free, +#endif + .map_phys = iounit_map_phys, + .unmap_phys = iounit_unmap_phys, + .map_sg = iounit_map_sg, + .unmap_sg = iounit_unmap_sg, +}; diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c new file mode 100644 index 000000000..df0eb99a4 --- /dev/null +++ b/arch/sparc/mm/iommu.c @@ -0,0 +1,457 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * iommu.c: IOMMU specific routines for memory management. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mm_32.h" + +/* + * This can be sized dynamically, but we will do this + * only when we have a guidance about actual I/O pressures. + */ +#define IOMMU_RNGE IOMMU_RNGE_256MB +#define IOMMU_START 0xF0000000 +#define IOMMU_WINSIZE (256*1024*1024U) +#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 256KB */ +#define IOMMU_ORDER 6 /* 4096 * (1<<6) */ + +static int viking_flush; +/* viking.S */ +extern void viking_flush_page(unsigned long page); +extern void viking_mxcc_flush_page(unsigned long page); + +/* + * Values precomputed according to CPU type. + */ +static unsigned int ioperm_noc; /* Consistent mapping iopte flags */ +static pgprot_t dvma_prot; /* Consistent mapping pte flags */ + +#define IOPERM (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID) +#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ) + +static const struct dma_map_ops sbus_iommu_dma_gflush_ops; +static const struct dma_map_ops sbus_iommu_dma_pflush_ops; + +static void __init sbus_iommu_init(struct platform_device *op) +{ + struct iommu_struct *iommu; + unsigned int impl, vers; + unsigned long *bitmap; + unsigned long control; + unsigned long base; + unsigned long tmp; + + iommu = kmalloc_obj(struct iommu_struct); + if (!iommu) { + prom_printf("Unable to allocate iommu structure\n"); + prom_halt(); + } + + iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3, + "iommu_regs"); + if (!iommu->regs) { + prom_printf("Cannot map IOMMU registers\n"); + prom_halt(); + } + + control = sbus_readl(&iommu->regs->control); + impl = (control & IOMMU_CTRL_IMPL) >> 28; + vers = (control & IOMMU_CTRL_VERS) >> 24; + control &= ~(IOMMU_CTRL_RNGE); + control |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB); + sbus_writel(control, &iommu->regs->control); + + iommu_invalidate(iommu->regs); + iommu->start = IOMMU_START; + iommu->end = 0xffffffff; + + /* Allocate IOMMU page table */ + /* Stupid alignment constraints give me a headache. + We need 256K or 512K or 1M or 2M area aligned to + its size and current gfp will fortunately give + it to us. */ + tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER); + if (!tmp) { + prom_printf("Unable to allocate iommu table [0x%lx]\n", + IOMMU_NPTES * sizeof(iopte_t)); + prom_halt(); + } + iommu->page_table = (iopte_t *)tmp; + + /* Initialize new table. */ + memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t)); + flush_cache_all(); + flush_tlb_all(); + + base = __pa((unsigned long)iommu->page_table) >> 4; + sbus_writel(base, &iommu->regs->base); + iommu_invalidate(iommu->regs); + + bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL); + if (!bitmap) { + prom_printf("Unable to allocate iommu bitmap [%d]\n", + (int)(IOMMU_NPTES>>3)); + prom_halt(); + } + bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES); + /* To be coherent on HyperSparc, the page color of DVMA + * and physical addresses must match. + */ + if (srmmu_modtype == HyperSparc) + iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT; + else + iommu->usemap.num_colors = 1; + + printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n", + impl, vers, iommu->page_table, + (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES); + + op->dev.archdata.iommu = iommu; + + if (flush_page_for_dma_global) + op->dev.dma_ops = &sbus_iommu_dma_gflush_ops; + else + op->dev.dma_ops = &sbus_iommu_dma_pflush_ops; +} + +static int __init iommu_init(void) +{ + struct device_node *dp; + + for_each_node_by_name(dp, "iommu") { + struct platform_device *op = of_find_device_by_node(dp); + + sbus_iommu_init(op); + of_propagate_archdata(op); + } + + return 0; +} + +subsys_initcall(iommu_init); + +/* Flush the iotlb entries to ram. */ +/* This could be better if we didn't have to flush whole pages. */ +static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) +{ + unsigned long start; + unsigned long end; + + start = (unsigned long)iopte; + end = PAGE_ALIGN(start + niopte*sizeof(iopte_t)); + start &= PAGE_MASK; + if (viking_mxcc_present) { + while(start < end) { + viking_mxcc_flush_page(start); + start += PAGE_SIZE; + } + } else if (viking_flush) { + while(start < end) { + viking_flush_page(start); + start += PAGE_SIZE; + } + } else { + while(start < end) { + __flush_page_to_ram(start); + start += PAGE_SIZE; + } + } +} + +static dma_addr_t __sbus_iommu_map_phys(struct device *dev, phys_addr_t paddr, + size_t len, bool per_page_flush, unsigned long attrs) +{ + struct iommu_struct *iommu = dev->archdata.iommu; + unsigned long off = offset_in_page(paddr); + unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; + unsigned long pfn = __phys_to_pfn(paddr); + unsigned int busa, busa0; + iopte_t *iopte, *iopte0; + int ioptex, i; + + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; + + /* XXX So what is maxphys for us and how do drivers know it? */ + if (!len || len > 256 * 1024) + return DMA_MAPPING_ERROR; + + /* + * We expect unmapped highmem pages to be not in the cache. + * XXX Is this a good assumption? + * XXX What if someone else unmaps it here and races us? + */ + if (per_page_flush && !PhysHighMem(paddr)) { + unsigned long vaddr, p; + + vaddr = (unsigned long)phys_to_virt(paddr); + for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE) + flush_page_for_dma(p); + } + + /* page color = pfn of page */ + ioptex = bit_map_string_get(&iommu->usemap, npages, pfn); + if (ioptex < 0) + panic("iommu out"); + busa0 = iommu->start + (ioptex << PAGE_SHIFT); + iopte0 = &iommu->page_table[ioptex]; + + busa = busa0; + iopte = iopte0; + for (i = 0; i < npages; i++) { + iopte_val(*iopte) = MKIOPTE(pfn, IOPERM); + iommu_invalidate_page(iommu->regs, busa); + busa += PAGE_SIZE; + iopte++; + pfn++; + } + + iommu_flush_iotlb(iopte0, npages); + return busa0 + off; +} + +static dma_addr_t sbus_iommu_map_phys_gflush(struct device *dev, + phys_addr_t phys, size_t len, enum dma_data_direction dir, + unsigned long attrs) +{ + flush_page_for_dma(0); + return __sbus_iommu_map_phys(dev, phys, len, false, attrs); +} + +static dma_addr_t sbus_iommu_map_phys_pflush(struct device *dev, + phys_addr_t phys, size_t len, enum dma_data_direction dir, + unsigned long attrs) +{ + return __sbus_iommu_map_phys(dev, phys, len, true, attrs); +} + +static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs, + bool per_page_flush) +{ + struct scatterlist *sg; + int j; + + for_each_sg(sgl, sg, nents, j) { + sg->dma_address = __sbus_iommu_map_phys(dev, sg_phys(sg), + sg->length, per_page_flush, attrs); + if (sg->dma_address == DMA_MAPPING_ERROR) + return -EIO; + sg->dma_length = sg->length; + } + + return nents; +} + +static int sbus_iommu_map_sg_gflush(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + flush_page_for_dma(0); + return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, false); +} + +static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true); +} + +static void sbus_iommu_unmap_phys(struct device *dev, dma_addr_t dma_addr, + size_t len, enum dma_data_direction dir, unsigned long attrs) +{ + struct iommu_struct *iommu = dev->archdata.iommu; + unsigned int busa = dma_addr & PAGE_MASK; + unsigned long off = dma_addr & ~PAGE_MASK; + unsigned int npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT; + unsigned int ioptex = (busa - iommu->start) >> PAGE_SHIFT; + unsigned int i; + + BUG_ON(busa < iommu->start); + for (i = 0; i < npages; i++) { + iopte_val(iommu->page_table[ioptex + i]) = 0; + iommu_invalidate_page(iommu->regs, busa); + busa += PAGE_SIZE; + } + bit_map_clear(&iommu->usemap, ioptex, npages); +} + +static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl, + int nents, enum dma_data_direction dir, unsigned long attrs) +{ + struct scatterlist *sg; + int i; + + for_each_sg(sgl, sg, nents, i) { + sbus_iommu_unmap_phys(dev, sg->dma_address, sg->length, dir, + attrs); + sg->dma_address = 0x21212121; + } +} + +#ifdef CONFIG_SBUS +static void *sbus_iommu_alloc(struct device *dev, size_t len, + dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) +{ + struct iommu_struct *iommu = dev->archdata.iommu; + unsigned long va, addr, page, end, ret; + iopte_t *iopte = iommu->page_table; + iopte_t *first; + int ioptex; + + /* XXX So what is maxphys for us and how do drivers know it? */ + if (!len || len > 256 * 1024) + return NULL; + + len = PAGE_ALIGN(len); + va = __get_free_pages(gfp | __GFP_ZERO, get_order(len)); + if (va == 0) + return NULL; + + addr = ret = sparc_dma_alloc_resource(dev, len); + if (!addr) + goto out_free_pages; + + BUG_ON((va & ~PAGE_MASK) != 0); + BUG_ON((addr & ~PAGE_MASK) != 0); + BUG_ON((len & ~PAGE_MASK) != 0); + + /* page color = physical address */ + ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT, + addr >> PAGE_SHIFT); + if (ioptex < 0) + panic("iommu out"); + + iopte += ioptex; + first = iopte; + end = addr + len; + while(addr < end) { + page = va; + { + pmd_t *pmdp; + pte_t *ptep; + + if (viking_mxcc_present) + viking_mxcc_flush_page(page); + else if (viking_flush) + viking_flush_page(page); + else + __flush_page_to_ram(page); + + pmdp = pmd_off_k(addr); + ptep = pte_offset_kernel(pmdp, addr); + + set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot)); + } + iopte_val(*iopte++) = + MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc); + addr += PAGE_SIZE; + va += PAGE_SIZE; + } + /* P3: why do we need this? + * + * DAVEM: Because there are several aspects, none of which + * are handled by a single interface. Some cpus are + * completely not I/O DMA coherent, and some have + * virtually indexed caches. The driver DMA flushing + * methods handle the former case, but here during + * IOMMU page table modifications, and usage of non-cacheable + * cpu mappings of pages potentially in the cpu caches, we have + * to handle the latter case as well. + */ + flush_cache_all(); + iommu_flush_iotlb(first, len >> PAGE_SHIFT); + flush_tlb_all(); + iommu_invalidate(iommu->regs); + + *dma_handle = iommu->start + (ioptex << PAGE_SHIFT); + return (void *)ret; + +out_free_pages: + free_pages(va, get_order(len)); + return NULL; +} + +static void sbus_iommu_free(struct device *dev, size_t len, void *cpu_addr, + dma_addr_t busa, unsigned long attrs) +{ + struct iommu_struct *iommu = dev->archdata.iommu; + iopte_t *iopte = iommu->page_table; + struct page *page = virt_to_page(cpu_addr); + int ioptex = (busa - iommu->start) >> PAGE_SHIFT; + unsigned long end; + + if (!sparc_dma_free_resource(cpu_addr, len)) + return; + + BUG_ON((busa & ~PAGE_MASK) != 0); + BUG_ON((len & ~PAGE_MASK) != 0); + + iopte += ioptex; + end = busa + len; + while (busa < end) { + iopte_val(*iopte++) = 0; + busa += PAGE_SIZE; + } + flush_tlb_all(); + iommu_invalidate(iommu->regs); + bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT); + + __free_pages(page, get_order(len)); +} +#endif + +static const struct dma_map_ops sbus_iommu_dma_gflush_ops = { +#ifdef CONFIG_SBUS + .alloc = sbus_iommu_alloc, + .free = sbus_iommu_free, +#endif + .map_phys = sbus_iommu_map_phys_gflush, + .unmap_phys = sbus_iommu_unmap_phys, + .map_sg = sbus_iommu_map_sg_gflush, + .unmap_sg = sbus_iommu_unmap_sg, +}; + +static const struct dma_map_ops sbus_iommu_dma_pflush_ops = { +#ifdef CONFIG_SBUS + .alloc = sbus_iommu_alloc, + .free = sbus_iommu_free, +#endif + .map_phys = sbus_iommu_map_phys_pflush, + .unmap_phys = sbus_iommu_unmap_phys, + .map_sg = sbus_iommu_map_sg_pflush, + .unmap_sg = sbus_iommu_unmap_sg, +}; + +void __init ld_mmu_iommu(void) +{ + if (viking_mxcc_present || srmmu_modtype == HyperSparc) { + dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV); + ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID; + } else { + dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV); + ioperm_noc = IOPTE_WRITE | IOPTE_VALID; + } +} diff --git a/arch/sparc/mm/leon_mm.c b/arch/sparc/mm/leon_mm.c new file mode 100644 index 000000000..1dc9b3d70 --- /dev/null +++ b/arch/sparc/mm/leon_mm.c @@ -0,0 +1,346 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/arch/sparc/mm/leon_m.c + * + * Copyright (C) 2004 Konrad Eisele (eiselekd@web.de, konrad@gaisler.com) Gaisler Research + * Copyright (C) 2009 Daniel Hellstrom (daniel@gaisler.com) Aeroflex Gaisler AB + * Copyright (C) 2009 Konrad Eisele (konrad@gaisler.com) Aeroflex Gaisler AB + * + * do srmmu probe in software + * + */ + +#include +#include +#include +#include +#include + +#include "mm_32.h" + +int leon_flush_during_switch = 1; +static int srmmu_swprobe_trace; + +static inline unsigned long leon_get_ctable_ptr(void) +{ + unsigned int retval; + + __asm__ __volatile__("lda [%1] %2, %0\n\t" : + "=r" (retval) : + "r" (SRMMU_CTXTBL_PTR), + "i" (ASI_LEON_MMUREGS)); + return (retval & SRMMU_CTX_PMASK) << 4; +} + + +unsigned long leon_swprobe(unsigned long vaddr, unsigned long *paddr) +{ + + unsigned int ctxtbl; + unsigned int pgd, pmd, ped; + unsigned int ptr; + unsigned int lvl, pte; + unsigned int ctx; + unsigned int paddr_calc; + + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: trace on\n"); + + ctxtbl = leon_get_ctable_ptr(); + if (!(ctxtbl)) { + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: leon_get_ctable_ptr returned 0=>0\n"); + return 0; + } + if (!_pfn_valid(PFN(ctxtbl))) { + if (srmmu_swprobe_trace) + printk(KERN_INFO + "swprobe: !_pfn_valid(%x)=>0\n", + PFN(ctxtbl)); + return 0; + } + + ctx = srmmu_get_context(); + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: --- ctx (%x) ---\n", ctx); + + pgd = LEON_BYPASS_LOAD_PA(ctxtbl + (ctx * 4)); + + if (((pgd & SRMMU_ET_MASK) == SRMMU_ET_PTE)) { + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: pgd is entry level 3\n"); + lvl = 3; + pte = pgd; + goto ready; + } + if (((pgd & SRMMU_ET_MASK) != SRMMU_ET_PTD)) { + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: pgd is invalid => 0\n"); + return 0; + } + + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: --- pgd (%x) ---\n", pgd); + + ptr = (pgd & SRMMU_PTD_PMASK) << 4; + ptr += ((((vaddr) >> LEON_PGD_SH) & LEON_PGD_M) * 4); + if (!_pfn_valid(PFN(ptr))) + return 0; + + pmd = LEON_BYPASS_LOAD_PA(ptr); + if (((pmd & SRMMU_ET_MASK) == SRMMU_ET_PTE)) { + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: pmd is entry level 2\n"); + lvl = 2; + pte = pmd; + goto ready; + } + if (((pmd & SRMMU_ET_MASK) != SRMMU_ET_PTD)) { + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: pmd is invalid => 0\n"); + return 0; + } + + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: --- pmd (%x) ---\n", pmd); + + ptr = (pmd & SRMMU_PTD_PMASK) << 4; + ptr += (((vaddr >> LEON_PMD_SH) & LEON_PMD_M) * 4); + if (!_pfn_valid(PFN(ptr))) { + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: !_pfn_valid(%x)=>0\n", + PFN(ptr)); + return 0; + } + + ped = LEON_BYPASS_LOAD_PA(ptr); + + if (((ped & SRMMU_ET_MASK) == SRMMU_ET_PTE)) { + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: ped is entry level 1\n"); + lvl = 1; + pte = ped; + goto ready; + } + if (((ped & SRMMU_ET_MASK) != SRMMU_ET_PTD)) { + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: ped is invalid => 0\n"); + return 0; + } + + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: --- ped (%x) ---\n", ped); + + ptr = (ped & SRMMU_PTD_PMASK) << 4; + ptr += (((vaddr >> LEON_PTE_SH) & LEON_PTE_M) * 4); + if (!_pfn_valid(PFN(ptr))) + return 0; + + ptr = LEON_BYPASS_LOAD_PA(ptr); + if (((ptr & SRMMU_ET_MASK) == SRMMU_ET_PTE)) { + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: ptr is entry level 0\n"); + lvl = 0; + pte = ptr; + goto ready; + } + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: ptr is invalid => 0\n"); + return 0; + +ready: + switch (lvl) { + case 0: + paddr_calc = + (vaddr & ~(-1 << LEON_PTE_SH)) | ((pte & ~0xff) << 4); + break; + case 1: + paddr_calc = + (vaddr & ~(-1 << LEON_PMD_SH)) | ((pte & ~0xff) << 4); + break; + case 2: + paddr_calc = + (vaddr & ~(-1 << LEON_PGD_SH)) | ((pte & ~0xff) << 4); + break; + default: + case 3: + paddr_calc = vaddr; + break; + } + if (srmmu_swprobe_trace) + printk(KERN_INFO "swprobe: padde %x\n", paddr_calc); + if (paddr) + *paddr = paddr_calc; + return pte; +} + +void leon_flush_icache_all(void) +{ + __asm__ __volatile__(" flush "); /*iflush*/ +} + +void leon_flush_dcache_all(void) +{ + __asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" : : + "i"(ASI_LEON_DFLUSH) : "memory"); +} + +void leon_flush_pcache_all(struct vm_area_struct *vma, unsigned long page) +{ + if (vma->vm_flags & VM_EXEC) + leon_flush_icache_all(); + leon_flush_dcache_all(); +} + +void leon_flush_cache_all(void) +{ + __asm__ __volatile__(" flush "); /*iflush*/ + __asm__ __volatile__("sta %%g0, [%%g0] %0\n\t" : : + "i"(ASI_LEON_DFLUSH) : "memory"); +} + +void leon_flush_tlb_all(void) +{ + leon_flush_cache_all(); + __asm__ __volatile__("sta %%g0, [%0] %1\n\t" : : "r"(0x400), + "i"(ASI_LEON_MMUFLUSH) : "memory"); +} + +/* get all cache regs */ +void leon3_getCacheRegs(struct leon3_cacheregs *regs) +{ + unsigned long ccr, iccr, dccr; + + if (!regs) + return; + /* Get Cache regs from "Cache ASI" address 0x0, 0x8 and 0xC */ + __asm__ __volatile__("lda [%%g0] %3, %0\n\t" + "mov 0x08, %%g1\n\t" + "lda [%%g1] %3, %1\n\t" + "mov 0x0c, %%g1\n\t" + "lda [%%g1] %3, %2\n\t" + : "=r"(ccr), "=r"(iccr), "=r"(dccr) + /* output */ + : "i"(ASI_LEON_CACHEREGS) /* input */ + : "g1" /* clobber list */ + ); + regs->ccr = ccr; + regs->iccr = iccr; + regs->dccr = dccr; +} + +/* Due to virtual cache we need to check cache configuration if + * it is possible to skip flushing in some cases. + * + * Leon2 and Leon3 differ in their way of telling cache information + * + */ +int __init leon_flush_needed(void) +{ + int flush_needed = -1; + unsigned int ssize, sets; + char *setStr[4] = + { "direct mapped", "2-way associative", "3-way associative", + "4-way associative" + }; + /* leon 3 */ + struct leon3_cacheregs cregs; + leon3_getCacheRegs(&cregs); + sets = (cregs.dccr & LEON3_XCCR_SETS_MASK) >> 24; + /* (ssize=>realsize) 0=>1k, 1=>2k, 2=>4k, 3=>8k ... */ + ssize = 1 << ((cregs.dccr & LEON3_XCCR_SSIZE_MASK) >> 20); + + printk(KERN_INFO "CACHE: %s cache, set size %dk\n", + sets > 3 ? "unknown" : setStr[sets], ssize); + if ((ssize <= (PAGE_SIZE / 1024)) && (sets == 0)) { + /* Set Size <= Page size ==> + flush on every context switch not needed. */ + flush_needed = 0; + printk(KERN_INFO "CACHE: not flushing on every context switch\n"); + } + return flush_needed; +} + +void leon_switch_mm(void) +{ + flush_tlb_mm((void *)0); + if (leon_flush_during_switch) + leon_flush_cache_all(); +} + +static void leon_flush_cache_mm(struct mm_struct *mm) +{ + leon_flush_cache_all(); +} + +static void leon_flush_cache_page(struct vm_area_struct *vma, unsigned long page) +{ + leon_flush_pcache_all(vma, page); +} + +static void leon_flush_cache_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + leon_flush_cache_all(); +} + +static void leon_flush_tlb_mm(struct mm_struct *mm) +{ + leon_flush_tlb_all(); +} + +static void leon_flush_tlb_page(struct vm_area_struct *vma, + unsigned long page) +{ + leon_flush_tlb_all(); +} + +static void leon_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + leon_flush_tlb_all(); +} + +static void leon_flush_page_to_ram(unsigned long page) +{ + leon_flush_cache_all(); +} + +static void leon_flush_sig_insns(struct mm_struct *mm, unsigned long page) +{ + leon_flush_cache_all(); +} + +static void leon_flush_page_for_dma(unsigned long page) +{ + leon_flush_dcache_all(); +} + +void __init poke_leonsparc(void) +{ +} + +static const struct sparc32_cachetlb_ops leon_ops = { + .cache_all = leon_flush_cache_all, + .cache_mm = leon_flush_cache_mm, + .cache_page = leon_flush_cache_page, + .cache_range = leon_flush_cache_range, + .tlb_all = leon_flush_tlb_all, + .tlb_mm = leon_flush_tlb_mm, + .tlb_page = leon_flush_tlb_page, + .tlb_range = leon_flush_tlb_range, + .page_to_ram = leon_flush_page_to_ram, + .sig_insns = leon_flush_sig_insns, + .page_for_dma = leon_flush_page_for_dma, +}; + +void __init init_leon(void) +{ + srmmu_name = "LEON"; + sparc32_cachetlb_ops = &leon_ops; + poke_srmmu = poke_leonsparc; + + leon_flush_during_switch = leon_flush_needed(); +} diff --git a/arch/sparc/mm/mm_32.h b/arch/sparc/mm/mm_32.h new file mode 100644 index 000000000..ee55f1080 --- /dev/null +++ b/arch/sparc/mm/mm_32.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* fault_32.c - visible as they are called from assembler */ +asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, + unsigned long address); + +void window_overflow_fault(void); +void window_underflow_fault(unsigned long sp); +void window_ret_fault(struct pt_regs *regs); + +/* srmmu.c */ +extern char *srmmu_name; +extern int viking_mxcc_present; +extern int flush_page_for_dma_global; + +extern void (*poke_srmmu)(void); + +void __init srmmu_paging_init(void); + +/* iommu.c */ +void ld_mmu_iommu(void); diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c new file mode 100644 index 000000000..9a74902ad --- /dev/null +++ b/arch/sparc/mm/srmmu.c @@ -0,0 +1,1773 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * srmmu.c: SRMMU specific routines for memory management. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Now the cpu specific definitions. */ +#include +#include +#include +#include +#include +#include +#include + +#include "mm_32.h" + +enum mbus_module srmmu_modtype; +static unsigned int hwbug_bitmask; +int vac_cache_size; +EXPORT_SYMBOL(vac_cache_size); +int vac_line_size; + +extern struct resource sparc_iomap; + +extern unsigned long last_valid_pfn; + +static pgd_t *srmmu_swapper_pg_dir; + +const struct sparc32_cachetlb_ops *sparc32_cachetlb_ops; +EXPORT_SYMBOL(sparc32_cachetlb_ops); + +#ifdef CONFIG_SMP +const struct sparc32_cachetlb_ops *local_ops; + +#define FLUSH_BEGIN(mm) +#define FLUSH_END +#else +#define FLUSH_BEGIN(mm) if ((mm)->context != NO_CONTEXT) { +#define FLUSH_END } +#endif + +int flush_page_for_dma_global = 1; + +char *srmmu_name; + +ctxd_t *srmmu_ctx_table_phys; +static ctxd_t *srmmu_context_table; + +int viking_mxcc_present; +static DEFINE_SPINLOCK(srmmu_context_spinlock); + +static int is_hypersparc; + +static int srmmu_cache_pagetables; + +/* these will be initialized in srmmu_nocache_calcsize() */ +static unsigned long srmmu_nocache_size; +static unsigned long srmmu_nocache_end; + +/* 1 bit <=> 256 bytes of nocache <=> 64 PTEs */ +#define SRMMU_NOCACHE_BITMAP_SHIFT (PAGE_SHIFT - 4) + +/* The context table is a nocache user with the biggest alignment needs. */ +#define SRMMU_NOCACHE_ALIGN_MAX (sizeof(ctxd_t)*SRMMU_MAX_CONTEXTS) + +void *srmmu_nocache_pool; +static struct bit_map srmmu_nocache_map; + +static inline int srmmu_pmd_none(pmd_t pmd) +{ return !(pmd_val(pmd) & 0xFFFFFFF); } + +/* XXX should we hyper_flush_whole_icache here - Anton */ +static inline void srmmu_ctxd_set(ctxd_t *ctxp, pgd_t *pgdp) +{ + pte_t pte; + + pte = __pte((SRMMU_ET_PTD | (__nocache_pa(pgdp) >> 4))); + set_pte((pte_t *)ctxp, pte); +} + +/* + * Locations of MSI Registers. + */ +#define MSI_MBUS_ARBEN 0xe0001008 /* MBus Arbiter Enable register */ + +/* + * Useful bits in the MSI Registers. + */ +#define MSI_ASYNC_MODE 0x80000000 /* Operate the MSI asynchronously */ + +static void msi_set_sync(void) +{ + __asm__ __volatile__ ("lda [%0] %1, %%g3\n\t" + "andn %%g3, %2, %%g3\n\t" + "sta %%g3, [%0] %1\n\t" : : + "r" (MSI_MBUS_ARBEN), + "i" (ASI_M_CTL), "r" (MSI_ASYNC_MODE) : "g3"); +} + +void pmd_set(pmd_t *pmdp, pte_t *ptep) +{ + unsigned long ptp = __nocache_pa(ptep) >> 4; + set_pte((pte_t *)&pmd_val(*pmdp), __pte(SRMMU_ET_PTD | ptp)); +} + +/* + * size: bytes to allocate in the nocache area. + * align: bytes, number to align at. + * Returns the virtual address of the allocated area. + */ +static void *__srmmu_get_nocache(int size, int align) +{ + int offset, minsz = 1 << SRMMU_NOCACHE_BITMAP_SHIFT; + unsigned long addr; + + if (size < minsz) { + printk(KERN_ERR "Size 0x%x too small for nocache request\n", + size); + size = minsz; + } + if (size & (minsz - 1)) { + printk(KERN_ERR "Size 0x%x unaligned in nocache request\n", + size); + size += minsz - 1; + } + BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX); + + offset = bit_map_string_get(&srmmu_nocache_map, + size >> SRMMU_NOCACHE_BITMAP_SHIFT, + align >> SRMMU_NOCACHE_BITMAP_SHIFT); + if (offset == -1) { + printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n", + size, (int) srmmu_nocache_size, + srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); + return NULL; + } + + addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT); + return (void *)addr; +} + +void *srmmu_get_nocache(int size, int align) +{ + void *tmp; + + tmp = __srmmu_get_nocache(size, align); + + if (tmp) + memset(tmp, 0, size); + + return tmp; +} + +void srmmu_free_nocache(void *addr, int size) +{ + unsigned long vaddr; + int offset; + + vaddr = (unsigned long)addr; + if (vaddr < SRMMU_NOCACHE_VADDR) { + printk("Vaddr %lx is smaller than nocache base 0x%lx\n", + vaddr, (unsigned long)SRMMU_NOCACHE_VADDR); + BUG(); + } + if (vaddr + size > srmmu_nocache_end) { + printk("Vaddr %lx is bigger than nocache end 0x%lx\n", + vaddr, srmmu_nocache_end); + BUG(); + } + if (!is_power_of_2(size)) { + printk("Size 0x%x is not a power of 2\n", size); + BUG(); + } + if (size < SRMMU_NOCACHE_BITMAP_SHIFT) { + printk("Size 0x%x is too small\n", size); + BUG(); + } + if (vaddr & (size - 1)) { + printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size); + BUG(); + } + + offset = (vaddr - SRMMU_NOCACHE_VADDR) >> SRMMU_NOCACHE_BITMAP_SHIFT; + size = size >> SRMMU_NOCACHE_BITMAP_SHIFT; + + bit_map_clear(&srmmu_nocache_map, offset, size); +} + +static void srmmu_early_allocate_ptable_skeleton(unsigned long start, + unsigned long end); + +/* Return how much physical memory we have. */ +static unsigned long __init probe_memory(void) +{ + unsigned long total = 0; + int i; + + for (i = 0; sp_banks[i].num_bytes; i++) + total += sp_banks[i].num_bytes; + + return total; +} + +/* + * Reserve nocache dynamically proportionally to the amount of + * system RAM. -- Tomas Szepe , June 2002 + */ +static void __init srmmu_nocache_calcsize(void) +{ + unsigned long sysmemavail = probe_memory() / 1024; + int srmmu_nocache_npages; + + srmmu_nocache_npages = + sysmemavail / SRMMU_NOCACHE_ALCRATIO / 1024 * 256; + + /* P3 XXX The 4x overuse: corroborated by /proc/meminfo. */ + // if (srmmu_nocache_npages < 256) srmmu_nocache_npages = 256; + if (srmmu_nocache_npages < SRMMU_MIN_NOCACHE_PAGES) + srmmu_nocache_npages = SRMMU_MIN_NOCACHE_PAGES; + + /* anything above 1280 blows up */ + if (srmmu_nocache_npages > SRMMU_MAX_NOCACHE_PAGES) + srmmu_nocache_npages = SRMMU_MAX_NOCACHE_PAGES; + + srmmu_nocache_size = srmmu_nocache_npages * PAGE_SIZE; + srmmu_nocache_end = SRMMU_NOCACHE_VADDR + srmmu_nocache_size; +} + +static void __init srmmu_nocache_init(void) +{ + void *srmmu_nocache_bitmap; + unsigned int bitmap_bits; + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + unsigned long paddr, vaddr; + unsigned long pteval; + + bitmap_bits = srmmu_nocache_size >> SRMMU_NOCACHE_BITMAP_SHIFT; + + srmmu_nocache_pool = memblock_alloc_or_panic(srmmu_nocache_size, + SRMMU_NOCACHE_ALIGN_MAX); + memset(srmmu_nocache_pool, 0, srmmu_nocache_size); + + srmmu_nocache_bitmap = + memblock_alloc_or_panic(BITS_TO_LONGS(bitmap_bits) * sizeof(long), + SMP_CACHE_BYTES); + bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); + + srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); + memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE); + init_mm.pgd = srmmu_swapper_pg_dir; + + srmmu_early_allocate_ptable_skeleton(SRMMU_NOCACHE_VADDR, srmmu_nocache_end); + + paddr = __pa((unsigned long)srmmu_nocache_pool); + vaddr = SRMMU_NOCACHE_VADDR; + + while (vaddr < srmmu_nocache_end) { + pgd = pgd_offset_k(vaddr); + p4d = p4d_offset(pgd, vaddr); + pud = pud_offset(p4d, vaddr); + pmd = pmd_offset(__nocache_fix(pud), vaddr); + pte = pte_offset_kernel(__nocache_fix(pmd), vaddr); + + pteval = ((paddr >> 4) | SRMMU_ET_PTE | SRMMU_PRIV); + + if (srmmu_cache_pagetables) + pteval |= SRMMU_CACHE; + + set_pte(__nocache_fix(pte), __pte(pteval)); + + vaddr += PAGE_SIZE; + paddr += PAGE_SIZE; + } + + flush_cache_all(); + flush_tlb_all(); +} + +pgd_t *get_pgd_fast(void) +{ + pgd_t *pgd = NULL; + + pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); + if (pgd) { + pgd_t *init = pgd_offset_k(0); + memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); + memcpy(pgd + USER_PTRS_PER_PGD, init + USER_PTRS_PER_PGD, + (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); + } + + return pgd; +} + +/* + * Hardware needs alignment to 256 only, but we align to whole page size + * to reduce fragmentation problems due to the buddy principle. + * XXX Provide actual fragmentation statistics in /proc. + * + * Alignments up to the page size are the same for physical and virtual + * addresses of the nocache area. + */ +pgtable_t pte_alloc_one(struct mm_struct *mm) +{ + pte_t *ptep; + struct page *page; + + if (!(ptep = pte_alloc_one_kernel(mm))) + return NULL; + page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); + spin_lock(&mm->page_table_lock); + if (page_ref_inc_return(page) == 2 && + !pagetable_pte_ctor(mm, page_ptdesc(page))) { + page_ref_dec(page); + ptep = NULL; + } + spin_unlock(&mm->page_table_lock); + + return ptep; +} + +void pte_free(struct mm_struct *mm, pgtable_t ptep) +{ + struct page *page; + + page = pfn_to_page(__nocache_pa((unsigned long)ptep) >> PAGE_SHIFT); + spin_lock(&mm->page_table_lock); + if (page_ref_dec_return(page) == 1) + pagetable_dtor(page_ptdesc(page)); + spin_unlock(&mm->page_table_lock); + + srmmu_free_nocache(ptep, SRMMU_PTE_TABLE_SIZE); +} + +/* context handling - a dynamically sized pool is used */ +#define NO_CONTEXT -1 + +struct ctx_list { + struct ctx_list *next; + struct ctx_list *prev; + unsigned int ctx_number; + struct mm_struct *ctx_mm; +}; + +static struct ctx_list *ctx_list_pool; +static struct ctx_list ctx_free; +static struct ctx_list ctx_used; + +/* At boot time we determine the number of contexts */ +static int num_contexts; + +static inline void remove_from_ctx_list(struct ctx_list *entry) +{ + entry->next->prev = entry->prev; + entry->prev->next = entry->next; +} + +static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry) +{ + entry->next = head; + (entry->prev = head->prev)->next = entry; + head->prev = entry; +} +#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry) +#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry) + + +static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) +{ + struct ctx_list *ctxp; + + ctxp = ctx_free.next; + if (ctxp != &ctx_free) { + remove_from_ctx_list(ctxp); + add_to_used_ctxlist(ctxp); + mm->context = ctxp->ctx_number; + ctxp->ctx_mm = mm; + return; + } + ctxp = ctx_used.next; + if (ctxp->ctx_mm == old_mm) + ctxp = ctxp->next; + if (ctxp == &ctx_used) + panic("out of mmu contexts"); + flush_cache_mm(ctxp->ctx_mm); + flush_tlb_mm(ctxp->ctx_mm); + remove_from_ctx_list(ctxp); + add_to_used_ctxlist(ctxp); + ctxp->ctx_mm->context = NO_CONTEXT; + ctxp->ctx_mm = mm; + mm->context = ctxp->ctx_number; +} + +static inline void free_context(int context) +{ + struct ctx_list *ctx_old; + + ctx_old = ctx_list_pool + context; + remove_from_ctx_list(ctx_old); + add_to_free_ctxlist(ctx_old); +} + +static void __init sparc_context_init(int numctx) +{ + int ctx; + unsigned long size; + + size = numctx * sizeof(struct ctx_list); + ctx_list_pool = memblock_alloc_or_panic(size, SMP_CACHE_BYTES); + + for (ctx = 0; ctx < numctx; ctx++) { + struct ctx_list *clist; + + clist = (ctx_list_pool + ctx); + clist->ctx_number = ctx; + clist->ctx_mm = NULL; + } + ctx_free.next = ctx_free.prev = &ctx_free; + ctx_used.next = ctx_used.prev = &ctx_used; + for (ctx = 0; ctx < numctx; ctx++) + add_to_free_ctxlist(ctx_list_pool + ctx); +} + +void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, + struct task_struct *tsk) +{ + unsigned long flags; + + if (mm->context == NO_CONTEXT) { + spin_lock_irqsave(&srmmu_context_spinlock, flags); + alloc_context(old_mm, mm); + spin_unlock_irqrestore(&srmmu_context_spinlock, flags); + srmmu_ctxd_set(&srmmu_context_table[mm->context], mm->pgd); + } + + if (sparc_cpu_model == sparc_leon) + leon_switch_mm(); + + if (is_hypersparc) + hyper_flush_whole_icache(); + + srmmu_set_context(mm->context); +} + +/* Low level IO area allocation on the SRMMU. */ +static inline void srmmu_mapioaddr(unsigned long physaddr, + unsigned long virt_addr, int bus_type) +{ + pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + unsigned long tmp; + + physaddr &= PAGE_MASK; + pgdp = pgd_offset_k(virt_addr); + p4dp = p4d_offset(pgdp, virt_addr); + pudp = pud_offset(p4dp, virt_addr); + pmdp = pmd_offset(pudp, virt_addr); + ptep = pte_offset_kernel(pmdp, virt_addr); + tmp = (physaddr >> 4) | SRMMU_ET_PTE; + + /* I need to test whether this is consistent over all + * sun4m's. The bus_type represents the upper 4 bits of + * 36-bit physical address on the I/O space lines... + */ + tmp |= (bus_type << 28); + tmp |= SRMMU_PRIV; + __flush_page_to_ram(virt_addr); + set_pte(ptep, __pte(tmp)); +} + +void srmmu_mapiorange(unsigned int bus, unsigned long xpa, + unsigned long xva, unsigned int len) +{ + while (len != 0) { + len -= PAGE_SIZE; + srmmu_mapioaddr(xpa, xva, bus); + xva += PAGE_SIZE; + xpa += PAGE_SIZE; + } + flush_tlb_all(); +} + +static inline void srmmu_unmapioaddr(unsigned long virt_addr) +{ + pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + + + pgdp = pgd_offset_k(virt_addr); + p4dp = p4d_offset(pgdp, virt_addr); + pudp = pud_offset(p4dp, virt_addr); + pmdp = pmd_offset(pudp, virt_addr); + ptep = pte_offset_kernel(pmdp, virt_addr); + + /* No need to flush uncacheable page. */ + __pte_clear(ptep); +} + +void srmmu_unmapiorange(unsigned long virt_addr, unsigned int len) +{ + while (len != 0) { + len -= PAGE_SIZE; + srmmu_unmapioaddr(virt_addr); + virt_addr += PAGE_SIZE; + } + flush_tlb_all(); +} + +/* tsunami.S */ +extern void tsunami_flush_cache_all(void); +extern void tsunami_flush_cache_mm(struct mm_struct *mm); +extern void tsunami_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +extern void tsunami_flush_cache_page(struct vm_area_struct *vma, unsigned long page); +extern void tsunami_flush_page_to_ram(unsigned long page); +extern void tsunami_flush_page_for_dma(unsigned long page); +extern void tsunami_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr); +extern void tsunami_flush_tlb_all(void); +extern void tsunami_flush_tlb_mm(struct mm_struct *mm); +extern void tsunami_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +extern void tsunami_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); +extern void tsunami_setup_blockops(void); + +/* swift.S */ +extern void swift_flush_cache_all(void); +extern void swift_flush_cache_mm(struct mm_struct *mm); +extern void swift_flush_cache_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end); +extern void swift_flush_cache_page(struct vm_area_struct *vma, unsigned long page); +extern void swift_flush_page_to_ram(unsigned long page); +extern void swift_flush_page_for_dma(unsigned long page); +extern void swift_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr); +extern void swift_flush_tlb_all(void); +extern void swift_flush_tlb_mm(struct mm_struct *mm); +extern void swift_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end); +extern void swift_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); + + +/* + * The following are all MBUS based SRMMU modules, and therefore could + * be found in a multiprocessor configuration. On the whole, these + * chips seems to be much more touchy about DVMA and page tables + * with respect to cache coherency. + */ + +/* viking.S */ +extern void viking_flush_cache_all(void); +extern void viking_flush_cache_mm(struct mm_struct *mm); +extern void viking_flush_cache_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +extern void viking_flush_cache_page(struct vm_area_struct *vma, unsigned long page); +extern void viking_flush_page_to_ram(unsigned long page); +extern void viking_flush_page_for_dma(unsigned long page); +extern void viking_flush_sig_insns(struct mm_struct *mm, unsigned long addr); +extern void viking_flush_page(unsigned long page); +extern void viking_mxcc_flush_page(unsigned long page); +extern void viking_flush_tlb_all(void); +extern void viking_flush_tlb_mm(struct mm_struct *mm); +extern void viking_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +extern void viking_flush_tlb_page(struct vm_area_struct *vma, + unsigned long page); +extern void sun4dsmp_flush_tlb_all(void); +extern void sun4dsmp_flush_tlb_mm(struct mm_struct *mm); +extern void sun4dsmp_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +extern void sun4dsmp_flush_tlb_page(struct vm_area_struct *vma, + unsigned long page); + +/* hypersparc.S */ +extern void hypersparc_flush_cache_all(void); +extern void hypersparc_flush_cache_mm(struct mm_struct *mm); +extern void hypersparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +extern void hypersparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page); +extern void hypersparc_flush_page_to_ram(unsigned long page); +extern void hypersparc_flush_page_for_dma(unsigned long page); +extern void hypersparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr); +extern void hypersparc_flush_tlb_all(void); +extern void hypersparc_flush_tlb_mm(struct mm_struct *mm); +extern void hypersparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); +extern void hypersparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page); +extern void hypersparc_setup_blockops(void); + +/* + * NOTE: All of this startup code assumes the low 16mb (approx.) of + * kernel mappings are done with one single contiguous chunk of + * ram. On small ram machines (classics mainly) we only get + * around 8mb mapped for us. + */ + +static void __init early_pgtable_allocfail(char *type) +{ + prom_printf("inherit_prom_mappings: Cannot alloc kernel %s.\n", type); + prom_halt(); +} + +static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, + unsigned long end) +{ + pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + + while (start < end) { + pgdp = pgd_offset_k(start); + p4dp = p4d_offset(pgdp, start); + pudp = pud_offset(p4dp, start); + if (pud_none(*__nocache_fix(pudp))) { + pmdp = __srmmu_get_nocache( + SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); + if (pmdp == NULL) + early_pgtable_allocfail("pmd"); + memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); + pud_set(__nocache_fix(pudp), pmdp); + } + pmdp = pmd_offset(__nocache_fix(pudp), start); + if (srmmu_pmd_none(*__nocache_fix(pmdp))) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); + if (ptep == NULL) + early_pgtable_allocfail("pte"); + memset(__nocache_fix(ptep), 0, PTE_SIZE); + pmd_set(__nocache_fix(pmdp), ptep); + } + if (start > (0xffffffffUL - PMD_SIZE)) + break; + start = (start + PMD_SIZE) & PMD_MASK; + } +} + +static void __init srmmu_allocate_ptable_skeleton(unsigned long start, + unsigned long end) +{ + pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + + while (start < end) { + pgdp = pgd_offset_k(start); + p4dp = p4d_offset(pgdp, start); + pudp = pud_offset(p4dp, start); + if (pud_none(*pudp)) { + pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); + if (pmdp == NULL) + early_pgtable_allocfail("pmd"); + memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE); + pud_set((pud_t *)pgdp, pmdp); + } + pmdp = pmd_offset(pudp, start); + if (srmmu_pmd_none(*pmdp)) { + ptep = __srmmu_get_nocache(PTE_SIZE, + PTE_SIZE); + if (ptep == NULL) + early_pgtable_allocfail("pte"); + memset(ptep, 0, PTE_SIZE); + pmd_set(pmdp, ptep); + } + if (start > (0xffffffffUL - PMD_SIZE)) + break; + start = (start + PMD_SIZE) & PMD_MASK; + } +} + +/* These flush types are not available on all chips... */ +static inline unsigned long srmmu_probe(unsigned long vaddr) +{ + unsigned long retval; + + if (sparc_cpu_model != sparc_leon) { + + vaddr &= PAGE_MASK; + __asm__ __volatile__("lda [%1] %2, %0\n\t" : + "=r" (retval) : + "r" (vaddr | 0x400), "i" (ASI_M_FLUSH_PROBE)); + } else { + retval = leon_swprobe(vaddr, NULL); + } + return retval; +} + +/* + * This is much cleaner than poking around physical address space + * looking at the prom's page table directly which is what most + * other OS's do. Yuck... this is much better. + */ +static void __init srmmu_inherit_prom_mappings(unsigned long start, + unsigned long end) +{ + unsigned long probed; + unsigned long addr; + pgd_t *pgdp; + p4d_t *p4dp; + pud_t *pudp; + pmd_t *pmdp; + pte_t *ptep; + int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ + + while (start <= end) { + if (start == 0) + break; /* probably wrap around */ + if (start == 0xfef00000) + start = KADB_DEBUGGER_BEGVM; + probed = srmmu_probe(start); + if (!probed) { + /* continue probing until we find an entry */ + start += PAGE_SIZE; + continue; + } + + /* A red snapper, see what it really is. */ + what = 0; + addr = start - PAGE_SIZE; + + if (!(start & ~(PMD_MASK))) { + if (srmmu_probe(addr + PMD_SIZE) == probed) + what = 1; + } + + if (!(start & ~(PGDIR_MASK))) { + if (srmmu_probe(addr + PGDIR_SIZE) == probed) + what = 2; + } + + pgdp = pgd_offset_k(start); + p4dp = p4d_offset(pgdp, start); + pudp = pud_offset(p4dp, start); + if (what == 2) { + *__nocache_fix(pgdp) = __pgd(probed); + start += PGDIR_SIZE; + continue; + } + if (pud_none(*__nocache_fix(pudp))) { + pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, + SRMMU_PMD_TABLE_SIZE); + if (pmdp == NULL) + early_pgtable_allocfail("pmd"); + memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); + pud_set(__nocache_fix(pudp), pmdp); + } + pmdp = pmd_offset(__nocache_fix(pudp), start); + if (what == 1) { + *(pmd_t *)__nocache_fix(pmdp) = __pmd(probed); + start += PMD_SIZE; + continue; + } + if (srmmu_pmd_none(*__nocache_fix(pmdp))) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); + if (ptep == NULL) + early_pgtable_allocfail("pte"); + memset(__nocache_fix(ptep), 0, PTE_SIZE); + pmd_set(__nocache_fix(pmdp), ptep); + } + ptep = pte_offset_kernel(__nocache_fix(pmdp), start); + *__nocache_fix(ptep) = __pte(probed); + start += PAGE_SIZE; + } +} + +#define KERNEL_PTE(page_shifted) ((page_shifted)|SRMMU_CACHE|SRMMU_PRIV|SRMMU_VALID) + +/* Create a third-level SRMMU 16MB page mapping. */ +static void __init do_large_mapping(unsigned long vaddr, unsigned long phys_base) +{ + pgd_t *pgdp = pgd_offset_k(vaddr); + unsigned long big_pte; + + big_pte = KERNEL_PTE(phys_base >> 4); + *__nocache_fix(pgdp) = __pgd(big_pte); +} + +/* Map sp_bank entry SP_ENTRY, starting at virtual address VBASE. */ +static unsigned long __init map_spbank(unsigned long vbase, int sp_entry) +{ + unsigned long pstart = (sp_banks[sp_entry].base_addr & PGDIR_MASK); + unsigned long vstart = (vbase & PGDIR_MASK); + unsigned long vend = PGDIR_ALIGN(vbase + sp_banks[sp_entry].num_bytes); + /* Map "low" memory only */ + const unsigned long min_vaddr = PAGE_OFFSET; + const unsigned long max_vaddr = PAGE_OFFSET + SRMMU_MAXMEM; + + if (vstart < min_vaddr || vstart >= max_vaddr) + return vstart; + + if (vend > max_vaddr || vend < min_vaddr) + vend = max_vaddr; + + while (vstart < vend) { + do_large_mapping(vstart, pstart); + vstart += PGDIR_SIZE; pstart += PGDIR_SIZE; + } + return vstart; +} + +static void __init map_kernel(void) +{ + int i; + + if (phys_base > 0) { + do_large_mapping(PAGE_OFFSET, phys_base); + } + + for (i = 0; sp_banks[i].num_bytes != 0; i++) { + map_spbank((unsigned long)__va(sp_banks[i].base_addr), i); + } +} + +void (*poke_srmmu)(void) = NULL; + +void __init arch_zone_limits_init(unsigned long *max_zone_pfns) +{ + max_zone_pfns[ZONE_DMA] = max_low_pfn; + max_zone_pfns[ZONE_NORMAL] = max_low_pfn; + max_zone_pfns[ZONE_HIGHMEM] = highend_pfn; +} + +void __init srmmu_paging_init(void) +{ + int i; + phandle cpunode; + char node_str[128]; + pgd_t *pgd; + p4d_t *p4d; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + unsigned long pages_avail; + + init_mm.context = (unsigned long) NO_CONTEXT; + sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */ + + if (sparc_cpu_model == sun4d) + num_contexts = 65536; /* We know it is Viking */ + else { + /* Find the number of contexts on the srmmu. */ + cpunode = prom_getchild(prom_root_node); + num_contexts = 0; + while (cpunode != 0) { + prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); + if (!strcmp(node_str, "cpu")) { + num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8); + break; + } + cpunode = prom_getsibling(cpunode); + } + } + + if (!num_contexts) { + prom_printf("Something wrong, can't find cpu node in paging_init.\n"); + prom_halt(); + } + + pages_avail = 0; + last_valid_pfn = bootmem_init(&pages_avail); + + srmmu_nocache_calcsize(); + srmmu_nocache_init(); + srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE)); + map_kernel(); + + /* ctx table has to be physically aligned to its size */ + srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t)); + srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa(srmmu_context_table); + + for (i = 0; i < num_contexts; i++) + srmmu_ctxd_set(__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); + + flush_cache_all(); + srmmu_set_ctable_ptr((unsigned long)srmmu_ctx_table_phys); +#ifdef CONFIG_SMP + /* Stop from hanging here... */ + local_ops->tlb_all(); +#else + flush_tlb_all(); +#endif + poke_srmmu(); + + srmmu_allocate_ptable_skeleton(sparc_iomap.start, IOBASE_END); + srmmu_allocate_ptable_skeleton(DVMA_VADDR, DVMA_END); + + srmmu_allocate_ptable_skeleton( + __fix_to_virt(__end_of_fixed_addresses - 1), FIXADDR_TOP); + srmmu_allocate_ptable_skeleton(PKMAP_BASE, PKMAP_END); + + pgd = pgd_offset_k(PKMAP_BASE); + p4d = p4d_offset(pgd, PKMAP_BASE); + pud = pud_offset(p4d, PKMAP_BASE); + pmd = pmd_offset(pud, PKMAP_BASE); + pte = pte_offset_kernel(pmd, PKMAP_BASE); + pkmap_page_table = pte; + + flush_cache_all(); + flush_tlb_all(); + + sparc_context_init(num_contexts); +} + +void mmu_info(struct seq_file *m) +{ + seq_printf(m, + "MMU type\t: %s\n" + "contexts\t: %d\n" + "nocache total\t: %ld\n" + "nocache used\t: %d\n", + srmmu_name, + num_contexts, + srmmu_nocache_size, + srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); +} + +int init_new_context(struct task_struct *tsk, struct mm_struct *mm) +{ + mm->context = NO_CONTEXT; + return 0; +} + +void destroy_context(struct mm_struct *mm) +{ + unsigned long flags; + + if (mm->context != NO_CONTEXT) { + flush_cache_mm(mm); + srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir); + flush_tlb_mm(mm); + spin_lock_irqsave(&srmmu_context_spinlock, flags); + free_context(mm->context); + spin_unlock_irqrestore(&srmmu_context_spinlock, flags); + mm->context = NO_CONTEXT; + } +} + +/* Init various srmmu chip types. */ +static void __init srmmu_is_bad(void) +{ + prom_printf("Could not determine SRMMU chip type.\n"); + prom_halt(); +} + +static void __init init_vac_layout(void) +{ + phandle nd; + int cache_lines; + char node_str[128]; +#ifdef CONFIG_SMP + int cpu = 0; + unsigned long max_size = 0; + unsigned long min_line_size = 0x10000000; +#endif + + nd = prom_getchild(prom_root_node); + while ((nd = prom_getsibling(nd)) != 0) { + prom_getstring(nd, "device_type", node_str, sizeof(node_str)); + if (!strcmp(node_str, "cpu")) { + vac_line_size = prom_getint(nd, "cache-line-size"); + if (vac_line_size == -1) { + prom_printf("can't determine cache-line-size, halting.\n"); + prom_halt(); + } + cache_lines = prom_getint(nd, "cache-nlines"); + if (cache_lines == -1) { + prom_printf("can't determine cache-nlines, halting.\n"); + prom_halt(); + } + + vac_cache_size = cache_lines * vac_line_size; +#ifdef CONFIG_SMP + if (vac_cache_size > max_size) + max_size = vac_cache_size; + if (vac_line_size < min_line_size) + min_line_size = vac_line_size; + //FIXME: cpus not contiguous!! + cpu++; + if (cpu >= nr_cpu_ids || !cpu_online(cpu)) + break; +#else + break; +#endif + } + } + if (nd == 0) { + prom_printf("No CPU nodes found, halting.\n"); + prom_halt(); + } +#ifdef CONFIG_SMP + vac_cache_size = max_size; + vac_line_size = min_line_size; +#endif + printk("SRMMU: Using VAC size of %d bytes, line size %d bytes.\n", + (int)vac_cache_size, (int)vac_line_size); +} + +static void poke_hypersparc(void) +{ + volatile unsigned long clear; + unsigned long mreg = srmmu_get_mmureg(); + + hyper_flush_unconditional_combined(); + + mreg &= ~(HYPERSPARC_CWENABLE); + mreg |= (HYPERSPARC_CENABLE | HYPERSPARC_WBENABLE); + mreg |= (HYPERSPARC_CMODE); + + srmmu_set_mmureg(mreg); + +#if 0 /* XXX I think this is bad news... -DaveM */ + hyper_clear_all_tags(); +#endif + + put_ross_icr(HYPERSPARC_ICCR_FTD | HYPERSPARC_ICCR_ICE); + hyper_flush_whole_icache(); + clear = srmmu_get_faddr(); + clear = srmmu_get_fstatus(); +} + +static const struct sparc32_cachetlb_ops hypersparc_ops = { + .cache_all = hypersparc_flush_cache_all, + .cache_mm = hypersparc_flush_cache_mm, + .cache_page = hypersparc_flush_cache_page, + .cache_range = hypersparc_flush_cache_range, + .tlb_all = hypersparc_flush_tlb_all, + .tlb_mm = hypersparc_flush_tlb_mm, + .tlb_page = hypersparc_flush_tlb_page, + .tlb_range = hypersparc_flush_tlb_range, + .page_to_ram = hypersparc_flush_page_to_ram, + .sig_insns = hypersparc_flush_sig_insns, + .page_for_dma = hypersparc_flush_page_for_dma, +}; + +static void __init init_hypersparc(void) +{ + srmmu_name = "ROSS HyperSparc"; + srmmu_modtype = HyperSparc; + + init_vac_layout(); + + is_hypersparc = 1; + sparc32_cachetlb_ops = &hypersparc_ops; + + poke_srmmu = poke_hypersparc; + + hypersparc_setup_blockops(); +} + +static void poke_swift(void) +{ + unsigned long mreg; + + /* Clear any crap from the cache or else... */ + swift_flush_cache_all(); + + /* Enable I & D caches */ + mreg = srmmu_get_mmureg(); + mreg |= (SWIFT_IE | SWIFT_DE); + /* + * The Swift branch folding logic is completely broken. At + * trap time, if things are just right, if can mistakenly + * think that a trap is coming from kernel mode when in fact + * it is coming from user mode (it mis-executes the branch in + * the trap code). So you see things like crashme completely + * hosing your machine which is completely unacceptable. Turn + * this shit off... nice job Fujitsu. + */ + mreg &= ~(SWIFT_BF); + srmmu_set_mmureg(mreg); +} + +static const struct sparc32_cachetlb_ops swift_ops = { + .cache_all = swift_flush_cache_all, + .cache_mm = swift_flush_cache_mm, + .cache_page = swift_flush_cache_page, + .cache_range = swift_flush_cache_range, + .tlb_all = swift_flush_tlb_all, + .tlb_mm = swift_flush_tlb_mm, + .tlb_page = swift_flush_tlb_page, + .tlb_range = swift_flush_tlb_range, + .page_to_ram = swift_flush_page_to_ram, + .sig_insns = swift_flush_sig_insns, + .page_for_dma = swift_flush_page_for_dma, +}; + +#define SWIFT_MASKID_ADDR 0x10003018 +static void __init init_swift(void) +{ + unsigned long swift_rev; + + __asm__ __volatile__("lda [%1] %2, %0\n\t" + "srl %0, 0x18, %0\n\t" : + "=r" (swift_rev) : + "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS)); + srmmu_name = "Fujitsu Swift"; + switch (swift_rev) { + case 0x11: + case 0x20: + case 0x23: + case 0x30: + srmmu_modtype = Swift_lots_o_bugs; + hwbug_bitmask |= (HWBUG_KERN_ACCBROKEN | HWBUG_KERN_CBITBROKEN); + /* + * Gee george, I wonder why Sun is so hush hush about + * this hardware bug... really braindamage stuff going + * on here. However I think we can find a way to avoid + * all of the workaround overhead under Linux. Basically, + * any page fault can cause kernel pages to become user + * accessible (the mmu gets confused and clears some of + * the ACC bits in kernel ptes). Aha, sounds pretty + * horrible eh? But wait, after extensive testing it appears + * that if you use pgd_t level large kernel pte's (like the + * 4MB pages on the Pentium) the bug does not get tripped + * at all. This avoids almost all of the major overhead. + * Welcome to a world where your vendor tells you to, + * "apply this kernel patch" instead of "sorry for the + * broken hardware, send it back and we'll give you + * properly functioning parts" + */ + break; + case 0x25: + case 0x31: + srmmu_modtype = Swift_bad_c; + hwbug_bitmask |= HWBUG_KERN_CBITBROKEN; + /* + * You see Sun allude to this hardware bug but never + * admit things directly, they'll say things like, + * "the Swift chip cache problems" or similar. + */ + break; + default: + srmmu_modtype = Swift_ok; + break; + } + + sparc32_cachetlb_ops = &swift_ops; + flush_page_for_dma_global = 0; + + /* + * Are you now convinced that the Swift is one of the + * biggest VLSI abortions of all time? Bravo Fujitsu! + * Fujitsu, the !#?!%$'d up processor people. I bet if + * you examined the microcode of the Swift you'd find + * XXX's all over the place. + */ + poke_srmmu = poke_swift; +} + +static void turbosparc_flush_cache_all(void) +{ + flush_user_windows(); + turbosparc_idflash_clear(); +} + +static void turbosparc_flush_cache_mm(struct mm_struct *mm) +{ + FLUSH_BEGIN(mm) + flush_user_windows(); + turbosparc_idflash_clear(); + FLUSH_END +} + +static void turbosparc_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) +{ + FLUSH_BEGIN(vma->vm_mm) + flush_user_windows(); + turbosparc_idflash_clear(); + FLUSH_END +} + +static void turbosparc_flush_cache_page(struct vm_area_struct *vma, unsigned long page) +{ + FLUSH_BEGIN(vma->vm_mm) + flush_user_windows(); + if (vma->vm_flags & VM_EXEC) + turbosparc_flush_icache(); + turbosparc_flush_dcache(); + FLUSH_END +} + +/* TurboSparc is copy-back, if we turn it on, but this does not work. */ +static void turbosparc_flush_page_to_ram(unsigned long page) +{ +#ifdef TURBOSPARC_WRITEBACK + volatile unsigned long clear; + + if (srmmu_probe(page)) + turbosparc_flush_page_cache(page); + clear = srmmu_get_fstatus(); +#endif +} + +static void turbosparc_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) +{ +} + +static void turbosparc_flush_page_for_dma(unsigned long page) +{ + turbosparc_flush_dcache(); +} + +static void turbosparc_flush_tlb_all(void) +{ + srmmu_flush_whole_tlb(); +} + +static void turbosparc_flush_tlb_mm(struct mm_struct *mm) +{ + FLUSH_BEGIN(mm) + srmmu_flush_whole_tlb(); + FLUSH_END +} + +static void turbosparc_flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) +{ + FLUSH_BEGIN(vma->vm_mm) + srmmu_flush_whole_tlb(); + FLUSH_END +} + +static void turbosparc_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + FLUSH_BEGIN(vma->vm_mm) + srmmu_flush_whole_tlb(); + FLUSH_END +} + + +static void poke_turbosparc(void) +{ + unsigned long mreg = srmmu_get_mmureg(); + unsigned long ccreg; + + /* Clear any crap from the cache or else... */ + turbosparc_flush_cache_all(); + /* Temporarily disable I & D caches */ + mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); + mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */ + srmmu_set_mmureg(mreg); + + ccreg = turbosparc_get_ccreg(); + +#ifdef TURBOSPARC_WRITEBACK + ccreg |= (TURBOSPARC_SNENABLE); /* Do DVMA snooping in Dcache */ + ccreg &= ~(TURBOSPARC_uS2 | TURBOSPARC_WTENABLE); + /* Write-back D-cache, emulate VLSI + * abortion number three, not number one */ +#else + /* For now let's play safe, optimize later */ + ccreg |= (TURBOSPARC_SNENABLE | TURBOSPARC_WTENABLE); + /* Do DVMA snooping in Dcache, Write-thru D-cache */ + ccreg &= ~(TURBOSPARC_uS2); + /* Emulate VLSI abortion number three, not number one */ +#endif + + switch (ccreg & 7) { + case 0: /* No SE cache */ + case 7: /* Test mode */ + break; + default: + ccreg |= (TURBOSPARC_SCENABLE); + } + turbosparc_set_ccreg(ccreg); + + mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */ + mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */ + srmmu_set_mmureg(mreg); +} + +static const struct sparc32_cachetlb_ops turbosparc_ops = { + .cache_all = turbosparc_flush_cache_all, + .cache_mm = turbosparc_flush_cache_mm, + .cache_page = turbosparc_flush_cache_page, + .cache_range = turbosparc_flush_cache_range, + .tlb_all = turbosparc_flush_tlb_all, + .tlb_mm = turbosparc_flush_tlb_mm, + .tlb_page = turbosparc_flush_tlb_page, + .tlb_range = turbosparc_flush_tlb_range, + .page_to_ram = turbosparc_flush_page_to_ram, + .sig_insns = turbosparc_flush_sig_insns, + .page_for_dma = turbosparc_flush_page_for_dma, +}; + +static void __init init_turbosparc(void) +{ + srmmu_name = "Fujitsu TurboSparc"; + srmmu_modtype = TurboSparc; + sparc32_cachetlb_ops = &turbosparc_ops; + poke_srmmu = poke_turbosparc; +} + +static void poke_tsunami(void) +{ + unsigned long mreg = srmmu_get_mmureg(); + + tsunami_flush_icache(); + tsunami_flush_dcache(); + mreg &= ~TSUNAMI_ITD; + mreg |= (TSUNAMI_IENAB | TSUNAMI_DENAB); + srmmu_set_mmureg(mreg); +} + +static const struct sparc32_cachetlb_ops tsunami_ops = { + .cache_all = tsunami_flush_cache_all, + .cache_mm = tsunami_flush_cache_mm, + .cache_page = tsunami_flush_cache_page, + .cache_range = tsunami_flush_cache_range, + .tlb_all = tsunami_flush_tlb_all, + .tlb_mm = tsunami_flush_tlb_mm, + .tlb_page = tsunami_flush_tlb_page, + .tlb_range = tsunami_flush_tlb_range, + .page_to_ram = tsunami_flush_page_to_ram, + .sig_insns = tsunami_flush_sig_insns, + .page_for_dma = tsunami_flush_page_for_dma, +}; + +static void __init init_tsunami(void) +{ + /* + * Tsunami's pretty sane, Sun and TI actually got it + * somewhat right this time. Fujitsu should have + * taken some lessons from them. + */ + + srmmu_name = "TI Tsunami"; + srmmu_modtype = Tsunami; + sparc32_cachetlb_ops = &tsunami_ops; + poke_srmmu = poke_tsunami; + + tsunami_setup_blockops(); +} + +static void poke_viking(void) +{ + unsigned long mreg = srmmu_get_mmureg(); + static int smp_catch; + + if (viking_mxcc_present) { + unsigned long mxcc_control = mxcc_get_creg(); + + mxcc_control |= (MXCC_CTL_ECE | MXCC_CTL_PRE | MXCC_CTL_MCE); + mxcc_control &= ~(MXCC_CTL_RRC); + mxcc_set_creg(mxcc_control); + + /* + * We don't need memory parity checks. + * XXX This is a mess, have to dig out later. ecd. + viking_mxcc_turn_off_parity(&mreg, &mxcc_control); + */ + + /* We do cache ptables on MXCC. */ + mreg |= VIKING_TCENABLE; + } else { + unsigned long bpreg; + + mreg &= ~(VIKING_TCENABLE); + if (smp_catch++) { + /* Must disable mixed-cmd mode here for other cpu's. */ + bpreg = viking_get_bpreg(); + bpreg &= ~(VIKING_ACTION_MIX); + viking_set_bpreg(bpreg); + + /* Just in case PROM does something funny. */ + msi_set_sync(); + } + } + + mreg |= VIKING_SPENABLE; + mreg |= (VIKING_ICENABLE | VIKING_DCENABLE); + mreg |= VIKING_SBENABLE; + mreg &= ~(VIKING_ACENABLE); + srmmu_set_mmureg(mreg); +} + +static struct sparc32_cachetlb_ops viking_ops __ro_after_init = { + .cache_all = viking_flush_cache_all, + .cache_mm = viking_flush_cache_mm, + .cache_page = viking_flush_cache_page, + .cache_range = viking_flush_cache_range, + .tlb_all = viking_flush_tlb_all, + .tlb_mm = viking_flush_tlb_mm, + .tlb_page = viking_flush_tlb_page, + .tlb_range = viking_flush_tlb_range, + .page_to_ram = viking_flush_page_to_ram, + .sig_insns = viking_flush_sig_insns, + .page_for_dma = viking_flush_page_for_dma, +}; + +#ifdef CONFIG_SMP +/* On sun4d the cpu broadcasts local TLB flushes, so we can just + * perform the local TLB flush and all the other cpus will see it. + * But, unfortunately, there is a bug in the sun4d XBUS backplane + * that requires that we add some synchronization to these flushes. + * + * The bug is that the fifo which keeps track of all the pending TLB + * broadcasts in the system is an entry or two too small, so if we + * have too many going at once we'll overflow that fifo and lose a TLB + * flush resulting in corruption. + * + * Our workaround is to take a global spinlock around the TLB flushes, + * which guarentees we won't ever have too many pending. It's a big + * hammer, but a semaphore like system to make sure we only have N TLB + * flushes going at once will require SMP locking anyways so there's + * no real value in trying any harder than this. + */ +static struct sparc32_cachetlb_ops viking_sun4d_smp_ops __ro_after_init = { + .cache_all = viking_flush_cache_all, + .cache_mm = viking_flush_cache_mm, + .cache_page = viking_flush_cache_page, + .cache_range = viking_flush_cache_range, + .tlb_all = sun4dsmp_flush_tlb_all, + .tlb_mm = sun4dsmp_flush_tlb_mm, + .tlb_page = sun4dsmp_flush_tlb_page, + .tlb_range = sun4dsmp_flush_tlb_range, + .page_to_ram = viking_flush_page_to_ram, + .sig_insns = viking_flush_sig_insns, + .page_for_dma = viking_flush_page_for_dma, +}; +#endif + +static void __init init_viking(void) +{ + unsigned long mreg = srmmu_get_mmureg(); + + /* Ahhh, the viking. SRMMU VLSI abortion number two... */ + if (mreg & VIKING_MMODE) { + srmmu_name = "TI Viking"; + viking_mxcc_present = 0; + msi_set_sync(); + + /* + * We need this to make sure old viking takes no hits + * on its cache for dma snoops to workaround the + * "load from non-cacheable memory" interrupt bug. + * This is only necessary because of the new way in + * which we use the IOMMU. + */ + viking_ops.page_for_dma = viking_flush_page; +#ifdef CONFIG_SMP + viking_sun4d_smp_ops.page_for_dma = viking_flush_page; +#endif + flush_page_for_dma_global = 0; + } else { + srmmu_name = "TI Viking/MXCC"; + viking_mxcc_present = 1; + srmmu_cache_pagetables = 1; + } + + sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) + &viking_ops; +#ifdef CONFIG_SMP + if (sparc_cpu_model == sun4d) + sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) + &viking_sun4d_smp_ops; +#endif + + poke_srmmu = poke_viking; +} + +/* Probe for the srmmu chip version. */ +static void __init get_srmmu_type(void) +{ + unsigned long mreg, psr; + unsigned long mod_typ, mod_rev, psr_typ, psr_vers; + + srmmu_modtype = SRMMU_INVAL_MOD; + hwbug_bitmask = 0; + + mreg = srmmu_get_mmureg(); psr = get_psr(); + mod_typ = (mreg & 0xf0000000) >> 28; + mod_rev = (mreg & 0x0f000000) >> 24; + psr_typ = (psr >> 28) & 0xf; + psr_vers = (psr >> 24) & 0xf; + + /* First, check for sparc-leon. */ + if (sparc_cpu_model == sparc_leon) { + init_leon(); + return; + } + + /* Second, check for HyperSparc or Cypress. */ + if (mod_typ == 1) { + switch (mod_rev) { + case 7: + /* UP or MP Hypersparc */ + init_hypersparc(); + break; + case 0: + case 2: + case 10: + case 11: + case 12: + case 13: + case 14: + case 15: + default: + prom_printf("Sparc-Linux Cypress support does not longer exit.\n"); + prom_halt(); + break; + } + return; + } + + /* Now Fujitsu TurboSparc. It might happen that it is + * in Swift emulation mode, so we will check later... + */ + if (psr_typ == 0 && psr_vers == 5) { + init_turbosparc(); + return; + } + + /* Next check for Fujitsu Swift. */ + if (psr_typ == 0 && psr_vers == 4) { + phandle cpunode; + char node_str[128]; + + /* Look if it is not a TurboSparc emulating Swift... */ + cpunode = prom_getchild(prom_root_node); + while ((cpunode = prom_getsibling(cpunode)) != 0) { + prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); + if (!strcmp(node_str, "cpu")) { + if (!prom_getintdefault(cpunode, "psr-implementation", 1) && + prom_getintdefault(cpunode, "psr-version", 1) == 5) { + init_turbosparc(); + return; + } + break; + } + } + + init_swift(); + return; + } + + /* Now the Viking family of srmmu. */ + if (psr_typ == 4 && + ((psr_vers == 0) || + ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) { + init_viking(); + return; + } + + /* Finally the Tsunami. */ + if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) { + init_tsunami(); + return; + } + + /* Oh well */ + srmmu_is_bad(); +} + +#ifdef CONFIG_SMP +/* Local cross-calls. */ +static void smp_flush_page_for_dma(unsigned long page) +{ + xc1(local_ops->page_for_dma, page); + local_ops->page_for_dma(page); +} + +static void smp_flush_cache_all(void) +{ + xc0(local_ops->cache_all); + local_ops->cache_all(); +} + +static void smp_flush_tlb_all(void) +{ + xc0(local_ops->tlb_all); + local_ops->tlb_all(); +} + +static bool any_other_mm_cpus(struct mm_struct *mm) +{ + return cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids; +} + +static void smp_flush_cache_mm(struct mm_struct *mm) +{ + if (mm->context != NO_CONTEXT) { + if (any_other_mm_cpus(mm)) + xc1(local_ops->cache_mm, (unsigned long)mm); + local_ops->cache_mm(mm); + } +} + +static void smp_flush_tlb_mm(struct mm_struct *mm) +{ + if (mm->context != NO_CONTEXT) { + if (any_other_mm_cpus(mm)) { + xc1(local_ops->tlb_mm, (unsigned long)mm); + if (atomic_read(&mm->mm_users) == 1 && current->active_mm == mm) + cpumask_copy(mm_cpumask(mm), + cpumask_of(smp_processor_id())); + } + local_ops->tlb_mm(mm); + } +} + +static void smp_flush_cache_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + + if (mm->context != NO_CONTEXT) { + if (any_other_mm_cpus(mm)) + xc3(local_ops->cache_range, (unsigned long)vma, start, + end); + local_ops->cache_range(vma, start, end); + } +} + +static void smp_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, + unsigned long end) +{ + struct mm_struct *mm = vma->vm_mm; + + if (mm->context != NO_CONTEXT) { + if (any_other_mm_cpus(mm)) + xc3(local_ops->tlb_range, (unsigned long)vma, start, + end); + local_ops->tlb_range(vma, start, end); + } +} + +static void smp_flush_cache_page(struct vm_area_struct *vma, unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + + if (mm->context != NO_CONTEXT) { + if (any_other_mm_cpus(mm)) + xc2(local_ops->cache_page, (unsigned long)vma, page); + local_ops->cache_page(vma, page); + } +} + +static void smp_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + struct mm_struct *mm = vma->vm_mm; + + if (mm->context != NO_CONTEXT) { + if (any_other_mm_cpus(mm)) + xc2(local_ops->tlb_page, (unsigned long)vma, page); + local_ops->tlb_page(vma, page); + } +} + +static void smp_flush_page_to_ram(unsigned long page) +{ + /* Current theory is that those who call this are the one's + * who have just dirtied their cache with the pages contents + * in kernel space, therefore we only run this on local cpu. + * + * XXX This experiment failed, research further... -DaveM + */ +#if 1 + xc1(local_ops->page_to_ram, page); +#endif + local_ops->page_to_ram(page); +} + +static void smp_flush_sig_insns(struct mm_struct *mm, unsigned long insn_addr) +{ + if (any_other_mm_cpus(mm)) + xc2(local_ops->sig_insns, (unsigned long)mm, insn_addr); + local_ops->sig_insns(mm, insn_addr); +} + +static struct sparc32_cachetlb_ops smp_cachetlb_ops __ro_after_init = { + .cache_all = smp_flush_cache_all, + .cache_mm = smp_flush_cache_mm, + .cache_page = smp_flush_cache_page, + .cache_range = smp_flush_cache_range, + .tlb_all = smp_flush_tlb_all, + .tlb_mm = smp_flush_tlb_mm, + .tlb_page = smp_flush_tlb_page, + .tlb_range = smp_flush_tlb_range, + .page_to_ram = smp_flush_page_to_ram, + .sig_insns = smp_flush_sig_insns, + .page_for_dma = smp_flush_page_for_dma, +}; +#endif + +/* Load up routines and constants for sun4m and sun4d mmu */ +void __init load_mmu(void) +{ + /* Functions */ + get_srmmu_type(); + +#ifdef CONFIG_SMP + /* El switcheroo... */ + local_ops = sparc32_cachetlb_ops; + + if (sparc_cpu_model == sun4d || sparc_cpu_model == sparc_leon) { + smp_cachetlb_ops.tlb_all = local_ops->tlb_all; + smp_cachetlb_ops.tlb_mm = local_ops->tlb_mm; + smp_cachetlb_ops.tlb_range = local_ops->tlb_range; + smp_cachetlb_ops.tlb_page = local_ops->tlb_page; + } + + if (poke_srmmu == poke_viking) { + /* Avoid unnecessary cross calls. */ + smp_cachetlb_ops.cache_all = local_ops->cache_all; + smp_cachetlb_ops.cache_mm = local_ops->cache_mm; + smp_cachetlb_ops.cache_range = local_ops->cache_range; + smp_cachetlb_ops.cache_page = local_ops->cache_page; + + smp_cachetlb_ops.page_to_ram = local_ops->page_to_ram; + smp_cachetlb_ops.sig_insns = local_ops->sig_insns; + smp_cachetlb_ops.page_for_dma = local_ops->page_for_dma; + } + + /* It really is const after this point. */ + sparc32_cachetlb_ops = (const struct sparc32_cachetlb_ops *) + &smp_cachetlb_ops; +#endif + + if (sparc_cpu_model != sun4d) + ld_mmu_iommu(); +#ifdef CONFIG_SMP + if (sparc_cpu_model == sun4d) + sun4d_init_smp(); + else if (sparc_cpu_model == sparc_leon) + leon_init_smp(); + else + sun4m_init_smp(); +#endif +} diff --git a/arch/sparc/mm/srmmu_access.S b/arch/sparc/mm/srmmu_access.S new file mode 100644 index 000000000..d8d2e644a --- /dev/null +++ b/arch/sparc/mm/srmmu_access.S @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Assembler variants of srmmu access functions. + * Implemented in assembler to allow run-time patching. + * LEON uses a different ASI for MMUREGS than SUN. + * + * The leon_1insn_patch infrastructure is used + * for the run-time patching. + */ + +#include + +#include +#include +#include + +/* unsigned int srmmu_get_mmureg(void) */ +ENTRY(srmmu_get_mmureg) +LEON_PI(lda [%g0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%g0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_mmureg) + +/* void srmmu_set_mmureg(unsigned long regval) */ +ENTRY(srmmu_set_mmureg) +LEON_PI(sta %o0, [%g0] ASI_LEON_MMUREGS) +SUN_PI_(sta %o0, [%g0] ASI_M_MMUREGS) + retl + nop +ENDPROC(srmmu_set_mmureg) + +/* void srmmu_set_ctable_ptr(unsigned long paddr) */ +ENTRY(srmmu_set_ctable_ptr) + /* paddr = ((paddr >> 4) & SRMMU_CTX_PMASK); */ + srl %o0, 4, %g1 + and %g1, SRMMU_CTX_PMASK, %g1 + + mov SRMMU_CTXTBL_PTR, %g2 +LEON_PI(sta %g1, [%g2] ASI_LEON_MMUREGS) +SUN_PI_(sta %g1, [%g2] ASI_M_MMUREGS) + retl + nop +ENDPROC(srmmu_set_ctable_ptr) + + +/* void srmmu_set_context(int context) */ +ENTRY(srmmu_set_context) + mov SRMMU_CTX_REG, %g1 +LEON_PI(sta %o0, [%g1] ASI_LEON_MMUREGS) +SUN_PI_(sta %o0, [%g1] ASI_M_MMUREGS) + retl + nop +ENDPROC(srmmu_set_context) + + +/* int srmmu_get_context(void) */ +ENTRY(srmmu_get_context) + mov SRMMU_CTX_REG, %o0 +LEON_PI(lda [%o0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%o0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_context) + + +/* unsigned int srmmu_get_fstatus(void) */ +ENTRY(srmmu_get_fstatus) + mov SRMMU_FAULT_STATUS, %o0 +LEON_PI(lda [%o0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%o0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_fstatus) + + +/* unsigned int srmmu_get_faddr(void) */ +ENTRY(srmmu_get_faddr) + mov SRMMU_FAULT_ADDR, %o0 +LEON_PI(lda [%o0] ASI_LEON_MMUREGS, %o0) +SUN_PI_(lda [%o0] ASI_M_MMUREGS, %o0) + retl + nop +ENDPROC(srmmu_get_faddr) diff --git a/arch/sparc/mm/swift.S b/arch/sparc/mm/swift.S new file mode 100644 index 000000000..f414bfd8d --- /dev/null +++ b/arch/sparc/mm/swift.S @@ -0,0 +1,256 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * swift.S: MicroSparc-II mmu/cache operations. + * + * Copyright (C) 1999 David S. Miller (davem@redhat.com) + */ + +#include +#include +#include +#include +#include + + .text + .align 4 + +#if 1 /* XXX screw this, I can't get the VAC flushes working + * XXX reliably... -DaveM + */ + .globl swift_flush_cache_all, swift_flush_cache_mm + .globl swift_flush_cache_range, swift_flush_cache_page + .globl swift_flush_page_for_dma + .globl swift_flush_page_to_ram + +swift_flush_cache_all: +swift_flush_cache_mm: +swift_flush_cache_range: +swift_flush_cache_page: +swift_flush_page_for_dma: +swift_flush_page_to_ram: + sethi %hi(0x2000), %o0 +1: subcc %o0, 0x10, %o0 + add %o0, %o0, %o1 + sta %g0, [%o0] ASI_M_DATAC_TAG + bne 1b + sta %g0, [%o1] ASI_M_TXTC_TAG + retl + nop +#else + + .globl swift_flush_cache_all +swift_flush_cache_all: + WINDOW_FLUSH(%g4, %g5) + + /* Just clear out all the tags. */ + sethi %hi(16 * 1024), %o0 +1: subcc %o0, 16, %o0 + sta %g0, [%o0] ASI_M_TXTC_TAG + bne 1b + sta %g0, [%o0] ASI_M_DATAC_TAG + retl + nop + + .globl swift_flush_cache_mm +swift_flush_cache_mm: + ld [%o0 + AOFF_mm_context], %g2 + cmp %g2, -1 + be swift_flush_cache_mm_out + WINDOW_FLUSH(%g4, %g5) + rd %psr, %g1 + andn %g1, PSR_ET, %g3 + wr %g3, 0x0, %psr + nop + nop + mov SRMMU_CTX_REG, %g7 + lda [%g7] ASI_M_MMUREGS, %g5 + sta %g2, [%g7] ASI_M_MMUREGS + +#if 1 + sethi %hi(0x2000), %o0 +1: subcc %o0, 0x10, %o0 + sta %g0, [%o0] ASI_M_FLUSH_CTX + bne 1b + nop +#else + clr %o0 + or %g0, 2048, %g7 + or %g0, 2048, %o1 + add %o1, 2048, %o2 + add %o2, 2048, %o3 + mov 16, %o4 + add %o4, 2048, %o5 + add %o5, 2048, %g2 + add %g2, 2048, %g3 +1: sta %g0, [%o0 ] ASI_M_FLUSH_CTX + sta %g0, [%o0 + %o1] ASI_M_FLUSH_CTX + sta %g0, [%o0 + %o2] ASI_M_FLUSH_CTX + sta %g0, [%o0 + %o3] ASI_M_FLUSH_CTX + sta %g0, [%o0 + %o4] ASI_M_FLUSH_CTX + sta %g0, [%o0 + %o5] ASI_M_FLUSH_CTX + sta %g0, [%o0 + %g2] ASI_M_FLUSH_CTX + sta %g0, [%o0 + %g3] ASI_M_FLUSH_CTX + subcc %g7, 32, %g7 + bne 1b + add %o0, 32, %o0 +#endif + + mov SRMMU_CTX_REG, %g7 + sta %g5, [%g7] ASI_M_MMUREGS + wr %g1, 0x0, %psr + nop + nop +swift_flush_cache_mm_out: + retl + nop + + .globl swift_flush_cache_range +swift_flush_cache_range: + ld [%o0 + VMA_VM_MM], %o0 + sub %o2, %o1, %o2 + sethi %hi(4096), %o3 + cmp %o2, %o3 + bgu swift_flush_cache_mm + nop + b 70f + nop + + .globl swift_flush_cache_page +swift_flush_cache_page: + ld [%o0 + VMA_VM_MM], %o0 +70: + ld [%o0 + AOFF_mm_context], %g2 + cmp %g2, -1 + be swift_flush_cache_page_out + WINDOW_FLUSH(%g4, %g5) + rd %psr, %g1 + andn %g1, PSR_ET, %g3 + wr %g3, 0x0, %psr + nop + nop + mov SRMMU_CTX_REG, %g7 + lda [%g7] ASI_M_MMUREGS, %g5 + sta %g2, [%g7] ASI_M_MMUREGS + + andn %o1, (PAGE_SIZE - 1), %o1 +#if 1 + sethi %hi(0x1000), %o0 +1: subcc %o0, 0x10, %o0 + sta %g0, [%o1 + %o0] ASI_M_FLUSH_PAGE + bne 1b + nop +#else + or %g0, 512, %g7 + or %g0, 512, %o0 + add %o0, 512, %o2 + add %o2, 512, %o3 + add %o3, 512, %o4 + add %o4, 512, %o5 + add %o5, 512, %g3 + add %g3, 512, %g4 +1: sta %g0, [%o1 ] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o0] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o2] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o3] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o4] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o5] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %g3] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %g4] ASI_M_FLUSH_PAGE + subcc %g7, 16, %g7 + bne 1b + add %o1, 16, %o1 +#endif + + mov SRMMU_CTX_REG, %g7 + sta %g5, [%g7] ASI_M_MMUREGS + wr %g1, 0x0, %psr + nop + nop +swift_flush_cache_page_out: + retl + nop + + /* Swift is write-thru, however it is not + * I/O nor TLB-walk coherent. Also it has + * caches which are virtually indexed and tagged. + */ + .globl swift_flush_page_for_dma + .globl swift_flush_page_to_ram +swift_flush_page_for_dma: +swift_flush_page_to_ram: + andn %o0, (PAGE_SIZE - 1), %o1 +#if 1 + sethi %hi(0x1000), %o0 +1: subcc %o0, 0x10, %o0 + sta %g0, [%o1 + %o0] ASI_M_FLUSH_PAGE + bne 1b + nop +#else + or %g0, 512, %g7 + or %g0, 512, %o0 + add %o0, 512, %o2 + add %o2, 512, %o3 + add %o3, 512, %o4 + add %o4, 512, %o5 + add %o5, 512, %g3 + add %g3, 512, %g4 +1: sta %g0, [%o1 ] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o0] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o2] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o3] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o4] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %o5] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %g3] ASI_M_FLUSH_PAGE + sta %g0, [%o1 + %g4] ASI_M_FLUSH_PAGE + subcc %g7, 16, %g7 + bne 1b + add %o1, 16, %o1 +#endif + retl + nop +#endif + + .globl swift_flush_sig_insns +swift_flush_sig_insns: + flush %o1 + retl + flush %o1 + 4 + + .globl swift_flush_tlb_mm + .globl swift_flush_tlb_range + .globl swift_flush_tlb_all +swift_flush_tlb_range: + ld [%o0 + VMA_VM_MM], %o0 +swift_flush_tlb_mm: + ld [%o0 + AOFF_mm_context], %g2 + cmp %g2, -1 + be swift_flush_tlb_all_out +swift_flush_tlb_all: + mov 0x400, %o1 + sta %g0, [%o1] ASI_M_FLUSH_PROBE +swift_flush_tlb_all_out: + retl + nop + + .globl swift_flush_tlb_page +swift_flush_tlb_page: + ld [%o0 + VMA_VM_MM], %o0 + mov SRMMU_CTX_REG, %g1 + ld [%o0 + AOFF_mm_context], %o3 + andn %o1, (PAGE_SIZE - 1), %o1 + cmp %o3, -1 + be swift_flush_tlb_page_out + nop +#if 1 + mov 0x400, %o1 + sta %g0, [%o1] ASI_M_FLUSH_PROBE +#else + lda [%g1] ASI_M_MMUREGS, %g5 + sta %o3, [%g1] ASI_M_MMUREGS + sta %g0, [%o1] ASI_M_FLUSH_PAGE /* rem. virt. cache. prot. */ + sta %g0, [%o1] ASI_M_FLUSH_PROBE + sta %g5, [%g1] ASI_M_MMUREGS +#endif +swift_flush_tlb_page_out: + retl + nop diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c new file mode 100644 index 000000000..6d9dd5eb1 --- /dev/null +++ b/arch/sparc/mm/tlb.c @@ -0,0 +1,314 @@ +// SPDX-License-Identifier: GPL-2.0 +/* arch/sparc64/mm/tlb.c + * + * Copyright (C) 2004 David S. Miller + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +/* Heavily inspired by the ppc64 code. */ + +static DEFINE_PER_CPU(struct tlb_batch, tlb_batch); + +void flush_tlb_pending(void) +{ + struct tlb_batch *tb = &get_cpu_var(tlb_batch); + struct mm_struct *mm = tb->mm; + + if (!tb->tlb_nr) + goto out; + + flush_tsb_user(tb); + + if (CTX_VALID(mm->context)) { + if (tb->tlb_nr == 1) { + global_flush_tlb_page(mm, tb->vaddrs[0]); + } else { +#ifdef CONFIG_SMP + smp_flush_tlb_pending(tb->mm, tb->tlb_nr, + &tb->vaddrs[0]); +#else + __flush_tlb_pending(CTX_HWBITS(tb->mm->context), + tb->tlb_nr, &tb->vaddrs[0]); +#endif + } + } + + tb->tlb_nr = 0; + +out: + put_cpu_var(tlb_batch); +} + +void arch_enter_lazy_mmu_mode(void) +{ + preempt_disable(); +} +/* For lazy_mmu_mode KUnit tests */ +EXPORT_SYMBOL_IF_KUNIT(arch_enter_lazy_mmu_mode); + +void arch_flush_lazy_mmu_mode(void) +{ + struct tlb_batch *tb = this_cpu_ptr(&tlb_batch); + + if (tb->tlb_nr) + flush_tlb_pending(); +} +EXPORT_SYMBOL_IF_KUNIT(arch_flush_lazy_mmu_mode); + +void arch_leave_lazy_mmu_mode(void) +{ + arch_flush_lazy_mmu_mode(); + preempt_enable(); +} +EXPORT_SYMBOL_IF_KUNIT(arch_leave_lazy_mmu_mode); + +static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr, + bool exec, unsigned int hugepage_shift) +{ + struct tlb_batch *tb = &get_cpu_var(tlb_batch); + unsigned long nr; + + vaddr &= PAGE_MASK; + if (exec) + vaddr |= 0x1UL; + + nr = tb->tlb_nr; + + if (unlikely(nr != 0 && mm != tb->mm)) { + flush_tlb_pending(); + nr = 0; + } + + if (!is_lazy_mmu_mode_active()) { + flush_tsb_user_page(mm, vaddr, hugepage_shift); + global_flush_tlb_page(mm, vaddr); + goto out; + } + + if (nr == 0) { + tb->mm = mm; + tb->hugepage_shift = hugepage_shift; + } + + if (tb->hugepage_shift != hugepage_shift) { + flush_tlb_pending(); + tb->hugepage_shift = hugepage_shift; + nr = 0; + } + + tb->vaddrs[nr] = vaddr; + tb->tlb_nr = ++nr; + if (nr >= TLB_BATCH_NR) + flush_tlb_pending(); + +out: + put_cpu_var(tlb_batch); +} + +void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr, + pte_t *ptep, pte_t orig, int fullmm, + unsigned int hugepage_shift) +{ + if (tlb_type != hypervisor && + pte_dirty(orig)) { + unsigned long paddr, pfn = pte_pfn(orig); + struct address_space *mapping; + struct page *page; + struct folio *folio; + + if (!pfn_valid(pfn)) + goto no_cache_flush; + + page = pfn_to_page(pfn); + if (PageReserved(page)) + goto no_cache_flush; + + /* A real file page? */ + folio = page_folio(page); + mapping = folio_flush_mapping(folio); + if (!mapping) + goto no_cache_flush; + + paddr = (unsigned long) page_address(page); + if ((paddr ^ vaddr) & (1 << 13)) + flush_dcache_folio_all(mm, folio); + } + +no_cache_flush: + if (!fullmm) + tlb_batch_add_one(mm, vaddr, pte_exec(orig), hugepage_shift); +} + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr, + pmd_t pmd) +{ + unsigned long end; + pte_t *pte; + + pte = pte_offset_map(&pmd, vaddr); + if (!pte) + return; + end = vaddr + HPAGE_SIZE; + while (vaddr < end) { + if (pte_val(*pte) & _PAGE_VALID) { + bool exec = pte_exec(*pte); + + tlb_batch_add_one(mm, vaddr, exec, PAGE_SHIFT); + } + pte++; + vaddr += PAGE_SIZE; + } + pte_unmap(pte); +} + + +static void __set_pmd_acct(struct mm_struct *mm, unsigned long addr, + pmd_t orig, pmd_t pmd) +{ + if (mm == &init_mm) + return; + + if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) { + /* + * Note that this routine only sets pmds for THP pages. + * Hugetlb pages are handled elsewhere. We need to check + * for huge zero page. Huge zero pages are like hugetlb + * pages in that there is no RSS, but there is the need + * for TSB entries. So, huge zero page counts go into + * hugetlb_pte_count. + */ + if (pmd_val(pmd) & _PAGE_PMD_HUGE) { + if (is_huge_zero_pmd(pmd)) + mm->context.hugetlb_pte_count++; + else + mm->context.thp_pte_count++; + } else { + if (is_huge_zero_pmd(orig)) + mm->context.hugetlb_pte_count--; + else + mm->context.thp_pte_count--; + } + + /* Do not try to allocate the TSB hash table if we + * don't have one already. We have various locks held + * and thus we'll end up doing a GFP_KERNEL allocation + * in an atomic context. + * + * Instead, we let the first TLB miss on a hugepage + * take care of this. + */ + } + + if (!pmd_none(orig)) { + addr &= HPAGE_MASK; + if (pmd_trans_huge(orig)) { + pte_t orig_pte = __pte(pmd_val(orig)); + bool exec = pte_exec(orig_pte); + + tlb_batch_add_one(mm, addr, exec, REAL_HPAGE_SHIFT); + tlb_batch_add_one(mm, addr + REAL_HPAGE_SIZE, exec, + REAL_HPAGE_SHIFT); + } else { + tlb_batch_pmd_scan(mm, addr, orig); + } + } +} + +void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + pmd_t orig = *pmdp; + + *pmdp = pmd; + __set_pmd_acct(mm, addr, orig, pmd); +} + +static inline pmd_t pmdp_establish(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp, pmd_t pmd) +{ + pmd_t old; + + do { + old = *pmdp; + } while (cmpxchg64(&pmdp->pmd, old.pmd, pmd.pmd) != old.pmd); + __set_pmd_acct(vma->vm_mm, address, old, pmd); + + return old; +} + +/* + * This routine is only called when splitting a THP + */ +pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, + pmd_t *pmdp) +{ + pmd_t old, entry; + + VM_WARN_ON_ONCE(!pmd_present(*pmdp)); + entry = __pmd(pmd_val(*pmdp) & ~_PAGE_VALID); + old = pmdp_establish(vma, address, pmdp, entry); + flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); + + /* + * set_pmd_at() will not be called in a way to decrement + * thp_pte_count when splitting a THP, so do it now. + * Sanity check pmd before doing the actual decrement. + */ + if ((pmd_val(entry) & _PAGE_PMD_HUGE) && + !is_huge_zero_pmd(entry)) + (vma->vm_mm)->context.thp_pte_count--; + + return old; +} + +void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, + pgtable_t pgtable) +{ + struct list_head *lh = (struct list_head *) pgtable; + + assert_spin_locked(&mm->page_table_lock); + + /* FIFO */ + if (!pmd_huge_pte(mm, pmdp)) + INIT_LIST_HEAD(lh); + else + list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); + pmd_huge_pte(mm, pmdp) = pgtable; +} + +pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) +{ + struct list_head *lh; + pgtable_t pgtable; + + assert_spin_locked(&mm->page_table_lock); + + /* FIFO */ + pgtable = pmd_huge_pte(mm, pmdp); + lh = (struct list_head *) pgtable; + if (list_empty(lh)) + pmd_huge_pte(mm, pmdp) = NULL; + else { + pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; + list_del(lh); + } + pte_val(pgtable[0]) = 0; + pte_val(pgtable[1]) = 0; + + return pgtable; +} +#endif /* CONFIG_TRANSPARENT_HUGEPAGE */ diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c new file mode 100644 index 000000000..5fe52a64c --- /dev/null +++ b/arch/sparc/mm/tsb.c @@ -0,0 +1,635 @@ +// SPDX-License-Identifier: GPL-2.0 +/* arch/sparc64/mm/tsb.c + * + * Copyright (C) 2006, 2008 David S. Miller + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES]; + +static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long hash_shift, unsigned long nentries) +{ + vaddr >>= hash_shift; + return vaddr & (nentries - 1); +} + +static inline int tag_compare(unsigned long tag, unsigned long vaddr) +{ + return (tag == (vaddr >> 22)); +} + +static void flush_tsb_kernel_range_scan(unsigned long start, unsigned long end) +{ + unsigned long idx; + + for (idx = 0; idx < KERNEL_TSB_NENTRIES; idx++) { + struct tsb *ent = &swapper_tsb[idx]; + unsigned long match = idx << 13; + + match |= (ent->tag << 22); + if (match >= start && match < end) + ent->tag = (1UL << TSB_TAG_INVALID_BIT); + } +} + +/* TSB flushes need only occur on the processor initiating the address + * space modification, not on each cpu the address space has run on. + * Only the TLB flush needs that treatment. + */ + +void flush_tsb_kernel_range(unsigned long start, unsigned long end) +{ + unsigned long v; + + if ((end - start) >> PAGE_SHIFT >= 2 * KERNEL_TSB_NENTRIES) + return flush_tsb_kernel_range_scan(start, end); + + for (v = start; v < end; v += PAGE_SIZE) { + unsigned long hash = tsb_hash(v, PAGE_SHIFT, + KERNEL_TSB_NENTRIES); + struct tsb *ent = &swapper_tsb[hash]; + + if (tag_compare(ent->tag, v)) + ent->tag = (1UL << TSB_TAG_INVALID_BIT); + } +} + +static void __flush_tsb_one_entry(unsigned long tsb, unsigned long v, + unsigned long hash_shift, + unsigned long nentries) +{ + unsigned long tag, ent, hash; + + v &= ~0x1UL; + hash = tsb_hash(v, hash_shift, nentries); + ent = tsb + (hash * sizeof(struct tsb)); + tag = (v >> 22UL); + + tsb_flush(ent, tag); +} + +static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, + unsigned long tsb, unsigned long nentries) +{ + unsigned long i; + + for (i = 0; i < tb->tlb_nr; i++) + __flush_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, nentries); +} + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +static void __flush_huge_tsb_one_entry(unsigned long tsb, unsigned long v, + unsigned long hash_shift, + unsigned long nentries, + unsigned int hugepage_shift) +{ + unsigned int hpage_entries; + unsigned int i; + + hpage_entries = 1 << (hugepage_shift - hash_shift); + for (i = 0; i < hpage_entries; i++) + __flush_tsb_one_entry(tsb, v + (i << hash_shift), hash_shift, + nentries); +} + +static void __flush_huge_tsb_one(struct tlb_batch *tb, unsigned long hash_shift, + unsigned long tsb, unsigned long nentries, + unsigned int hugepage_shift) +{ + unsigned long i; + + for (i = 0; i < tb->tlb_nr; i++) + __flush_huge_tsb_one_entry(tsb, tb->vaddrs[i], hash_shift, + nentries, hugepage_shift); +} +#endif + +void flush_tsb_user(struct tlb_batch *tb) +{ + struct mm_struct *mm = tb->mm; + unsigned long nentries, base, flags; + + spin_lock_irqsave(&mm->context.lock, flags); + + if (tb->hugepage_shift < REAL_HPAGE_SHIFT) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; + nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + if (tb->hugepage_shift == PAGE_SHIFT) + __flush_tsb_one(tb, PAGE_SHIFT, base, nentries); +#if defined(CONFIG_HUGETLB_PAGE) + else + __flush_huge_tsb_one(tb, PAGE_SHIFT, base, nentries, + tb->hugepage_shift); +#endif + } +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; + nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_huge_tsb_one(tb, REAL_HPAGE_SHIFT, base, nentries, + tb->hugepage_shift); + } +#endif + spin_unlock_irqrestore(&mm->context.lock, flags); +} + +void flush_tsb_user_page(struct mm_struct *mm, unsigned long vaddr, + unsigned int hugepage_shift) +{ + unsigned long nentries, base, flags; + + spin_lock_irqsave(&mm->context.lock, flags); + + if (hugepage_shift < REAL_HPAGE_SHIFT) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb; + nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + if (hugepage_shift == PAGE_SHIFT) + __flush_tsb_one_entry(base, vaddr, PAGE_SHIFT, + nentries); +#if defined(CONFIG_HUGETLB_PAGE) + else + __flush_huge_tsb_one_entry(base, vaddr, PAGE_SHIFT, + nentries, hugepage_shift); +#endif + } +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + else if (mm->context.tsb_block[MM_TSB_HUGE].tsb) { + base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb; + nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries; + if (tlb_type == cheetah_plus || tlb_type == hypervisor) + base = __pa(base); + __flush_huge_tsb_one_entry(base, vaddr, REAL_HPAGE_SHIFT, + nentries, hugepage_shift); + } +#endif + spin_unlock_irqrestore(&mm->context.lock, flags); +} + +#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K +#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) +#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB +#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB +#endif + +static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes) +{ + unsigned long tsb_reg, base, tsb_paddr; + unsigned long page_sz, tte; + + mm->context.tsb_block[tsb_idx].tsb_nentries = + tsb_bytes / sizeof(struct tsb); + + switch (tsb_idx) { + case MM_TSB_BASE: + base = TSBMAP_8K_BASE; + break; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + case MM_TSB_HUGE: + base = TSBMAP_4M_BASE; + break; +#endif + default: + BUG(); + } + + tte = pgprot_val(PAGE_KERNEL_LOCKED); + tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb); + BUG_ON(tsb_paddr & (tsb_bytes - 1UL)); + + /* Use the smallest page size that can map the whole TSB + * in one TLB entry. + */ + switch (tsb_bytes) { + case 8192 << 0: + tsb_reg = 0x0UL; +#ifdef DCACHE_ALIASING_POSSIBLE + base += (tsb_paddr & 8192); +#endif + page_sz = 8192; + break; + + case 8192 << 1: + tsb_reg = 0x1UL; + page_sz = 64 * 1024; + break; + + case 8192 << 2: + tsb_reg = 0x2UL; + page_sz = 64 * 1024; + break; + + case 8192 << 3: + tsb_reg = 0x3UL; + page_sz = 64 * 1024; + break; + + case 8192 << 4: + tsb_reg = 0x4UL; + page_sz = 512 * 1024; + break; + + case 8192 << 5: + tsb_reg = 0x5UL; + page_sz = 512 * 1024; + break; + + case 8192 << 6: + tsb_reg = 0x6UL; + page_sz = 512 * 1024; + break; + + case 8192 << 7: + tsb_reg = 0x7UL; + page_sz = 4 * 1024 * 1024; + break; + + default: + printk(KERN_ERR "TSB[%s:%d]: Impossible TSB size %lu, killing process.\n", + current->comm, current->pid, tsb_bytes); + BUG(); + } + tte |= pte_sz_bits(page_sz); + + if (tlb_type == cheetah_plus || tlb_type == hypervisor) { + /* Physical mapping, no locked TLB entry for TSB. */ + tsb_reg |= tsb_paddr; + + mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg; + mm->context.tsb_block[tsb_idx].tsb_map_vaddr = 0; + mm->context.tsb_block[tsb_idx].tsb_map_pte = 0; + } else { + tsb_reg |= base; + tsb_reg |= (tsb_paddr & (page_sz - 1UL)); + tte |= (tsb_paddr & ~(page_sz - 1UL)); + + mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg; + mm->context.tsb_block[tsb_idx].tsb_map_vaddr = base; + mm->context.tsb_block[tsb_idx].tsb_map_pte = tte; + } + + /* Setup the Hypervisor TSB descriptor. */ + if (tlb_type == hypervisor) { + struct hv_tsb_descr *hp = &mm->context.tsb_descr[tsb_idx]; + + switch (tsb_idx) { + case MM_TSB_BASE: + hp->pgsz_idx = HV_PGSZ_IDX_BASE; + break; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + case MM_TSB_HUGE: + hp->pgsz_idx = HV_PGSZ_IDX_HUGE; + break; +#endif + default: + BUG(); + } + hp->assoc = 1; + hp->num_ttes = tsb_bytes / 16; + hp->ctx_idx = 0; + switch (tsb_idx) { + case MM_TSB_BASE: + hp->pgsz_mask = HV_PGSZ_MASK_BASE; + break; +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + case MM_TSB_HUGE: + hp->pgsz_mask = HV_PGSZ_MASK_HUGE; + break; +#endif + default: + BUG(); + } + hp->tsb_base = tsb_paddr; + hp->resv = 0; + } +} + +struct kmem_cache *pgtable_cache __read_mostly; + +static struct kmem_cache *tsb_caches[8] __read_mostly; + +static const char *tsb_cache_names[8] = { + "tsb_8KB", + "tsb_16KB", + "tsb_32KB", + "tsb_64KB", + "tsb_128KB", + "tsb_256KB", + "tsb_512KB", + "tsb_1MB", +}; + +void __init pgtable_cache_init(void) +{ + unsigned long i; + + pgtable_cache = kmem_cache_create("pgtable_cache", + PAGE_SIZE, PAGE_SIZE, + 0, + _clear_page); + if (!pgtable_cache) { + prom_printf("pgtable_cache_init(): Could not create!\n"); + prom_halt(); + } + + for (i = 0; i < ARRAY_SIZE(tsb_cache_names); i++) { + unsigned long size = 8192 << i; + const char *name = tsb_cache_names[i]; + + tsb_caches[i] = kmem_cache_create(name, + size, size, + 0, NULL); + if (!tsb_caches[i]) { + prom_printf("Could not create %s cache\n", name); + prom_halt(); + } + } +} + +int sysctl_tsb_ratio = -2; + +static unsigned long tsb_size_to_rss_limit(unsigned long new_size) +{ + unsigned long num_ents = (new_size / sizeof(struct tsb)); + + if (sysctl_tsb_ratio < 0) + return num_ents - (num_ents >> -sysctl_tsb_ratio); + else + return num_ents + (num_ents >> sysctl_tsb_ratio); +} + +/* When the RSS of an address space exceeds tsb_rss_limit for a TSB, + * do_sparc64_fault() invokes this routine to try and grow it. + * + * When we reach the maximum TSB size supported, we stick ~0UL into + * tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault() + * will not trigger any longer. + * + * The TSB can be anywhere from 8K to 1MB in size, in increasing powers + * of two. The TSB must be aligned to its size, so f.e. a 512K TSB + * must be 512K aligned. It also must be physically contiguous, so we + * cannot use vmalloc(). + * + * The idea here is to grow the TSB when the RSS of the process approaches + * the number of entries that the current TSB can hold at once. Currently, + * we trigger when the RSS hits 3/4 of the TSB capacity. + */ +void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss) +{ + unsigned long max_tsb_size = 1 * 1024 * 1024; + unsigned long new_size, old_size, flags; + struct tsb *old_tsb, *new_tsb; + unsigned long new_cache_index, old_cache_index; + unsigned long new_rss_limit; + gfp_t gfp_flags; + + if (max_tsb_size > PAGE_SIZE << MAX_PAGE_ORDER) + max_tsb_size = PAGE_SIZE << MAX_PAGE_ORDER; + + new_cache_index = 0; + for (new_size = 8192; new_size < max_tsb_size; new_size <<= 1UL) { + new_rss_limit = tsb_size_to_rss_limit(new_size); + if (new_rss_limit > rss) + break; + new_cache_index++; + } + + if (new_size == max_tsb_size) + new_rss_limit = ~0UL; + +retry_tsb_alloc: + gfp_flags = GFP_KERNEL; + if (new_size > (PAGE_SIZE * 2)) + gfp_flags |= __GFP_NOWARN | __GFP_NORETRY; + + new_tsb = kmem_cache_alloc_node(tsb_caches[new_cache_index], + gfp_flags, numa_node_id()); + if (unlikely(!new_tsb)) { + /* Not being able to fork due to a high-order TSB + * allocation failure is very bad behavior. Just back + * down to a 0-order allocation and force no TSB + * growing for this address space. + */ + if (mm->context.tsb_block[tsb_index].tsb == NULL && + new_cache_index > 0) { + new_cache_index = 0; + new_size = 8192; + new_rss_limit = ~0UL; + goto retry_tsb_alloc; + } + + /* If we failed on a TSB grow, we are under serious + * memory pressure so don't try to grow any more. + */ + if (mm->context.tsb_block[tsb_index].tsb != NULL) + mm->context.tsb_block[tsb_index].tsb_rss_limit = ~0UL; + return; + } + + /* Mark all tags as invalid. */ + tsb_init(new_tsb, new_size); + + /* Ok, we are about to commit the changes. If we are + * growing an existing TSB the locking is very tricky, + * so WATCH OUT! + * + * We have to hold mm->context.lock while committing to the + * new TSB, this synchronizes us with processors in + * flush_tsb_user() and switch_mm() for this address space. + * + * But even with that lock held, processors run asynchronously + * accessing the old TSB via TLB miss handling. This is OK + * because those actions are just propagating state from the + * Linux page tables into the TSB, page table mappings are not + * being changed. If a real fault occurs, the processor will + * synchronize with us when it hits flush_tsb_user(), this is + * also true for the case where vmscan is modifying the page + * tables. The only thing we need to be careful with is to + * skip any locked TSB entries during copy_tsb(). + * + * When we finish committing to the new TSB, we have to drop + * the lock and ask all other cpus running this address space + * to run tsb_context_switch() to see the new TSB table. + */ + spin_lock_irqsave(&mm->context.lock, flags); + + old_tsb = mm->context.tsb_block[tsb_index].tsb; + old_cache_index = + (mm->context.tsb_block[tsb_index].tsb_reg_val & 0x7UL); + old_size = (mm->context.tsb_block[tsb_index].tsb_nentries * + sizeof(struct tsb)); + + + /* Handle multiple threads trying to grow the TSB at the same time. + * One will get in here first, and bump the size and the RSS limit. + * The others will get in here next and hit this check. + */ + if (unlikely(old_tsb && + (rss < mm->context.tsb_block[tsb_index].tsb_rss_limit))) { + spin_unlock_irqrestore(&mm->context.lock, flags); + + kmem_cache_free(tsb_caches[new_cache_index], new_tsb); + return; + } + + mm->context.tsb_block[tsb_index].tsb_rss_limit = new_rss_limit; + + if (old_tsb) { + extern void copy_tsb(unsigned long old_tsb_base, + unsigned long old_tsb_size, + unsigned long new_tsb_base, + unsigned long new_tsb_size, + unsigned long page_size_shift); + unsigned long old_tsb_base = (unsigned long) old_tsb; + unsigned long new_tsb_base = (unsigned long) new_tsb; + + if (tlb_type == cheetah_plus || tlb_type == hypervisor) { + old_tsb_base = __pa(old_tsb_base); + new_tsb_base = __pa(new_tsb_base); + } + copy_tsb(old_tsb_base, old_size, new_tsb_base, new_size, + tsb_index == MM_TSB_BASE ? + PAGE_SHIFT : REAL_HPAGE_SHIFT); + } + + mm->context.tsb_block[tsb_index].tsb = new_tsb; + setup_tsb_params(mm, tsb_index, new_size); + + spin_unlock_irqrestore(&mm->context.lock, flags); + + /* If old_tsb is NULL, we're being invoked for the first time + * from init_new_context(). + */ + if (old_tsb) { + /* Reload it on the local cpu. */ + tsb_context_switch(mm); + + /* Now force other processors to do the same. */ + preempt_disable(); + smp_tsb_sync(mm); + preempt_enable(); + + /* Now it is safe to free the old tsb. */ + kmem_cache_free(tsb_caches[old_cache_index], old_tsb); + } +} + +int init_new_context(struct task_struct *tsk, struct mm_struct *mm) +{ + unsigned long mm_rss = get_mm_rss(mm); +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + unsigned long saved_hugetlb_pte_count; + unsigned long saved_thp_pte_count; +#endif + unsigned int i; + + spin_lock_init(&mm->context.lock); + + mm->context.sparc64_ctx_val = 0UL; + + mm->context.tag_store = NULL; + spin_lock_init(&mm->context.tag_lock); + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + /* We reset them to zero because the fork() page copying + * will re-increment the counters as the parent PTEs are + * copied into the child address space. + */ + saved_hugetlb_pte_count = mm->context.hugetlb_pte_count; + saved_thp_pte_count = mm->context.thp_pte_count; + mm->context.hugetlb_pte_count = 0; + mm->context.thp_pte_count = 0; + + mm_rss -= saved_thp_pte_count * (HPAGE_SIZE / PAGE_SIZE); +#endif + + /* copy_mm() copies over the parent's mm_struct before calling + * us, so we need to zero out the TSB pointer or else tsb_grow() + * will be confused and think there is an older TSB to free up. + */ + for (i = 0; i < MM_NUM_TSBS; i++) + mm->context.tsb_block[i].tsb = NULL; + + /* If this is fork, inherit the parent's TSB size. We would + * grow it to that size on the first page fault anyways. + */ + tsb_grow(mm, MM_TSB_BASE, mm_rss); + +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (unlikely(saved_hugetlb_pte_count + saved_thp_pte_count)) + tsb_grow(mm, MM_TSB_HUGE, + (saved_hugetlb_pte_count + saved_thp_pte_count) * + REAL_HPAGE_PER_HPAGE); +#endif + + if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb)) + return -ENOMEM; + + return 0; +} + +static void tsb_destroy_one(struct tsb_config *tp) +{ + unsigned long cache_index; + + if (!tp->tsb) + return; + cache_index = tp->tsb_reg_val & 0x7UL; + kmem_cache_free(tsb_caches[cache_index], tp->tsb); + tp->tsb = NULL; + tp->tsb_reg_val = 0UL; +} + +void destroy_context(struct mm_struct *mm) +{ + unsigned long flags, i; + + for (i = 0; i < MM_NUM_TSBS; i++) + tsb_destroy_one(&mm->context.tsb_block[i]); + + spin_lock_irqsave(&ctx_alloc_lock, flags); + + if (CTX_VALID(mm->context)) { + unsigned long nr = CTX_NRBITS(mm->context); + mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63)); + } + + spin_unlock_irqrestore(&ctx_alloc_lock, flags); + + /* If ADI tag storage was allocated for this task, free it */ + if (mm->context.tag_store) { + tag_storage_desc_t *tag_desc; + unsigned long max_desc; + unsigned char *tags; + + tag_desc = mm->context.tag_store; + max_desc = PAGE_SIZE/sizeof(tag_storage_desc_t); + for (i = 0; i < max_desc; i++) { + tags = tag_desc->tags; + tag_desc->tags = NULL; + kfree(tags); + tag_desc++; + } + kfree(mm->context.tag_store); + mm->context.tag_store = NULL; + } +} diff --git a/arch/sparc/mm/tsunami.S b/arch/sparc/mm/tsunami.S new file mode 100644 index 000000000..62b742df6 --- /dev/null +++ b/arch/sparc/mm/tsunami.S @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * tsunami.S: High speed MicroSparc-I mmu/cache operations. + * + * Copyright (C) 1997 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include + + .text + .align 4 + + .globl tsunami_flush_cache_all, tsunami_flush_cache_mm + .globl tsunami_flush_cache_range, tsunami_flush_cache_page + .globl tsunami_flush_page_to_ram, tsunami_flush_page_for_dma + .globl tsunami_flush_sig_insns + .globl tsunami_flush_tlb_all, tsunami_flush_tlb_mm + .globl tsunami_flush_tlb_range, tsunami_flush_tlb_page + + /* Sliiick... */ +tsunami_flush_cache_page: +tsunami_flush_cache_range: + ld [%o0 + VMA_VM_MM], %o0 +tsunami_flush_cache_mm: + ld [%o0 + AOFF_mm_context], %g2 + cmp %g2, -1 + be tsunami_flush_cache_out +tsunami_flush_cache_all: + WINDOW_FLUSH(%g4, %g5) +tsunami_flush_page_for_dma: + sta %g0, [%g0] ASI_M_IC_FLCLEAR + sta %g0, [%g0] ASI_M_DC_FLCLEAR +tsunami_flush_cache_out: +tsunami_flush_page_to_ram: + retl + nop + +tsunami_flush_sig_insns: + flush %o1 + retl + flush %o1 + 4 + + /* More slick stuff... */ +tsunami_flush_tlb_range: + ld [%o0 + VMA_VM_MM], %o0 +tsunami_flush_tlb_mm: + ld [%o0 + AOFF_mm_context], %g2 + cmp %g2, -1 + be tsunami_flush_tlb_out +tsunami_flush_tlb_all: + mov 0x400, %o1 + sta %g0, [%o1] ASI_M_FLUSH_PROBE + nop + nop + nop + nop + nop +tsunami_flush_tlb_out: + retl + nop + + /* This one can be done in a fine grained manner... */ +tsunami_flush_tlb_page: + ld [%o0 + VMA_VM_MM], %o0 + mov SRMMU_CTX_REG, %g1 + ld [%o0 + AOFF_mm_context], %o3 + andn %o1, (PAGE_SIZE - 1), %o1 + cmp %o3, -1 + be tsunami_flush_tlb_page_out + lda [%g1] ASI_M_MMUREGS, %g5 + sta %o3, [%g1] ASI_M_MMUREGS + sta %g0, [%o1] ASI_M_FLUSH_PROBE + nop + nop + nop + nop + nop +tsunami_flush_tlb_page_out: + retl + sta %g5, [%g1] ASI_M_MMUREGS + +#define MIRROR_BLOCK(dst, src, offset, t0, t1, t2, t3) \ + ldd [src + offset + 0x18], t0; \ + std t0, [dst + offset + 0x18]; \ + ldd [src + offset + 0x10], t2; \ + std t2, [dst + offset + 0x10]; \ + ldd [src + offset + 0x08], t0; \ + std t0, [dst + offset + 0x08]; \ + ldd [src + offset + 0x00], t2; \ + std t2, [dst + offset + 0x00]; + +tsunami_copy_1page: +/* NOTE: This routine has to be shorter than 70insns --jj */ + or %g0, (PAGE_SIZE >> 8), %g1 +1: + MIRROR_BLOCK(%o0, %o1, 0x00, %o2, %o3, %o4, %o5) + MIRROR_BLOCK(%o0, %o1, 0x20, %o2, %o3, %o4, %o5) + MIRROR_BLOCK(%o0, %o1, 0x40, %o2, %o3, %o4, %o5) + MIRROR_BLOCK(%o0, %o1, 0x60, %o2, %o3, %o4, %o5) + MIRROR_BLOCK(%o0, %o1, 0x80, %o2, %o3, %o4, %o5) + MIRROR_BLOCK(%o0, %o1, 0xa0, %o2, %o3, %o4, %o5) + MIRROR_BLOCK(%o0, %o1, 0xc0, %o2, %o3, %o4, %o5) + MIRROR_BLOCK(%o0, %o1, 0xe0, %o2, %o3, %o4, %o5) + subcc %g1, 1, %g1 + add %o0, 0x100, %o0 + bne 1b + add %o1, 0x100, %o1 + + .globl tsunami_setup_blockops +tsunami_setup_blockops: + sethi %hi(__copy_1page), %o0 + or %o0, %lo(__copy_1page), %o0 + sethi %hi(tsunami_copy_1page), %o1 + or %o1, %lo(tsunami_copy_1page), %o1 + sethi %hi(tsunami_setup_blockops), %o2 + or %o2, %lo(tsunami_setup_blockops), %o2 + ld [%o1], %o4 +1: add %o1, 4, %o1 + st %o4, [%o0] + add %o0, 4, %o0 + cmp %o1, %o2 + bne 1b + ld [%o1], %o4 + sta %g0, [%g0] ASI_M_IC_FLCLEAR + sta %g0, [%g0] ASI_M_DC_FLCLEAR + retl + nop diff --git a/arch/sparc/mm/ultra.S b/arch/sparc/mm/ultra.S new file mode 100644 index 000000000..70e658d10 --- /dev/null +++ b/arch/sparc/mm/ultra.S @@ -0,0 +1,1102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ultra.S: Don't expand these all over the place... + * + * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + /* Basically, most of the Spitfire vs. Cheetah madness + * has to do with the fact that Cheetah does not support + * IMMU flushes out of the secondary context. Someone needs + * to throw a south lake birthday party for the folks + * in Microelectronics who refused to fix this shit. + */ + + /* This file is meant to be read efficiently by the CPU, not humans. + * Staraj sie tego nikomu nie pierdolnac... + */ + .text + .align 32 + .globl __flush_tlb_mm +__flush_tlb_mm: /* 19 insns */ + /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */ + ldxa [%o1] ASI_DMMU, %g2 + cmp %g2, %o0 + bne,pn %icc, __spitfire_flush_tlb_mm_slow + mov 0x50, %g3 + stxa %g0, [%g3] ASI_DMMU_DEMAP + stxa %g0, [%g3] ASI_IMMU_DEMAP + sethi %hi(KERNBASE), %g3 + flush %g3 + retl + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + + .align 32 + .globl __flush_tlb_page +__flush_tlb_page: /* 22 insns */ + /* %o0 = context, %o1 = vaddr */ + rdpr %pstate, %g7 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, %pstate + mov SECONDARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + stxa %o0, [%o4] ASI_DMMU + andcc %o1, 1, %g0 + andn %o1, 1, %o3 + be,pn %icc, 1f + or %o3, 0x10, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +1: stxa %g0, [%o3] ASI_DMMU_DEMAP + membar #Sync + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 + retl + wrpr %g7, 0x0, %pstate + nop + nop + nop + nop + + .align 32 + .globl __flush_tlb_pending +__flush_tlb_pending: /* 27 insns */ + /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ + rdpr %pstate, %g7 + sllx %o1, 3, %o1 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, %pstate + mov SECONDARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + stxa %o0, [%o4] ASI_DMMU +1: sub %o1, (1 << 3), %o1 + ldx [%o2 + %o1], %o3 + andcc %o3, 1, %g0 + andn %o3, 1, %o3 + be,pn %icc, 2f + or %o3, 0x10, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +2: stxa %g0, [%o3] ASI_DMMU_DEMAP + membar #Sync + brnz,pt %o1, 1b + nop + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 + retl + wrpr %g7, 0x0, %pstate + nop + nop + nop + nop + + .align 32 + .globl __flush_tlb_kernel_range +__flush_tlb_kernel_range: /* 31 insns */ + /* %o0=start, %o1=end */ + cmp %o0, %o1 + be,pn %xcc, 2f + sub %o1, %o0, %o3 + srlx %o3, 18, %o4 + brnz,pn %o4, __spitfire_flush_tlb_kernel_range_slow + sethi %hi(PAGE_SIZE), %o4 + sub %o3, %o4, %o3 + or %o0, 0x20, %o0 ! Nucleus +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %o3, 1b + sub %o3, %o4, %o3 +2: sethi %hi(KERNBASE), %o3 + flush %o3 + retl + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + +__spitfire_flush_tlb_kernel_range_slow: + mov 63 * 8, %o4 +1: ldxa [%o4] ASI_ITLB_DATA_ACCESS, %o3 + andcc %o3, 0x40, %g0 /* _PAGE_L_4U */ + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %o3 + stxa %g0, [%o3] ASI_IMMU + stxa %g0, [%o4] ASI_ITLB_DATA_ACCESS + membar #Sync +2: ldxa [%o4] ASI_DTLB_DATA_ACCESS, %o3 + andcc %o3, 0x40, %g0 + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %o3 + stxa %g0, [%o3] ASI_DMMU + stxa %g0, [%o4] ASI_DTLB_DATA_ACCESS + membar #Sync +2: sub %o4, 8, %o4 + brgez,pt %o4, 1b + nop + retl + nop + +__spitfire_flush_tlb_mm_slow: + rdpr %pstate, %g1 + wrpr %g1, PSTATE_IE, %pstate + stxa %o0, [%o1] ASI_DMMU + stxa %g0, [%g3] ASI_DMMU_DEMAP + stxa %g0, [%g3] ASI_IMMU_DEMAP + flush %g6 + stxa %g2, [%o1] ASI_DMMU + sethi %hi(KERNBASE), %o1 + flush %o1 + retl + wrpr %g1, 0, %pstate + +/* + * The following code flushes one page_size worth. + */ + .section .kprobes.text, "ax" + .align 32 + .globl __flush_icache_page +__flush_icache_page: /* %o0 = phys_page */ + srlx %o0, PAGE_SHIFT, %o0 + sethi %hi(PAGE_OFFSET), %g1 + sllx %o0, PAGE_SHIFT, %o0 + sethi %hi(PAGE_SIZE), %g2 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 + add %o0, %g1, %o0 +1: subcc %g2, 32, %g2 + bne,pt %icc, 1b + flush %o0 + %g2 + retl + nop + +#ifdef DCACHE_ALIASING_POSSIBLE + +#if (PAGE_SHIFT != 13) +#error only page shift of 13 is supported by dcache flush +#endif + +#define DTAG_MASK 0x3 + + /* This routine is Spitfire specific so the hardcoded + * D-cache size and line-size are OK. + */ + .align 64 + .globl __flush_dcache_page +__flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */ + sethi %hi(PAGE_OFFSET), %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 + sub %o0, %g1, %o0 ! physical address + srlx %o0, 11, %o0 ! make D-cache TAG + sethi %hi(1 << 14), %o2 ! D-cache size + sub %o2, (1 << 5), %o2 ! D-cache line size +1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG + andcc %o3, DTAG_MASK, %g0 ! Valid? + be,pn %xcc, 2f ! Nope, branch + andn %o3, DTAG_MASK, %o3 ! Clear valid bits + cmp %o3, %o0 ! TAG match? + bne,pt %xcc, 2f ! Nope, branch + nop + stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG + membar #Sync +2: brnz,pt %o2, 1b + sub %o2, (1 << 5), %o2 ! D-cache line size + + /* The I-cache does not snoop local stores so we + * better flush that too when necessary. + */ + brnz,pt %o1, __flush_icache_page + sllx %o0, 11, %o0 + retl + nop + +#endif /* DCACHE_ALIASING_POSSIBLE */ + + .previous + + /* Cheetah specific versions, patched at boot time. */ +__cheetah_flush_tlb_mm: /* 19 insns */ + rdpr %pstate, %g7 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, 0x0, %pstate + wrpr %g0, 1, %tl + mov PRIMARY_CONTEXT, %o2 + mov 0x40, %g3 + ldxa [%o2] ASI_DMMU, %g2 + srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1 + sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1 + or %o0, %o1, %o0 /* Preserve nucleus page size fields */ + stxa %o0, [%o2] ASI_DMMU + stxa %g0, [%g3] ASI_DMMU_DEMAP + stxa %g0, [%g3] ASI_IMMU_DEMAP + stxa %g2, [%o2] ASI_DMMU + sethi %hi(KERNBASE), %o2 + flush %o2 + wrpr %g0, 0, %tl + retl + wrpr %g7, 0x0, %pstate + +__cheetah_flush_tlb_page: /* 22 insns */ + /* %o0 = context, %o1 = vaddr */ + rdpr %pstate, %g7 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, 0x0, %pstate + wrpr %g0, 1, %tl + mov PRIMARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 + sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 + or %o0, %o3, %o0 /* Preserve nucleus page size fields */ + stxa %o0, [%o4] ASI_DMMU + andcc %o1, 1, %g0 + be,pn %icc, 1f + andn %o1, 1, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +1: stxa %g0, [%o3] ASI_DMMU_DEMAP + membar #Sync + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 + wrpr %g0, 0, %tl + retl + wrpr %g7, 0x0, %pstate + +__cheetah_flush_tlb_pending: /* 27 insns */ + /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ + rdpr %pstate, %g7 + sllx %o1, 3, %o1 + andn %g7, PSTATE_IE, %g2 + wrpr %g2, 0x0, %pstate + wrpr %g0, 1, %tl + mov PRIMARY_CONTEXT, %o4 + ldxa [%o4] ASI_DMMU, %g2 + srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3 + sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3 + or %o0, %o3, %o0 /* Preserve nucleus page size fields */ + stxa %o0, [%o4] ASI_DMMU +1: sub %o1, (1 << 3), %o1 + ldx [%o2 + %o1], %o3 + andcc %o3, 1, %g0 + be,pn %icc, 2f + andn %o3, 1, %o3 + stxa %g0, [%o3] ASI_IMMU_DEMAP +2: stxa %g0, [%o3] ASI_DMMU_DEMAP + membar #Sync + brnz,pt %o1, 1b + nop + stxa %g2, [%o4] ASI_DMMU + sethi %hi(KERNBASE), %o4 + flush %o4 + wrpr %g0, 0, %tl + retl + wrpr %g7, 0x0, %pstate + +__cheetah_flush_tlb_kernel_range: /* 31 insns */ + /* %o0=start, %o1=end */ + cmp %o0, %o1 + be,pn %xcc, 2f + sub %o1, %o0, %o3 + srlx %o3, 18, %o4 + brnz,pn %o4, 3f + sethi %hi(PAGE_SIZE), %o4 + sub %o3, %o4, %o3 + or %o0, 0x20, %o0 ! Nucleus +1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP + stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %o3, 1b + sub %o3, %o4, %o3 +2: sethi %hi(KERNBASE), %o3 + flush %o3 + retl + nop +3: mov 0x80, %o4 + stxa %g0, [%o4] ASI_DMMU_DEMAP + membar #Sync + stxa %g0, [%o4] ASI_IMMU_DEMAP + membar #Sync + retl + nop + nop + nop + nop + nop + nop + nop + nop + +#ifdef DCACHE_ALIASING_POSSIBLE +__cheetah_flush_dcache_page: /* 11 insns */ + sethi %hi(PAGE_OFFSET), %g1 + ldx [%g1 + %lo(PAGE_OFFSET)], %g1 + sub %o0, %g1, %o0 + sethi %hi(PAGE_SIZE), %o4 +1: subcc %o4, (1 << 5), %o4 + stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE + membar #Sync + bne,pt %icc, 1b + nop + retl /* I-cache flush never needed on Cheetah, see callers. */ + nop +#endif /* DCACHE_ALIASING_POSSIBLE */ + + /* Hypervisor specific versions, patched at boot time. */ +__hypervisor_tlb_tl0_error: + save %sp, -192, %sp + mov %i0, %o0 + call hypervisor_tlbop_error + mov %i1, %o1 + ret + restore + +__hypervisor_flush_tlb_mm: /* 19 insns */ + mov %o0, %o2 /* ARG2: mmu context */ + mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 1f + mov HV_FAST_MMU_DEMAP_CTX, %o1 + retl + nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o5 + jmpl %o5 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop + +__hypervisor_flush_tlb_page: /* 22 insns */ + /* %o0 = context, %o1 = vaddr */ + mov %o0, %g2 + mov %o1, %o0 /* ARG0: vaddr + IMMU-bit */ + mov %g2, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + srlx %o0, PAGE_SHIFT, %o0 + sllx %o0, PAGE_SHIFT, %o0 + ta HV_MMU_UNMAP_ADDR_TRAP + brnz,pn %o0, 1f + mov HV_MMU_UNMAP_ADDR_TRAP, %o1 + retl + nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop + nop + nop + +__hypervisor_flush_tlb_pending: /* 27 insns */ + /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */ + sllx %o1, 3, %g1 + mov %o2, %g2 + mov %o0, %g3 +1: sub %g1, (1 << 3), %g1 + ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */ + mov %g3, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + srlx %o0, PAGE_SHIFT, %o0 + sllx %o0, PAGE_SHIFT, %o0 + ta HV_MMU_UNMAP_ADDR_TRAP + brnz,pn %o0, 1f + mov HV_MMU_UNMAP_ADDR_TRAP, %o1 + brnz,pt %g1, 1b + nop + retl + nop +1: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop + nop + nop + nop + nop + nop + nop + nop + nop + +__hypervisor_flush_tlb_kernel_range: /* 31 insns */ + /* %o0=start, %o1=end */ + cmp %o0, %o1 + be,pn %xcc, 2f + sub %o1, %o0, %g2 + srlx %g2, 18, %g3 + brnz,pn %g3, 4f + mov %o0, %g1 + sethi %hi(PAGE_SIZE), %g3 + sub %g2, %g3, %g2 +1: add %g1, %g2, %o0 /* ARG0: virtual address */ + mov 0, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + ta HV_MMU_UNMAP_ADDR_TRAP + brnz,pn %o0, 3f + mov HV_MMU_UNMAP_ADDR_TRAP, %o1 + brnz,pt %g2, 1b + sub %g2, %g3, %g2 +2: retl + nop +3: sethi %hi(__hypervisor_tlb_tl0_error), %o2 + jmpl %o2 + %lo(__hypervisor_tlb_tl0_error), %g0 + nop +4: mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov 0, %o2 /* ARG2: mmu context == nucleus */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 3b + mov HV_FAST_MMU_DEMAP_CTX, %o1 + retl + nop + +#ifdef DCACHE_ALIASING_POSSIBLE + /* XXX Niagara and friends have an 8K cache, so no aliasing is + * XXX possible, but nothing explicit in the Hypervisor API + * XXX guarantees this. + */ +__hypervisor_flush_dcache_page: /* 2 insns */ + retl + nop +#endif + +tlb_patch_one: +1: lduw [%o1], %g1 + stw %g1, [%o0] + flush %o0 + subcc %o2, 1, %o2 + add %o1, 4, %o1 + bne,pt %icc, 1b + add %o0, 4, %o0 + retl + nop + +#ifdef CONFIG_SMP + /* These are all called by the slaves of a cross call, at + * trap level 1, with interrupts fully disabled. + * + * Register usage: + * %g5 mm->context (all tlb flushes) + * %g1 address arg 1 (tlb page and range flushes) + * %g7 address arg 2 (tlb range flush only) + * + * %g6 scratch 1 + * %g2 scratch 2 + * %g3 scratch 3 + * %g4 scratch 4 + */ + .align 32 + .globl xcall_flush_tlb_mm +xcall_flush_tlb_mm: /* 24 insns */ + mov PRIMARY_CONTEXT, %g2 + ldxa [%g2] ASI_DMMU, %g3 + srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4 + sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 + or %g5, %g4, %g5 /* Preserve nucleus page size fields */ + stxa %g5, [%g2] ASI_DMMU + mov 0x40, %g4 + stxa %g0, [%g4] ASI_DMMU_DEMAP + stxa %g0, [%g4] ASI_IMMU_DEMAP + stxa %g3, [%g2] ASI_DMMU + retry + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + + .globl xcall_flush_tlb_page +xcall_flush_tlb_page: /* 20 insns */ + /* %g5=context, %g1=vaddr */ + mov PRIMARY_CONTEXT, %g4 + ldxa [%g4] ASI_DMMU, %g2 + srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4 + sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4 + or %g5, %g4, %g5 + mov PRIMARY_CONTEXT, %g4 + stxa %g5, [%g4] ASI_DMMU + andcc %g1, 0x1, %g0 + be,pn %icc, 2f + andn %g1, 0x1, %g5 + stxa %g0, [%g5] ASI_IMMU_DEMAP +2: stxa %g0, [%g5] ASI_DMMU_DEMAP + membar #Sync + stxa %g2, [%g4] ASI_DMMU + retry + nop + nop + nop + nop + nop + + .globl xcall_flush_tlb_kernel_range +xcall_flush_tlb_kernel_range: /* 44 insns */ + sethi %hi(PAGE_SIZE - 1), %g2 + or %g2, %lo(PAGE_SIZE - 1), %g2 + andn %g1, %g2, %g1 + andn %g7, %g2, %g7 + sub %g7, %g1, %g3 + srlx %g3, 18, %g2 + brnz,pn %g2, 2f + sethi %hi(PAGE_SIZE), %g2 + sub %g3, %g2, %g3 + or %g1, 0x20, %g1 ! Nucleus +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %g3, 1b + sub %g3, %g2, %g3 + retry +2: mov 63 * 8, %g1 +1: ldxa [%g1] ASI_ITLB_DATA_ACCESS, %g2 + andcc %g2, 0x40, %g0 /* _PAGE_L_4U */ + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %g2 + stxa %g0, [%g2] ASI_IMMU + stxa %g0, [%g1] ASI_ITLB_DATA_ACCESS + membar #Sync +2: ldxa [%g1] ASI_DTLB_DATA_ACCESS, %g2 + andcc %g2, 0x40, %g0 + bne,pn %xcc, 2f + mov TLB_TAG_ACCESS, %g2 + stxa %g0, [%g2] ASI_DMMU + stxa %g0, [%g1] ASI_DTLB_DATA_ACCESS + membar #Sync +2: sub %g1, 8, %g1 + brgez,pt %g1, 1b + nop + retry + nop + nop + nop + nop + nop + nop + nop + nop + nop + + /* This runs in a very controlled environment, so we do + * not need to worry about BH races etc. + */ + .globl xcall_sync_tick +xcall_sync_tick: + +661: rdpr %pstate, %g2 + wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate + .section .sun4v_2insn_patch, "ax" + .word 661b + nop + nop + .previous + + rdpr %pil, %g2 + wrpr %g0, PIL_NORMAL_MAX, %pil + sethi %hi(109f), %g7 + b,pt %xcc, etrap_irq +109: or %g7, %lo(109b), %g7 +#ifdef CONFIG_TRACE_IRQFLAGS + call trace_hardirqs_off + nop +#endif + call smp_synchronize_tick_client + nop + b rtrap_xcall + ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1 + + .globl xcall_fetch_glob_regs +xcall_fetch_glob_regs: + sethi %hi(global_cpu_snapshot), %g1 + or %g1, %lo(global_cpu_snapshot), %g1 + __GET_CPUID(%g2) + sllx %g2, 6, %g3 + add %g1, %g3, %g1 + rdpr %tstate, %g7 + stx %g7, [%g1 + GR_SNAP_TSTATE] + rdpr %tpc, %g7 + stx %g7, [%g1 + GR_SNAP_TPC] + rdpr %tnpc, %g7 + stx %g7, [%g1 + GR_SNAP_TNPC] + stx %o7, [%g1 + GR_SNAP_O7] + stx %i7, [%g1 + GR_SNAP_I7] + /* Don't try this at home kids... */ + rdpr %cwp, %g3 + sub %g3, 1, %g7 + wrpr %g7, %cwp + mov %i7, %g7 + wrpr %g3, %cwp + stx %g7, [%g1 + GR_SNAP_RPC] + sethi %hi(trap_block), %g7 + or %g7, %lo(trap_block), %g7 + sllx %g2, TRAP_BLOCK_SZ_SHIFT, %g2 + add %g7, %g2, %g7 + ldx [%g7 + TRAP_PER_CPU_THREAD], %g3 + stx %g3, [%g1 + GR_SNAP_THREAD] + retry + + .globl xcall_fetch_glob_pmu +xcall_fetch_glob_pmu: + sethi %hi(global_cpu_snapshot), %g1 + or %g1, %lo(global_cpu_snapshot), %g1 + __GET_CPUID(%g2) + sllx %g2, 6, %g3 + add %g1, %g3, %g1 + rd %pic, %g7 + stx %g7, [%g1 + (4 * 8)] + rd %pcr, %g7 + stx %g7, [%g1 + (0 * 8)] + retry + + .globl xcall_fetch_glob_pmu_n4 +xcall_fetch_glob_pmu_n4: + sethi %hi(global_cpu_snapshot), %g1 + or %g1, %lo(global_cpu_snapshot), %g1 + __GET_CPUID(%g2) + sllx %g2, 6, %g3 + add %g1, %g3, %g1 + + ldxa [%g0] ASI_PIC, %g7 + stx %g7, [%g1 + (4 * 8)] + mov 0x08, %g3 + ldxa [%g3] ASI_PIC, %g7 + stx %g7, [%g1 + (5 * 8)] + mov 0x10, %g3 + ldxa [%g3] ASI_PIC, %g7 + stx %g7, [%g1 + (6 * 8)] + mov 0x18, %g3 + ldxa [%g3] ASI_PIC, %g7 + stx %g7, [%g1 + (7 * 8)] + + mov %o0, %g2 + mov %o1, %g3 + mov %o5, %g7 + + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 3, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (3 * 8)] + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 2, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (2 * 8)] + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 1, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (1 * 8)] + mov HV_FAST_VT_GET_PERFREG, %o5 + mov 0, %o0 + ta HV_FAST_TRAP + stx %o1, [%g1 + (0 * 8)] + + mov %g2, %o0 + mov %g3, %o1 + mov %g7, %o5 + + retry + +__cheetah_xcall_flush_tlb_kernel_range: /* 44 insns */ + sethi %hi(PAGE_SIZE - 1), %g2 + or %g2, %lo(PAGE_SIZE - 1), %g2 + andn %g1, %g2, %g1 + andn %g7, %g2, %g7 + sub %g7, %g1, %g3 + srlx %g3, 18, %g2 + brnz,pn %g2, 2f + sethi %hi(PAGE_SIZE), %g2 + sub %g3, %g2, %g3 + or %g1, 0x20, %g1 ! Nucleus +1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP + stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP + membar #Sync + brnz,pt %g3, 1b + sub %g3, %g2, %g3 + retry +2: mov 0x80, %g2 + stxa %g0, [%g2] ASI_DMMU_DEMAP + membar #Sync + stxa %g0, [%g2] ASI_IMMU_DEMAP + membar #Sync + retry + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + nop + +#ifdef DCACHE_ALIASING_POSSIBLE + .align 32 + .globl xcall_flush_dcache_page_cheetah +xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */ + sethi %hi(PAGE_SIZE), %g3 +1: subcc %g3, (1 << 5), %g3 + stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE + membar #Sync + bne,pt %icc, 1b + nop + retry + nop +#endif /* DCACHE_ALIASING_POSSIBLE */ + + .globl xcall_flush_dcache_page_spitfire +xcall_flush_dcache_page_spitfire: /* %g1 == physical page address + %g7 == kernel page virtual address + %g5 == (page->mapping != NULL) */ +#ifdef DCACHE_ALIASING_POSSIBLE + srlx %g1, (13 - 2), %g1 ! Form tag comparitor + sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K + sub %g3, (1 << 5), %g3 ! D$ linesize == 32 +1: ldxa [%g3] ASI_DCACHE_TAG, %g2 + andcc %g2, 0x3, %g0 + be,pn %xcc, 2f + andn %g2, 0x3, %g2 + cmp %g2, %g1 + + bne,pt %xcc, 2f + nop + stxa %g0, [%g3] ASI_DCACHE_TAG + membar #Sync +2: cmp %g3, 0 + bne,pt %xcc, 1b + sub %g3, (1 << 5), %g3 + + brz,pn %g5, 2f +#endif /* DCACHE_ALIASING_POSSIBLE */ + sethi %hi(PAGE_SIZE), %g3 + +1: flush %g7 + subcc %g3, (1 << 5), %g3 + bne,pt %icc, 1b + add %g7, (1 << 5), %g7 + +2: retry + nop + nop + + /* %g5: error + * %g6: tlb op + */ +__hypervisor_tlb_xcall_error: + mov %g5, %g4 + mov %g6, %g5 + ba,pt %xcc, etrap + rd %pc, %g7 + mov %l4, %o0 + call hypervisor_tlbop_error_xcall + mov %l5, %o1 + ba,a,pt %xcc, rtrap + + .globl __hypervisor_xcall_flush_tlb_mm +__hypervisor_xcall_flush_tlb_mm: /* 24 insns */ + /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */ + mov %o0, %g2 + mov %o1, %g3 + mov %o2, %g4 + mov %o3, %g1 + mov %o5, %g7 + clr %o0 /* ARG0: CPU lists unimplemented */ + clr %o1 /* ARG1: CPU lists unimplemented */ + mov %g5, %o2 /* ARG2: mmu context */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + mov HV_FAST_MMU_DEMAP_CTX, %g6 + brnz,pn %o0, 1f + mov %o0, %g5 + mov %g2, %o0 + mov %g3, %o1 + mov %g4, %o2 + mov %g1, %o3 + mov %g7, %o5 + membar #Sync + retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop + + .globl __hypervisor_xcall_flush_tlb_page +__hypervisor_xcall_flush_tlb_page: /* 20 insns */ + /* %g5=ctx, %g1=vaddr */ + mov %o0, %g2 + mov %o1, %g3 + mov %o2, %g4 + mov %g1, %o0 /* ARG0: virtual address */ + mov %g5, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + srlx %o0, PAGE_SHIFT, %o0 + sllx %o0, PAGE_SHIFT, %o0 + ta HV_MMU_UNMAP_ADDR_TRAP + mov HV_MMU_UNMAP_ADDR_TRAP, %g6 + brnz,a,pn %o0, 1f + mov %o0, %g5 + mov %g2, %o0 + mov %g3, %o1 + mov %g4, %o2 + membar #Sync + retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop + + .globl __hypervisor_xcall_flush_tlb_kernel_range +__hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */ + /* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */ + sethi %hi(PAGE_SIZE - 1), %g2 + or %g2, %lo(PAGE_SIZE - 1), %g2 + andn %g1, %g2, %g1 + andn %g7, %g2, %g7 + sub %g7, %g1, %g3 + srlx %g3, 18, %g7 + add %g2, 1, %g2 + sub %g3, %g2, %g3 + mov %o0, %g2 + mov %o1, %g4 + brnz,pn %g7, 2f + mov %o2, %g7 +1: add %g1, %g3, %o0 /* ARG0: virtual address */ + mov 0, %o1 /* ARG1: mmu context */ + mov HV_MMU_ALL, %o2 /* ARG2: flags */ + ta HV_MMU_UNMAP_ADDR_TRAP + mov HV_MMU_UNMAP_ADDR_TRAP, %g6 + brnz,pn %o0, 1f + mov %o0, %g5 + sethi %hi(PAGE_SIZE), %o2 + brnz,pt %g3, 1b + sub %g3, %o2, %g3 +5: mov %g2, %o0 + mov %g4, %o1 + mov %g7, %o2 + membar #Sync + retry +1: sethi %hi(__hypervisor_tlb_xcall_error), %g4 + jmpl %g4 + %lo(__hypervisor_tlb_xcall_error), %g0 + nop +2: mov %o3, %g1 + mov %o5, %g3 + mov 0, %o0 /* ARG0: CPU lists unimplemented */ + mov 0, %o1 /* ARG1: CPU lists unimplemented */ + mov 0, %o2 /* ARG2: mmu context == nucleus */ + mov HV_MMU_ALL, %o3 /* ARG3: flags */ + mov HV_FAST_MMU_DEMAP_CTX, %o5 + ta HV_FAST_TRAP + mov %g1, %o3 + brz,pt %o0, 5b + mov %g3, %o5 + mov HV_FAST_MMU_DEMAP_CTX, %g6 + ba,pt %xcc, 1b + clr %g5 + + /* These just get rescheduled to PIL vectors. */ + .globl xcall_call_function +xcall_call_function: + wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint + retry + + .globl xcall_call_function_single +xcall_call_function_single: + wr %g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint + retry + + .globl xcall_receive_signal +xcall_receive_signal: + wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint + retry + + .globl xcall_capture +xcall_capture: + wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint + retry + +#ifdef CONFIG_KGDB + .globl xcall_kgdb_capture +xcall_kgdb_capture: + wr %g0, (1 << PIL_KGDB_CAPTURE), %set_softint + retry +#endif + +#endif /* CONFIG_SMP */ + + .globl cheetah_patch_cachetlbops +cheetah_patch_cachetlbops: + save %sp, -128, %sp + + sethi %hi(__flush_tlb_mm), %o0 + or %o0, %lo(__flush_tlb_mm), %o0 + sethi %hi(__cheetah_flush_tlb_mm), %o1 + or %o1, %lo(__cheetah_flush_tlb_mm), %o1 + call tlb_patch_one + mov 19, %o2 + + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__cheetah_flush_tlb_page), %o1 + or %o1, %lo(__cheetah_flush_tlb_page), %o1 + call tlb_patch_one + mov 22, %o2 + + sethi %hi(__flush_tlb_pending), %o0 + or %o0, %lo(__flush_tlb_pending), %o0 + sethi %hi(__cheetah_flush_tlb_pending), %o1 + or %o1, %lo(__cheetah_flush_tlb_pending), %o1 + call tlb_patch_one + mov 27, %o2 + + sethi %hi(__flush_tlb_kernel_range), %o0 + or %o0, %lo(__flush_tlb_kernel_range), %o0 + sethi %hi(__cheetah_flush_tlb_kernel_range), %o1 + or %o1, %lo(__cheetah_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 31, %o2 + +#ifdef DCACHE_ALIASING_POSSIBLE + sethi %hi(__flush_dcache_page), %o0 + or %o0, %lo(__flush_dcache_page), %o0 + sethi %hi(__cheetah_flush_dcache_page), %o1 + or %o1, %lo(__cheetah_flush_dcache_page), %o1 + call tlb_patch_one + mov 11, %o2 +#endif /* DCACHE_ALIASING_POSSIBLE */ + +#ifdef CONFIG_SMP + sethi %hi(xcall_flush_tlb_kernel_range), %o0 + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 + sethi %hi(__cheetah_xcall_flush_tlb_kernel_range), %o1 + or %o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 44, %o2 +#endif /* CONFIG_SMP */ + + ret + restore + + .globl hypervisor_patch_cachetlbops +hypervisor_patch_cachetlbops: + save %sp, -128, %sp + + sethi %hi(__flush_tlb_mm), %o0 + or %o0, %lo(__flush_tlb_mm), %o0 + sethi %hi(__hypervisor_flush_tlb_mm), %o1 + or %o1, %lo(__hypervisor_flush_tlb_mm), %o1 + call tlb_patch_one + mov 19, %o2 + + sethi %hi(__flush_tlb_page), %o0 + or %o0, %lo(__flush_tlb_page), %o0 + sethi %hi(__hypervisor_flush_tlb_page), %o1 + or %o1, %lo(__hypervisor_flush_tlb_page), %o1 + call tlb_patch_one + mov 22, %o2 + + sethi %hi(__flush_tlb_pending), %o0 + or %o0, %lo(__flush_tlb_pending), %o0 + sethi %hi(__hypervisor_flush_tlb_pending), %o1 + or %o1, %lo(__hypervisor_flush_tlb_pending), %o1 + call tlb_patch_one + mov 27, %o2 + + sethi %hi(__flush_tlb_kernel_range), %o0 + or %o0, %lo(__flush_tlb_kernel_range), %o0 + sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1 + or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 31, %o2 + +#ifdef DCACHE_ALIASING_POSSIBLE + sethi %hi(__flush_dcache_page), %o0 + or %o0, %lo(__flush_dcache_page), %o0 + sethi %hi(__hypervisor_flush_dcache_page), %o1 + or %o1, %lo(__hypervisor_flush_dcache_page), %o1 + call tlb_patch_one + mov 2, %o2 +#endif /* DCACHE_ALIASING_POSSIBLE */ + +#ifdef CONFIG_SMP + sethi %hi(xcall_flush_tlb_mm), %o0 + or %o0, %lo(xcall_flush_tlb_mm), %o0 + sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1 + or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1 + call tlb_patch_one + mov 24, %o2 + + sethi %hi(xcall_flush_tlb_page), %o0 + or %o0, %lo(xcall_flush_tlb_page), %o0 + sethi %hi(__hypervisor_xcall_flush_tlb_page), %o1 + or %o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1 + call tlb_patch_one + mov 20, %o2 + + sethi %hi(xcall_flush_tlb_kernel_range), %o0 + or %o0, %lo(xcall_flush_tlb_kernel_range), %o0 + sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1 + or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1 + call tlb_patch_one + mov 44, %o2 +#endif /* CONFIG_SMP */ + + ret + restore diff --git a/arch/sparc/mm/viking.S b/arch/sparc/mm/viking.S new file mode 100644 index 000000000..48f062de7 --- /dev/null +++ b/arch/sparc/mm/viking.S @@ -0,0 +1,284 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * viking.S: High speed Viking cache/mmu operations + * + * Copyright (C) 1997 Eddie C. Dost (ecd@skynet.be) + * Copyright (C) 1997,1998,1999 Jakub Jelinek (jj@ultra.linux.cz) + * Copyright (C) 1999 Pavel Semerad (semerad@ss1000.ms.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_SMP + .data + .align 4 +sun4dsmp_flush_tlb_spin: + .word 0 +#endif + + .text + .align 4 + + .globl viking_flush_cache_all, viking_flush_cache_mm + .globl viking_flush_cache_range, viking_flush_cache_page + .globl viking_flush_page, viking_mxcc_flush_page + .globl viking_flush_page_for_dma, viking_flush_page_to_ram + .globl viking_flush_sig_insns + .globl viking_flush_tlb_all, viking_flush_tlb_mm + .globl viking_flush_tlb_range, viking_flush_tlb_page + +viking_flush_page: + sethi %hi(PAGE_OFFSET), %g2 + sub %o0, %g2, %g3 + srl %g3, 12, %g1 ! ppage >> 12 + + clr %o1 ! set counter, 0 - 127 + sethi %hi(PAGE_OFFSET + PAGE_SIZE - 0x80000000), %o3 + sethi %hi(0x80000000), %o4 + sethi %hi(VIKING_PTAG_VALID), %o5 + sethi %hi(2*PAGE_SIZE), %o0 + sethi %hi(PAGE_SIZE), %g7 + clr %o2 ! block counter, 0 - 3 +5: + sll %o1, 5, %g4 + or %g4, %o4, %g4 ! 0x80000000 | (set << 5) + + sll %o2, 26, %g5 ! block << 26 +6: + or %g5, %g4, %g5 + ldda [%g5] ASI_M_DATAC_TAG, %g2 + cmp %g3, %g1 ! ptag == ppage? + bne 7f + inc %o2 + + andcc %g2, %o5, %g0 ! ptag VALID? + be 7f + add %g4, %o3, %g2 ! (PAGE_OFFSET + PAGE_SIZE) | (set << 5) + ld [%g2], %g3 + ld [%g2 + %g7], %g3 + add %g2, %o0, %g2 + ld [%g2], %g3 + ld [%g2 + %g7], %g3 + add %g2, %o0, %g2 + ld [%g2], %g3 + ld [%g2 + %g7], %g3 + add %g2, %o0, %g2 + ld [%g2], %g3 + b 8f + ld [%g2 + %g7], %g3 + +7: + cmp %o2, 3 + ble 6b + sll %o2, 26, %g5 ! block << 26 + +8: inc %o1 + cmp %o1, 0x7f + ble 5b + clr %o2 + +9: retl + nop + +viking_mxcc_flush_page: + sethi %hi(PAGE_OFFSET), %g2 + sub %o0, %g2, %g3 + sub %g3, -PAGE_SIZE, %g3 ! ppage + PAGE_SIZE + sethi %hi(MXCC_SRCSTREAM), %o3 ! assume %hi(MXCC_SRCSTREAM) == %hi(MXCC_DESTSTREAM) + mov 0x10, %g2 ! set cacheable bit + or %o3, %lo(MXCC_SRCSTREAM), %o2 + or %o3, %lo(MXCC_DESSTREAM), %o3 + sub %g3, MXCC_STREAM_SIZE, %g3 +6: + stda %g2, [%o2] ASI_M_MXCC + stda %g2, [%o3] ASI_M_MXCC + andncc %g3, PAGE_MASK, %g0 + bne 6b + sub %g3, MXCC_STREAM_SIZE, %g3 + +9: retl + nop + +viking_flush_cache_page: +viking_flush_cache_range: +#ifndef CONFIG_SMP + ld [%o0 + VMA_VM_MM], %o0 +#endif +viking_flush_cache_mm: +#ifndef CONFIG_SMP + ld [%o0 + AOFF_mm_context], %g1 + cmp %g1, -1 + bne viking_flush_cache_all + nop + b,a viking_flush_cache_out +#endif +viking_flush_cache_all: + WINDOW_FLUSH(%g4, %g5) +viking_flush_cache_out: + retl + nop + +viking_flush_tlb_all: + mov 0x400, %g1 + retl + sta %g0, [%g1] ASI_M_FLUSH_PROBE + +viking_flush_tlb_mm: + mov SRMMU_CTX_REG, %g1 + ld [%o0 + AOFF_mm_context], %o1 + lda [%g1] ASI_M_MMUREGS, %g5 +#ifndef CONFIG_SMP + cmp %o1, -1 + be 1f +#endif + mov 0x300, %g2 + sta %o1, [%g1] ASI_M_MMUREGS + sta %g0, [%g2] ASI_M_FLUSH_PROBE + retl + sta %g5, [%g1] ASI_M_MMUREGS +#ifndef CONFIG_SMP +1: retl + nop +#endif + +viking_flush_tlb_range: + ld [%o0 + VMA_VM_MM], %o0 + mov SRMMU_CTX_REG, %g1 + ld [%o0 + AOFF_mm_context], %o3 + lda [%g1] ASI_M_MMUREGS, %g5 +#ifndef CONFIG_SMP + cmp %o3, -1 + be 2f +#endif + sethi %hi(~((1 << PGDIR_SHIFT) - 1)), %o4 + sta %o3, [%g1] ASI_M_MMUREGS + and %o1, %o4, %o1 + add %o1, 0x200, %o1 + sta %g0, [%o1] ASI_M_FLUSH_PROBE +1: sub %o1, %o4, %o1 + cmp %o1, %o2 + blu,a 1b + sta %g0, [%o1] ASI_M_FLUSH_PROBE + retl + sta %g5, [%g1] ASI_M_MMUREGS +#ifndef CONFIG_SMP +2: retl + nop +#endif + +viking_flush_tlb_page: + ld [%o0 + VMA_VM_MM], %o0 + mov SRMMU_CTX_REG, %g1 + ld [%o0 + AOFF_mm_context], %o3 + lda [%g1] ASI_M_MMUREGS, %g5 +#ifndef CONFIG_SMP + cmp %o3, -1 + be 1f +#endif + and %o1, PAGE_MASK, %o1 + sta %o3, [%g1] ASI_M_MMUREGS + sta %g0, [%o1] ASI_M_FLUSH_PROBE + retl + sta %g5, [%g1] ASI_M_MMUREGS +#ifndef CONFIG_SMP +1: retl + nop +#endif + +viking_flush_page_to_ram: +viking_flush_page_for_dma: +viking_flush_sig_insns: + retl + nop + +#ifdef CONFIG_SMP + .globl sun4dsmp_flush_tlb_all, sun4dsmp_flush_tlb_mm + .globl sun4dsmp_flush_tlb_range, sun4dsmp_flush_tlb_page +sun4dsmp_flush_tlb_all: + sethi %hi(sun4dsmp_flush_tlb_spin), %g3 +1: ldstub [%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5 + tst %g5 + bne 2f + mov 0x400, %g1 + sta %g0, [%g1] ASI_M_FLUSH_PROBE + retl + stb %g0, [%g3 + %lo(sun4dsmp_flush_tlb_spin)] +2: tst %g5 + bne,a 2b + ldub [%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5 + b,a 1b + +sun4dsmp_flush_tlb_mm: + sethi %hi(sun4dsmp_flush_tlb_spin), %g3 +1: ldstub [%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5 + tst %g5 + bne 2f + mov SRMMU_CTX_REG, %g1 + ld [%o0 + AOFF_mm_context], %o1 + lda [%g1] ASI_M_MMUREGS, %g5 + mov 0x300, %g2 + sta %o1, [%g1] ASI_M_MMUREGS + sta %g0, [%g2] ASI_M_FLUSH_PROBE + sta %g5, [%g1] ASI_M_MMUREGS + retl + stb %g0, [%g3 + %lo(sun4dsmp_flush_tlb_spin)] +2: tst %g5 + bne,a 2b + ldub [%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5 + b,a 1b + +sun4dsmp_flush_tlb_range: + sethi %hi(sun4dsmp_flush_tlb_spin), %g3 +1: ldstub [%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5 + tst %g5 + bne 3f + mov SRMMU_CTX_REG, %g1 + ld [%o0 + VMA_VM_MM], %o0 + ld [%o0 + AOFF_mm_context], %o3 + lda [%g1] ASI_M_MMUREGS, %g5 + sethi %hi(~((1 << PGDIR_SHIFT) - 1)), %o4 + sta %o3, [%g1] ASI_M_MMUREGS + and %o1, %o4, %o1 + add %o1, 0x200, %o1 + sta %g0, [%o1] ASI_M_FLUSH_PROBE +2: sub %o1, %o4, %o1 + cmp %o1, %o2 + blu,a 2b + sta %g0, [%o1] ASI_M_FLUSH_PROBE + sta %g5, [%g1] ASI_M_MMUREGS + retl + stb %g0, [%g3 + %lo(sun4dsmp_flush_tlb_spin)] +3: tst %g5 + bne,a 3b + ldub [%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5 + b,a 1b + +sun4dsmp_flush_tlb_page: + sethi %hi(sun4dsmp_flush_tlb_spin), %g3 +1: ldstub [%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5 + tst %g5 + bne 2f + mov SRMMU_CTX_REG, %g1 + ld [%o0 + VMA_VM_MM], %o0 + ld [%o0 + AOFF_mm_context], %o3 + lda [%g1] ASI_M_MMUREGS, %g5 + and %o1, PAGE_MASK, %o1 + sta %o3, [%g1] ASI_M_MMUREGS + sta %g0, [%o1] ASI_M_FLUSH_PROBE + sta %g5, [%g1] ASI_M_MMUREGS + retl + stb %g0, [%g3 + %lo(sun4dsmp_flush_tlb_spin)] +2: tst %g5 + bne,a 2b + ldub [%g3 + %lo(sun4dsmp_flush_tlb_spin)], %g5 + b,a 1b + nop +#endif diff --git a/arch/sparc/net/Makefile b/arch/sparc/net/Makefile new file mode 100644 index 000000000..806267de3 --- /dev/null +++ b/arch/sparc/net/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Arch-specific network modules +# +obj-$(CONFIG_BPF_JIT) += bpf_jit_comp_$(BITS).o +ifeq ($(BITS),32) +obj-$(CONFIG_BPF_JIT) += bpf_jit_asm_32.o +endif diff --git a/arch/sparc/net/bpf_jit_32.h b/arch/sparc/net/bpf_jit_32.h new file mode 100644 index 000000000..cfd6a8be0 --- /dev/null +++ b/arch/sparc/net/bpf_jit_32.h @@ -0,0 +1,69 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BPF_JIT_H +#define _BPF_JIT_H + +/* Conventions: + * %g1 : temporary + * %g2 : Secondary temporary used by SKB data helper stubs. + * %g3 : packet offset passed into SKB data helper stubs. + * %o0 : pointer to skb (first argument given to JIT function) + * %o1 : BPF A accumulator + * %o2 : BPF X accumulator + * %o3 : Holds saved %o7 so we can call helper functions without needing + * to allocate a register window. + * %o4 : skb->len - skb->data_len + * %o5 : skb->data + */ + +#ifndef __ASSEMBLER__ +#define G0 0x00 +#define G1 0x01 +#define G3 0x03 +#define G6 0x06 +#define O0 0x08 +#define O1 0x09 +#define O2 0x0a +#define O3 0x0b +#define O4 0x0c +#define O5 0x0d +#define SP 0x0e +#define O7 0x0f +#define FP 0x1e + +#define r_SKB O0 +#define r_A O1 +#define r_X O2 +#define r_saved_O7 O3 +#define r_HEADLEN O4 +#define r_SKB_DATA O5 +#define r_TMP G1 +#define r_TMP2 G2 +#define r_OFF G3 + +/* assembly code in arch/sparc/net/bpf_jit_asm_32.S */ +extern u32 bpf_jit_load_word[]; +extern u32 bpf_jit_load_half[]; +extern u32 bpf_jit_load_byte[]; +extern u32 bpf_jit_load_byte_msh[]; +extern u32 bpf_jit_load_word_positive_offset[]; +extern u32 bpf_jit_load_half_positive_offset[]; +extern u32 bpf_jit_load_byte_positive_offset[]; +extern u32 bpf_jit_load_byte_msh_positive_offset[]; +extern u32 bpf_jit_load_word_negative_offset[]; +extern u32 bpf_jit_load_half_negative_offset[]; +extern u32 bpf_jit_load_byte_negative_offset[]; +extern u32 bpf_jit_load_byte_msh_negative_offset[]; + +#else +#define r_SKB %o0 +#define r_A %o1 +#define r_X %o2 +#define r_saved_O7 %o3 +#define r_HEADLEN %o4 +#define r_SKB_DATA %o5 +#define r_TMP %g1 +#define r_TMP2 %g2 +#define r_OFF %g3 +#endif + +#endif /* _BPF_JIT_H */ diff --git a/arch/sparc/net/bpf_jit_64.h b/arch/sparc/net/bpf_jit_64.h new file mode 100644 index 000000000..fbc836f1c --- /dev/null +++ b/arch/sparc/net/bpf_jit_64.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _BPF_JIT_H +#define _BPF_JIT_H + +#ifndef __ASSEMBLER__ +#define G0 0x00 +#define G1 0x01 +#define G2 0x02 +#define G3 0x03 +#define G6 0x06 +#define G7 0x07 +#define O0 0x08 +#define O1 0x09 +#define O2 0x0a +#define O3 0x0b +#define O4 0x0c +#define O5 0x0d +#define SP 0x0e +#define O7 0x0f +#define L0 0x10 +#define L1 0x11 +#define L2 0x12 +#define L3 0x13 +#define L4 0x14 +#define L5 0x15 +#define L6 0x16 +#define L7 0x17 +#define I0 0x18 +#define I1 0x19 +#define I2 0x1a +#define I3 0x1b +#define I4 0x1c +#define I5 0x1d +#define FP 0x1e +#define I7 0x1f +#endif + +#endif /* _BPF_JIT_H */ diff --git a/arch/sparc/net/bpf_jit_asm_32.S b/arch/sparc/net/bpf_jit_asm_32.S new file mode 100644 index 000000000..a2e28e046 --- /dev/null +++ b/arch/sparc/net/bpf_jit_asm_32.S @@ -0,0 +1,202 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include + +#include "bpf_jit_32.h" + +#define SAVE_SZ 96 +#define SCRATCH_OFF 72 +#define BE_PTR(label) be label +#define SIGN_EXTEND(reg) + +#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ + + .text + .globl bpf_jit_load_word +bpf_jit_load_word: + cmp r_OFF, 0 + bl bpf_slow_path_word_neg + nop + .globl bpf_jit_load_word_positive_offset +bpf_jit_load_word_positive_offset: + sub r_HEADLEN, r_OFF, r_TMP + cmp r_TMP, 3 + ble bpf_slow_path_word + add r_SKB_DATA, r_OFF, r_TMP + andcc r_TMP, 3, %g0 + bne load_word_unaligned + nop + retl + ld [r_TMP], r_A +load_word_unaligned: + ldub [r_TMP + 0x0], r_OFF + ldub [r_TMP + 0x1], r_TMP2 + sll r_OFF, 8, r_OFF + or r_OFF, r_TMP2, r_OFF + ldub [r_TMP + 0x2], r_TMP2 + sll r_OFF, 8, r_OFF + or r_OFF, r_TMP2, r_OFF + ldub [r_TMP + 0x3], r_TMP2 + sll r_OFF, 8, r_OFF + retl + or r_OFF, r_TMP2, r_A + + .globl bpf_jit_load_half +bpf_jit_load_half: + cmp r_OFF, 0 + bl bpf_slow_path_half_neg + nop + .globl bpf_jit_load_half_positive_offset +bpf_jit_load_half_positive_offset: + sub r_HEADLEN, r_OFF, r_TMP + cmp r_TMP, 1 + ble bpf_slow_path_half + add r_SKB_DATA, r_OFF, r_TMP + andcc r_TMP, 1, %g0 + bne load_half_unaligned + nop + retl + lduh [r_TMP], r_A +load_half_unaligned: + ldub [r_TMP + 0x0], r_OFF + ldub [r_TMP + 0x1], r_TMP2 + sll r_OFF, 8, r_OFF + retl + or r_OFF, r_TMP2, r_A + + .globl bpf_jit_load_byte +bpf_jit_load_byte: + cmp r_OFF, 0 + bl bpf_slow_path_byte_neg + nop + .globl bpf_jit_load_byte_positive_offset +bpf_jit_load_byte_positive_offset: + cmp r_OFF, r_HEADLEN + bge bpf_slow_path_byte + nop + retl + ldub [r_SKB_DATA + r_OFF], r_A + + .globl bpf_jit_load_byte_msh +bpf_jit_load_byte_msh: + cmp r_OFF, 0 + bl bpf_slow_path_byte_msh_neg + nop + .globl bpf_jit_load_byte_msh_positive_offset +bpf_jit_load_byte_msh_positive_offset: + cmp r_OFF, r_HEADLEN + bge bpf_slow_path_byte_msh + nop + ldub [r_SKB_DATA + r_OFF], r_OFF + and r_OFF, 0xf, r_OFF + retl + sll r_OFF, 2, r_X + +#define bpf_slow_path_common(LEN) \ + save %sp, -SAVE_SZ, %sp; \ + mov %i0, %o0; \ + mov r_OFF, %o1; \ + add %fp, SCRATCH_OFF, %o2; \ + call skb_copy_bits; \ + mov (LEN), %o3; \ + cmp %o0, 0; \ + restore; + +bpf_slow_path_word: + bpf_slow_path_common(4) + bl bpf_error + ld [%sp + SCRATCH_OFF], r_A + retl + nop +bpf_slow_path_half: + bpf_slow_path_common(2) + bl bpf_error + lduh [%sp + SCRATCH_OFF], r_A + retl + nop +bpf_slow_path_byte: + bpf_slow_path_common(1) + bl bpf_error + ldub [%sp + SCRATCH_OFF], r_A + retl + nop +bpf_slow_path_byte_msh: + bpf_slow_path_common(1) + bl bpf_error + ldub [%sp + SCRATCH_OFF], r_A + and r_OFF, 0xf, r_OFF + retl + sll r_OFF, 2, r_X + +#define bpf_negative_common(LEN) \ + save %sp, -SAVE_SZ, %sp; \ + mov %i0, %o0; \ + mov r_OFF, %o1; \ + SIGN_EXTEND(%o1); \ + call bpf_internal_load_pointer_neg_helper; \ + mov (LEN), %o2; \ + mov %o0, r_TMP; \ + cmp %o0, 0; \ + BE_PTR(bpf_error); \ + restore; + +bpf_slow_path_word_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_word_negative_offset +bpf_jit_load_word_negative_offset: + bpf_negative_common(4) + andcc r_TMP, 3, %g0 + bne load_word_unaligned + nop + retl + ld [r_TMP], r_A + +bpf_slow_path_half_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_half_negative_offset +bpf_jit_load_half_negative_offset: + bpf_negative_common(2) + andcc r_TMP, 1, %g0 + bne load_half_unaligned + nop + retl + lduh [r_TMP], r_A + +bpf_slow_path_byte_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_byte_negative_offset +bpf_jit_load_byte_negative_offset: + bpf_negative_common(1) + retl + ldub [r_TMP], r_A + +bpf_slow_path_byte_msh_neg: + sethi %hi(SKF_MAX_NEG_OFF), r_TMP + cmp r_OFF, r_TMP + bl bpf_error + nop + .globl bpf_jit_load_byte_msh_negative_offset +bpf_jit_load_byte_msh_negative_offset: + bpf_negative_common(1) + ldub [r_TMP], r_OFF + and r_OFF, 0xf, r_OFF + retl + sll r_OFF, 2, r_X + +bpf_error: + /* Make the JIT program return zero. The JIT epilogue + * stores away the original %o7 into r_saved_O7. The + * normal leaf function return is to use "retl" which + * would evalute to "jmpl %o7 + 8, %g0" but we want to + * use the saved value thus the sequence you see here. + */ + jmpl r_saved_O7 + 8, %g0 + clr %o0 diff --git a/arch/sparc/net/bpf_jit_comp_32.c b/arch/sparc/net/bpf_jit_comp_32.c new file mode 100644 index 000000000..bda2dbd3f --- /dev/null +++ b/arch/sparc/net/bpf_jit_comp_32.c @@ -0,0 +1,764 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "bpf_jit_32.h" + +static inline bool is_simm13(unsigned int value) +{ + return value + 0x1000 < 0x2000; +} + +#define SEEN_DATAREF 1 /* might call external helpers */ +#define SEEN_XREG 2 /* ebx is used */ +#define SEEN_MEM 4 /* use mem[] for temporary storage */ + +#define S13(X) ((X) & 0x1fff) +#define IMMED 0x00002000 +#define RD(X) ((X) << 25) +#define RS1(X) ((X) << 14) +#define RS2(X) ((X)) +#define OP(X) ((X) << 30) +#define OP2(X) ((X) << 22) +#define OP3(X) ((X) << 19) +#define COND(X) ((X) << 25) +#define F1(X) OP(X) +#define F2(X, Y) (OP(X) | OP2(Y)) +#define F3(X, Y) (OP(X) | OP3(Y)) + +#define CONDN COND(0x0) +#define CONDE COND(0x1) +#define CONDLE COND(0x2) +#define CONDL COND(0x3) +#define CONDLEU COND(0x4) +#define CONDCS COND(0x5) +#define CONDNEG COND(0x6) +#define CONDVC COND(0x7) +#define CONDA COND(0x8) +#define CONDNE COND(0x9) +#define CONDG COND(0xa) +#define CONDGE COND(0xb) +#define CONDGU COND(0xc) +#define CONDCC COND(0xd) +#define CONDPOS COND(0xe) +#define CONDVS COND(0xf) + +#define CONDGEU CONDCC +#define CONDLU CONDCS + +#define WDISP22(X) (((X) >> 2) & 0x3fffff) + +#define BA (F2(0, 2) | CONDA) +#define BGU (F2(0, 2) | CONDGU) +#define BLEU (F2(0, 2) | CONDLEU) +#define BGEU (F2(0, 2) | CONDGEU) +#define BLU (F2(0, 2) | CONDLU) +#define BE (F2(0, 2) | CONDE) +#define BNE (F2(0, 2) | CONDNE) + +#define BE_PTR BE + +#define SETHI(K, REG) \ + (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) +#define OR_LO(K, REG) \ + (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG)) + +#define ADD F3(2, 0x00) +#define AND F3(2, 0x01) +#define ANDCC F3(2, 0x11) +#define OR F3(2, 0x02) +#define XOR F3(2, 0x03) +#define SUB F3(2, 0x04) +#define SUBCC F3(2, 0x14) +#define MUL F3(2, 0x0a) /* umul */ +#define DIV F3(2, 0x0e) /* udiv */ +#define SLL F3(2, 0x25) +#define SRL F3(2, 0x26) +#define JMPL F3(2, 0x38) +#define CALL F1(1) +#define BR F2(0, 0x01) +#define RD_Y F3(2, 0x28) +#define WR_Y F3(2, 0x30) + +#define LD32 F3(3, 0x00) +#define LD8 F3(3, 0x01) +#define LD16 F3(3, 0x02) +#define LD64 F3(3, 0x0b) +#define ST32 F3(3, 0x04) + +#define LDPTR LD32 +#define BASE_STACKFRAME 96 + +#define LD32I (LD32 | IMMED) +#define LD8I (LD8 | IMMED) +#define LD16I (LD16 | IMMED) +#define LD64I (LD64 | IMMED) +#define LDPTRI (LDPTR | IMMED) +#define ST32I (ST32 | IMMED) + +#define emit_nop() \ +do { \ + *prog++ = SETHI(0, G0); \ +} while (0) + +#define emit_neg() \ +do { /* sub %g0, r_A, r_A */ \ + *prog++ = SUB | RS1(G0) | RS2(r_A) | RD(r_A); \ +} while (0) + +#define emit_reg_move(FROM, TO) \ +do { /* or %g0, FROM, TO */ \ + *prog++ = OR | RS1(G0) | RS2(FROM) | RD(TO); \ +} while (0) + +#define emit_clear(REG) \ +do { /* or %g0, %g0, REG */ \ + *prog++ = OR | RS1(G0) | RS2(G0) | RD(REG); \ +} while (0) + +#define emit_set_const(K, REG) \ +do { /* sethi %hi(K), REG */ \ + *prog++ = SETHI(K, REG); \ + /* or REG, %lo(K), REG */ \ + *prog++ = OR_LO(K, REG); \ +} while (0) + + /* Emit + * + * OP r_A, r_X, r_A + */ +#define emit_alu_X(OPCODE) \ +do { \ + seen |= SEEN_XREG; \ + *prog++ = OPCODE | RS1(r_A) | RS2(r_X) | RD(r_A); \ +} while (0) + + /* Emit either: + * + * OP r_A, K, r_A + * + * or + * + * sethi %hi(K), r_TMP + * or r_TMP, %lo(K), r_TMP + * OP r_A, r_TMP, r_A + * + * depending upon whether K fits in a signed 13-bit + * immediate instruction field. Emit nothing if K + * is zero. + */ +#define emit_alu_K(OPCODE, K) \ +do { \ + if (K || OPCODE == AND || OPCODE == MUL) { \ + unsigned int _insn = OPCODE; \ + _insn |= RS1(r_A) | RD(r_A); \ + if (is_simm13(K)) { \ + *prog++ = _insn | IMMED | S13(K); \ + } else { \ + emit_set_const(K, r_TMP); \ + *prog++ = _insn | RS2(r_TMP); \ + } \ + } \ +} while (0) + +#define emit_loadimm(K, DEST) \ +do { \ + if (is_simm13(K)) { \ + /* or %g0, K, DEST */ \ + *prog++ = OR | IMMED | RS1(G0) | S13(K) | RD(DEST); \ + } else { \ + emit_set_const(K, DEST); \ + } \ +} while (0) + +#define emit_loadptr(BASE, STRUCT, FIELD, DEST) \ +do { unsigned int _off = offsetof(STRUCT, FIELD); \ + BUILD_BUG_ON(sizeof_field(STRUCT, FIELD) != sizeof(void *)); \ + *prog++ = LDPTRI | RS1(BASE) | S13(_off) | RD(DEST); \ +} while (0) + +#define emit_load32(BASE, STRUCT, FIELD, DEST) \ +do { unsigned int _off = offsetof(STRUCT, FIELD); \ + BUILD_BUG_ON(sizeof_field(STRUCT, FIELD) != sizeof(u32)); \ + *prog++ = LD32I | RS1(BASE) | S13(_off) | RD(DEST); \ +} while (0) + +#define emit_load16(BASE, STRUCT, FIELD, DEST) \ +do { unsigned int _off = offsetof(STRUCT, FIELD); \ + BUILD_BUG_ON(sizeof_field(STRUCT, FIELD) != sizeof(u16)); \ + *prog++ = LD16I | RS1(BASE) | S13(_off) | RD(DEST); \ +} while (0) + +#define __emit_load8(BASE, STRUCT, FIELD, DEST) \ +do { unsigned int _off = offsetof(STRUCT, FIELD); \ + *prog++ = LD8I | RS1(BASE) | S13(_off) | RD(DEST); \ +} while (0) + +#define emit_load8(BASE, STRUCT, FIELD, DEST) \ +do { BUILD_BUG_ON(sizeof_field(STRUCT, FIELD) != sizeof(u8)); \ + __emit_load8(BASE, STRUCT, FIELD, DEST); \ +} while (0) + +#define BIAS (-4) + +#define emit_ldmem(OFF, DEST) \ +do { *prog++ = LD32I | RS1(SP) | S13(BIAS - (OFF)) | RD(DEST); \ +} while (0) + +#define emit_stmem(OFF, SRC) \ +do { *prog++ = ST32I | RS1(SP) | S13(BIAS - (OFF)) | RD(SRC); \ +} while (0) + +#ifdef CONFIG_SMP +#define emit_load_cpu(REG) \ + emit_load32(G6, struct thread_info, cpu, REG) +#else +#define emit_load_cpu(REG) emit_clear(REG) +#endif + +#define emit_skb_loadptr(FIELD, DEST) \ + emit_loadptr(r_SKB, struct sk_buff, FIELD, DEST) +#define emit_skb_load32(FIELD, DEST) \ + emit_load32(r_SKB, struct sk_buff, FIELD, DEST) +#define emit_skb_load16(FIELD, DEST) \ + emit_load16(r_SKB, struct sk_buff, FIELD, DEST) +#define __emit_skb_load8(FIELD, DEST) \ + __emit_load8(r_SKB, struct sk_buff, FIELD, DEST) +#define emit_skb_load8(FIELD, DEST) \ + emit_load8(r_SKB, struct sk_buff, FIELD, DEST) + +#define emit_jmpl(BASE, IMM_OFF, LREG) \ + *prog++ = (JMPL | IMMED | RS1(BASE) | S13(IMM_OFF) | RD(LREG)) + +#define emit_call(FUNC) \ +do { void *_here = image + addrs[i] - 8; \ + unsigned int _off = (void *)(FUNC) - _here; \ + *prog++ = CALL | (((_off) >> 2) & 0x3fffffff); \ + emit_nop(); \ +} while (0) + +#define emit_branch(BR_OPC, DEST) \ +do { unsigned int _here = addrs[i] - 8; \ + *prog++ = BR_OPC | WDISP22((DEST) - _here); \ +} while (0) + +#define emit_branch_off(BR_OPC, OFF) \ +do { *prog++ = BR_OPC | WDISP22(OFF); \ +} while (0) + +#define emit_jump(DEST) emit_branch(BA, DEST) + +#define emit_read_y(REG) *prog++ = RD_Y | RD(REG) +#define emit_write_y(REG) *prog++ = WR_Y | IMMED | RS1(REG) | S13(0) + +#define emit_cmp(R1, R2) \ + *prog++ = (SUBCC | RS1(R1) | RS2(R2) | RD(G0)) + +#define emit_cmpi(R1, IMM) \ + *prog++ = (SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0)); + +#define emit_btst(R1, R2) \ + *prog++ = (ANDCC | RS1(R1) | RS2(R2) | RD(G0)) + +#define emit_btsti(R1, IMM) \ + *prog++ = (ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0)); + +#define emit_sub(R1, R2, R3) \ + *prog++ = (SUB | RS1(R1) | RS2(R2) | RD(R3)) + +#define emit_subi(R1, IMM, R3) \ + *prog++ = (SUB | IMMED | RS1(R1) | S13(IMM) | RD(R3)) + +#define emit_add(R1, R2, R3) \ + *prog++ = (ADD | RS1(R1) | RS2(R2) | RD(R3)) + +#define emit_addi(R1, IMM, R3) \ + *prog++ = (ADD | IMMED | RS1(R1) | S13(IMM) | RD(R3)) + +#define emit_and(R1, R2, R3) \ + *prog++ = (AND | RS1(R1) | RS2(R2) | RD(R3)) + +#define emit_andi(R1, IMM, R3) \ + *prog++ = (AND | IMMED | RS1(R1) | S13(IMM) | RD(R3)) + +#define emit_alloc_stack(SZ) \ + *prog++ = (SUB | IMMED | RS1(SP) | S13(SZ) | RD(SP)) + +#define emit_release_stack(SZ) \ + *prog++ = (ADD | IMMED | RS1(SP) | S13(SZ) | RD(SP)) + +/* A note about branch offset calculations. The addrs[] array, + * indexed by BPF instruction, records the address after all the + * sparc instructions emitted for that BPF instruction. + * + * The most common case is to emit a branch at the end of such + * a code sequence. So this would be two instructions, the + * branch and its delay slot. + * + * Therefore by default the branch emitters calculate the branch + * offset field as: + * + * destination - (addrs[i] - 8) + * + * This "addrs[i] - 8" is the address of the branch itself or + * what "." would be in assembler notation. The "8" part is + * how we take into consideration the branch and its delay + * slot mentioned above. + * + * Sometimes we need to emit a branch earlier in the code + * sequence. And in these situations we adjust "destination" + * to accommodate this difference. For example, if we needed + * to emit a branch (and its delay slot) right before the + * final instruction emitted for a BPF opcode, we'd use + * "destination + 4" instead of just plain "destination" above. + * + * This is why you see all of these funny emit_branch() and + * emit_jump() calls with adjusted offsets. + */ + +void bpf_jit_compile(struct bpf_prog *fp) +{ + unsigned int cleanup_addr, proglen, oldproglen = 0; + u32 temp[8], *prog, *func, seen = 0, pass; + const struct sock_filter *filter = fp->insns; + int i, flen = fp->len, pc_ret0 = -1; + unsigned int *addrs; + void *image; + + if (!bpf_jit_enable) + return; + + addrs = kmalloc_array(flen, sizeof(*addrs), GFP_KERNEL); + if (addrs == NULL) + return; + + /* Before first pass, make a rough estimation of addrs[] + * each bpf instruction is translated to less than 64 bytes + */ + for (proglen = 0, i = 0; i < flen; i++) { + proglen += 64; + addrs[i] = proglen; + } + cleanup_addr = proglen; /* epilogue address */ + image = NULL; + for (pass = 0; pass < 10; pass++) { + u8 seen_or_pass0 = (pass == 0) ? (SEEN_XREG | SEEN_DATAREF | SEEN_MEM) : seen; + + /* no prologue/epilogue for trivial filters (RET something) */ + proglen = 0; + prog = temp; + + /* Prologue */ + if (seen_or_pass0) { + if (seen_or_pass0 & SEEN_MEM) { + unsigned int sz = BASE_STACKFRAME; + sz += BPF_MEMWORDS * sizeof(u32); + emit_alloc_stack(sz); + } + + /* Make sure we dont leek kernel memory. */ + if (seen_or_pass0 & SEEN_XREG) + emit_clear(r_X); + + /* If this filter needs to access skb data, + * load %o4 and %o5 with: + * %o4 = skb->len - skb->data_len + * %o5 = skb->data + * And also back up %o7 into r_saved_O7 so we can + * invoke the stubs using 'call'. + */ + if (seen_or_pass0 & SEEN_DATAREF) { + emit_load32(r_SKB, struct sk_buff, len, r_HEADLEN); + emit_load32(r_SKB, struct sk_buff, data_len, r_TMP); + emit_sub(r_HEADLEN, r_TMP, r_HEADLEN); + emit_loadptr(r_SKB, struct sk_buff, data, r_SKB_DATA); + } + } + emit_reg_move(O7, r_saved_O7); + + /* Make sure we dont leak kernel information to the user. */ + if (bpf_needs_clear_a(&filter[0])) + emit_clear(r_A); /* A = 0 */ + + for (i = 0; i < flen; i++) { + unsigned int K = filter[i].k; + unsigned int t_offset; + unsigned int f_offset; + u32 t_op, f_op; + u16 code = bpf_anc_helper(&filter[i]); + int ilen; + + switch (code) { + case BPF_ALU | BPF_ADD | BPF_X: /* A += X; */ + emit_alu_X(ADD); + break; + case BPF_ALU | BPF_ADD | BPF_K: /* A += K; */ + emit_alu_K(ADD, K); + break; + case BPF_ALU | BPF_SUB | BPF_X: /* A -= X; */ + emit_alu_X(SUB); + break; + case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ + emit_alu_K(SUB, K); + break; + case BPF_ALU | BPF_AND | BPF_X: /* A &= X */ + emit_alu_X(AND); + break; + case BPF_ALU | BPF_AND | BPF_K: /* A &= K */ + emit_alu_K(AND, K); + break; + case BPF_ALU | BPF_OR | BPF_X: /* A |= X */ + emit_alu_X(OR); + break; + case BPF_ALU | BPF_OR | BPF_K: /* A |= K */ + emit_alu_K(OR, K); + break; + case BPF_ANC | SKF_AD_ALU_XOR_X: /* A ^= X; */ + case BPF_ALU | BPF_XOR | BPF_X: + emit_alu_X(XOR); + break; + case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */ + emit_alu_K(XOR, K); + break; + case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X */ + emit_alu_X(SLL); + break; + case BPF_ALU | BPF_LSH | BPF_K: /* A <<= K */ + emit_alu_K(SLL, K); + break; + case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X */ + emit_alu_X(SRL); + break; + case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K */ + emit_alu_K(SRL, K); + break; + case BPF_ALU | BPF_MUL | BPF_X: /* A *= X; */ + emit_alu_X(MUL); + break; + case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ + emit_alu_K(MUL, K); + break; + case BPF_ALU | BPF_DIV | BPF_K: /* A /= K with K != 0*/ + if (K == 1) + break; + emit_write_y(G0); + /* The Sparc v8 architecture requires + * three instructions between a %y + * register write and the first use. + */ + emit_nop(); + emit_nop(); + emit_nop(); + emit_alu_K(DIV, K); + break; + case BPF_ALU | BPF_DIV | BPF_X: /* A /= X; */ + emit_cmpi(r_X, 0); + if (pc_ret0 > 0) { + t_offset = addrs[pc_ret0 - 1]; + emit_branch(BE, t_offset + 20); + emit_nop(); /* delay slot */ + } else { + emit_branch_off(BNE, 16); + emit_nop(); + emit_jump(cleanup_addr + 20); + emit_clear(r_A); + } + emit_write_y(G0); + /* The Sparc v8 architecture requires + * three instructions between a %y + * register write and the first use. + */ + emit_nop(); + emit_nop(); + emit_nop(); + emit_alu_X(DIV); + break; + case BPF_ALU | BPF_NEG: + emit_neg(); + break; + case BPF_RET | BPF_K: + if (!K) { + if (pc_ret0 == -1) + pc_ret0 = i; + emit_clear(r_A); + } else { + emit_loadimm(K, r_A); + } + fallthrough; + case BPF_RET | BPF_A: + if (seen_or_pass0) { + if (i != flen - 1) { + emit_jump(cleanup_addr); + emit_nop(); + break; + } + if (seen_or_pass0 & SEEN_MEM) { + unsigned int sz = BASE_STACKFRAME; + sz += BPF_MEMWORDS * sizeof(u32); + emit_release_stack(sz); + } + } + /* jmpl %r_saved_O7 + 8, %g0 */ + emit_jmpl(r_saved_O7, 8, G0); + emit_reg_move(r_A, O0); /* delay slot */ + break; + case BPF_MISC | BPF_TAX: + seen |= SEEN_XREG; + emit_reg_move(r_A, r_X); + break; + case BPF_MISC | BPF_TXA: + seen |= SEEN_XREG; + emit_reg_move(r_X, r_A); + break; + case BPF_ANC | SKF_AD_CPU: + emit_load_cpu(r_A); + break; + case BPF_ANC | SKF_AD_PROTOCOL: + emit_skb_load16(protocol, r_A); + break; + case BPF_ANC | SKF_AD_PKTTYPE: + __emit_skb_load8(__pkt_type_offset, r_A); + emit_andi(r_A, PKT_TYPE_MAX, r_A); + emit_alu_K(SRL, 5); + break; + case BPF_ANC | SKF_AD_IFINDEX: + emit_skb_loadptr(dev, r_A); + emit_cmpi(r_A, 0); + emit_branch(BE_PTR, cleanup_addr + 4); + emit_nop(); + emit_load32(r_A, struct net_device, ifindex, r_A); + break; + case BPF_ANC | SKF_AD_MARK: + emit_skb_load32(mark, r_A); + break; + case BPF_ANC | SKF_AD_QUEUE: + emit_skb_load16(queue_mapping, r_A); + break; + case BPF_ANC | SKF_AD_HATYPE: + emit_skb_loadptr(dev, r_A); + emit_cmpi(r_A, 0); + emit_branch(BE_PTR, cleanup_addr + 4); + emit_nop(); + emit_load16(r_A, struct net_device, type, r_A); + break; + case BPF_ANC | SKF_AD_RXHASH: + emit_skb_load32(hash, r_A); + break; + case BPF_ANC | SKF_AD_VLAN_TAG: + emit_skb_load16(vlan_tci, r_A); + break; + case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: + emit_skb_load32(vlan_all, r_A); + emit_cmpi(r_A, 0); + emit_branch_off(BE, 12); + emit_nop(); + emit_loadimm(1, r_A); + break; + case BPF_LD | BPF_W | BPF_LEN: + emit_skb_load32(len, r_A); + break; + case BPF_LDX | BPF_W | BPF_LEN: + emit_skb_load32(len, r_X); + break; + case BPF_LD | BPF_IMM: + emit_loadimm(K, r_A); + break; + case BPF_LDX | BPF_IMM: + emit_loadimm(K, r_X); + break; + case BPF_LD | BPF_MEM: + seen |= SEEN_MEM; + emit_ldmem(K * 4, r_A); + break; + case BPF_LDX | BPF_MEM: + seen |= SEEN_MEM | SEEN_XREG; + emit_ldmem(K * 4, r_X); + break; + case BPF_ST: + seen |= SEEN_MEM; + emit_stmem(K * 4, r_A); + break; + case BPF_STX: + seen |= SEEN_MEM | SEEN_XREG; + emit_stmem(K * 4, r_X); + break; + +#define CHOOSE_LOAD_FUNC(K, func) \ + ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative_offset : func) : func##_positive_offset) + + case BPF_LD | BPF_W | BPF_ABS: + func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_word); +common_load: seen |= SEEN_DATAREF; + emit_loadimm(K, r_OFF); + emit_call(func); + break; + case BPF_LD | BPF_H | BPF_ABS: + func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_half); + goto common_load; + case BPF_LD | BPF_B | BPF_ABS: + func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_byte); + goto common_load; + case BPF_LDX | BPF_B | BPF_MSH: + func = CHOOSE_LOAD_FUNC(K, bpf_jit_load_byte_msh); + goto common_load; + case BPF_LD | BPF_W | BPF_IND: + func = bpf_jit_load_word; +common_load_ind: seen |= SEEN_DATAREF | SEEN_XREG; + if (K) { + if (is_simm13(K)) { + emit_addi(r_X, K, r_OFF); + } else { + emit_loadimm(K, r_TMP); + emit_add(r_X, r_TMP, r_OFF); + } + } else { + emit_reg_move(r_X, r_OFF); + } + emit_call(func); + break; + case BPF_LD | BPF_H | BPF_IND: + func = bpf_jit_load_half; + goto common_load_ind; + case BPF_LD | BPF_B | BPF_IND: + func = bpf_jit_load_byte; + goto common_load_ind; + case BPF_JMP | BPF_JA: + emit_jump(addrs[i + K]); + emit_nop(); + break; + +#define COND_SEL(CODE, TOP, FOP) \ + case CODE: \ + t_op = TOP; \ + f_op = FOP; \ + goto cond_branch + + COND_SEL(BPF_JMP | BPF_JGT | BPF_K, BGU, BLEU); + COND_SEL(BPF_JMP | BPF_JGE | BPF_K, BGEU, BLU); + COND_SEL(BPF_JMP | BPF_JEQ | BPF_K, BE, BNE); + COND_SEL(BPF_JMP | BPF_JSET | BPF_K, BNE, BE); + COND_SEL(BPF_JMP | BPF_JGT | BPF_X, BGU, BLEU); + COND_SEL(BPF_JMP | BPF_JGE | BPF_X, BGEU, BLU); + COND_SEL(BPF_JMP | BPF_JEQ | BPF_X, BE, BNE); + COND_SEL(BPF_JMP | BPF_JSET | BPF_X, BNE, BE); + +cond_branch: f_offset = addrs[i + filter[i].jf]; + t_offset = addrs[i + filter[i].jt]; + + /* same targets, can avoid doing the test :) */ + if (filter[i].jt == filter[i].jf) { + emit_jump(t_offset); + emit_nop(); + break; + } + + switch (code) { + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JEQ | BPF_X: + seen |= SEEN_XREG; + emit_cmp(r_A, r_X); + break; + case BPF_JMP | BPF_JSET | BPF_X: + seen |= SEEN_XREG; + emit_btst(r_A, r_X); + break; + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: + if (is_simm13(K)) { + emit_cmpi(r_A, K); + } else { + emit_loadimm(K, r_TMP); + emit_cmp(r_A, r_TMP); + } + break; + case BPF_JMP | BPF_JSET | BPF_K: + if (is_simm13(K)) { + emit_btsti(r_A, K); + } else { + emit_loadimm(K, r_TMP); + emit_btst(r_A, r_TMP); + } + break; + } + if (filter[i].jt != 0) { + if (filter[i].jf) + t_offset += 8; + emit_branch(t_op, t_offset); + emit_nop(); /* delay slot */ + if (filter[i].jf) { + emit_jump(f_offset); + emit_nop(); + } + break; + } + emit_branch(f_op, f_offset); + emit_nop(); /* delay slot */ + break; + + default: + /* hmm, too complex filter, give up with jit compiler */ + goto out; + } + ilen = (void *) prog - (void *) temp; + if (image) { + if (unlikely(proglen + ilen > oldproglen)) { + pr_err("bpb_jit_compile fatal error\n"); + kfree(addrs); + execmem_free(image); + return; + } + memcpy(image + proglen, temp, ilen); + } + proglen += ilen; + addrs[i] = proglen; + prog = temp; + } + /* last bpf instruction is always a RET : + * use it to give the cleanup instruction(s) addr + */ + cleanup_addr = proglen - 8; /* jmpl; mov r_A,%o0; */ + if (seen_or_pass0 & SEEN_MEM) + cleanup_addr -= 4; /* add %sp, X, %sp; */ + + if (image) { + if (proglen != oldproglen) + pr_err("bpb_jit_compile proglen=%u != oldproglen=%u\n", + proglen, oldproglen); + break; + } + if (proglen == oldproglen) { + image = execmem_alloc(EXECMEM_BPF, proglen); + if (!image) + goto out; + } + oldproglen = proglen; + } + + if (bpf_jit_enable > 1) + bpf_jit_dump(flen, proglen, pass + 1, image); + + if (image) { + fp->bpf_func = (void *)image; + fp->jited = 1; + } +out: + kfree(addrs); + return; +} + +void bpf_jit_free(struct bpf_prog *fp) +{ + if (fp->jited) + execmem_free(fp->bpf_func); + + bpf_prog_unlock_free(fp); +} diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c new file mode 100644 index 000000000..2fa0e9375 --- /dev/null +++ b/arch/sparc/net/bpf_jit_comp_64.c @@ -0,0 +1,1613 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "bpf_jit_64.h" + +static inline bool is_simm13(unsigned int value) +{ + return value + 0x1000 < 0x2000; +} + +static inline bool is_simm10(unsigned int value) +{ + return value + 0x200 < 0x400; +} + +static inline bool is_simm5(unsigned int value) +{ + return value + 0x10 < 0x20; +} + +static inline bool is_sethi(unsigned int value) +{ + return (value & ~0x3fffff) == 0; +} + +static void bpf_flush_icache(void *start_, void *end_) +{ + /* Cheetah's I-cache is fully coherent. */ + if (tlb_type == spitfire) { + unsigned long start = (unsigned long) start_; + unsigned long end = (unsigned long) end_; + + start &= ~7UL; + end = (end + 7UL) & ~7UL; + while (start < end) { + flushi(start); + start += 32; + } + } +} + +#define S13(X) ((X) & 0x1fff) +#define S5(X) ((X) & 0x1f) +#define IMMED 0x00002000 +#define RD(X) ((X) << 25) +#define RS1(X) ((X) << 14) +#define RS2(X) ((X)) +#define OP(X) ((X) << 30) +#define OP2(X) ((X) << 22) +#define OP3(X) ((X) << 19) +#define COND(X) (((X) & 0xf) << 25) +#define CBCOND(X) (((X) & 0x1f) << 25) +#define F1(X) OP(X) +#define F2(X, Y) (OP(X) | OP2(Y)) +#define F3(X, Y) (OP(X) | OP3(Y)) +#define ASI(X) (((X) & 0xff) << 5) + +#define CONDN COND(0x0) +#define CONDE COND(0x1) +#define CONDLE COND(0x2) +#define CONDL COND(0x3) +#define CONDLEU COND(0x4) +#define CONDCS COND(0x5) +#define CONDNEG COND(0x6) +#define CONDVC COND(0x7) +#define CONDA COND(0x8) +#define CONDNE COND(0x9) +#define CONDG COND(0xa) +#define CONDGE COND(0xb) +#define CONDGU COND(0xc) +#define CONDCC COND(0xd) +#define CONDPOS COND(0xe) +#define CONDVS COND(0xf) + +#define CONDGEU CONDCC +#define CONDLU CONDCS + +#define WDISP22(X) (((X) >> 2) & 0x3fffff) +#define WDISP19(X) (((X) >> 2) & 0x7ffff) + +/* The 10-bit branch displacement for CBCOND is split into two fields */ +static u32 WDISP10(u32 off) +{ + u32 ret = ((off >> 2) & 0xff) << 5; + + ret |= ((off >> (2 + 8)) & 0x03) << 19; + + return ret; +} + +#define CBCONDE CBCOND(0x09) +#define CBCONDLE CBCOND(0x0a) +#define CBCONDL CBCOND(0x0b) +#define CBCONDLEU CBCOND(0x0c) +#define CBCONDCS CBCOND(0x0d) +#define CBCONDN CBCOND(0x0e) +#define CBCONDVS CBCOND(0x0f) +#define CBCONDNE CBCOND(0x19) +#define CBCONDG CBCOND(0x1a) +#define CBCONDGE CBCOND(0x1b) +#define CBCONDGU CBCOND(0x1c) +#define CBCONDCC CBCOND(0x1d) +#define CBCONDPOS CBCOND(0x1e) +#define CBCONDVC CBCOND(0x1f) + +#define CBCONDGEU CBCONDCC +#define CBCONDLU CBCONDCS + +#define ANNUL (1 << 29) +#define XCC (1 << 21) + +#define BRANCH (F2(0, 1) | XCC) +#define CBCOND_OP (F2(0, 3) | XCC) + +#define BA (BRANCH | CONDA) +#define BG (BRANCH | CONDG) +#define BL (BRANCH | CONDL) +#define BLE (BRANCH | CONDLE) +#define BGU (BRANCH | CONDGU) +#define BLEU (BRANCH | CONDLEU) +#define BGE (BRANCH | CONDGE) +#define BGEU (BRANCH | CONDGEU) +#define BLU (BRANCH | CONDLU) +#define BE (BRANCH | CONDE) +#define BNE (BRANCH | CONDNE) + +#define SETHI(K, REG) \ + (F2(0, 0x4) | RD(REG) | (((K) >> 10) & 0x3fffff)) +#define OR_LO(K, REG) \ + (F3(2, 0x02) | IMMED | RS1(REG) | ((K) & 0x3ff) | RD(REG)) + +#define ADD F3(2, 0x00) +#define AND F3(2, 0x01) +#define ANDCC F3(2, 0x11) +#define OR F3(2, 0x02) +#define XOR F3(2, 0x03) +#define SUB F3(2, 0x04) +#define SUBCC F3(2, 0x14) +#define MUL F3(2, 0x0a) +#define MULX F3(2, 0x09) +#define UDIVX F3(2, 0x0d) +#define DIV F3(2, 0x0e) +#define SLL F3(2, 0x25) +#define SLLX (F3(2, 0x25)|(1<<12)) +#define SRA F3(2, 0x27) +#define SRAX (F3(2, 0x27)|(1<<12)) +#define SRL F3(2, 0x26) +#define SRLX (F3(2, 0x26)|(1<<12)) +#define JMPL F3(2, 0x38) +#define SAVE F3(2, 0x3c) +#define RESTORE F3(2, 0x3d) +#define CALL F1(1) +#define BR F2(0, 0x01) +#define RD_Y F3(2, 0x28) +#define WR_Y F3(2, 0x30) + +#define LD32 F3(3, 0x00) +#define LD8 F3(3, 0x01) +#define LD16 F3(3, 0x02) +#define LD64 F3(3, 0x0b) +#define LD64A F3(3, 0x1b) +#define ST8 F3(3, 0x05) +#define ST16 F3(3, 0x06) +#define ST32 F3(3, 0x04) +#define ST64 F3(3, 0x0e) + +#define CAS F3(3, 0x3c) +#define CASX F3(3, 0x3e) + +#define LDPTR LD64 +#define BASE_STACKFRAME 176 + +#define LD32I (LD32 | IMMED) +#define LD8I (LD8 | IMMED) +#define LD16I (LD16 | IMMED) +#define LD64I (LD64 | IMMED) +#define LDPTRI (LDPTR | IMMED) +#define ST32I (ST32 | IMMED) + +struct jit_ctx { + struct bpf_prog *prog; + unsigned int *offset; + int idx; + int epilogue_offset; + bool tmp_1_used; + bool tmp_2_used; + bool tmp_3_used; + bool saw_frame_pointer; + bool saw_call; + bool saw_tail_call; + u32 *image; +}; + +#define TMP_REG_1 (MAX_BPF_JIT_REG + 0) +#define TMP_REG_2 (MAX_BPF_JIT_REG + 1) +#define TMP_REG_3 (MAX_BPF_JIT_REG + 2) + +/* Map BPF registers to SPARC registers */ +static const int bpf2sparc[] = { + /* return value from in-kernel function, and exit value from eBPF */ + [BPF_REG_0] = O5, + + /* arguments from eBPF program to in-kernel function */ + [BPF_REG_1] = O0, + [BPF_REG_2] = O1, + [BPF_REG_3] = O2, + [BPF_REG_4] = O3, + [BPF_REG_5] = O4, + + /* callee saved registers that in-kernel function will preserve */ + [BPF_REG_6] = L0, + [BPF_REG_7] = L1, + [BPF_REG_8] = L2, + [BPF_REG_9] = L3, + + /* read-only frame pointer to access stack */ + [BPF_REG_FP] = L6, + + [BPF_REG_AX] = G7, + + /* temporary register for BPF JIT */ + [TMP_REG_1] = G1, + [TMP_REG_2] = G2, + [TMP_REG_3] = G3, +}; + +static void emit(const u32 insn, struct jit_ctx *ctx) +{ + if (ctx->image != NULL) + ctx->image[ctx->idx] = insn; + + ctx->idx++; +} + +static void emit_call(u32 *func, struct jit_ctx *ctx) +{ + if (ctx->image != NULL) { + void *here = &ctx->image[ctx->idx]; + unsigned int off; + + off = (void *)func - here; + ctx->image[ctx->idx] = CALL | ((off >> 2) & 0x3fffffff); + } + ctx->idx++; +} + +static void emit_nop(struct jit_ctx *ctx) +{ + emit(SETHI(0, G0), ctx); +} + +static void emit_reg_move(u32 from, u32 to, struct jit_ctx *ctx) +{ + emit(OR | RS1(G0) | RS2(from) | RD(to), ctx); +} + +/* Emit 32-bit constant, zero extended. */ +static void emit_set_const(s32 K, u32 reg, struct jit_ctx *ctx) +{ + emit(SETHI(K, reg), ctx); + emit(OR_LO(K, reg), ctx); +} + +/* Emit 32-bit constant, sign extended. */ +static void emit_set_const_sext(s32 K, u32 reg, struct jit_ctx *ctx) +{ + if (K >= 0) { + emit(SETHI(K, reg), ctx); + emit(OR_LO(K, reg), ctx); + } else { + u32 hbits = ~(u32) K; + u32 lbits = -0x400 | (u32) K; + + emit(SETHI(hbits, reg), ctx); + emit(XOR | IMMED | RS1(reg) | S13(lbits) | RD(reg), ctx); + } +} + +static void emit_alu(u32 opcode, u32 src, u32 dst, struct jit_ctx *ctx) +{ + emit(opcode | RS1(dst) | RS2(src) | RD(dst), ctx); +} + +static void emit_alu3(u32 opcode, u32 a, u32 b, u32 c, struct jit_ctx *ctx) +{ + emit(opcode | RS1(a) | RS2(b) | RD(c), ctx); +} + +static void emit_alu_K(unsigned int opcode, unsigned int dst, unsigned int imm, + struct jit_ctx *ctx) +{ + bool small_immed = is_simm13(imm); + unsigned int insn = opcode; + + insn |= RS1(dst) | RD(dst); + if (small_immed) { + emit(insn | IMMED | S13(imm), ctx); + } else { + unsigned int tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_set_const_sext(imm, tmp, ctx); + emit(insn | RS2(tmp), ctx); + } +} + +static void emit_alu3_K(unsigned int opcode, unsigned int src, unsigned int imm, + unsigned int dst, struct jit_ctx *ctx) +{ + bool small_immed = is_simm13(imm); + unsigned int insn = opcode; + + insn |= RS1(src) | RD(dst); + if (small_immed) { + emit(insn | IMMED | S13(imm), ctx); + } else { + unsigned int tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_set_const_sext(imm, tmp, ctx); + emit(insn | RS2(tmp), ctx); + } +} + +static void emit_loadimm32(s32 K, unsigned int dest, struct jit_ctx *ctx) +{ + if (K >= 0 && is_simm13(K)) { + /* or %g0, K, DEST */ + emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); + } else { + emit_set_const(K, dest, ctx); + } +} + +static void emit_loadimm(s32 K, unsigned int dest, struct jit_ctx *ctx) +{ + if (is_simm13(K)) { + /* or %g0, K, DEST */ + emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); + } else { + emit_set_const(K, dest, ctx); + } +} + +static void emit_loadimm_sext(s32 K, unsigned int dest, struct jit_ctx *ctx) +{ + if (is_simm13(K)) { + /* or %g0, K, DEST */ + emit(OR | IMMED | RS1(G0) | S13(K) | RD(dest), ctx); + } else { + emit_set_const_sext(K, dest, ctx); + } +} + +static void analyze_64bit_constant(u32 high_bits, u32 low_bits, + int *hbsp, int *lbsp, int *abbasp) +{ + int lowest_bit_set, highest_bit_set, all_bits_between_are_set; + int i; + + lowest_bit_set = highest_bit_set = -1; + i = 0; + do { + if ((lowest_bit_set == -1) && ((low_bits >> i) & 1)) + lowest_bit_set = i; + if ((highest_bit_set == -1) && ((high_bits >> (32 - i - 1)) & 1)) + highest_bit_set = (64 - i - 1); + } while (++i < 32 && (highest_bit_set == -1 || + lowest_bit_set == -1)); + if (i == 32) { + i = 0; + do { + if (lowest_bit_set == -1 && ((high_bits >> i) & 1)) + lowest_bit_set = i + 32; + if (highest_bit_set == -1 && + ((low_bits >> (32 - i - 1)) & 1)) + highest_bit_set = 32 - i - 1; + } while (++i < 32 && (highest_bit_set == -1 || + lowest_bit_set == -1)); + } + + all_bits_between_are_set = 1; + for (i = lowest_bit_set; i <= highest_bit_set; i++) { + if (i < 32) { + if ((low_bits & (1 << i)) != 0) + continue; + } else { + if ((high_bits & (1 << (i - 32))) != 0) + continue; + } + all_bits_between_are_set = 0; + break; + } + *hbsp = highest_bit_set; + *lbsp = lowest_bit_set; + *abbasp = all_bits_between_are_set; +} + +static unsigned long create_simple_focus_bits(unsigned long high_bits, + unsigned long low_bits, + int lowest_bit_set, int shift) +{ + long hi, lo; + + if (lowest_bit_set < 32) { + lo = (low_bits >> lowest_bit_set) << shift; + hi = ((high_bits << (32 - lowest_bit_set)) << shift); + } else { + lo = 0; + hi = ((high_bits >> (lowest_bit_set - 32)) << shift); + } + return hi | lo; +} + +static bool const64_is_2insns(unsigned long high_bits, + unsigned long low_bits) +{ + int highest_bit_set, lowest_bit_set, all_bits_between_are_set; + + if (high_bits == 0 || high_bits == 0xffffffff) + return true; + + analyze_64bit_constant(high_bits, low_bits, + &highest_bit_set, &lowest_bit_set, + &all_bits_between_are_set); + + if ((highest_bit_set == 63 || lowest_bit_set == 0) && + all_bits_between_are_set != 0) + return true; + + if (highest_bit_set - lowest_bit_set < 21) + return true; + + return false; +} + +static void sparc_emit_set_const64_quick2(unsigned long high_bits, + unsigned long low_imm, + unsigned int dest, + int shift_count, struct jit_ctx *ctx) +{ + emit_loadimm32(high_bits, dest, ctx); + + /* Now shift it up into place. */ + emit_alu_K(SLLX, dest, shift_count, ctx); + + /* If there is a low immediate part piece, finish up by + * putting that in as well. + */ + if (low_imm != 0) + emit(OR | IMMED | RS1(dest) | S13(low_imm) | RD(dest), ctx); +} + +static void emit_loadimm64(u64 K, unsigned int dest, struct jit_ctx *ctx) +{ + int all_bits_between_are_set, lowest_bit_set, highest_bit_set; + unsigned int tmp = bpf2sparc[TMP_REG_1]; + u32 low_bits = (K & 0xffffffff); + u32 high_bits = (K >> 32); + + /* These two tests also take care of all of the one + * instruction cases. + */ + if (high_bits == 0xffffffff && (low_bits & 0x80000000)) + return emit_loadimm_sext(K, dest, ctx); + if (high_bits == 0x00000000) + return emit_loadimm32(K, dest, ctx); + + analyze_64bit_constant(high_bits, low_bits, &highest_bit_set, + &lowest_bit_set, &all_bits_between_are_set); + + /* 1) mov -1, %reg + * sllx %reg, shift, %reg + * 2) mov -1, %reg + * srlx %reg, shift, %reg + * 3) mov some_small_const, %reg + * sllx %reg, shift, %reg + */ + if (((highest_bit_set == 63 || lowest_bit_set == 0) && + all_bits_between_are_set != 0) || + ((highest_bit_set - lowest_bit_set) < 12)) { + int shift = lowest_bit_set; + long the_const = -1; + + if ((highest_bit_set != 63 && lowest_bit_set != 0) || + all_bits_between_are_set == 0) { + the_const = + create_simple_focus_bits(high_bits, low_bits, + lowest_bit_set, 0); + } else if (lowest_bit_set == 0) + shift = -(63 - highest_bit_set); + + emit(OR | IMMED | RS1(G0) | S13(the_const) | RD(dest), ctx); + if (shift > 0) + emit_alu_K(SLLX, dest, shift, ctx); + else if (shift < 0) + emit_alu_K(SRLX, dest, -shift, ctx); + + return; + } + + /* Now a range of 22 or less bits set somewhere. + * 1) sethi %hi(focus_bits), %reg + * sllx %reg, shift, %reg + * 2) sethi %hi(focus_bits), %reg + * srlx %reg, shift, %reg + */ + if ((highest_bit_set - lowest_bit_set) < 21) { + unsigned long focus_bits = + create_simple_focus_bits(high_bits, low_bits, + lowest_bit_set, 10); + + emit(SETHI(focus_bits, dest), ctx); + + /* If lowest_bit_set == 10 then a sethi alone could + * have done it. + */ + if (lowest_bit_set < 10) + emit_alu_K(SRLX, dest, 10 - lowest_bit_set, ctx); + else if (lowest_bit_set > 10) + emit_alu_K(SLLX, dest, lowest_bit_set - 10, ctx); + return; + } + + /* Ok, now 3 instruction sequences. */ + if (low_bits == 0) { + emit_loadimm32(high_bits, dest, ctx); + emit_alu_K(SLLX, dest, 32, ctx); + return; + } + + /* We may be able to do something quick + * when the constant is negated, so try that. + */ + if (const64_is_2insns((~high_bits) & 0xffffffff, + (~low_bits) & 0xfffffc00)) { + /* NOTE: The trailing bits get XOR'd so we need the + * non-negated bits, not the negated ones. + */ + unsigned long trailing_bits = low_bits & 0x3ff; + + if ((((~high_bits) & 0xffffffff) == 0 && + ((~low_bits) & 0x80000000) == 0) || + (((~high_bits) & 0xffffffff) == 0xffffffff && + ((~low_bits) & 0x80000000) != 0)) { + unsigned long fast_int = (~low_bits & 0xffffffff); + + if ((is_sethi(fast_int) && + (~high_bits & 0xffffffff) == 0)) { + emit(SETHI(fast_int, dest), ctx); + } else if (is_simm13(fast_int)) { + emit(OR | IMMED | RS1(G0) | S13(fast_int) | RD(dest), ctx); + } else { + emit_loadimm64(fast_int, dest, ctx); + } + } else { + u64 n = ((~low_bits) & 0xfffffc00) | + (((unsigned long)((~high_bits) & 0xffffffff))<<32); + emit_loadimm64(n, dest, ctx); + } + + low_bits = -0x400 | trailing_bits; + + emit(XOR | IMMED | RS1(dest) | S13(low_bits) | RD(dest), ctx); + return; + } + + /* 1) sethi %hi(xxx), %reg + * or %reg, %lo(xxx), %reg + * sllx %reg, yyy, %reg + */ + if ((highest_bit_set - lowest_bit_set) < 32) { + unsigned long focus_bits = + create_simple_focus_bits(high_bits, low_bits, + lowest_bit_set, 0); + + /* So what we know is that the set bits straddle the + * middle of the 64-bit word. + */ + sparc_emit_set_const64_quick2(focus_bits, 0, dest, + lowest_bit_set, ctx); + return; + } + + /* 1) sethi %hi(high_bits), %reg + * or %reg, %lo(high_bits), %reg + * sllx %reg, 32, %reg + * or %reg, low_bits, %reg + */ + if (is_simm13(low_bits) && ((int)low_bits > 0)) { + sparc_emit_set_const64_quick2(high_bits, low_bits, + dest, 32, ctx); + return; + } + + /* Oh well, we tried... Do a full 64-bit decomposition. */ + ctx->tmp_1_used = true; + + emit_loadimm32(high_bits, tmp, ctx); + emit_loadimm32(low_bits, dest, ctx); + emit_alu_K(SLLX, tmp, 32, ctx); + emit(OR | RS1(dest) | RS2(tmp) | RD(dest), ctx); +} + +static void emit_branch(unsigned int br_opc, unsigned int from_idx, unsigned int to_idx, + struct jit_ctx *ctx) +{ + unsigned int off = to_idx - from_idx; + + if (br_opc & XCC) + emit(br_opc | WDISP19(off << 2), ctx); + else + emit(br_opc | WDISP22(off << 2), ctx); +} + +static void emit_cbcond(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, + const u8 dst, const u8 src, struct jit_ctx *ctx) +{ + unsigned int off = to_idx - from_idx; + + emit(cb_opc | WDISP10(off << 2) | RS1(dst) | RS2(src), ctx); +} + +static void emit_cbcondi(unsigned int cb_opc, unsigned int from_idx, unsigned int to_idx, + const u8 dst, s32 imm, struct jit_ctx *ctx) +{ + unsigned int off = to_idx - from_idx; + + emit(cb_opc | IMMED | WDISP10(off << 2) | RS1(dst) | S5(imm), ctx); +} + +#define emit_read_y(REG, CTX) emit(RD_Y | RD(REG), CTX) +#define emit_write_y(REG, CTX) emit(WR_Y | IMMED | RS1(REG) | S13(0), CTX) + +#define emit_cmp(R1, R2, CTX) \ + emit(SUBCC | RS1(R1) | RS2(R2) | RD(G0), CTX) + +#define emit_cmpi(R1, IMM, CTX) \ + emit(SUBCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) + +#define emit_btst(R1, R2, CTX) \ + emit(ANDCC | RS1(R1) | RS2(R2) | RD(G0), CTX) + +#define emit_btsti(R1, IMM, CTX) \ + emit(ANDCC | IMMED | RS1(R1) | S13(IMM) | RD(G0), CTX) + +static int emit_compare_and_branch(const u8 code, const u8 dst, u8 src, + const s32 imm, bool is_imm, int branch_dst, + struct jit_ctx *ctx) +{ + bool use_cbcond = (sparc64_elf_hwcap & AV_SPARC_CBCOND) != 0; + const u8 tmp = bpf2sparc[TMP_REG_1]; + + branch_dst = ctx->offset[branch_dst]; + + if (!is_simm10(branch_dst - ctx->idx) || + BPF_OP(code) == BPF_JSET) + use_cbcond = false; + + if (is_imm) { + bool fits = true; + + if (use_cbcond) { + if (!is_simm5(imm)) + fits = false; + } else if (!is_simm13(imm)) { + fits = false; + } + if (!fits) { + ctx->tmp_1_used = true; + emit_loadimm_sext(imm, tmp, ctx); + src = tmp; + is_imm = false; + } + } + + if (!use_cbcond) { + u32 br_opcode; + + if (BPF_OP(code) == BPF_JSET) { + if (is_imm) + emit_btsti(dst, imm, ctx); + else + emit_btst(dst, src, ctx); + } else { + if (is_imm) + emit_cmpi(dst, imm, ctx); + else + emit_cmp(dst, src, ctx); + } + switch (BPF_OP(code)) { + case BPF_JEQ: + br_opcode = BE; + break; + case BPF_JGT: + br_opcode = BGU; + break; + case BPF_JLT: + br_opcode = BLU; + break; + case BPF_JGE: + br_opcode = BGEU; + break; + case BPF_JLE: + br_opcode = BLEU; + break; + case BPF_JSET: + case BPF_JNE: + br_opcode = BNE; + break; + case BPF_JSGT: + br_opcode = BG; + break; + case BPF_JSLT: + br_opcode = BL; + break; + case BPF_JSGE: + br_opcode = BGE; + break; + case BPF_JSLE: + br_opcode = BLE; + break; + default: + /* Make sure we dont leak kernel information to the + * user. + */ + return -EFAULT; + } + emit_branch(br_opcode, ctx->idx, branch_dst, ctx); + emit_nop(ctx); + } else { + u32 cbcond_opcode; + + switch (BPF_OP(code)) { + case BPF_JEQ: + cbcond_opcode = CBCONDE; + break; + case BPF_JGT: + cbcond_opcode = CBCONDGU; + break; + case BPF_JLT: + cbcond_opcode = CBCONDLU; + break; + case BPF_JGE: + cbcond_opcode = CBCONDGEU; + break; + case BPF_JLE: + cbcond_opcode = CBCONDLEU; + break; + case BPF_JNE: + cbcond_opcode = CBCONDNE; + break; + case BPF_JSGT: + cbcond_opcode = CBCONDG; + break; + case BPF_JSLT: + cbcond_opcode = CBCONDL; + break; + case BPF_JSGE: + cbcond_opcode = CBCONDGE; + break; + case BPF_JSLE: + cbcond_opcode = CBCONDLE; + break; + default: + /* Make sure we dont leak kernel information to the + * user. + */ + return -EFAULT; + } + cbcond_opcode |= CBCOND_OP; + if (is_imm) + emit_cbcondi(cbcond_opcode, ctx->idx, branch_dst, + dst, imm, ctx); + else + emit_cbcond(cbcond_opcode, ctx->idx, branch_dst, + dst, src, ctx); + } + return 0; +} + +/* Just skip the save instruction and the ctx register move. */ +#define BPF_TAILCALL_PROLOGUE_SKIP 32 +#define BPF_TAILCALL_CNT_SP_OFF (STACK_BIAS + 128) + +static void build_prologue(struct jit_ctx *ctx) +{ + s32 stack_needed = BASE_STACKFRAME; + + if (ctx->saw_frame_pointer || ctx->saw_tail_call) { + struct bpf_prog *prog = ctx->prog; + u32 stack_depth; + + stack_depth = prog->aux->stack_depth; + stack_needed += round_up(stack_depth, 16); + } + + if (ctx->saw_tail_call) + stack_needed += 8; + + /* save %sp, -176, %sp */ + emit(SAVE | IMMED | RS1(SP) | S13(-stack_needed) | RD(SP), ctx); + + /* tail_call_cnt = 0 */ + if (ctx->saw_tail_call) { + u32 off = BPF_TAILCALL_CNT_SP_OFF; + + emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(G0), ctx); + } else { + emit_nop(ctx); + } + if (ctx->saw_frame_pointer) { + const u8 vfp = bpf2sparc[BPF_REG_FP]; + + emit(ADD | IMMED | RS1(FP) | S13(STACK_BIAS) | RD(vfp), ctx); + } else { + emit_nop(ctx); + } + + emit_reg_move(I0, O0, ctx); + emit_reg_move(I1, O1, ctx); + emit_reg_move(I2, O2, ctx); + emit_reg_move(I3, O3, ctx); + emit_reg_move(I4, O4, ctx); + /* If you add anything here, adjust BPF_TAILCALL_PROLOGUE_SKIP above. */ +} + +static void build_epilogue(struct jit_ctx *ctx) +{ + ctx->epilogue_offset = ctx->idx; + + /* ret (jmpl %i7 + 8, %g0) */ + emit(JMPL | IMMED | RS1(I7) | S13(8) | RD(G0), ctx); + + /* restore %i5, %g0, %o0 */ + emit(RESTORE | RS1(bpf2sparc[BPF_REG_0]) | RS2(G0) | RD(O0), ctx); +} + +static void emit_tail_call(struct jit_ctx *ctx) +{ + const u8 bpf_array = bpf2sparc[BPF_REG_2]; + const u8 bpf_index = bpf2sparc[BPF_REG_3]; + const u8 tmp = bpf2sparc[TMP_REG_1]; + u32 off; + + ctx->saw_tail_call = true; + + off = offsetof(struct bpf_array, map.max_entries); + emit(LD32 | IMMED | RS1(bpf_array) | S13(off) | RD(tmp), ctx); + emit_cmp(bpf_index, tmp, ctx); +#define OFFSET1 17 + emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET1, ctx); + emit_nop(ctx); + + off = BPF_TAILCALL_CNT_SP_OFF; + emit(LD32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); + emit_cmpi(tmp, MAX_TAIL_CALL_CNT, ctx); +#define OFFSET2 13 + emit_branch(BGEU, ctx->idx, ctx->idx + OFFSET2, ctx); + emit_nop(ctx); + + emit_alu_K(ADD, tmp, 1, ctx); + off = BPF_TAILCALL_CNT_SP_OFF; + emit(ST32 | IMMED | RS1(SP) | S13(off) | RD(tmp), ctx); + + emit_alu3_K(SLL, bpf_index, 3, tmp, ctx); + emit_alu(ADD, bpf_array, tmp, ctx); + off = offsetof(struct bpf_array, ptrs); + emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); + + emit_cmpi(tmp, 0, ctx); +#define OFFSET3 5 + emit_branch(BE, ctx->idx, ctx->idx + OFFSET3, ctx); + emit_nop(ctx); + + off = offsetof(struct bpf_prog, bpf_func); + emit(LD64 | IMMED | RS1(tmp) | S13(off) | RD(tmp), ctx); + + off = BPF_TAILCALL_PROLOGUE_SKIP; + emit(JMPL | IMMED | RS1(tmp) | S13(off) | RD(G0), ctx); + emit_nop(ctx); +} + +static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx) +{ + const u8 code = insn->code; + const u8 dst = bpf2sparc[insn->dst_reg]; + const u8 src = bpf2sparc[insn->src_reg]; + const int i = insn - ctx->prog->insnsi; + const s16 off = insn->off; + const s32 imm = insn->imm; + + if (insn->src_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + switch (code) { + /* dst = src */ + case BPF_ALU | BPF_MOV | BPF_X: + emit_alu3_K(SRL, src, 0, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_MOV | BPF_X: + emit_reg_move(src, dst, ctx); + break; + /* dst = dst OP src */ + case BPF_ALU | BPF_ADD | BPF_X: + case BPF_ALU64 | BPF_ADD | BPF_X: + emit_alu(ADD, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_SUB | BPF_X: + case BPF_ALU64 | BPF_SUB | BPF_X: + emit_alu(SUB, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_AND | BPF_X: + case BPF_ALU64 | BPF_AND | BPF_X: + emit_alu(AND, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_OR | BPF_X: + case BPF_ALU64 | BPF_OR | BPF_X: + emit_alu(OR, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_XOR | BPF_X: + case BPF_ALU64 | BPF_XOR | BPF_X: + emit_alu(XOR, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_MUL | BPF_X: + emit_alu(MUL, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_MUL | BPF_X: + emit_alu(MULX, src, dst, ctx); + break; + case BPF_ALU | BPF_DIV | BPF_X: + emit_write_y(G0, ctx); + emit_alu(DIV, src, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_DIV | BPF_X: + emit_alu(UDIVX, src, dst, ctx); + break; + case BPF_ALU | BPF_MOD | BPF_X: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_write_y(G0, ctx); + emit_alu3(DIV, dst, src, tmp, ctx); + emit_alu3(MULX, tmp, src, tmp, ctx); + emit_alu3(SUB, dst, tmp, dst, ctx); + goto do_alu32_trunc; + } + case BPF_ALU64 | BPF_MOD | BPF_X: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_alu3(UDIVX, dst, src, tmp, ctx); + emit_alu3(MULX, tmp, src, tmp, ctx); + emit_alu3(SUB, dst, tmp, dst, ctx); + break; + } + case BPF_ALU | BPF_LSH | BPF_X: + emit_alu(SLL, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_LSH | BPF_X: + emit_alu(SLLX, src, dst, ctx); + break; + case BPF_ALU | BPF_RSH | BPF_X: + emit_alu(SRL, src, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_RSH | BPF_X: + emit_alu(SRLX, src, dst, ctx); + break; + case BPF_ALU | BPF_ARSH | BPF_X: + emit_alu(SRA, src, dst, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_ARSH | BPF_X: + emit_alu(SRAX, src, dst, ctx); + break; + + /* dst = -dst */ + case BPF_ALU | BPF_NEG: + case BPF_ALU64 | BPF_NEG: + emit(SUB | RS1(0) | RS2(dst) | RD(dst), ctx); + goto do_alu32_trunc; + + case BPF_ALU | BPF_END | BPF_FROM_BE: + switch (imm) { + case 16: + emit_alu_K(SLL, dst, 16, ctx); + emit_alu_K(SRL, dst, 16, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case 32: + if (!ctx->prog->aux->verifier_zext) + emit_alu_K(SRL, dst, 0, ctx); + break; + case 64: + /* nop */ + break; + + } + break; + + /* dst = BSWAP##imm(dst) */ + case BPF_ALU | BPF_END | BPF_FROM_LE: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + + ctx->tmp_1_used = true; + switch (imm) { + case 16: + emit_alu3_K(AND, dst, 0xff, tmp, ctx); + emit_alu3_K(SRL, dst, 8, dst, ctx); + emit_alu3_K(AND, dst, 0xff, dst, ctx); + emit_alu3_K(SLL, tmp, 8, tmp, ctx); + emit_alu(OR, tmp, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + + case 32: + ctx->tmp_2_used = true; + emit_alu3_K(SRL, dst, 24, tmp, ctx); /* tmp = dst >> 24 */ + emit_alu3_K(SRL, dst, 16, tmp2, ctx); /* tmp2 = dst >> 16 */ + emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ + emit_alu3_K(SLL, tmp2, 8, tmp2, ctx); /* tmp2 = tmp2 << 8 */ + emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ + emit_alu3_K(SRL, dst, 8, tmp2, ctx); /* tmp2 = dst >> 8 */ + emit_alu3_K(AND, tmp2, 0xff, tmp2, ctx);/* tmp2 = tmp2 & 0xff */ + emit_alu3_K(SLL, tmp2, 16, tmp2, ctx); /* tmp2 = tmp2 << 16 */ + emit_alu(OR, tmp2, tmp, ctx); /* tmp = tmp | tmp2 */ + emit_alu3_K(AND, dst, 0xff, dst, ctx); /* dst = dst & 0xff */ + emit_alu3_K(SLL, dst, 24, dst, ctx); /* dst = dst << 24 */ + emit_alu(OR, tmp, dst, ctx); /* dst = dst | tmp */ + if (insn_is_zext(&insn[1])) + return 1; + break; + + case 64: + emit_alu3_K(ADD, SP, STACK_BIAS + 128, tmp, ctx); + emit(ST64 | RS1(tmp) | RS2(G0) | RD(dst), ctx); + emit(LD64A | ASI(ASI_PL) | RS1(tmp) | RS2(G0) | RD(dst), ctx); + break; + } + break; + } + /* dst = imm */ + case BPF_ALU | BPF_MOV | BPF_K: + emit_loadimm32(imm, dst, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_MOV | BPF_K: + emit_loadimm_sext(imm, dst, ctx); + break; + /* dst = dst OP imm */ + case BPF_ALU | BPF_ADD | BPF_K: + case BPF_ALU64 | BPF_ADD | BPF_K: + emit_alu_K(ADD, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_SUB | BPF_K: + case BPF_ALU64 | BPF_SUB | BPF_K: + emit_alu_K(SUB, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_AND | BPF_K: + case BPF_ALU64 | BPF_AND | BPF_K: + emit_alu_K(AND, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_OR | BPF_K: + case BPF_ALU64 | BPF_OR | BPF_K: + emit_alu_K(OR, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_XOR | BPF_K: + case BPF_ALU64 | BPF_XOR | BPF_K: + emit_alu_K(XOR, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU | BPF_MUL | BPF_K: + emit_alu_K(MUL, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_MUL | BPF_K: + emit_alu_K(MULX, dst, imm, ctx); + break; + case BPF_ALU | BPF_DIV | BPF_K: + if (imm == 0) + return -EINVAL; + + emit_write_y(G0, ctx); + emit_alu_K(DIV, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_DIV | BPF_K: + if (imm == 0) + return -EINVAL; + + emit_alu_K(UDIVX, dst, imm, ctx); + break; + case BPF_ALU64 | BPF_MOD | BPF_K: + case BPF_ALU | BPF_MOD | BPF_K: { + const u8 tmp = bpf2sparc[TMP_REG_2]; + unsigned int div; + + if (imm == 0) + return -EINVAL; + + div = (BPF_CLASS(code) == BPF_ALU64) ? UDIVX : DIV; + + ctx->tmp_2_used = true; + + if (BPF_CLASS(code) != BPF_ALU64) + emit_write_y(G0, ctx); + if (is_simm13(imm)) { + emit(div | IMMED | RS1(dst) | S13(imm) | RD(tmp), ctx); + emit(MULX | IMMED | RS1(tmp) | S13(imm) | RD(tmp), ctx); + emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); + } else { + const u8 tmp1 = bpf2sparc[TMP_REG_1]; + + ctx->tmp_1_used = true; + + emit_set_const_sext(imm, tmp1, ctx); + emit(div | RS1(dst) | RS2(tmp1) | RD(tmp), ctx); + emit(MULX | RS1(tmp) | RS2(tmp1) | RD(tmp), ctx); + emit(SUB | RS1(dst) | RS2(tmp) | RD(dst), ctx); + } + goto do_alu32_trunc; + } + case BPF_ALU | BPF_LSH | BPF_K: + emit_alu_K(SLL, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_LSH | BPF_K: + emit_alu_K(SLLX, dst, imm, ctx); + break; + case BPF_ALU | BPF_RSH | BPF_K: + emit_alu_K(SRL, dst, imm, ctx); + if (insn_is_zext(&insn[1])) + return 1; + break; + case BPF_ALU64 | BPF_RSH | BPF_K: + emit_alu_K(SRLX, dst, imm, ctx); + break; + case BPF_ALU | BPF_ARSH | BPF_K: + emit_alu_K(SRA, dst, imm, ctx); + goto do_alu32_trunc; + case BPF_ALU64 | BPF_ARSH | BPF_K: + emit_alu_K(SRAX, dst, imm, ctx); + break; + + do_alu32_trunc: + if (BPF_CLASS(code) == BPF_ALU && + !ctx->prog->aux->verifier_zext) + emit_alu_K(SRL, dst, 0, ctx); + break; + + /* JUMP off */ + case BPF_JMP | BPF_JA: + emit_branch(BA, ctx->idx, ctx->offset[i + off], ctx); + emit_nop(ctx); + break; + /* IF (dst COND src) JUMP off */ + case BPF_JMP | BPF_JEQ | BPF_X: + case BPF_JMP | BPF_JGT | BPF_X: + case BPF_JMP | BPF_JLT | BPF_X: + case BPF_JMP | BPF_JGE | BPF_X: + case BPF_JMP | BPF_JLE | BPF_X: + case BPF_JMP | BPF_JNE | BPF_X: + case BPF_JMP | BPF_JSGT | BPF_X: + case BPF_JMP | BPF_JSLT | BPF_X: + case BPF_JMP | BPF_JSGE | BPF_X: + case BPF_JMP | BPF_JSLE | BPF_X: + case BPF_JMP | BPF_JSET | BPF_X: { + int err; + + err = emit_compare_and_branch(code, dst, src, 0, false, i + off, ctx); + if (err) + return err; + break; + } + /* IF (dst COND imm) JUMP off */ + case BPF_JMP | BPF_JEQ | BPF_K: + case BPF_JMP | BPF_JGT | BPF_K: + case BPF_JMP | BPF_JLT | BPF_K: + case BPF_JMP | BPF_JGE | BPF_K: + case BPF_JMP | BPF_JLE | BPF_K: + case BPF_JMP | BPF_JNE | BPF_K: + case BPF_JMP | BPF_JSGT | BPF_K: + case BPF_JMP | BPF_JSLT | BPF_K: + case BPF_JMP | BPF_JSGE | BPF_K: + case BPF_JMP | BPF_JSLE | BPF_K: + case BPF_JMP | BPF_JSET | BPF_K: { + int err; + + err = emit_compare_and_branch(code, dst, 0, imm, true, i + off, ctx); + if (err) + return err; + break; + } + + /* function call */ + case BPF_JMP | BPF_CALL: + { + u8 *func = ((u8 *)__bpf_call_base) + imm; + + ctx->saw_call = true; + + emit_call((u32 *)func, ctx); + emit_nop(ctx); + + emit_reg_move(O0, bpf2sparc[BPF_REG_0], ctx); + break; + } + + /* tail call */ + case BPF_JMP | BPF_TAIL_CALL: + emit_tail_call(ctx); + break; + + /* function return */ + case BPF_JMP | BPF_EXIT: + /* Optimization: when last instruction is EXIT, + simply fallthrough to epilogue. */ + if (i == ctx->prog->len - 1) + break; + emit_branch(BA, ctx->idx, ctx->epilogue_offset, ctx); + emit_nop(ctx); + break; + + /* dst = imm64 */ + case BPF_LD | BPF_IMM | BPF_DW: + { + const struct bpf_insn insn1 = insn[1]; + u64 imm64; + + imm64 = (u64)insn1.imm << 32 | (u32)imm; + emit_loadimm64(imm64, dst, ctx); + + return 1; + } + + /* LDX: dst = *(size *)(src + off) */ + case BPF_LDX | BPF_MEM | BPF_W: + case BPF_LDX | BPF_MEM | BPF_H: + case BPF_LDX | BPF_MEM | BPF_B: + case BPF_LDX | BPF_MEM | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + u32 opcode = 0, rs2; + + ctx->tmp_1_used = true; + switch (BPF_SIZE(code)) { + case BPF_W: + opcode = LD32; + break; + case BPF_H: + opcode = LD16; + break; + case BPF_B: + opcode = LD8; + break; + case BPF_DW: + opcode = LD64; + break; + } + + if (is_simm13(off)) { + opcode |= IMMED; + rs2 = S13(off); + } else { + emit_loadimm(off, tmp, ctx); + rs2 = RS2(tmp); + } + emit(opcode | RS1(src) | rs2 | RD(dst), ctx); + if (opcode != LD64 && insn_is_zext(&insn[1])) + return 1; + break; + } + /* speculation barrier */ + case BPF_ST | BPF_NOSPEC: + break; + /* ST: *(size *)(dst + off) = imm */ + case BPF_ST | BPF_MEM | BPF_W: + case BPF_ST | BPF_MEM | BPF_H: + case BPF_ST | BPF_MEM | BPF_B: + case BPF_ST | BPF_MEM | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + u32 opcode = 0, rs2; + + if (insn->dst_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + ctx->tmp_2_used = true; + emit_loadimm(imm, tmp2, ctx); + + switch (BPF_SIZE(code)) { + case BPF_W: + opcode = ST32; + break; + case BPF_H: + opcode = ST16; + break; + case BPF_B: + opcode = ST8; + break; + case BPF_DW: + opcode = ST64; + break; + } + + if (is_simm13(off)) { + opcode |= IMMED; + rs2 = S13(off); + } else { + ctx->tmp_1_used = true; + emit_loadimm(off, tmp, ctx); + rs2 = RS2(tmp); + } + emit(opcode | RS1(dst) | rs2 | RD(tmp2), ctx); + break; + } + + /* STX: *(size *)(dst + off) = src */ + case BPF_STX | BPF_MEM | BPF_W: + case BPF_STX | BPF_MEM | BPF_H: + case BPF_STX | BPF_MEM | BPF_B: + case BPF_STX | BPF_MEM | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + u32 opcode = 0, rs2; + + if (insn->dst_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + switch (BPF_SIZE(code)) { + case BPF_W: + opcode = ST32; + break; + case BPF_H: + opcode = ST16; + break; + case BPF_B: + opcode = ST8; + break; + case BPF_DW: + opcode = ST64; + break; + } + if (is_simm13(off)) { + opcode |= IMMED; + rs2 = S13(off); + } else { + ctx->tmp_1_used = true; + emit_loadimm(off, tmp, ctx); + rs2 = RS2(tmp); + } + emit(opcode | RS1(dst) | rs2 | RD(src), ctx); + break; + } + + case BPF_STX | BPF_ATOMIC | BPF_W: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + const u8 tmp3 = bpf2sparc[TMP_REG_3]; + + if (insn->imm != BPF_ADD) { + pr_err_once("unknown atomic op %02x\n", insn->imm); + return -EINVAL; + } + + /* lock *(u32 *)(dst + off) += src */ + + if (insn->dst_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + ctx->tmp_1_used = true; + ctx->tmp_2_used = true; + ctx->tmp_3_used = true; + emit_loadimm(off, tmp, ctx); + emit_alu3(ADD, dst, tmp, tmp, ctx); + + emit(LD32 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); + emit_alu3(ADD, tmp2, src, tmp3, ctx); + emit(CAS | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); + emit_cmp(tmp2, tmp3, ctx); + emit_branch(BNE, 4, 0, ctx); + emit_nop(ctx); + break; + } + /* STX XADD: lock *(u64 *)(dst + off) += src */ + case BPF_STX | BPF_ATOMIC | BPF_DW: { + const u8 tmp = bpf2sparc[TMP_REG_1]; + const u8 tmp2 = bpf2sparc[TMP_REG_2]; + const u8 tmp3 = bpf2sparc[TMP_REG_3]; + + if (insn->imm != BPF_ADD) { + pr_err_once("unknown atomic op %02x\n", insn->imm); + return -EINVAL; + } + + if (insn->dst_reg == BPF_REG_FP) + ctx->saw_frame_pointer = true; + + ctx->tmp_1_used = true; + ctx->tmp_2_used = true; + ctx->tmp_3_used = true; + emit_loadimm(off, tmp, ctx); + emit_alu3(ADD, dst, tmp, tmp, ctx); + + emit(LD64 | RS1(tmp) | RS2(G0) | RD(tmp2), ctx); + emit_alu3(ADD, tmp2, src, tmp3, ctx); + emit(CASX | ASI(ASI_P) | RS1(tmp) | RS2(tmp2) | RD(tmp3), ctx); + emit_cmp(tmp2, tmp3, ctx); + emit_branch(BNE, 4, 0, ctx); + emit_nop(ctx); + break; + } + + default: + pr_err_once("unknown opcode %02x\n", code); + return -EINVAL; + } + + return 0; +} + +static int build_body(struct jit_ctx *ctx) +{ + const struct bpf_prog *prog = ctx->prog; + int i; + + for (i = 0; i < prog->len; i++) { + const struct bpf_insn *insn = &prog->insnsi[i]; + int ret; + + ret = build_insn(insn, ctx); + + if (ret > 0) { + i++; + ctx->offset[i] = ctx->idx; + continue; + } + ctx->offset[i] = ctx->idx; + if (ret) + return ret; + } + return 0; +} + +static void jit_fill_hole(void *area, unsigned int size) +{ + u32 *ptr; + /* We are guaranteed to have aligned memory. */ + for (ptr = area; size >= sizeof(u32); size -= sizeof(u32)) + *ptr++ = 0x91d02005; /* ta 5 */ +} + +bool bpf_jit_needs_zext(void) +{ + return true; +} + +struct sparc64_jit_data { + struct bpf_binary_header *header; + u8 *image; + struct jit_ctx ctx; +}; + +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog) +{ + struct sparc64_jit_data *jit_data; + struct bpf_binary_header *header; + u32 prev_image_size, image_size; + bool extra_pass = false; + struct jit_ctx ctx; + u8 *image_ptr; + int pass, i; + + if (!prog->jit_requested) + return prog; + + jit_data = prog->aux->jit_data; + if (!jit_data) { + jit_data = kzalloc_obj(*jit_data); + if (!jit_data) + return prog; + prog->aux->jit_data = jit_data; + } + if (jit_data->ctx.offset) { + ctx = jit_data->ctx; + image_ptr = jit_data->image; + header = jit_data->header; + extra_pass = true; + image_size = sizeof(u32) * ctx.idx; + prev_image_size = image_size; + pass = 1; + goto skip_init_ctx; + } + + memset(&ctx, 0, sizeof(ctx)); + ctx.prog = prog; + + ctx.offset = kmalloc_array(prog->len, sizeof(unsigned int), GFP_KERNEL); + if (ctx.offset == NULL) + goto out_err; + + /* Longest sequence emitted is for bswap32, 12 instructions. Pre-cook + * the offset array so that we converge faster. + */ + for (i = 0; i < prog->len; i++) + ctx.offset[i] = i * (12 * 4); + + prev_image_size = ~0U; + for (pass = 1; pass < 40; pass++) { + ctx.idx = 0; + + build_prologue(&ctx); + if (build_body(&ctx)) + goto out_err; + build_epilogue(&ctx); + + if (bpf_jit_enable > 1) + pr_info("Pass %d: size = %u, seen = [%c%c%c%c%c%c]\n", pass, + ctx.idx * 4, + ctx.tmp_1_used ? '1' : ' ', + ctx.tmp_2_used ? '2' : ' ', + ctx.tmp_3_used ? '3' : ' ', + ctx.saw_frame_pointer ? 'F' : ' ', + ctx.saw_call ? 'C' : ' ', + ctx.saw_tail_call ? 'T' : ' '); + + if (ctx.idx * 4 == prev_image_size) + break; + prev_image_size = ctx.idx * 4; + cond_resched(); + } + + /* Now we know the actual image size. */ + image_size = sizeof(u32) * ctx.idx; + header = bpf_jit_binary_alloc(image_size, &image_ptr, + sizeof(u32), jit_fill_hole); + if (header == NULL) + goto out_err; + + ctx.image = (u32 *)image_ptr; +skip_init_ctx: + ctx.idx = 0; + + build_prologue(&ctx); + + if (build_body(&ctx)) { + bpf_jit_binary_free(header); + goto out_err; + } + + build_epilogue(&ctx); + + if (ctx.idx * 4 != prev_image_size) { + pr_err("bpf_jit: Failed to converge, prev_size=%u size=%d\n", + prev_image_size, ctx.idx * 4); + bpf_jit_binary_free(header); + goto out_err; + } + + if (bpf_jit_enable > 1) + bpf_jit_dump(prog->len, image_size, pass, ctx.image); + + bpf_flush_icache(header, (u8 *)header + header->size); + + if (!prog->is_func || extra_pass) { + if (bpf_jit_binary_lock_ro(header)) { + bpf_jit_binary_free(header); + goto out_err; + } + } else { + jit_data->ctx = ctx; + jit_data->image = image_ptr; + jit_data->header = header; + } + + prog->bpf_func = (void *)ctx.image; + prog->jited = 1; + prog->jited_len = image_size; + + if (!prog->is_func || extra_pass) { + bpf_prog_fill_jited_linfo(prog, ctx.offset); +out_off: + kfree(ctx.offset); + kfree(jit_data); + prog->aux->jit_data = NULL; + } + + return prog; + +out_err: + if (extra_pass) { + prog->bpf_func = NULL; + prog->jited = 0; + prog->jited_len = 0; + } + goto out_off; +} diff --git a/arch/sparc/power/Makefile b/arch/sparc/power/Makefile new file mode 100644 index 000000000..d8f75e7cb --- /dev/null +++ b/arch/sparc/power/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Makefile for Sparc-specific hibernate files. + +obj-$(CONFIG_HIBERNATION) += hibernate.o hibernate_asm.o diff --git a/arch/sparc/power/hibernate.c b/arch/sparc/power/hibernate.c new file mode 100644 index 000000000..da8e2bc2e --- /dev/null +++ b/arch/sparc/power/hibernate.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * hibernate.c: Hibernaton support specific for sparc64. + * + * Copyright (C) 2013 Kirill V Tkhai (tkhai@yandex.ru) + */ + +#include +#include + +#include +#include +#include +#include +#include + +struct saved_context saved_context; + +/* + * pfn_is_nosave - check if given pfn is in the 'nosave' section + */ + +int pfn_is_nosave(unsigned long pfn) +{ + unsigned long nosave_begin_pfn = PFN_DOWN((unsigned long)&__nosave_begin); + unsigned long nosave_end_pfn = PFN_DOWN((unsigned long)&__nosave_end); + + return (pfn >= nosave_begin_pfn) && (pfn < nosave_end_pfn); +} + +void save_processor_state(void) +{ + save_and_clear_fpu(); +} + +void restore_processor_state(void) +{ + struct mm_struct *mm = current->active_mm; + + tsb_context_switch_ctx(mm, CTX_HWBITS(mm->context)); +} diff --git a/arch/sparc/power/hibernate_asm.S b/arch/sparc/power/hibernate_asm.S new file mode 100644 index 000000000..8cfaf5b6a --- /dev/null +++ b/arch/sparc/power/hibernate_asm.S @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * hibernate_asm.S: Hibernaton support specific for sparc64. + * + * Copyright (C) 2013 Kirill V Tkhai (tkhai@yandex.ru) + */ + +#include + +#include +#include +#include + +ENTRY(swsusp_arch_suspend) + save %sp, -128, %sp + save %sp, -128, %sp + flushw + + setuw saved_context, %g3 + + /* Save window regs */ + rdpr %cwp, %g2 + stx %g2, [%g3 + SC_REG_CWP] + rdpr %wstate, %g2 + stx %g2, [%g3 + SC_REG_WSTATE] + stx %fp, [%g3 + SC_REG_FP] + + /* Save state regs */ + rdpr %tick, %g2 + stx %g2, [%g3 + SC_REG_TICK] + rdpr %pstate, %g2 + stx %g2, [%g3 + SC_REG_PSTATE] + + /* Save global regs */ + stx %g4, [%g3 + SC_REG_G4] + stx %g5, [%g3 + SC_REG_G5] + stx %g6, [%g3 + SC_REG_G6] + + call swsusp_save + nop + + mov %o0, %i0 + restore + + mov %o0, %i0 + ret + restore + +ENTRY(swsusp_arch_resume) + /* Write restore_pblist to %l0 */ + sethi %hi(restore_pblist), %l0 + ldx [%l0 + %lo(restore_pblist)], %l0 + + call __flush_tlb_all + nop + + /* Write PAGE_OFFSET to %g7 */ + sethi %hi(PAGE_OFFSET), %g7 + ldx [%g7 + %lo(PAGE_OFFSET)], %g7 + + setuw (PAGE_SIZE-8), %g3 + + /* Use MMU Bypass */ + rd %asi, %g1 + wr %g0, ASI_PHYS_USE_EC, %asi + + ba fill_itlb + nop + +pbe_loop: + cmp %l0, %g0 + be restore_ctx + sub %l0, %g7, %l0 + + ldxa [%l0 ] %asi, %l1 /* address */ + ldxa [%l0 + 8] %asi, %l2 /* orig_address */ + + /* phys addr */ + sub %l1, %g7, %l1 + sub %l2, %g7, %l2 + + mov %g3, %l3 /* PAGE_SIZE-8 */ +copy_loop: + ldxa [%l1 + %l3] ASI_PHYS_USE_EC, %g2 + stxa %g2, [%l2 + %l3] ASI_PHYS_USE_EC + cmp %l3, %g0 + bne copy_loop + sub %l3, 8, %l3 + + /* next pbe */ + ba pbe_loop + ldxa [%l0 + 16] %asi, %l0 + +restore_ctx: + setuw saved_context, %g3 + + /* Restore window regs */ + wrpr %g0, 0, %canrestore + wrpr %g0, 0, %otherwin + wrpr %g0, 6, %cansave + wrpr %g0, 0, %cleanwin + + ldxa [%g3 + SC_REG_CWP] %asi, %g2 + wrpr %g2, %cwp + ldxa [%g3 + SC_REG_WSTATE] %asi, %g2 + wrpr %g2, %wstate + ldxa [%g3 + SC_REG_FP] %asi, %fp + + /* Restore state regs */ + ldxa [%g3 + SC_REG_PSTATE] %asi, %g2 + wrpr %g2, %pstate + ldxa [%g3 + SC_REG_TICK] %asi, %g2 + wrpr %g2, %tick + + /* Restore global regs */ + ldxa [%g3 + SC_REG_G4] %asi, %g4 + ldxa [%g3 + SC_REG_G5] %asi, %g5 + ldxa [%g3 + SC_REG_G6] %asi, %g6 + + wr %g1, %g0, %asi + + restore + restore + + wrpr %g0, 14, %pil + + retl + mov %g0, %o0 + +fill_itlb: + ba pbe_loop + wrpr %g0, 15, %pil diff --git a/arch/sparc/prom/Makefile b/arch/sparc/prom/Makefile new file mode 100644 index 000000000..92db8bb4a --- /dev/null +++ b/arch/sparc/prom/Makefile @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: GPL-2.0 +# Makefile for the Sun Boot PROM interface library under +# Linux. +# + +lib-y := bootstr_$(BITS).o +lib-y += init_$(BITS).o +lib-$(CONFIG_SPARC32) += memory.o +lib-y += misc_$(BITS).o +lib-$(CONFIG_SPARC32) += mp.o +lib-$(CONFIG_SPARC32) += ranges.o +lib-y += console_$(BITS).o +lib-y += printf.o +lib-y += tree_$(BITS).o +lib-$(CONFIG_SPARC64) += p1275.o +lib-$(CONFIG_SPARC64) += cif.o diff --git a/arch/sparc/prom/bootstr_32.c b/arch/sparc/prom/bootstr_32.c new file mode 100644 index 000000000..1c7cd258b --- /dev/null +++ b/arch/sparc/prom/bootstr_32.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * bootstr.c: Boot string/argument acquisition from the PROM. + * + * Copyright(C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include + +#define BARG_LEN 256 +static char barg_buf[BARG_LEN] = { 0 }; +static char fetched __initdata = 0; + +char * __init +prom_getbootargs(void) +{ + int iter; + char *cp, *arg; + + /* This check saves us from a panic when bootfd patches args. */ + if (fetched) { + return barg_buf; + } + + switch (prom_vers) { + case PROM_V0: + cp = barg_buf; + /* Start from 1 and go over fd(0,0,0)kernel */ + for (iter = 1; iter < 8; iter++) { + arg = (*(romvec->pv_v0bootargs))->argv[iter]; + if (arg == NULL) + break; + while (*arg != 0) { + /* Leave place for space and null. */ + if (cp >= barg_buf + BARG_LEN - 2) + /* We might issue a warning here. */ + break; + *cp++ = *arg++; + } + *cp++ = ' '; + if (cp >= barg_buf + BARG_LEN - 1) + /* We might issue a warning here. */ + break; + } + *cp = 0; + break; + case PROM_V2: + case PROM_V3: + /* + * V3 PROM cannot supply as with more than 128 bytes + * of an argument. But a smart bootstrap loader can. + */ + strscpy(barg_buf, *romvec->pv_v2bootargs.bootargs, sizeof(barg_buf)); + break; + default: + break; + } + + fetched = 1; + return barg_buf; +} diff --git a/arch/sparc/prom/bootstr_64.c b/arch/sparc/prom/bootstr_64.c new file mode 100644 index 000000000..f1cc34d99 --- /dev/null +++ b/arch/sparc/prom/bootstr_64.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * bootstr.c: Boot string/argument acquisition from the PROM. + * + * Copyright(C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright(C) 1996,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include + +/* WARNING: The boot loader knows that these next three variables come one right + * after another in the .data section. Do not move this stuff into + * the .bss section or it will break things. + */ + +/* We limit BARG_LEN to 1024 because this is the size of the + * 'barg_out' command line buffer in the SILO bootloader. + */ +#define BARG_LEN 1024 +struct { + int bootstr_len; + int bootstr_valid; + char bootstr_buf[BARG_LEN]; +} bootstr_info = { + .bootstr_len = BARG_LEN, +#ifdef CONFIG_CMDLINE + .bootstr_valid = 1, + .bootstr_buf = CONFIG_CMDLINE, +#endif +}; + +char * __init +prom_getbootargs(void) +{ + /* This check saves us from a panic when bootfd patches args. */ + if (bootstr_info.bootstr_valid) + return bootstr_info.bootstr_buf; + prom_getstring(prom_chosen_node, "bootargs", + bootstr_info.bootstr_buf, BARG_LEN); + bootstr_info.bootstr_valid = 1; + return bootstr_info.bootstr_buf; +} diff --git a/arch/sparc/prom/cif.S b/arch/sparc/prom/cif.S new file mode 100644 index 000000000..dd06bb1fb --- /dev/null +++ b/arch/sparc/prom/cif.S @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* cif.S: PROM entry/exit assembler trampolines. + * + * Copyright (C) 1996, 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (C) 2005, 2006 David S. Miller + */ + +#include +#include +#include + + .text + .globl prom_cif_direct +prom_cif_direct: + save %sp, -192, %sp + sethi %hi(p1275buf), %o1 + or %o1, %lo(p1275buf), %o1 + ldx [%o1 + 0x0008], %l2 ! prom_cif_handler + mov %g4, %l0 + mov %g5, %l1 + mov %g6, %l3 + call %l2 + mov %i0, %o0 ! prom_args + mov %l0, %g4 + mov %l1, %g5 + mov %l3, %g6 + ret + restore + + .globl prom_cif_callback +prom_cif_callback: + sethi %hi(p1275buf), %o1 + or %o1, %lo(p1275buf), %o1 + save %sp, -192, %sp + TRAP_LOAD_THREAD_REG(%g6, %g1) + LOAD_PER_CPU_BASE(%g5, %g6, %g4, %g3, %o0) + ldx [%g6 + TI_TASK], %g4 + call prom_world + mov 0, %o0 + ldx [%i1 + 0x000], %l2 + call %l2 + mov %i0, %o0 + mov %o0, %l1 + call prom_world + mov 1, %o0 + ret + restore %l1, 0, %o0 + diff --git a/arch/sparc/prom/console_32.c b/arch/sparc/prom/console_32.c new file mode 100644 index 000000000..6404e5bbb --- /dev/null +++ b/arch/sparc/prom/console_32.c @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * console.c: Routines that deal with sending and receiving IO + * to/from the current console device using the PROM. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1998 Pete Zaitcev + */ + +#include +#include +#include +#include +#include +#include + +extern void restore_current(void); + +/* Non blocking put character to console device, returns -1 if + * unsuccessful. + */ +static int prom_nbputchar(const char *buf) +{ + unsigned long flags; + int i = -1; + + spin_lock_irqsave(&prom_lock, flags); + switch(prom_vers) { + case PROM_V0: + if ((*(romvec->pv_nbputchar))(*buf)) + i = 1; + break; + case PROM_V2: + case PROM_V3: + if ((*(romvec->pv_v2devops).v2_dev_write)(*romvec->pv_v2bootargs.fd_stdout, + buf, 0x1) == 1) + i = 1; + break; + default: + break; + } + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + return i; /* Ugh, we could spin forever on unsupported proms ;( */ +} + +void prom_console_write_buf(const char *buf, int len) +{ + while (len) { + int n = prom_nbputchar(buf); + if (n < 0) + continue; + len--; + buf++; + } +} + diff --git a/arch/sparc/prom/console_64.c b/arch/sparc/prom/console_64.c new file mode 100644 index 000000000..86dace787 --- /dev/null +++ b/arch/sparc/prom/console_64.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +/* console.c: Routines that deal with sending and receiving IO + * to/from the current console device using the PROM. + * + * Copyright (C) 1995 David S. Miller (davem@davemloft.net) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include + +static int __prom_console_write_buf(const char *buf, int len) +{ + unsigned long args[7]; + int ret; + + args[0] = (unsigned long) "write"; + args[1] = 3; + args[2] = 1; + args[3] = (unsigned int) prom_stdout; + args[4] = (unsigned long) buf; + args[5] = (unsigned int) len; + args[6] = (unsigned long) -1; + + p1275_cmd_direct(args); + + ret = (int) args[6]; + if (ret < 0) + return -1; + return ret; +} + +void prom_console_write_buf(const char *buf, int len) +{ + while (len) { + int n = __prom_console_write_buf(buf, len); + if (n < 0) + continue; + len -= n; + buf += len; + } +} diff --git a/arch/sparc/prom/init_32.c b/arch/sparc/prom/init_32.c new file mode 100644 index 000000000..d20470166 --- /dev/null +++ b/arch/sparc/prom/init_32.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * init.c: Initialize internal variables used by the PROM + * library functions. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include + +#include +#include + +struct linux_romvec *romvec; +EXPORT_SYMBOL(romvec); + +enum prom_major_version prom_vers; +unsigned int prom_rev, prom_prev; + +/* The root node of the prom device tree. */ +phandle prom_root_node; +EXPORT_SYMBOL(prom_root_node); + +/* Pointer to the device tree operations structure. */ +struct linux_nodeops *prom_nodeops; + +/* You must call prom_init() before you attempt to use any of the + * routines in the prom library. + * It gets passed the pointer to the PROM vector. + */ + +void __init prom_init(struct linux_romvec *rp) +{ + romvec = rp; + + switch(romvec->pv_romvers) { + case 0: + prom_vers = PROM_V0; + break; + case 2: + prom_vers = PROM_V2; + break; + case 3: + prom_vers = PROM_V3; + break; + default: + prom_printf("PROMLIB: Bad PROM version %d\n", + romvec->pv_romvers); + prom_halt(); + break; + } + + prom_rev = romvec->pv_plugin_revision; + prom_prev = romvec->pv_printrev; + prom_nodeops = romvec->pv_nodeops; + + prom_root_node = prom_getsibling(0); + if ((prom_root_node == 0) || ((s32)prom_root_node == -1)) + prom_halt(); + + if((((unsigned long) prom_nodeops) == 0) || + (((unsigned long) prom_nodeops) == -1)) + prom_halt(); + + prom_meminit(); + + prom_ranges_init(); + + printk("PROMLIB: Sun Boot Prom Version %d Revision %d\n", + romvec->pv_romvers, prom_rev); + + /* Initialization successful. */ +} diff --git a/arch/sparc/prom/init_64.c b/arch/sparc/prom/init_64.c new file mode 100644 index 000000000..f7b8a1a86 --- /dev/null +++ b/arch/sparc/prom/init_64.c @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * init.c: Initialize internal variables used by the PROM + * library functions. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include + +#include +#include + +/* OBP version string. */ +char prom_version[80]; + +/* The root node of the prom device tree. */ +int prom_stdout; +phandle prom_chosen_node; + +/* You must call prom_init() before you attempt to use any of the + * routines in the prom library. + * It gets passed the pointer to the PROM vector. + */ +void __init prom_init(void *cif_handler) +{ + phandle node; + + prom_cif_init(cif_handler); + + prom_chosen_node = prom_finddevice(prom_chosen_path); + if (!prom_chosen_node || (s32)prom_chosen_node == -1) + prom_halt(); + + prom_stdout = prom_getint(prom_chosen_node, "stdout"); + + node = prom_finddevice("/openprom"); + if (!node || (s32)node == -1) + prom_halt(); + + prom_getstring(node, "version", prom_version, sizeof(prom_version)); + + prom_printf("\n"); +} + +void __init prom_init_report(void) +{ + printk("PROMLIB: Sun IEEE Boot Prom '%s'\n", prom_version); + printk("PROMLIB: Root node compatible: %s\n", prom_root_compatible); +} diff --git a/arch/sparc/prom/memory.c b/arch/sparc/prom/memory.c new file mode 100644 index 000000000..269d6ab5e --- /dev/null +++ b/arch/sparc/prom/memory.c @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +/* memory.c: Prom routine for acquiring various bits of information + * about RAM on the machine, both virtual and physical. + * + * Copyright (C) 1995, 2008 David S. Miller (davem@davemloft.net) + * Copyright (C) 1997 Michael A. Griffith (grif@acm.org) + */ + +#include +#include +#include + +#include +#include +#include + +static int __init prom_meminit_v0(void) +{ + struct linux_mlist_v0 *p; + int index; + + index = 0; + for (p = *(romvec->pv_v0mem.v0_available); p; p = p->theres_more) { + sp_banks[index].base_addr = (unsigned long) p->start_adr; + sp_banks[index].num_bytes = p->num_bytes; + index++; + } + + return index; +} + +static int __init prom_meminit_v2(void) +{ + struct linux_prom_registers reg[64]; + phandle node; + int size, num_ents, i; + + node = prom_searchsiblings(prom_getchild(prom_root_node), "memory"); + size = prom_getproperty(node, "available", (char *) reg, sizeof(reg)); + num_ents = size / sizeof(struct linux_prom_registers); + + for (i = 0; i < num_ents; i++) { + sp_banks[i].base_addr = reg[i].phys_addr; + sp_banks[i].num_bytes = reg[i].reg_size; + } + + return num_ents; +} + +static int sp_banks_cmp(const void *a, const void *b) +{ + const struct sparc_phys_banks *x = a, *y = b; + + if (x->base_addr > y->base_addr) + return 1; + if (x->base_addr < y->base_addr) + return -1; + return 0; +} + +/* Initialize the memory lists based upon the prom version. */ +void __init prom_meminit(void) +{ + int i, num_ents = 0; + + switch (prom_vers) { + case PROM_V0: + num_ents = prom_meminit_v0(); + break; + + case PROM_V2: + case PROM_V3: + num_ents = prom_meminit_v2(); + break; + + default: + break; + } + sort(sp_banks, num_ents, sizeof(struct sparc_phys_banks), + sp_banks_cmp, NULL); + + /* Sentinel. */ + sp_banks[num_ents].base_addr = 0xdeadbeef; + sp_banks[num_ents].num_bytes = 0; + + for (i = 0; i < num_ents; i++) + sp_banks[i].num_bytes &= PAGE_MASK; +} diff --git a/arch/sparc/prom/misc_32.c b/arch/sparc/prom/misc_32.c new file mode 100644 index 000000000..625750924 --- /dev/null +++ b/arch/sparc/prom/misc_32.c @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * misc.c: Miscellaneous prom functions that don't belong + * anywhere else. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include + +#include +#include +#include + +extern void restore_current(void); + +DEFINE_SPINLOCK(prom_lock); + +/* Reset and reboot the machine with the command 'bcommand'. */ +void +prom_reboot(char *bcommand) +{ + unsigned long flags; + spin_lock_irqsave(&prom_lock, flags); + (*(romvec->pv_reboot))(bcommand); + /* Never get here. */ + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); +} + +/* Forth evaluate the expression contained in 'fstring'. */ +void +prom_feval(char *fstring) +{ + unsigned long flags; + if(!fstring || fstring[0] == 0) + return; + spin_lock_irqsave(&prom_lock, flags); + if(prom_vers == PROM_V0) + (*(romvec->pv_fortheval.v0_eval))(strlen(fstring), fstring); + else + (*(romvec->pv_fortheval.v2_eval))(fstring); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); +} +EXPORT_SYMBOL(prom_feval); + +/* Drop into the prom, with the chance to continue with the 'go' + * prom command. + */ +void +prom_cmdline(void) +{ + unsigned long flags; + + spin_lock_irqsave(&prom_lock, flags); + (*(romvec->pv_abort))(); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + set_auxio(AUXIO_LED, 0); +} + +/* Drop into the prom, but completely terminate the program. + * No chance of continuing. + */ +void __noreturn +prom_halt(void) +{ + unsigned long flags; +again: + spin_lock_irqsave(&prom_lock, flags); + (*(romvec->pv_halt))(); + /* Never get here. */ + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + goto again; /* PROM is out to get me -DaveM */ +} + +typedef void (*sfunc_t)(void); + +/* Set prom sync handler to call function 'funcp'. */ +void +prom_setsync(sfunc_t funcp) +{ + if(!funcp) return; + *romvec->pv_synchook = funcp; +} + +/* Get the idprom and stuff it into buffer 'idbuf'. Returns the + * format type. 'num_bytes' is the number of bytes that your idbuf + * has space for. Returns 0xff on error. + */ +unsigned char +prom_get_idprom(char *idbuf, int num_bytes) +{ + int len; + + len = prom_getproplen(prom_root_node, "idprom"); + if((len>num_bytes) || (len==-1)) return 0xff; + if(!prom_getproperty(prom_root_node, "idprom", idbuf, num_bytes)) + return idbuf[0]; + + return 0xff; +} + +/* Get the major prom version number. */ +int +prom_version(void) +{ + return romvec->pv_romvers; +} + +/* Get the prom plugin-revision. */ +int +prom_getrev(void) +{ + return prom_rev; +} + +/* Get the prom firmware print revision. */ +int +prom_getprev(void) +{ + return prom_prev; +} diff --git a/arch/sparc/prom/misc_64.c b/arch/sparc/prom/misc_64.c new file mode 100644 index 000000000..3792736ff --- /dev/null +++ b/arch/sparc/prom/misc_64.c @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * misc.c: Miscellaneous prom functions that don't belong + * anywhere else. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static int prom_service_exists(const char *service_name) +{ + unsigned long args[5]; + + args[0] = (unsigned long) "test"; + args[1] = 1; + args[2] = 1; + args[3] = (unsigned long) service_name; + args[4] = (unsigned long) -1; + + p1275_cmd_direct(args); + + if (args[4]) + return 0; + return 1; +} + +void prom_sun4v_guest_soft_state(void) +{ + const char *svc = "SUNW,soft-state-supported"; + unsigned long args[3]; + + if (!prom_service_exists(svc)) + return; + args[0] = (unsigned long) svc; + args[1] = 0; + args[2] = 0; + p1275_cmd_direct(args); +} + +/* Reset and reboot the machine with the command 'bcommand'. */ +void prom_reboot(const char *bcommand) +{ + unsigned long args[4]; + +#ifdef CONFIG_SUN_LDOMS + if (ldom_domaining_enabled) + ldom_reboot(bcommand); +#endif + args[0] = (unsigned long) "boot"; + args[1] = 1; + args[2] = 0; + args[3] = (unsigned long) bcommand; + + p1275_cmd_direct(args); +} + +/* Forth evaluate the expression contained in 'fstring'. */ +void prom_feval(const char *fstring) +{ + unsigned long args[5]; + + if (!fstring || fstring[0] == 0) + return; + args[0] = (unsigned long) "interpret"; + args[1] = 1; + args[2] = 1; + args[3] = (unsigned long) fstring; + args[4] = (unsigned long) -1; + + p1275_cmd_direct(args); +} +EXPORT_SYMBOL(prom_feval); + +/* Drop into the prom, with the chance to continue with the 'go' + * prom command. + */ +void prom_cmdline(void) +{ + unsigned long args[3]; + unsigned long flags; + + local_irq_save(flags); + +#ifdef CONFIG_SMP + smp_capture(); +#endif + + args[0] = (unsigned long) "enter"; + args[1] = 0; + args[2] = 0; + + p1275_cmd_direct(args); + +#ifdef CONFIG_SMP + smp_release(); +#endif + + local_irq_restore(flags); +} + +/* Drop into the prom, but completely terminate the program. + * No chance of continuing. + */ +void notrace prom_halt(void) +{ + unsigned long args[3]; + +#ifdef CONFIG_SUN_LDOMS + if (ldom_domaining_enabled) + ldom_power_off(); +#endif +again: + args[0] = (unsigned long) "exit"; + args[1] = 0; + args[2] = 0; + p1275_cmd_direct(args); + goto again; /* PROM is out to get me -DaveM */ +} + +void prom_halt_power_off(void) +{ + unsigned long args[3]; + +#ifdef CONFIG_SUN_LDOMS + if (ldom_domaining_enabled) + ldom_power_off(); +#endif + args[0] = (unsigned long) "SUNW,power-off"; + args[1] = 0; + args[2] = 0; + p1275_cmd_direct(args); + + /* if nothing else helps, we just halt */ + prom_halt(); +} + +/* Get the idprom and stuff it into buffer 'idbuf'. Returns the + * format type. 'num_bytes' is the number of bytes that your idbuf + * has space for. Returns 0xff on error. + */ +unsigned char prom_get_idprom(char *idbuf, int num_bytes) +{ + int len; + + len = prom_getproplen(prom_root_node, "idprom"); + if ((len >num_bytes) || (len == -1)) + return 0xff; + if (!prom_getproperty(prom_root_node, "idprom", idbuf, num_bytes)) + return idbuf[0]; + + return 0xff; +} + +static int prom_get_mmu_ihandle(void) +{ + phandle node; + int ret; + + if (prom_mmu_ihandle_cache != 0) + return prom_mmu_ihandle_cache; + + node = prom_finddevice(prom_chosen_path); + ret = prom_getint(node, prom_mmu_name); + if (ret == -1 || ret == 0) + prom_mmu_ihandle_cache = -1; + else + prom_mmu_ihandle_cache = ret; + + return ret; +} + +static int prom_get_memory_ihandle(void) +{ + static int memory_ihandle_cache; + phandle node; + int ret; + + if (memory_ihandle_cache != 0) + return memory_ihandle_cache; + + node = prom_finddevice("/chosen"); + ret = prom_getint(node, "memory"); + if (ret == -1 || ret == 0) + memory_ihandle_cache = -1; + else + memory_ihandle_cache = ret; + + return ret; +} + +/* Load explicit I/D TLB entries. */ +static long tlb_load(const char *type, unsigned long index, + unsigned long tte_data, unsigned long vaddr) +{ + unsigned long args[9]; + + args[0] = (unsigned long) prom_callmethod_name; + args[1] = 5; + args[2] = 1; + args[3] = (unsigned long) type; + args[4] = (unsigned int) prom_get_mmu_ihandle(); + args[5] = vaddr; + args[6] = tte_data; + args[7] = index; + args[8] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return (long) args[8]; +} + +long prom_itlb_load(unsigned long index, + unsigned long tte_data, + unsigned long vaddr) +{ + return tlb_load("SUNW,itlb-load", index, tte_data, vaddr); +} + +long prom_dtlb_load(unsigned long index, + unsigned long tte_data, + unsigned long vaddr) +{ + return tlb_load("SUNW,dtlb-load", index, tte_data, vaddr); +} + +int prom_map(int mode, unsigned long size, + unsigned long vaddr, unsigned long paddr) +{ + unsigned long args[11]; + int ret; + + args[0] = (unsigned long) prom_callmethod_name; + args[1] = 7; + args[2] = 1; + args[3] = (unsigned long) prom_map_name; + args[4] = (unsigned int) prom_get_mmu_ihandle(); + args[5] = (unsigned int) mode; + args[6] = size; + args[7] = vaddr; + args[8] = 0; + args[9] = paddr; + args[10] = (unsigned long) -1; + + p1275_cmd_direct(args); + + ret = (int) args[10]; + if (ret == 0) + ret = -1; + return ret; +} + +void prom_unmap(unsigned long size, unsigned long vaddr) +{ + unsigned long args[7]; + + args[0] = (unsigned long) prom_callmethod_name; + args[1] = 4; + args[2] = 0; + args[3] = (unsigned long) prom_unmap_name; + args[4] = (unsigned int) prom_get_mmu_ihandle(); + args[5] = size; + args[6] = vaddr; + + p1275_cmd_direct(args); +} + +/* Set aside physical memory which is not touched or modified + * across soft resets. + */ +int prom_retain(const char *name, unsigned long size, + unsigned long align, unsigned long *paddr) +{ + unsigned long args[11]; + + args[0] = (unsigned long) prom_callmethod_name; + args[1] = 5; + args[2] = 3; + args[3] = (unsigned long) "SUNW,retain"; + args[4] = (unsigned int) prom_get_memory_ihandle(); + args[5] = align; + args[6] = size; + args[7] = (unsigned long) name; + args[8] = (unsigned long) -1; + args[9] = (unsigned long) -1; + args[10] = (unsigned long) -1; + + p1275_cmd_direct(args); + + if (args[8]) + return (int) args[8]; + + /* Next we get "phys_high" then "phys_low". On 64-bit + * the phys_high cell is don't care since the phys_low + * cell has the full value. + */ + *paddr = args[10]; + + return 0; +} + +/* Get "Unumber" string for the SIMM at the given + * memory address. Usually this will be of the form + * "Uxxxx" where xxxx is a decimal number which is + * etched into the motherboard next to the SIMM slot + * in question. + */ +int prom_getunumber(int syndrome_code, + unsigned long phys_addr, + char *buf, int buflen) +{ + unsigned long args[12]; + + args[0] = (unsigned long) prom_callmethod_name; + args[1] = 7; + args[2] = 2; + args[3] = (unsigned long) "SUNW,get-unumber"; + args[4] = (unsigned int) prom_get_memory_ihandle(); + args[5] = buflen; + args[6] = (unsigned long) buf; + args[7] = 0; + args[8] = phys_addr; + args[9] = (unsigned int) syndrome_code; + args[10] = (unsigned long) -1; + args[11] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return (int) args[10]; +} + +/* Power management extensions. */ +void prom_sleepself(void) +{ + unsigned long args[3]; + + args[0] = (unsigned long) "SUNW,sleep-self"; + args[1] = 0; + args[2] = 0; + p1275_cmd_direct(args); +} + +int prom_sleepsystem(void) +{ + unsigned long args[4]; + + args[0] = (unsigned long) "SUNW,sleep-system"; + args[1] = 0; + args[2] = 1; + args[3] = (unsigned long) -1; + p1275_cmd_direct(args); + + return (int) args[3]; +} + +int prom_wakeupsystem(void) +{ + unsigned long args[4]; + + args[0] = (unsigned long) "SUNW,wakeup-system"; + args[1] = 0; + args[2] = 1; + args[3] = (unsigned long) -1; + p1275_cmd_direct(args); + + return (int) args[3]; +} + +#ifdef CONFIG_SMP +void prom_startcpu(int cpunode, unsigned long pc, unsigned long arg) +{ + unsigned long args[6]; + + args[0] = (unsigned long) "SUNW,start-cpu"; + args[1] = 3; + args[2] = 0; + args[3] = (unsigned int) cpunode; + args[4] = pc; + args[5] = arg; + p1275_cmd_direct(args); +} + +void prom_startcpu_cpuid(int cpuid, unsigned long pc, unsigned long arg) +{ + unsigned long args[6]; + + args[0] = (unsigned long) "SUNW,start-cpu-by-cpuid"; + args[1] = 3; + args[2] = 0; + args[3] = (unsigned int) cpuid; + args[4] = pc; + args[5] = arg; + p1275_cmd_direct(args); +} + +void prom_stopcpu_cpuid(int cpuid) +{ + unsigned long args[4]; + + args[0] = (unsigned long) "SUNW,stop-cpu-by-cpuid"; + args[1] = 1; + args[2] = 0; + args[3] = (unsigned int) cpuid; + p1275_cmd_direct(args); +} + +void prom_stopself(void) +{ + unsigned long args[3]; + + args[0] = (unsigned long) "SUNW,stop-self"; + args[1] = 0; + args[2] = 0; + p1275_cmd_direct(args); +} + +void prom_idleself(void) +{ + unsigned long args[3]; + + args[0] = (unsigned long) "SUNW,idle-self"; + args[1] = 0; + args[2] = 0; + p1275_cmd_direct(args); +} + +void prom_resumecpu(int cpunode) +{ + unsigned long args[4]; + + args[0] = (unsigned long) "SUNW,resume-cpu"; + args[1] = 1; + args[2] = 0; + args[3] = (unsigned int) cpunode; + p1275_cmd_direct(args); +} +#endif diff --git a/arch/sparc/prom/mp.c b/arch/sparc/prom/mp.c new file mode 100644 index 000000000..67cf0e957 --- /dev/null +++ b/arch/sparc/prom/mp.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * mp.c: OpenBoot Prom Multiprocessor support routines. Don't call + * these on a UP or else you will halt and catch fire. ;) + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include + +#include +#include + +extern void restore_current(void); + +/* Start cpu with prom-tree node 'cpunode' using context described + * by 'ctable_reg' in context 'ctx' at program counter 'pc'. + * + * XXX Have to look into what the return values mean. XXX + */ +int +prom_startcpu(int cpunode, struct linux_prom_registers *ctable_reg, int ctx, char *pc) +{ + int ret; + unsigned long flags; + + spin_lock_irqsave(&prom_lock, flags); + switch(prom_vers) { + case PROM_V0: + case PROM_V2: + default: + ret = -1; + break; + case PROM_V3: + ret = (*(romvec->v3_cpustart))(cpunode, (int) ctable_reg, ctx, pc); + break; + } + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + + return ret; +} diff --git a/arch/sparc/prom/p1275.c b/arch/sparc/prom/p1275.c new file mode 100644 index 000000000..51c3f984b --- /dev/null +++ b/arch/sparc/prom/p1275.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * p1275.c: Sun IEEE 1275 PROM low level interface routines + * + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +struct { + long prom_callback; /* 0x00 */ + void (*prom_cif_handler)(long *); /* 0x08 */ +} p1275buf; + +extern void prom_world(int); + +extern void prom_cif_direct(unsigned long *args); +extern void prom_cif_callback(void); + +/* + * This provides SMP safety on the p1275buf. + */ +DEFINE_RAW_SPINLOCK(prom_entry_lock); + +void p1275_cmd_direct(unsigned long *args) +{ + unsigned long flags; + + local_save_flags(flags); + local_irq_restore((unsigned long)PIL_NMI); + raw_spin_lock(&prom_entry_lock); + + prom_world(1); + prom_cif_direct(args); + prom_world(0); + + raw_spin_unlock(&prom_entry_lock); + local_irq_restore(flags); +} + +void prom_cif_init(void *cif_handler) +{ + p1275buf.prom_cif_handler = (void (*)(long *))cif_handler; +} diff --git a/arch/sparc/prom/printf.c b/arch/sparc/prom/printf.c new file mode 100644 index 000000000..dcee3dfa6 --- /dev/null +++ b/arch/sparc/prom/printf.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * printf.c: Internal prom library printf facility. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + * Copyright (c) 2002 Pete Zaitcev (zaitcev@yahoo.com) + * + * We used to warn all over the code: DO NOT USE prom_printf(), + * and yet people do. Anton's banking code was outputting banks + * with prom_printf for most of the 2.4 lifetime. Since an effective + * stick is not available, we deployed a carrot: an early printk + * through PROM by means of -p boot option. This ought to fix it. + * USE printk; if you need, deploy -p. + */ + +#include +#include +#include + +#include +#include + +#define CONSOLE_WRITE_BUF_SIZE 1024 + +static char ppbuf[1024]; +static char console_write_buf[CONSOLE_WRITE_BUF_SIZE]; +static DEFINE_RAW_SPINLOCK(console_write_lock); + +void notrace prom_write(const char *buf, unsigned int n) +{ + unsigned int dest_len; + unsigned long flags; + char *dest; + + dest = console_write_buf; + raw_spin_lock_irqsave(&console_write_lock, flags); + + dest_len = 0; + while (n-- != 0) { + char ch = *buf++; + if (ch == '\n') { + *dest++ = '\r'; + dest_len++; + } + *dest++ = ch; + dest_len++; + if (dest_len >= CONSOLE_WRITE_BUF_SIZE - 1) { + prom_console_write_buf(console_write_buf, dest_len); + dest = console_write_buf; + dest_len = 0; + } + } + if (dest_len) + prom_console_write_buf(console_write_buf, dest_len); + + raw_spin_unlock_irqrestore(&console_write_lock, flags); +} + +void notrace prom_printf(const char *fmt, ...) +{ + va_list args; + int i; + + va_start(args, fmt); + i = vscnprintf(ppbuf, sizeof(ppbuf), fmt, args); + va_end(args); + + prom_write(ppbuf, i); +} diff --git a/arch/sparc/prom/ranges.c b/arch/sparc/prom/ranges.c new file mode 100644 index 000000000..20cb828bc --- /dev/null +++ b/arch/sparc/prom/ranges.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ranges.c: Handle ranges in newer proms for obio/sbus. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include + +#include +#include +#include + +static struct linux_prom_ranges promlib_obio_ranges[PROMREG_MAX]; +static int num_obio_ranges; + +/* Adjust register values based upon the ranges parameters. */ +static void prom_adjust_regs(struct linux_prom_registers *regp, int nregs, + struct linux_prom_ranges *rangep, int nranges) +{ + int regc, rngc; + + for (regc = 0; regc < nregs; regc++) { + for (rngc = 0; rngc < nranges; rngc++) + if (regp[regc].which_io == rangep[rngc].ot_child_space) + break; /* Fount it */ + if (rngc == nranges) /* oops */ + prom_printf("adjust_regs: Could not find range with matching bus type...\n"); + regp[regc].which_io = rangep[rngc].ot_parent_space; + regp[regc].phys_addr -= rangep[rngc].ot_child_base; + regp[regc].phys_addr += rangep[rngc].ot_parent_base; + } +} + +static void prom_adjust_ranges(struct linux_prom_ranges *ranges1, int nranges1, + struct linux_prom_ranges *ranges2, int nranges2) +{ + int rng1c, rng2c; + + for (rng1c = 0; rng1c < nranges1; rng1c++) { + for (rng2c = 0; rng2c < nranges2; rng2c++) + if (ranges1[rng1c].ot_parent_space == ranges2[rng2c].ot_child_space && + ranges1[rng1c].ot_parent_base >= ranges2[rng2c].ot_child_base && + ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size - ranges1[rng1c].ot_parent_base > 0U) + break; + if (rng2c == nranges2) /* oops */ + prom_printf("adjust_ranges: Could not find matching bus type...\n"); + else if (ranges1[rng1c].ot_parent_base + ranges1[rng1c].or_size > ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size) + ranges1[rng1c].or_size = ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size - ranges1[rng1c].ot_parent_base; + ranges1[rng1c].ot_parent_space = ranges2[rng2c].ot_parent_space; + ranges1[rng1c].ot_parent_base += ranges2[rng2c].ot_parent_base; + } +} + +/* Apply probed obio ranges to registers passed, if no ranges return. */ +void prom_apply_obio_ranges(struct linux_prom_registers *regs, int nregs) +{ + if (num_obio_ranges) + prom_adjust_regs(regs, nregs, promlib_obio_ranges, num_obio_ranges); +} +EXPORT_SYMBOL(prom_apply_obio_ranges); + +void __init prom_ranges_init(void) +{ + phandle node, obio_node; + int success; + + num_obio_ranges = 0; + + /* Check for obio and sbus ranges. */ + node = prom_getchild(prom_root_node); + obio_node = prom_searchsiblings(node, "obio"); + + if (obio_node) { + success = prom_getproperty(obio_node, "ranges", + (char *) promlib_obio_ranges, + sizeof(promlib_obio_ranges)); + if (success != -1) + num_obio_ranges = (success / sizeof(struct linux_prom_ranges)); + } + + if (num_obio_ranges) + prom_printf("PROMLIB: obio_ranges %d\n", num_obio_ranges); +} + +void prom_apply_generic_ranges(phandle node, phandle parent, + struct linux_prom_registers *regs, int nregs) +{ + int success; + int num_ranges; + struct linux_prom_ranges ranges[PROMREG_MAX]; + + success = prom_getproperty(node, "ranges", + (char *) ranges, + sizeof(ranges)); + if (success != -1) { + num_ranges = (success / sizeof(struct linux_prom_ranges)); + if (parent) { + struct linux_prom_ranges parent_ranges[PROMREG_MAX]; + int num_parent_ranges; + + success = prom_getproperty(parent, "ranges", + (char *) parent_ranges, + sizeof(parent_ranges)); + if (success != -1) { + num_parent_ranges = (success / sizeof(struct linux_prom_ranges)); + prom_adjust_ranges(ranges, num_ranges, parent_ranges, num_parent_ranges); + } + } + prom_adjust_regs(regs, nregs, ranges, num_ranges); + } +} diff --git a/arch/sparc/prom/tree_32.c b/arch/sparc/prom/tree_32.c new file mode 100644 index 000000000..0fed89375 --- /dev/null +++ b/arch/sparc/prom/tree_32.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * tree.c: Basic device tree traversal/scanning for the Linux + * prom library. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +extern void restore_current(void); + +static char promlib_buf[128]; + +/* Internal version of prom_getchild that does not alter return values. */ +static phandle __prom_getchild(phandle node) +{ + unsigned long flags; + phandle cnode; + + spin_lock_irqsave(&prom_lock, flags); + cnode = prom_nodeops->no_child(node); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + + return cnode; +} + +/* Return the child of node 'node' or zero if no this node has no + * direct descendent. + */ +phandle prom_getchild(phandle node) +{ + phandle cnode; + + if ((s32)node == -1) + return 0; + + cnode = __prom_getchild(node); + if (cnode == 0 || (s32)cnode == -1) + return 0; + + return cnode; +} +EXPORT_SYMBOL(prom_getchild); + +/* Internal version of prom_getsibling that does not alter return values. */ +static phandle __prom_getsibling(phandle node) +{ + unsigned long flags; + phandle cnode; + + spin_lock_irqsave(&prom_lock, flags); + cnode = prom_nodeops->no_nextnode(node); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + + return cnode; +} + +/* Return the next sibling of node 'node' or zero if no more siblings + * at this level of depth in the tree. + */ +phandle prom_getsibling(phandle node) +{ + phandle sibnode; + + if ((s32)node == -1) + return 0; + + sibnode = __prom_getsibling(node); + if (sibnode == 0 || (s32)sibnode == -1) + return 0; + + return sibnode; +} +EXPORT_SYMBOL(prom_getsibling); + +/* Return the length in bytes of property 'prop' at node 'node'. + * Return -1 on error. + */ +int prom_getproplen(phandle node, const char *prop) +{ + int ret; + unsigned long flags; + + if((!node) || (!prop)) + return -1; + + spin_lock_irqsave(&prom_lock, flags); + ret = prom_nodeops->no_proplen(node, prop); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + return ret; +} +EXPORT_SYMBOL(prom_getproplen); + +/* Acquire a property 'prop' at node 'node' and place it in + * 'buffer' which has a size of 'bufsize'. If the acquisition + * was successful the length will be returned, else -1 is returned. + */ +int prom_getproperty(phandle node, const char *prop, char *buffer, int bufsize) +{ + int plen, ret; + unsigned long flags; + + plen = prom_getproplen(node, prop); + if((plen > bufsize) || (plen == 0) || (plen == -1)) + return -1; + /* Ok, things seem all right. */ + spin_lock_irqsave(&prom_lock, flags); + ret = prom_nodeops->no_getprop(node, prop, buffer); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + return ret; +} +EXPORT_SYMBOL(prom_getproperty); + +/* Acquire an integer property and return its value. Returns -1 + * on failure. + */ +int prom_getint(phandle node, char *prop) +{ + static int intprop; + + if(prom_getproperty(node, prop, (char *) &intprop, sizeof(int)) != -1) + return intprop; + + return -1; +} +EXPORT_SYMBOL(prom_getint); + +/* Acquire an integer property, upon error return the passed default + * integer. + */ +int prom_getintdefault(phandle node, char *property, int deflt) +{ + int retval; + + retval = prom_getint(node, property); + if(retval == -1) return deflt; + + return retval; +} +EXPORT_SYMBOL(prom_getintdefault); + +/* Acquire a boolean property, 1=TRUE 0=FALSE. */ +int prom_getbool(phandle node, char *prop) +{ + int retval; + + retval = prom_getproplen(node, prop); + if(retval == -1) return 0; + return 1; +} +EXPORT_SYMBOL(prom_getbool); + +/* Acquire a property whose value is a string, returns a null + * string on error. The char pointer is the user supplied string + * buffer. + */ +void prom_getstring(phandle node, char *prop, char *user_buf, int ubuf_size) +{ + int len; + + len = prom_getproperty(node, prop, user_buf, ubuf_size); + if(len != -1) return; + user_buf[0] = 0; +} +EXPORT_SYMBOL(prom_getstring); + + +/* Search siblings at 'node_start' for a node with name + * 'nodename'. Return node if successful, zero if not. + */ +phandle prom_searchsiblings(phandle node_start, char *nodename) +{ + + phandle thisnode; + int error; + + for(thisnode = node_start; thisnode; + thisnode=prom_getsibling(thisnode)) { + error = prom_getproperty(thisnode, "name", promlib_buf, + sizeof(promlib_buf)); + /* Should this ever happen? */ + if(error == -1) continue; + if(strcmp(nodename, promlib_buf)==0) return thisnode; + } + + return 0; +} +EXPORT_SYMBOL(prom_searchsiblings); + +/* Interal version of nextprop that does not alter return values. */ +static char *__prom_nextprop(phandle node, char * oprop) +{ + unsigned long flags; + char *prop; + + spin_lock_irqsave(&prom_lock, flags); + prop = prom_nodeops->no_nextprop(node, oprop); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + + return prop; +} + +/* Return the property type string after property type 'oprop' + * at node 'node' . Returns empty string if no more + * property types for this node. + */ +char *prom_nextprop(phandle node, char *oprop, char *buffer) +{ + if (node == 0 || (s32)node == -1) + return ""; + + return __prom_nextprop(node, oprop); +} +EXPORT_SYMBOL(prom_nextprop); + +phandle prom_finddevice(char *name) +{ + char nbuf[128]; + char *s = name, *d; + phandle node = prom_root_node, node2; + unsigned int which_io, phys_addr; + struct linux_prom_registers reg[PROMREG_MAX]; + + while (*s++) { + if (!*s) return node; /* path '.../' is legal */ + node = prom_getchild(node); + + for (d = nbuf; *s != 0 && *s != '@' && *s != '/';) + *d++ = *s++; + *d = 0; + + node = prom_searchsiblings(node, nbuf); + if (!node) + return 0; + + if (*s == '@') { + if (isxdigit(s[1]) && s[2] == ',') { + which_io = simple_strtoul(s+1, NULL, 16); + phys_addr = simple_strtoul(s+3, &d, 16); + if (d != s + 3 && (!*d || *d == '/') + && d <= s + 3 + 8) { + node2 = node; + while (node2 && (s32)node2 != -1) { + if (prom_getproperty (node2, "reg", (char *)reg, sizeof (reg)) > 0) { + if (which_io == reg[0].which_io && phys_addr == reg[0].phys_addr) { + node = node2; + break; + } + } + node2 = prom_getsibling(node2); + if (!node2 || (s32)node2 == -1) + break; + node2 = prom_searchsiblings(prom_getsibling(node2), nbuf); + } + } + } + while (*s != 0 && *s != '/') s++; + } + } + return node; +} +EXPORT_SYMBOL(prom_finddevice); + +/* Set property 'pname' at node 'node' to value 'value' which has a length + * of 'size' bytes. Return the number of bytes the prom accepted. + */ +int prom_setprop(phandle node, const char *pname, char *value, int size) +{ + unsigned long flags; + int ret; + + if (size == 0) + return 0; + if ((pname == NULL) || (value == NULL)) + return 0; + spin_lock_irqsave(&prom_lock, flags); + ret = prom_nodeops->no_setprop(node, pname, value, size); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + return ret; +} +EXPORT_SYMBOL(prom_setprop); + +phandle prom_inst2pkg(int inst) +{ + phandle node; + unsigned long flags; + + spin_lock_irqsave(&prom_lock, flags); + node = (*romvec->pv_v2devops.v2_inst2pkg)(inst); + restore_current(); + spin_unlock_irqrestore(&prom_lock, flags); + if ((s32)node == -1) + return 0; + return node; +} diff --git a/arch/sparc/prom/tree_64.c b/arch/sparc/prom/tree_64.c new file mode 100644 index 000000000..7388339bb --- /dev/null +++ b/arch/sparc/prom/tree_64.c @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * tree.c: Basic device tree traversal/scanning for the Linux + * prom library. + * + * Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996,1997 Jakub Jelinek (jj@sunsite.mff.cuni.cz) + */ + +#include +#include +#include +#include +#include + +#include +#include +#include + +static phandle prom_node_to_node(const char *type, phandle node) +{ + unsigned long args[5]; + + args[0] = (unsigned long) type; + args[1] = 1; + args[2] = 1; + args[3] = (unsigned int) node; + args[4] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return (phandle) args[4]; +} + +/* Return the child of node 'node' or zero if no this node has no + * direct descendent. + */ +inline phandle __prom_getchild(phandle node) +{ + return prom_node_to_node("child", node); +} + +phandle prom_getchild(phandle node) +{ + phandle cnode; + + if ((s32)node == -1) + return 0; + cnode = __prom_getchild(node); + if ((s32)cnode == -1) + return 0; + return cnode; +} +EXPORT_SYMBOL(prom_getchild); + +inline phandle prom_getparent(phandle node) +{ + phandle cnode; + + if ((s32)node == -1) + return 0; + cnode = prom_node_to_node("parent", node); + if ((s32)cnode == -1) + return 0; + return cnode; +} + +/* Return the next sibling of node 'node' or zero if no more siblings + * at this level of depth in the tree. + */ +inline phandle __prom_getsibling(phandle node) +{ + return prom_node_to_node(prom_peer_name, node); +} + +phandle prom_getsibling(phandle node) +{ + phandle sibnode; + + if ((s32)node == -1) + return 0; + sibnode = __prom_getsibling(node); + if ((s32)sibnode == -1) + return 0; + + return sibnode; +} +EXPORT_SYMBOL(prom_getsibling); + +/* Return the length in bytes of property 'prop' at node 'node'. + * Return -1 on error. + */ +int prom_getproplen(phandle node, const char *prop) +{ + unsigned long args[6]; + + if (!node || !prop) + return -1; + + args[0] = (unsigned long) "getproplen"; + args[1] = 2; + args[2] = 1; + args[3] = (unsigned int) node; + args[4] = (unsigned long) prop; + args[5] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return (int) args[5]; +} +EXPORT_SYMBOL(prom_getproplen); + +/* Acquire a property 'prop' at node 'node' and place it in + * 'buffer' which has a size of 'bufsize'. If the acquisition + * was successful the length will be returned, else -1 is returned. + */ +int prom_getproperty(phandle node, const char *prop, + char *buffer, int bufsize) +{ + unsigned long args[8]; + int plen; + + plen = prom_getproplen(node, prop); + if ((plen > bufsize) || (plen == 0) || (plen == -1)) + return -1; + + args[0] = (unsigned long) prom_getprop_name; + args[1] = 4; + args[2] = 1; + args[3] = (unsigned int) node; + args[4] = (unsigned long) prop; + args[5] = (unsigned long) buffer; + args[6] = bufsize; + args[7] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return (int) args[7]; +} +EXPORT_SYMBOL(prom_getproperty); + +/* Acquire an integer property and return its value. Returns -1 + * on failure. + */ +int prom_getint(phandle node, const char *prop) +{ + int intprop; + + if (prom_getproperty(node, prop, (char *) &intprop, sizeof(int)) != -1) + return intprop; + + return -1; +} +EXPORT_SYMBOL(prom_getint); + +/* Acquire an integer property, upon error return the passed default + * integer. + */ + +int prom_getintdefault(phandle node, const char *property, int deflt) +{ + int retval; + + retval = prom_getint(node, property); + if (retval == -1) + return deflt; + + return retval; +} +EXPORT_SYMBOL(prom_getintdefault); + +/* Acquire a boolean property, 1=TRUE 0=FALSE. */ +int prom_getbool(phandle node, const char *prop) +{ + int retval; + + retval = prom_getproplen(node, prop); + if (retval == -1) + return 0; + return 1; +} +EXPORT_SYMBOL(prom_getbool); + +/* Acquire a property whose value is a string, returns a null + * string on error. The char pointer is the user supplied string + * buffer. + */ +void prom_getstring(phandle node, const char *prop, char *user_buf, + int ubuf_size) +{ + int len; + + len = prom_getproperty(node, prop, user_buf, ubuf_size); + if (len != -1) + return; + user_buf[0] = 0; +} +EXPORT_SYMBOL(prom_getstring); + +/* Does the device at node 'node' have name 'name'? + * YES = 1 NO = 0 + */ +int prom_nodematch(phandle node, const char *name) +{ + char namebuf[128]; + prom_getproperty(node, "name", namebuf, sizeof(namebuf)); + if (strcmp(namebuf, name) == 0) + return 1; + return 0; +} + +/* Search siblings at 'node_start' for a node with name + * 'nodename'. Return node if successful, zero if not. + */ +phandle prom_searchsiblings(phandle node_start, const char *nodename) +{ + phandle thisnode; + int error; + char promlib_buf[128]; + + for(thisnode = node_start; thisnode; + thisnode=prom_getsibling(thisnode)) { + error = prom_getproperty(thisnode, "name", promlib_buf, + sizeof(promlib_buf)); + /* Should this ever happen? */ + if(error == -1) continue; + if(strcmp(nodename, promlib_buf)==0) return thisnode; + } + + return 0; +} +EXPORT_SYMBOL(prom_searchsiblings); + +static const char *prom_nextprop_name = "nextprop"; + +/* Return the first property type for node 'node'. + * buffer should be at least 32B in length + */ +char *prom_firstprop(phandle node, char *buffer) +{ + unsigned long args[7]; + + *buffer = 0; + if ((s32)node == -1) + return buffer; + + args[0] = (unsigned long) prom_nextprop_name; + args[1] = 3; + args[2] = 1; + args[3] = (unsigned int) node; + args[4] = 0; + args[5] = (unsigned long) buffer; + args[6] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return buffer; +} +EXPORT_SYMBOL(prom_firstprop); + +/* Return the property type string after property type 'oprop' + * at node 'node' . Returns NULL string if no more + * property types for this node. + */ +char *prom_nextprop(phandle node, const char *oprop, char *buffer) +{ + unsigned long args[7]; + char buf[32]; + + if ((s32)node == -1) { + *buffer = 0; + return buffer; + } + if (oprop == buffer) { + strscpy(buf, oprop); + oprop = buf; + } + + args[0] = (unsigned long) prom_nextprop_name; + args[1] = 3; + args[2] = 1; + args[3] = (unsigned int) node; + args[4] = (unsigned long) oprop; + args[5] = (unsigned long) buffer; + args[6] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return buffer; +} +EXPORT_SYMBOL(prom_nextprop); + +phandle prom_finddevice(const char *name) +{ + unsigned long args[5]; + + if (!name) + return 0; + args[0] = (unsigned long) "finddevice"; + args[1] = 1; + args[2] = 1; + args[3] = (unsigned long) name; + args[4] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return (int) args[4]; +} +EXPORT_SYMBOL(prom_finddevice); + +int prom_node_has_property(phandle node, const char *prop) +{ + char buf [32]; + + *buf = 0; + do { + prom_nextprop(node, buf, buf); + if (!strcmp(buf, prop)) + return 1; + } while (*buf); + return 0; +} +EXPORT_SYMBOL(prom_node_has_property); + +/* Set property 'pname' at node 'node' to value 'value' which has a length + * of 'size' bytes. Return the number of bytes the prom accepted. + */ +int +prom_setprop(phandle node, const char *pname, char *value, int size) +{ + unsigned long args[8]; + + if (size == 0) + return 0; + if ((pname == NULL) || (value == NULL)) + return 0; + +#ifdef CONFIG_SUN_LDOMS + if (ldom_domaining_enabled) { + ldom_set_var(pname, value); + return 0; + } +#endif + args[0] = (unsigned long) "setprop"; + args[1] = 4; + args[2] = 1; + args[3] = (unsigned int) node; + args[4] = (unsigned long) pname; + args[5] = (unsigned long) value; + args[6] = size; + args[7] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return (int) args[7]; +} +EXPORT_SYMBOL(prom_setprop); + +inline phandle prom_inst2pkg(int inst) +{ + unsigned long args[5]; + phandle node; + + args[0] = (unsigned long) "instance-to-package"; + args[1] = 1; + args[2] = 1; + args[3] = (unsigned int) inst; + args[4] = (unsigned long) -1; + + p1275_cmd_direct(args); + + node = (int) args[4]; + if ((s32)node == -1) + return 0; + return node; +} + +int prom_ihandle2path(int handle, char *buffer, int bufsize) +{ + unsigned long args[7]; + + args[0] = (unsigned long) "instance-to-path"; + args[1] = 3; + args[2] = 1; + args[3] = (unsigned int) handle; + args[4] = (unsigned long) buffer; + args[5] = bufsize; + args[6] = (unsigned long) -1; + + p1275_cmd_direct(args); + + return (int) args[6]; +} diff --git a/arch/sparc/vdso/.gitignore b/arch/sparc/vdso/.gitignore new file mode 100644 index 000000000..8d4ebc990 --- /dev/null +++ b/arch/sparc/vdso/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +vdso.lds +vdso-image-*.c +vdso2c diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile new file mode 100644 index 000000000..83fb2aca5 --- /dev/null +++ b/arch/sparc/vdso/Makefile @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Building vDSO images for sparc. +# + +# Include the generic Makefile to check the built vDSO: +include $(srctree)/lib/vdso/Makefile.include + +# files to link into the vdso +vobjs-y := vdso-note.o vclock_gettime.o + +# files to link into kernel +obj-y += vma.o + +# vDSO images to build +obj-$(CONFIG_SPARC64) += vdso-image-64.o +obj-$(CONFIG_COMPAT) += vdso-image-32.o + +vobjs := $(addprefix $(obj)/, $(vobjs-y)) + +$(obj)/vdso.o: $(obj)/vdso.so + +targets += vdso.lds $(vobjs-y) +targets += $(foreach x, 32 64, vdso-image-$(x).c vdso$(x).so vdso$(x).so.dbg) + +CPPFLAGS_vdso.lds += -P -C + +VDSO_LDFLAGS_vdso.lds = -m elf64_sparc -soname linux-vdso.so.1 \ + -z max-page-size=8192 + +$(obj)/vdso64.so.dbg: $(obj)/vdso.lds $(vobjs) FORCE + $(call if_changed,vdso) + +HOST_EXTRACFLAGS += -I$(srctree)/tools/include +hostprogs += vdso2c + +quiet_cmd_vdso2c = VDSO2C $@ + cmd_vdso2c = $(obj)/vdso2c $< $(<:%.dbg=%) $@ + +$(obj)/vdso-image-%.c: $(obj)/vdso%.so.dbg $(obj)/vdso%.so $(obj)/vdso2c FORCE + $(call if_changed,vdso2c) + +# +# Don't omit frame pointers for ease of userspace debugging, but do +# optimize sibling calls. +# +CFL := $(PROFILING) -mcmodel=medlow -fPIC -O2 -fasynchronous-unwind-tables -m64 \ + $(filter -g%,$(KBUILD_CFLAGS)) -fno-stack-protector \ + -fno-omit-frame-pointer -foptimize-sibling-calls \ + -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO + +SPARC_REG_CFLAGS = -ffixed-g4 -ffixed-g5 $(call cc-option,-fcall-used-g5) $(call cc-option,-fcall-used-g7) + +$(vobjs): KBUILD_CFLAGS := $(filter-out $(RANDSTRUCT_CFLAGS) $(KSTACK_ERASE_CFLAGS) $(GCC_PLUGINS_CFLAGS) $(SPARC_REG_CFLAGS),$(KBUILD_CFLAGS)) $(CFL) + +# +# vDSO code runs in userspace and -pg doesn't help with profiling anyway. +# +CFLAGS_REMOVE_vclock_gettime.o = -pg +CFLAGS_REMOVE_vdso32/vclock_gettime.o = -pg + +$(obj)/%.so: OBJCOPYFLAGS := -S +$(obj)/%.so: $(obj)/%.so.dbg FORCE + $(call if_changed,objcopy) + +CPPFLAGS_vdso32/vdso32.lds = $(CPPFLAGS_vdso.lds) +VDSO_LDFLAGS_vdso32.lds = -m elf32_sparc -soname linux-gate.so.1 + +#This makes sure the $(obj) subdirectory exists even though vdso32/ +#is not a kbuild sub-make subdirectory +override obj-dirs = $(dir $(obj)) $(obj)/vdso32/ + +targets += vdso32/vdso32.lds +targets += vdso32/vdso-note.o +targets += vdso32/vclock_gettime.o + +KBUILD_AFLAGS_32 := $(filter-out -m64,$(KBUILD_AFLAGS)) -DBUILD_VDSO +$(obj)/vdso32.so.dbg: KBUILD_AFLAGS = $(KBUILD_AFLAGS_32) +$(obj)/vdso32.so.dbg: asflags-$(CONFIG_SPARC64) += -m32 + +KBUILD_CFLAGS_32 := $(filter-out -m64,$(KBUILD_CFLAGS)) +KBUILD_CFLAGS_32 := $(filter-out -mcmodel=medlow,$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out -fno-pic,$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(RANDSTRUCT_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(KSTACK_ERASE_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(GCC_PLUGINS_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 := $(filter-out $(SPARC_REG_CFLAGS),$(KBUILD_CFLAGS_32)) +KBUILD_CFLAGS_32 += -m32 -msoft-float -fpic +KBUILD_CFLAGS_32 += -fno-stack-protector +KBUILD_CFLAGS_32 += $(call cc-option, -foptimize-sibling-calls) +KBUILD_CFLAGS_32 += -fno-omit-frame-pointer +KBUILD_CFLAGS_32 += -DDISABLE_BRANCH_PROFILING +KBUILD_CFLAGS_32 += -mv8plus +$(obj)/vdso32.so.dbg: KBUILD_CFLAGS = $(KBUILD_CFLAGS_32) + +CHECKFLAGS_32 := $(filter-out -m64 -D__sparc_v9__ -D__arch64__, $(CHECKFLAGS)) -m32 +$(obj)/vdso32.so.dbg: CHECKFLAGS = $(CHECKFLAGS_32) + +$(obj)/vdso32.so.dbg: FORCE \ + $(obj)/vdso32/vdso32.lds \ + $(obj)/vdso32/vclock_gettime.o \ + $(obj)/vdso32/vdso-note.o + $(call if_changed,vdso) + +# +# The DSO images are built using a special linker script. +# +quiet_cmd_vdso = VDSO $@ + cmd_vdso = $(LD) -nostdlib -o $@ \ + $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ + -T $(filter %.lds,$^) $(filter %.o,$^); \ + $(cmd_vdso_check) + +VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined -z noexecstack diff --git a/arch/sparc/vdso/vclock_gettime.c b/arch/sparc/vdso/vclock_gettime.c new file mode 100644 index 000000000..221bd4ed1 --- /dev/null +++ b/arch/sparc/vdso/vclock_gettime.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright 2006 Andi Kleen, SUSE Labs. + * + * Fast user context implementation of clock_gettime, gettimeofday, and time. + * + * The code should have no internal unresolved relocations. + * Check with readelf after changing. + * Also alternative() doesn't work. + */ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#include +#include + +#include + +#include + +#include "../../../../lib/vdso/gettimeofday.c" + +#if defined(CONFIG_SPARC64) || defined(CONFIG_COMPAT_32BIT_TIME) +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +int gettimeofday(struct __kernel_old_timeval *, struct timezone *) + __weak __alias(__vdso_gettimeofday); +#endif + +#if defined(CONFIG_SPARC64) +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +int clock_gettime(clockid_t, struct __kernel_timespec *) + __weak __alias(__vdso_clock_gettime); + +#else + +#if defined(CONFIG_COMPAT_32BIT_TIME) +int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts) +{ + return __cvdso_clock_gettime32(clock, ts); +} + +int clock_gettime(clockid_t, struct old_timespec32 *) + __weak __alias(__vdso_clock_gettime); +#endif + +int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +int clock_gettime64(clockid_t, struct __kernel_timespec *) + __weak __alias(__vdso_clock_gettime64); + +#endif diff --git a/arch/sparc/vdso/vdso-layout.lds.S b/arch/sparc/vdso/vdso-layout.lds.S new file mode 100644 index 000000000..180e5d0ee --- /dev/null +++ b/arch/sparc/vdso/vdso-layout.lds.S @@ -0,0 +1,80 @@ +/* + * Linker script for vDSO. This is an ELF shared object prelinked to + * its virtual address, and with only one read-only segment. + * This script controls its layout. + */ + +#include +#include +#include + +SECTIONS +{ + /* + * User/kernel shared data is before the vDSO. This may be a little + * uglier than putting it after the vDSO, but it avoids issues with + * non-allocatable things that dangle past the end of the PT_LOAD + * segment. Page size is 8192 for both 64-bit and 32-bit vdso binaries + */ + + VDSO_VVAR_SYMS + + . = SIZEOF_HEADERS; + + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } + .dynsym : { *(.dynsym) } + .dynstr : { *(.dynstr) } + .gnu.version : { *(.gnu.version) } + .gnu.version_d : { *(.gnu.version_d) } + .gnu.version_r : { *(.gnu.version_r) } + + .dynamic : { *(.dynamic) } :text :dynamic + + .rodata : { + *(.rodata*) + *(.data*) + *(.sdata*) + *(.got.plt) *(.got) + *(.gnu.linkonce.d.*) + *(.bss*) + *(.dynbss*) + *(.gnu.linkonce.b.*) + } :text + + .note : { *(.note.*) } :text :note + + .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr + .eh_frame : { KEEP (*(.eh_frame)) } :text + + + /* + * Text is well-separated from actual data: there's plenty of + * stuff that isn't used at runtime in between. + */ + + .text : { *(.text*) } :text =0x90909090, + + /DISCARD/ : { + *(.discard) + *(.discard.*) + *(__bug_table) + } +} + +/* + * Very old versions of ld do not recognize this name token; use the constant. + */ +#define PT_GNU_EH_FRAME 0x6474e550 + +/* + * We must supply the ELF program headers explicitly to get just one + * PT_LOAD segment, and set the flags explicitly to make segments read-only. + */ +PHDRS +{ + text PT_LOAD FLAGS(5) FILEHDR PHDRS; /* PF_R|PF_X */ + dynamic PT_DYNAMIC FLAGS(4); /* PF_R */ + note PT_NOTE FLAGS(4); /* PF_R */ + eh_frame_hdr PT_GNU_EH_FRAME; +} diff --git a/arch/sparc/vdso/vdso-note.S b/arch/sparc/vdso/vdso-note.S new file mode 100644 index 000000000..79a071e43 --- /dev/null +++ b/arch/sparc/vdso/vdso-note.S @@ -0,0 +1,12 @@ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO text. + * Here we can supply some information useful to userland. + */ + +#include +#include +#include + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/sparc/vdso/vdso.lds.S b/arch/sparc/vdso/vdso.lds.S new file mode 100644 index 000000000..f3caa29a3 --- /dev/null +++ b/arch/sparc/vdso/vdso.lds.S @@ -0,0 +1,25 @@ +/* + * Linker script for 64-bit vDSO. + * We #include the file to define the layout details. + * + * This file defines the version script giving the user-exported symbols in + * the DSO. + */ + +#define BUILD_VDSO64 + +#include "vdso-layout.lds.S" + +/* + * This controls what userland symbols we export from the vDSO. + */ +VERSION { + LINUX_2.6 { + global: + clock_gettime; + __vdso_clock_gettime; + gettimeofday; + __vdso_gettimeofday; + local: *; + }; +} diff --git a/arch/sparc/vdso/vdso2c.c b/arch/sparc/vdso/vdso2c.c new file mode 100644 index 000000000..e5c61214a --- /dev/null +++ b/arch/sparc/vdso/vdso2c.c @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vdso2c - A vdso image preparation tool + * Copyright (c) 2014 Andy Lutomirski and others + * + * vdso2c requires stripped and unstripped input. It would be trivial + * to fully strip the input in here, but, for reasons described below, + * we need to write a section table. Doing this is more or less + * equivalent to dropping all non-allocatable sections, but it's + * easier to let objcopy handle that instead of doing it ourselves. + * If we ever need to do something fancier than what objcopy provides, + * it would be straightforward to add here. + * + * We keep a section table for a few reasons: + * + * Binutils has issues debugging the vDSO: it reads the section table to + * find SHT_NOTE; it won't look at PT_NOTE for the in-memory vDSO, which + * would break build-id if we removed the section table. Binutils + * also requires that shstrndx != 0. See: + * https://sourceware.org/bugzilla/show_bug.cgi?id=17064 + * + * elfutils might not look for PT_NOTE if there is a section table at + * all. I don't know whether this matters for any practical purpose. + * + * For simplicity, rather than hacking up a partial section table, we + * just write a mostly complete one. We omit non-dynamic symbols, + * though, since they're rather large. + * + * Once binutils gets fixed, we might be able to drop this for all but + * the 64-bit vdso, since build-id only works in kernel RPMs, and + * systems that update to new enough kernel RPMs will likely update + * binutils in sync. build-id has never worked for home-built kernel + * RPMs without manual symlinking, and I suspect that no one ever does + * that. + */ + +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +const char *outfilename; + +__attribute__((format(printf, 1, 2))) __attribute__((noreturn)) +static void fail(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + fprintf(stderr, "Error: "); + vfprintf(stderr, format, ap); + if (outfilename) + unlink(outfilename); + exit(1); + va_end(ap); +} + +/* + * Evil macros for big-endian reads and writes + */ +#define GBE(x, bits, ifnot) \ + __builtin_choose_expr( \ + (sizeof(*(x)) == bits/8), \ + (__typeof__(*(x)))get_unaligned_be##bits(x), ifnot) + +#define LAST_GBE(x) \ + __builtin_choose_expr(sizeof(*(x)) == 1, *(x), (void)(0)) + +#define GET_BE(x) \ + GBE(x, 64, GBE(x, 32, GBE(x, 16, LAST_GBE(x)))) + +#define PBE(x, val, bits, ifnot) \ + __builtin_choose_expr( \ + (sizeof(*(x)) == bits/8), \ + put_unaligned_be##bits((val), (x)), ifnot) + +#define LAST_PBE(x, val) \ + __builtin_choose_expr(sizeof(*(x)) == 1, *(x) = (val), (void)(0)) + +#define PUT_BE(x, val) \ + PBE(x, val, 64, PBE(x, val, 32, PBE(x, val, 16, LAST_PBE(x, val)))) + +#define BITSFUNC3(name, bits, suffix) name##bits##suffix +#define BITSFUNC2(name, bits, suffix) BITSFUNC3(name, bits, suffix) +#define BITSFUNC(name) BITSFUNC2(name, ELF_BITS, ) + +#define INT_BITS BITSFUNC2(int, ELF_BITS, _t) + +#define ELF_BITS_XFORM2(bits, x) Elf##bits##_##x +#define ELF_BITS_XFORM(bits, x) ELF_BITS_XFORM2(bits, x) +#define ELF(x) ELF_BITS_XFORM(ELF_BITS, x) + +#define ELF_BITS 64 +#include "vdso2c.h" +#undef ELF_BITS + +#define ELF_BITS 32 +#include "vdso2c.h" +#undef ELF_BITS + +static void go(void *raw_addr, size_t raw_len, + void *stripped_addr, size_t stripped_len, + FILE *outfile, const char *name) +{ + Elf64_Ehdr *hdr = (Elf64_Ehdr *)raw_addr; + + if (hdr->e_ident[EI_CLASS] == ELFCLASS64) { + go64(raw_addr, raw_len, stripped_addr, stripped_len, + outfile, name); + } else if (hdr->e_ident[EI_CLASS] == ELFCLASS32) { + go32(raw_addr, raw_len, stripped_addr, stripped_len, + outfile, name); + } else { + fail("unknown ELF class\n"); + } +} + +static void map_input(const char *name, void **addr, size_t *len, int prot) +{ + off_t tmp_len; + + int fd = open(name, O_RDONLY); + + if (fd == -1) + err(1, "%s", name); + + tmp_len = lseek(fd, 0, SEEK_END); + if (tmp_len == (off_t)-1) + err(1, "lseek"); + *len = (size_t)tmp_len; + + *addr = mmap(NULL, tmp_len, prot, MAP_PRIVATE, fd, 0); + if (*addr == MAP_FAILED) + err(1, "mmap"); + + close(fd); +} + +int main(int argc, char **argv) +{ + size_t raw_len, stripped_len; + void *raw_addr, *stripped_addr; + FILE *outfile; + char *name, *tmp; + int namelen; + + if (argc != 4) { + printf("Usage: vdso2c RAW_INPUT STRIPPED_INPUT OUTPUT\n"); + return 1; + } + + /* + * Figure out the struct name. If we're writing to a .so file, + * generate raw output insted. + */ + name = strdup(argv[3]); + namelen = strlen(name); + if (namelen >= 3 && !strcmp(name + namelen - 3, ".so")) { + name = NULL; + } else { + tmp = strrchr(name, '/'); + if (tmp) + name = tmp + 1; + tmp = strchr(name, '.'); + if (tmp) + *tmp = '\0'; + for (tmp = name; *tmp; tmp++) + if (*tmp == '-') + *tmp = '_'; + } + + map_input(argv[1], &raw_addr, &raw_len, PROT_READ); + map_input(argv[2], &stripped_addr, &stripped_len, PROT_READ); + + outfilename = argv[3]; + outfile = fopen(outfilename, "w"); + if (!outfile) + err(1, "%s", argv[2]); + + go(raw_addr, raw_len, stripped_addr, stripped_len, outfile, name); + + munmap(raw_addr, raw_len); + munmap(stripped_addr, stripped_len); + fclose(outfile); + + return 0; +} diff --git a/arch/sparc/vdso/vdso2c.h b/arch/sparc/vdso/vdso2c.h new file mode 100644 index 000000000..bad6a0593 --- /dev/null +++ b/arch/sparc/vdso/vdso2c.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +/* + * This file is included up to twice from vdso2c.c. It generates code for + * 32-bit and 64-bit vDSOs. We will eventually need both for 64-bit builds, + * since 32-bit vDSOs will then be built for 32-bit userspace. + */ + +static void BITSFUNC(go)(void *raw_addr, size_t raw_len, + void *stripped_addr, size_t stripped_len, + FILE *outfile, const char *name) +{ + int found_load = 0; + unsigned long load_size = -1; /* Work around bogus warning */ + unsigned long mapping_size; + int i; + unsigned long j; + ELF(Shdr) *symtab_hdr = NULL; + ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr; + ELF(Dyn) *dyn = 0, *dyn_end = 0; + ELF(Phdr) *pt = (ELF(Phdr) *)(raw_addr + GET_BE(&hdr->e_phoff)); + + /* Walk the segment table. */ + for (i = 0; i < GET_BE(&hdr->e_phnum); i++) { + if (GET_BE(&pt[i].p_type) == PT_LOAD) { + if (found_load) + fail("multiple PT_LOAD segs\n"); + + if (GET_BE(&pt[i].p_offset) != 0 || + GET_BE(&pt[i].p_vaddr) != 0) + fail("PT_LOAD in wrong place\n"); + + if (GET_BE(&pt[i].p_memsz) != GET_BE(&pt[i].p_filesz)) + fail("cannot handle memsz != filesz\n"); + + load_size = GET_BE(&pt[i].p_memsz); + found_load = 1; + } else if (GET_BE(&pt[i].p_type) == PT_DYNAMIC) { + dyn = raw_addr + GET_BE(&pt[i].p_offset); + dyn_end = raw_addr + GET_BE(&pt[i].p_offset) + + GET_BE(&pt[i].p_memsz); + } + } + if (!found_load) + fail("no PT_LOAD seg\n"); + + if (stripped_len < load_size) + fail("stripped input is too short\n"); + + /* Walk the dynamic table */ + for (i = 0; dyn + i < dyn_end && + GET_BE(&dyn[i].d_tag) != DT_NULL; i++) { + typeof(dyn[i].d_tag) tag = GET_BE(&dyn[i].d_tag); + typeof(dyn[i].d_un.d_val) val = GET_BE(&dyn[i].d_un.d_val); + + if ((tag == DT_RELSZ || tag == DT_RELASZ) && (val != 0)) + fail("vdso image contains dynamic relocations\n"); + } + + /* Walk the section table */ + for (i = 0; i < GET_BE(&hdr->e_shnum); i++) { + ELF(Shdr) *sh = raw_addr + GET_BE(&hdr->e_shoff) + + GET_BE(&hdr->e_shentsize) * i; + if (GET_BE(&sh->sh_type) == SHT_SYMTAB) + symtab_hdr = sh; + } + + if (!symtab_hdr) + fail("no symbol table\n"); + + if (!name) { + fwrite(stripped_addr, stripped_len, 1, outfile); + return; + } + + mapping_size = (stripped_len + 8191) / 8192 * 8192; + + fprintf(outfile, "/* AUTOMATICALLY GENERATED -- DO NOT EDIT */\n\n"); + fprintf(outfile, "#include \n"); + fprintf(outfile, "#include \n"); + fprintf(outfile, "\n"); + fprintf(outfile, + "static unsigned char raw_data[%lu] __ro_after_init __aligned(8192)= {", + mapping_size); + for (j = 0; j < stripped_len; j++) { + if (j % 10 == 0) + fprintf(outfile, "\n\t"); + fprintf(outfile, "0x%02X, ", + (int)((unsigned char *)stripped_addr)[j]); + } + fprintf(outfile, "\n};\n\n"); + + fprintf(outfile, "const struct vdso_image %s_builtin = {\n", name); + fprintf(outfile, "\t.data = raw_data,\n"); + fprintf(outfile, "\t.size = %lu,\n", mapping_size); + fprintf(outfile, "};\n"); +} diff --git a/arch/sparc/vdso/vdso32/.gitignore b/arch/sparc/vdso/vdso32/.gitignore new file mode 100644 index 000000000..516738484 --- /dev/null +++ b/arch/sparc/vdso/vdso32/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +vdso32.lds diff --git a/arch/sparc/vdso/vdso32/vclock_gettime.c b/arch/sparc/vdso/vdso32/vclock_gettime.c new file mode 100644 index 000000000..d7f99e674 --- /dev/null +++ b/arch/sparc/vdso/vdso32/vclock_gettime.c @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#define BUILD_VDSO32 + +#ifdef CONFIG_SPARC64 + +/* + * in case of a 32 bit VDSO for a 64 bit kernel fake a 32 bit kernel + * configuration + */ +#undef CONFIG_64BIT +#undef CONFIG_SPARC64 +#define BUILD_VDSO32_64 +#define CONFIG_32BIT +#undef CONFIG_QUEUED_RWLOCKS +#undef CONFIG_QUEUED_SPINLOCKS + +#endif + +#include "../vclock_gettime.c" diff --git a/arch/sparc/vdso/vdso32/vdso-note.S b/arch/sparc/vdso/vdso32/vdso-note.S new file mode 100644 index 000000000..e234983cf --- /dev/null +++ b/arch/sparc/vdso/vdso32/vdso-note.S @@ -0,0 +1,12 @@ +/* + * This supplies .note.* sections to go into the PT_NOTE inside the vDSO + * text. Here we can supply some information useful to userland. + */ + +#include +#include +#include + +ELFNOTE_START(Linux, 0, "a") + .long LINUX_VERSION_CODE +ELFNOTE_END diff --git a/arch/sparc/vdso/vdso32/vdso32.lds.S b/arch/sparc/vdso/vdso32/vdso32.lds.S new file mode 100644 index 000000000..28052168b --- /dev/null +++ b/arch/sparc/vdso/vdso32/vdso32.lds.S @@ -0,0 +1,28 @@ +/* + * Linker script for sparc32 vDSO + * We #include the file to define the layout details. + * + * This file defines the version script giving the user-exported symbols in + * the DSO. + */ + +#define BUILD_VDSO32 +#include "../vdso-layout.lds.S" + +/* + * This controls what userland symbols we export from the vDSO. + */ +VERSION { + LINUX_2.6 { + global: +#ifdef CONFIG_COMPAT_32BIT_TIME + clock_gettime; + __vdso_clock_gettime; + gettimeofday; + __vdso_gettimeofday; +#endif + clock_gettime64; + __vdso_clock_gettime64; + local: *; + }; +} diff --git a/arch/sparc/vdso/vma.c b/arch/sparc/vdso/vma.c new file mode 100644 index 000000000..60029d60f --- /dev/null +++ b/arch/sparc/vdso/vma.c @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Set up the VMAs to tell the VM about the vDSO. + * Copyright 2007 Andi Kleen, SUSE Labs. + */ + +/* + * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +unsigned int __read_mostly vdso_enabled = 1; + +#ifdef CONFIG_SPARC64 +static struct vm_special_mapping vdso_mapping64 = { + .name = "[vdso]" +}; +#endif + +#ifdef CONFIG_COMPAT +static struct vm_special_mapping vdso_mapping32 = { + .name = "[vdso]" +}; +#endif + +/* + * Allocate pages for the vdso and copy in the vdso text from the + * kernel image. + */ +static int __init init_vdso_image(const struct vdso_image *image, + struct vm_special_mapping *vdso_mapping, + bool elf64) +{ + int cnpages = (image->size) / PAGE_SIZE; + struct page *cp, **cpp = NULL; + int i; + + /* + * First, the vdso text. This is initialied data, an integral number of + * pages long. + */ + if (WARN_ON(image->size % PAGE_SIZE != 0)) + goto oom; + + cpp = kzalloc_objs(struct page *, cnpages); + vdso_mapping->pages = cpp; + + if (!cpp) + goto oom; + + for (i = 0; i < cnpages; i++) { + cp = alloc_page(GFP_KERNEL); + if (!cp) + goto oom; + cpp[i] = cp; + copy_page(page_address(cp), image->data + i * PAGE_SIZE); + } + + return 0; + oom: + if (cpp != NULL) { + for (i = 0; i < cnpages; i++) { + if (cpp[i] != NULL) + __free_page(cpp[i]); + } + kfree(cpp); + vdso_mapping->pages = NULL; + } + + pr_warn("Cannot allocate vdso\n"); + vdso_enabled = 0; + return -ENOMEM; +} + +static int __init init_vdso(void) +{ + int err = 0; +#ifdef CONFIG_SPARC64 + err = init_vdso_image(&vdso_image_64_builtin, &vdso_mapping64, true); + if (err) + return err; +#endif + +#ifdef CONFIG_COMPAT + err = init_vdso_image(&vdso_image_32_builtin, &vdso_mapping32, false); +#endif + return err; + +} +subsys_initcall(init_vdso); + +struct linux_binprm; + +/* Shuffle the vdso up a bit, randomly. */ +static unsigned long vdso_addr(unsigned long start, unsigned int len) +{ + unsigned int offset; + + /* This loses some more bits than a modulo, but is cheaper */ + offset = get_random_u32_below(PTRS_PER_PTE); + return start + (offset << PAGE_SHIFT); +} + +static_assert(VDSO_NR_PAGES == __VDSO_PAGES); + +static int map_vdso(const struct vdso_image *image, + struct vm_special_mapping *vdso_mapping) +{ + const size_t area_size = image->size + VDSO_NR_PAGES * PAGE_SIZE; + struct mm_struct *mm = current->mm; + struct vm_area_struct *vma; + unsigned long text_start, addr = 0; + int ret = 0; + + mmap_write_lock(mm); + + /* + * First, get an unmapped region: then randomize it, and make sure that + * region is free. + */ + if (current->flags & PF_RANDOMIZE) { + addr = get_unmapped_area(NULL, 0, area_size, 0, 0); + if (IS_ERR_VALUE(addr)) { + ret = addr; + goto up_fail; + } + addr = vdso_addr(addr, area_size); + } + addr = get_unmapped_area(NULL, addr, area_size, 0, 0); + if (IS_ERR_VALUE(addr)) { + ret = addr; + goto up_fail; + } + + text_start = addr + VDSO_NR_PAGES * PAGE_SIZE; + current->mm->context.vdso = (void __user *)text_start; + + /* + * MAYWRITE to allow gdb to COW and set breakpoints + */ + vma = _install_special_mapping(mm, + text_start, + image->size, + VM_READ|VM_EXEC| + VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, + vdso_mapping); + + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + goto up_fail; + } + + vma = vdso_install_vvar_mapping(mm, addr); + + if (IS_ERR(vma)) { + ret = PTR_ERR(vma); + do_munmap(mm, text_start, image->size, NULL); + } + +up_fail: + if (ret) + current->mm->context.vdso = NULL; + + mmap_write_unlock(mm); + return ret; +} + +int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) +{ + + if (!vdso_enabled) + return 0; + +#if defined CONFIG_COMPAT + if (!(is_32bit_task())) + return map_vdso(&vdso_image_64_builtin, &vdso_mapping64); + else + return map_vdso(&vdso_image_32_builtin, &vdso_mapping32); +#else + return map_vdso(&vdso_image_64_builtin, &vdso_mapping64); +#endif + +} + +static __init int vdso_setup(char *s) +{ + int err; + unsigned long val; + + err = kstrtoul(s, 10, &val); + if (!err) + vdso_enabled = val; + return 1; +} +__setup("vdso=", vdso_setup); diff --git a/arch/sparc/video/Makefile b/arch/sparc/video/Makefile new file mode 100644 index 000000000..dcfbe7a59 --- /dev/null +++ b/arch/sparc/video/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-y += video-common.o diff --git a/arch/sparc/video/video-common.c b/arch/sparc/video/video-common.c new file mode 100644 index 000000000..2414380ca --- /dev/null +++ b/arch/sparc/video/video-common.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include + +#include +#include + +bool video_is_primary_device(struct device *dev) +{ + struct device_node *node = dev->of_node; + + if (console_set_on_cmdline) + return false; + + if (node && node == of_console_device) + return true; + + return false; +} +EXPORT_SYMBOL(video_is_primary_device); + +MODULE_DESCRIPTION("Sparc video helpers"); +MODULE_LICENSE("GPL"); -- cgit v1.3.1