path: root/arch/powerpc/mm/book3s64/hash_native.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * native hashtable management.
 *
 * SMP scalability work:
 *    Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
 */

#undef DEBUG_LOW

#include <linux/spinlock.h>
#include <linux/bitops.h>
#include <linux/of.h>
#include <linux/processor.h>
#include <linux/threads.h>
#include <linux/smp.h>
#include <linux/pgtable.h>

#include <asm/machdep.h>
#include <asm/mmu.h>
#include <asm/mmu_context.h>
#include <asm/trace.h>
#include <asm/tlb.h>
#include <asm/cputable.h>
#include <asm/udbg.h>
#include <asm/kexec.h>
#include <asm/ppc-opcode.h>
#include <asm/feature-fixups.h>

#ifdef DEBUG_LOW
#define DBG_LOW(fmt...) udbg_printf(fmt)
#else
#define DBG_LOW(fmt...)
#endif

#ifdef __BIG_ENDIAN__
#define HPTE_LOCK_BIT 3
#else
#define HPTE_LOCK_BIT (56+3)
#endif

static DEFINE_RAW_SPINLOCK(native_tlbie_lock);

#ifdef CONFIG_LOCKDEP
static struct lockdep_map hpte_lock_map =
	STATIC_LOCKDEP_MAP_INIT("hpte_lock", &hpte_lock_map);

static void acquire_hpte_lock(void)
{
	lock_map_acquire(&hpte_lock_map);
}

static void release_hpte_lock(void)
{
	lock_map_release(&hpte_lock_map);
}
#else
static void acquire_hpte_lock(void)
{
}

static void release_hpte_lock(void)
{
}
#endif

static inline unsigned long  ___tlbie(unsigned long vpn, int psize,
						int apsize, int ssize)
{
	unsigned long va;
	unsigned int penc;
	unsigned long sllp;

	/*
	 * We need 14 to 65 bits of va for a tlibe of 4K page
	 * With vpn we ignore the lower VPN_SHIFT bits already.
	 * And top two bits are already ignored because we can
	 * only accomodate 76 bits in a 64 bit vpn with a VPN_SHIFT
	 * of 12.
	 */
	va = vpn << VPN_SHIFT;
	/*
	 * clear top 16 bits of 64bit va, non SLS segment
	 * Older versions of the architecture (2.02 and earler) require the
	 * masking of the top 16 bits.
	 */
	if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
		va &= ~(0xffffULL << 48);

	switch (psize) {
	case MMU_PAGE_4K:
		/* clear out bits after (52) [0....52.....63] */
		va &= ~((1ul << (64 - 52)) - 1);
		va |= ssize << 8;
		sllp = get_sllp_encoding(apsize);
		va |= sllp << 5;
		asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2)
			     : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
			     : "memory");
		break;
	default:
		/* We need 14 to 14 + i bits of va */
		penc = mmu_psize_defs[psize].penc[apsize];
		va &= ~((1ul << mmu_psize_defs[apsize].shift) - 1);
		va |= penc << 12;
		va |= ssize << 8;
		/*
		 * AVAL bits:
		 * We don't need all the bits, but rest of the bits
		 * must be ignored by the processor.
		 * vpn cover upto 65 bits of va. (0...65) and we need
		 * 58..64 bits of va.
		 */
		va |= (vpn & 0xfe); /* AVAL */
		va |= 1; /* L */
		asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2)
			     : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206)
			     : "memory");
		break;
	}
	return va;
}

static inline void fixup_tlbie_vpn(unsigned long vpn, int psize,
				   int apsize, int ssize)
{
	if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) {
		/* Radix flush for a hash guest */

		unsigned long rb,rs,prs,r,ric;

		rb = PPC_BIT(52); /* IS = 2 */
		rs = 0;  /* lpid = 0 */
		prs = 0; /* partition scoped */
		r = 1;   /* radix format */
		ric = 0; /* RIC_FLSUH_TLB */

		/*
		 * Need the extra ptesync to make sure we don't
		 * re-order the tlbie
		 */
		asm volatile("ptesync": : :"memory");
		asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
			     : : "r"(rb), "i"(r), "i"(prs),
			       "i"(ric), "r"(rs) : "memory");
	}


	if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) {
		/* Need the extra ptesync to ensure we don't reorder tlbie*/
		asm volatile("ptesync": : :"memory");
		___tlbie(vpn, psize, apsize, ssize);
	}
}

static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
{
	unsigned long rb;

	rb = ___tlbie(vpn, psize, apsize, ssize);
	trace_tlbie(0, 0, rb, 0, 0, 0, 0);
}

static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize)
{
	unsigned long va;
	unsigned int penc;
	unsigned long sllp;

	/* VPN_SHIFT can be atmost 12 */
	va = vpn << VPN_SHIFT;
	/*
	 * clear top 16 bits of 64 bit va, non SLS segment
	 * Older versions of the architecture (2.02 and earler) require the
	 * masking of the top 16 bits.
	 */
	if (mmu_has_feature(MMU_FTR_TLBIE_CROP_VA))
		va &= ~(0xffffULL << 48);

	switch (psize) {
	case MMU_PAGE_4K:
		/* clear out b