path: root/drivers/infiniband/sw/rdmavt/mr.c

// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
/*
 * Copyright(c) 2016 Intel Corporation.
 */

#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <rdma/ib_umem.h>
#include <rdma/rdma_vt.h>
#include "vt.h"
#include "mr.h"
#include "trace.h"

/**
 * rvt_driver_mr_init - Init MR resources per driver
 * @rdi: rvt dev struct
 *
 * Do any intilization needed when a driver registers with rdmavt.
 *
 * Return: 0 on success or errno on failure
 */
int rvt_driver_mr_init(struct rvt_dev_info *rdi)
{
	unsigned int lkey_table_size = rdi->dparms.lkey_table_size;
	unsigned lk_tab_size;
	int i;

	/*
	 * The top hfi1_lkey_table_size bits are used to index the
	 * table.  The lower 8 bits can be owned by the user (copied from
	 * the LKEY).  The remaining bits act as a generation number or tag.
	 */
	if (!lkey_table_size)
		return -EINVAL;

	spin_lock_init(&rdi->lkey_table.lock);

	/* ensure generation is at least 4 bits */
	if (lkey_table_size > RVT_MAX_LKEY_TABLE_BITS) {
		rvt_pr_warn(rdi, "lkey bits %u too large, reduced to %u\n",
			    lkey_table_size, RVT_MAX_LKEY_TABLE_BITS);
		rdi->dparms.lkey_table_size = RVT_MAX_LKEY_TABLE_BITS;
		lkey_table_size = rdi->dparms.lkey_table_size;
	}
	rdi->lkey_table.max = 1 << lkey_table_size;
	rdi->lkey_table.shift = 32 - lkey_table_size;
	lk_tab_size = rdi->lkey_table.max * sizeof(*rdi->lkey_table.table);
	rdi->lkey_table.table = (struct rvt_mregion __rcu **)
			       vmalloc_node(lk_tab_size, rdi->dparms.node);
	if (!rdi->lkey_table.table)
		return -ENOMEM;

	RCU_INIT_POINTER(rdi->dma_mr, NULL);
	for (i = 0; i < rdi->lkey_table.max; i++)
		RCU_INIT_POINTER(rdi->lkey_table.table[i], NULL);

	rdi->dparms.props.max_mr = rdi->lkey_table.max;
	return 0;
}

/**
 * rvt_mr_exit - clean up MR
 * @rdi: rvt dev structure
 *
 * called when drivers have unregistered or perhaps failed to register with us
 */
void rvt_mr_exit(struct rvt_dev_info *rdi)
{
	if (rdi->dma_mr)
		rvt_pr_err(rdi, "DMA MR not null!\n");

	vfree(rdi->lkey_table.table);
}

static void rvt_deinit_mregion(struct rvt_mregion *mr)
{
	int i = mr->mapsz;

	mr->mapsz = 0;
	while (i)
		kfree(mr->map[--i]);
	percpu_ref_exit(&mr->refcount);
}

static void __rvt_mregion_complete(struct percpu_ref *ref)
{
	struct rvt_mregion *mr = container_of(ref, struct rvt_mregion,
					      refcount);

	complete(&mr->comp);
}

static int rvt_init_mregion(struct rvt_mregion *mr, struct ib_pd *pd,
			    int count, unsigned int percpu_flags)
{
	int m, i = 0;
	struct rvt_dev_info *dev = ib_to_rvt(pd->device);

	mr->mapsz = 0;
	m = (count + RVT_SEGSZ - 1) / RVT_SEGSZ;
	for (; i < m; i++) {
		mr->map[i] = kzalloc_node(sizeof(*mr->map[0]), GFP_KERNEL,
					  dev->dparms.node);
		if (!mr->map[i])
			goto bail;
		mr->mapsz++;
	}
	init_completion(&mr->comp);
	/* count returning the ptr to user */
	if (percpu_ref_init(&mr->refcount, &__rvt_mregion_complete,
			    percpu_flags, GFP_KERNEL))
		goto bail;

	atomic_set(&mr->lkey_invalid, 0);
	mr->pd = pd;
	mr->max_segs = count;
	return 0;
bail:
	rvt_deinit_mregion(mr);
	return -ENOMEM;
}

/**
 * rvt_alloc_lkey - allocate an lkey
 * @mr: memory region that this lkey protects
 * @dma_region: 0->normal key, 1->restricted DMA key
 *
 * Returns 0 if successful, otherwise returns -errno.
 *
 * Increments mr reference count as required.
 *
 * Sets the lkey field mr for non-dma regions.
 *
 */
static int rvt_alloc_lkey(struct rvt_mregion *mr, int dma_region)
{
	unsigned long flags;
	u32 r;
	u32 n;
	int ret = 0;
	struct rvt_dev_info *dev = ib_to_rvt(mr->pd->device);
	struct rvt_lkey_table *rkt = &dev->lkey_table;

	rvt_get_mr(mr);
	spin_lock_irqsave(&rkt->lock, flags);

	/* special case for dma_mr lkey == 0 */
	if (dma_region) {
		struct rvt_mregion *tmr;

		tmr = rcu_access_pointer(dev->dma_mr);
		if (!tmr) {
			mr->lkey_published = 1;
			/* Insure published written first */
			rcu_assign_pointer(dev->dma_mr, mr);
			rvt_get_mr(mr);
		}
		goto success;