path: root/drivers/vdpa/mlx5/core/mr.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2020 Mellanox Technologies Ltd. */

#include <linux/vhost_types.h>
#include <linux/vdpa.h>
#include <linux/gcd.h>
#include <linux/string.h>
#include <linux/mlx5/qp.h>
#include "mlx5_vdpa.h"

/* DIV_ROUND_UP where the divider is a power of 2 give by its log base 2 value */
#define MLX5_DIV_ROUND_UP_POW2(_n, _s) \
({ \
	u64 __s = _s; \
	u64 _res; \
	_res = (((_n) + (1 << (__s)) - 1) >> (__s)); \
	_res; \
})

static int get_octo_len(u64 len, int page_shift)
{
	u64 page_size = 1ULL << page_shift;
	int npages;

	npages = ALIGN(len, page_size) >> page_shift;
	return (npages + 1) / 2;
}

static void mlx5_set_access_mode(void *mkc, int mode)
{
	MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
	MLX5_SET(mkc, mkc, access_mode_4_2, mode >> 2);
}

static void populate_mtts(struct mlx5_vdpa_direct_mr *mr, __be64 *mtt)
{
	struct scatterlist *sg;
	int nsg = mr->nsg;
	u64 dma_addr;
	u64 dma_len;
	int j = 0;
	int i;

	for_each_sg(mr->sg_head.sgl, sg, mr->nent, i) {
		for (dma_addr = sg_dma_address(sg), dma_len = sg_dma_len(sg);
		     nsg && dma_len;
		     nsg--, dma_addr += BIT(mr->log_size), dma_len -= BIT(mr->log_size))
			mtt[j++] = cpu_to_be64(dma_addr);
	}
}

struct mlx5_create_mkey_mem {
	u8 out[MLX5_ST_SZ_BYTES(create_mkey_out)];
	u8 in[MLX5_ST_SZ_BYTES(create_mkey_in)];
	__be64 mtt[];
};

struct mlx5_destroy_mkey_mem {
	u8 out[MLX5_ST_SZ_BYTES(destroy_mkey_out)];
	u8 in[MLX5_ST_SZ_BYTES(destroy_mkey_in)];
};

static void fill_create_direct_mr(struct mlx5_vdpa_dev *mvdev,
				  struct mlx5_vdpa_direct_mr *mr,
				  struct mlx5_create_mkey_mem *mem)
{
	void *in = &mem->in;
	void *mkc;

	MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
	MLX5_SET(mkc, mkc, lw, !!(mr->perm & VHOST_MAP_WO));
	MLX5_SET(mkc, mkc, lr, !!(mr->perm & VHOST_MAP_RO));
	mlx5_set_access_mode(mkc, MLX5_MKC_ACCESS_MODE_MTT);
	MLX5_SET(mkc, mkc, qpn, 0xffffff);
	MLX5_SET(mkc, mkc, pd, mvdev->res.pdn);
	MLX5_SET64(mkc, mkc, start_addr, mr->offset);
	MLX5_SET64(mkc, mkc, len, mr->end - mr->start);
	MLX5_SET(mkc, mkc, log_page_size, mr->log_size);
	MLX5_SET(mkc, mkc, translations_octword_size,
		 get_octo_len(mr->end - mr->start, mr->log_size));
	MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
		 get_octo_len(mr->end - mr->start, mr->log_size));
	populate_mtts(mr, MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt));

	MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
	MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
}

static void create_direct_mr_end(struct mlx5_vdpa_dev *mvdev,
				 struct mlx5_vdpa_direct_mr *mr,
				 struct mlx5_create_mkey_mem *mem)
{
	u32 mkey_index = MLX5_GET(create_mkey_out, mem->out, mkey_index);

	mr->mr = mlx5_idx_to_mkey(mkey_index);
}

static void fill_destroy_direct_mr(struct mlx5_vdpa_dev *mvdev,
				   struct mlx5_vdpa_direct_mr *mr,
				   struct mlx5_destroy_mkey_mem *mem)
{
	void *in = &mem->in;

	MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid);
	MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
	MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mr->mr));
}

static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
{
	if (!mr->mr)
		return;

	mlx5_vdpa_destroy_mkey(mvdev, mr->mr);
}

static u64 map_start(struct vhost_iotlb_map *map, struct mlx5_vdpa_direct_mr *mr)
{
	return max_t(u64, map->start, mr->start);
}

static<