path: root/drivers/s390/cio/vfio_ccw_cp.c

// SPDX-License-Identifier: GPL-2.0
/*
 * channel program interfaces
 *
 * Copyright IBM Corp. 2017
 *
 * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
 *            Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
 */

#include <linux/ratelimit.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/highmem.h>
#include <linux/iommu.h>
#include <linux/vfio.h>
#include <asm/idals.h>

#include "vfio_ccw_cp.h"
#include "vfio_ccw_private.h"

struct page_array {
	/* Array that stores pages need to pin. */
	dma_addr_t		*pa_iova;
	/* Array that receives the pinned pages. */
	struct page		**pa_page;
	/* Number of pages pinned from @pa_iova. */
	int			pa_nr;
};

struct ccwchain {
	struct list_head	next;
	struct ccw1		*ch_ccw;
	/* Guest physical address of the current chain. */
	u64			ch_iova;
	/* Count of the valid ccws in chain. */
	int			ch_len;
	/* Pinned PAGEs for the original data. */
	struct page_array	*ch_pa;
};

/*
 * page_array_alloc() - alloc memory for page array
 * @pa: page_array on which to perform the operation
 * @len: number of pages that should be pinned from @iova
 *
 * Attempt to allocate memory for page array.
 *
 * Usage of page_array:
 * We expect (pa_nr == 0) and (pa_iova == NULL), any field in
 * this structure will be filled in by this function.
 *
 * Returns:
 *         0 if page array is allocated
 *   -EINVAL if pa->pa_nr is not initially zero, or pa->pa_iova is not NULL
 *   -ENOMEM if alloc failed
 */
static int page_array_alloc(struct page_array *pa, unsigned int len)
{
	if (pa->pa_nr || pa->pa_iova)
		return -EINVAL;

	if (len == 0)
		return -EINVAL;

	pa->pa_nr = len;

	pa->pa_iova = kzalloc_objs(*pa->pa_iova, len);
	if (!pa->pa_iova)
		return -ENOMEM;

	pa->pa_page = kzalloc_objs(*pa->pa_page, len);
	if (!pa->pa_page) {
		kfree(pa->pa_iova);
		return -ENOMEM;
	}

	return 0;
}

/*
 * page_array_unpin() - Unpin user pages in memory
 * @pa: page_array on which to perform the operation
 * @vdev: the vfio device to perform the operation
 * @pa_nr: number of user pages to unpin
 * @unaligned: were pages unaligned on the pin request
 *
 * Only unpin if any pages were pinned to begin with, i.e. pa_nr > 0,
 * otherwise only clear pa->pa_nr
 */
static void page_array_unpin(struct page_array *pa,
			     struct vfio_device *vdev, int pa_nr, bool unaligned)
{
	int unpinned = 0, npage = 1;

	while (unpinned < pa_nr) {
		dma_addr_t *first = &pa->pa_iova[unpinned];
		dma_addr_t *last = &first[npage];

		if (unpinned + npage < pa_nr &&
		    *first + npage * PAGE_SIZE == *last &&
		    !unaligned) {
			npage++;
			continue;
		}

		vfio_unpin_pages(vdev, *first, npage);
		unpinned += npage;
		npage = 1;
	}

	pa->pa_nr = 0;
}

/*
 * page_array_pin() - Pin user pages in memory
 * @pa: page_array on which to perform the operation
 * @vdev: the vfio device to perform pin operations
 * @unaligned: are pages aligned to 4K boundary?
 *
 * Returns number of pages pinned upon success.
 * If the pin request partially succeeds, or fails completely,
 * all pages are left unpinned and a negative error value is returned.
 *
 * Requests to pin "aligned" pages can be coalesced into a single
 * vfio_pin_pages request for the sake of efficiency, based on the
 * expectation of 4K page requests. Unaligned requests are probably
 * dealing with 2K "pages", and cannot be coalesced without
 * reworking this logic to incorporate that math.
 */
static int page_array_pin(struct page_array *pa, struct vfio_device *vdev, bool unaligned)
{
	int pinned = 0, npage = 1;
	int ret = 0;

	while (pinned < pa->pa_nr) {
		dma_addr_t *first = &pa->pa_iova[pinned];
		dma_addr_t *last = &first[npage];

		if (pinned + npage < pa->pa_nr &&
		    *first + npage * PAGE_SIZE == *last &&
		    !unaligned) {
			npage++;
			continue;
		}

		ret = vfio_pin_pages(vdev, *first, npage,
				     IOMMU_READ | IOMMU_WRITE,
				     &pa->pa_page[pinned]);
		if (ret < 0) {
			goto err_out;
		} else if (ret > 0 && ret != npage) {
			pinned += ret;
			ret = -EINVAL;
			goto err_out;
		}
		pinned += npage;
		npage = 1;
	}

	return ret;

err_out:
	page_array_unpin(pa, vdev, pinned, unaligned);
	return ret;
}

/* Unpin the pages before releasing the memory. */
static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vdev, bool unaligned)
{
	page_array_unpin(pa, vdev, pa->pa_nr, unaligned);
	kfree(pa->pa_page);
	kfree(pa->pa_iova);
}

st