path: root/drivers/infiniband/hw/ionic/ionic_ibdev.h

/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */

#ifndef _IONIC_IBDEV_H_
#define _IONIC_IBDEV_H_

#include <rdma/ib_umem.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_pack.h>
#include <rdma/uverbs_ioctl.h>

#include <rdma/ionic-abi.h>
#include <ionic_api.h>
#include <ionic_regs.h>

#include "ionic_fw.h"
#include "ionic_queue.h"
#include "ionic_res.h"

#include "ionic_lif_cfg.h"

/* Config knobs */
#define IONIC_EQ_DEPTH 511
#define IONIC_EQ_COUNT 32
#define IONIC_AQ_DEPTH 63
#define IONIC_AQ_COUNT 4
#define IONIC_EQ_ISR_BUDGET 10
#define IONIC_EQ_WORK_BUDGET 1000
#define IONIC_MAX_RD_ATOM 16
#define IONIC_PKEY_TBL_LEN 1
#define IONIC_GID_TBL_LEN 256

#define IONIC_MAX_QPID 0xffffff
#define IONIC_SPEC_HIGH 8
#define IONIC_MAX_PD 1024
#define IONIC_SPEC_HIGH 8
#define IONIC_SQCMB_ORDER 5
#define IONIC_RQCMB_ORDER 0

#define IONIC_META_LAST		((void *)1ul)
#define IONIC_META_POSTED	((void *)2ul)

#define IONIC_CQ_GRACE 100

#define IONIC_ROCE_UDP_SPORT	28272
#define IONIC_DMA_LKEY		0
#define IONIC_DMA_RKEY		IONIC_DMA_LKEY

#define IONIC_CMB_SUPPORTED \
	(IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \
	 IONIC_CMB_WC | IONIC_CMB_UC)

/* resource is not reserved on the device, indicated in tbl_order */
#define IONIC_RES_INVALID	-1

struct ionic_aq;
struct ionic_cq;
struct ionic_eq;
struct ionic_vcq;

enum ionic_admin_state {
	IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */
	IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */
	IONIC_ADMIN_KILLED, /* not submitting, locally completed */
};

enum ionic_admin_flags {
	IONIC_ADMIN_F_BUSYWAIT  = BIT(0),	/* Don't sleep */
	IONIC_ADMIN_F_TEARDOWN  = BIT(1),	/* In destroy path */
	IONIC_ADMIN_F_INTERRUPT = BIT(2),	/* Interruptible w/timeout */
};

enum ionic_mmap_flag {
	IONIC_MMAP_WC = BIT(0),
};

struct ionic_mmap_entry {
	struct rdma_user_mmap_entry rdma_entry;
	unsigned long size;
	unsigned long pfn;
	u8 mmap_flags;
};

struct ionic_ibdev {
	struct ib_device	ibdev;

	struct ionic_lif_cfg	lif_cfg;

	struct xarray		qp_tbl;
	struct xarray		cq_tbl;

	struct ionic_resid_bits	inuse_dbid;
	struct ionic_resid_bits	inuse_pdid;
	struct ionic_resid_bits	inuse_ahid;
	struct ionic_resid_bits	inuse_mrid;
	struct ionic_resid_bits	inuse_qpid;
	struct ionic_resid_bits	inuse_cqid;

	u8			half_cqid_udma_shift;
	u8			half_qpid_udma_shift;
	u8			next_qpid_udma_idx;
	u8			next_mrkey;

	struct work_struct	reset_work;
	bool			reset_posted;
	u32			reset_cnt;

	struct delayed_work	admin_dwork;
	struct ionic_aq		**aq_vec;
	atomic_t		admin_state;

	struct ionic_eq		**eq_vec;

	struct ionic_v1_stat	*hw_stats;
	void			*hw_stats_buf;
	struct rdma_stat_desc	*hw_stats_hdrs;
	struct ionic_counter_stats *counter_stats;
	int			hw_stats_count;
};

struct ionic_eq {
	struct ionic_ibdev	*dev;

	u32			eqid;
	u32			intr;

	struct ionic_queue	q;

	int			armed;
	bool			enable;

	struct work_struct	work;

	int			irq;
	char			name[32];
};

struct ionic_admin_wr {
	struct completion		work;
	struct list_head		aq_ent;
	struct ionic_v1_admin_wqe	wqe;
	struct ionic_v1_cqe		cqe;
	struct ionic_aq			*aq;
	int				status;
};

struct ionic_admin_wr_q {
	struct ionic_admin_wr	*wr;
	int			wqe_strides;
};

struct ionic_aq {
	struct ionic_ibdev	*dev;
	struct ionic_vcq	*vcq;

	struct work_struct	work;

	atomic_t		admin_state;
	unsigned long		stamp;
	bool			armed;

	u32			aqid;
	u32			cqid;

	spinlock_t		lock; /* for posting */
	struct ionic_queue	q;
	struct ionic_admin_wr_q	*q_wr;
	struct list_head	wr_prod;
	struct list_head	wr_post;
};

struct ionic_ctx {
	struct ib_ucontext	ibctx;
	u32			dbid;
	struct rdma_user_mmap_entry	*mmap_dbell;
};

struct ionic_tbl_buf {
	u32		tbl_limit;
	u32		tbl_pages;
	size_t		tbl_size;
	__le64		*tbl_buf;
	dma_addr_t	tbl_dma;
	u8		page_size_log2;
};

struct ionic_pd {
	struct ib_pd		ibpd;

	u32			pdid;
	u32			flags;
};

struct ionic_cq {
	struct ionic_vcq	*vcq;

	u32			cqid;
	u32			eqid;

	spinlock_t		lock; /* for polling */
	struct list_head	poll_sq;
	bool			flush;
	struct list_head	flush_sq;
	struct list_head	flush_rq;
	struct list_head	ibkill_flush_ent;

	struct ionic_queue	q;
	bool			color;
	int			credit;
	u16			arm_any_prod;
	u16			arm_sol_prod;

	struct kref		cq_kref;
	struct completion	cq_rel_comp;

	/* infrequently accessed, keep at end */
	struct ib_umem		*umem;
};

struct ionic_vcq {
	struct ib_cq		ibcq;
	struct ionic_cq		cq[2];
	u8			udma_mask;
	u8			poll_idx;
};

struct ionic_sq_meta {
	u64			wrid;
	u32			len;
	u16			seq;
	u8			ibop;
	u8			ibsts;
	u8			remote:1;
	u8			signal:1;
	u8			local_comp:1;
};

struct ionic_rq_meta {
	struct ionic_rq_meta	*next;
	u64			wrid;
};

struct ionic_qp {
	struct ib_qp		ibqp;
	enum ib_qp_state	state;

	u32			qpid;
	u32			ahid;
	u32			sq_cqid;
	u32			rq_cqid;
	u8			udma_idx;
	u8			has_ah:1;
	u8			has_sq:1;
	u8			has_rq:1;
	u8			sig_all:1;

	struct list_head	qp_list_counter;

	struct list_head	cq_poll_sq;
	struct list_head	cq_flush_sq;
	struct list_head	cq_flush_rq;
	struct list_head	ibkill_flush_ent;

	spinlock_t		sq_lock; /* for posting and polling */
	struct ionic_queue	sq;
	struct ionic_sq_meta	*sq_meta;
	u16			*sq_msn_idx;
	int			sq_spec;
	u16			sq_old_prod;
	u16			sq_msn_prod;
	u16			sq_msn_cons;
	u8			sq_cmb;
	bool			sq_flush;
	bool			sq_flush_rcvd;

	spinlock_t		rq_lock; /* for posting and polling */
	struct ionic_queue	rq;
	struct ionic_rq_meta	*rq_meta;
	struct ionic_rq_meta	*rq_meta_head;
	int			rq_spec;
	u16			rq_old_prod;
	u8			rq_cmb;
	bool			rq_flush;

	struct kref		qp_kref;
	struct completion	qp_rel_comp;

	/* infrequently accessed, keep at end */
	int			sgid_index;
	int			sq_cmb_order;
	u32			sq_cmb_pgid;
	phys_addr_t		sq_cmb_addr;
	struct rdma_user_mmap_entry *mmap_sq_cmb;

	struct ib_umem		*sq_umem;

	int			rq_cmb_order;
	u32			rq_cmb_pgid;
	phys_addr_t		rq_cmb_addr;
	struct rdma_user_mmap_entry *mmap_rq_cmb;

	struct ib_umem		*rq_umem;

	int			dcqcn_profile;

	struct ib_ud_header	*hdr;
};

struct ionic_ah {
	struct ib_ah		ibah;
	u32			ahid;
	int			sgid_index;
	struct ib_ud_header	hdr;
};

struct ionic_mr {
	union {
		struct ib_mr	ibmr;
		struct ib_mw	ibmw;
	};

	u32			mrid;
	int			flags;

	struct ib_umem		*umem;
	struct ionic_tbl_buf	buf;
	bool			created;
};

struct ionic_counter_stats {
	int queue_stats_count;
	struct ionic_v1_stat *hdr;
	struct rdma_stat_desc *stats_hdrs;
	struct xarray xa_counters;
};

struct ionic_counter {
	void *vals;
	struct list_head qp_list;
};

static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev)
{
	return container_of(ibdev, struct ionic_ibdev, ibdev);
}

sta