aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c2
-rw-r--r--drivers/net/ethernet/marvell/sky2.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx4/en_rx.c2
-rw-r--r--drivers/net/ethernet/sun/cassini.c4
-rw-r--r--drivers/net/veth.c2
-rw-r--r--include/linux/skbuff_ref.h64
-rw-r--r--include/net/netmem.h24
-rw-r--r--net/core/net_test.c1222
-rw-r--r--net/core/skbuff.c157
-rw-r--r--net/ipv4/esp4.c4
-rw-r--r--net/ipv4/tcp.c6
-rw-r--r--net/ipv6/esp6.c4
-rw-r--r--net/tls/tls_device.c2
-rw-r--r--net/tls/tls_device_fallback.c2
-rw-r--r--net/tls/tls_strp.c2
-rw-r--r--net/xfrm/xfrm_iptfs.c3
16 files changed, 1387 insertions, 115 deletions
diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
index dbec95b49777..4d645771596a 100644
--- a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
+++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c
@@ -1660,7 +1660,7 @@ static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
for (i = 0; i < record->num_frags; i++) {
skb_shinfo(nskb)->frags[i] = record->frags[i];
/* increase the frag ref count */
- __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
+ skb_frag_ref(nskb, i);
}
skb_shinfo(nskb)->nr_frags = record->num_frags;
diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c
index 30facdece3e2..1d42a82db8f2 100644
--- a/drivers/net/ethernet/marvell/sky2.c
+++ b/drivers/net/ethernet/marvell/sky2.c
@@ -2501,7 +2501,7 @@ static void skb_put_frags(struct sk_buff *skb, unsigned int hdr_space,
if (length == 0) {
/* don't need this page */
- __skb_frag_unref(frag, false);
+ skb_frag_unref(skb, i);
--skb_shinfo(skb)->nr_frags;
} else {
size = min(length, (unsigned) PAGE_SIZE);
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index 96f97fa0f548..fe18606c2ab5 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -497,7 +497,7 @@ static int mlx4_en_complete_rx_desc(struct mlx4_en_priv *priv,
fail:
while (nr > 0) {
nr--;
- __skb_frag_unref(skb_shinfo(skb)->frags + nr, false);
+ skb_frag_unref(skb, nr);
}
return 0;
}
diff --git a/drivers/net/ethernet/sun/cassini.c b/drivers/net/ethernet/sun/cassini.c
index 9a8b5cf2d648..02158a1bb2f7 100644
--- a/drivers/net/ethernet/sun/cassini.c
+++ b/drivers/net/ethernet/sun/cassini.c
@@ -1998,7 +1998,7 @@ static int cas_rx_process_pkt(struct cas *cp, struct cas_rx_comp *rxc,
skb->len += hlen - swivel;
skb_frag_fill_page_desc(frag, page->buffer, off, hlen - swivel);
- __skb_frag_ref(frag);
+ skb_frag_ref(skb, 0);
/* any more data? */
if ((words[0] & RX_COMP1_SPLIT_PKT) && ((dlen -= hlen) > 0)) {
@@ -2022,7 +2022,7 @@ static int cas_rx_process_pkt(struct cas *cp, struct cas_rx_comp *rxc,
frag++;
skb_frag_fill_page_desc(frag, page->buffer, 0, hlen);
- __skb_frag_ref(frag);
+ skb_frag_ref(skb, 1);
RX_USED_ADD(page, hlen + cp->crc_size);
}
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 38f263d4f06b..c273073b99f5 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -751,7 +751,7 @@ static void veth_xdp_get(struct xdp_buff *xdp)
return;
for (i = 0; i < sinfo->nr_frags; i++)
- __skb_frag_ref(&sinfo->frags[i]);
+ get_netmem(skb_frag_netmem(&sinfo->frags[i]));
}
static int veth_convert_skb_to_xdp_buff(struct veth_rq *rq,
diff --git a/include/linux/skbuff_ref.h b/include/linux/skbuff_ref.h
index 05c8486bafac..f768a3fa1d60 100644
--- a/include/linux/skbuff_ref.h
+++ b/include/linux/skbuff_ref.h
@@ -9,32 +9,53 @@
#include <linux/skbuff.h>
+bool napi_pp_get_page(netmem_ref netmem);
+
/**
- * __skb_frag_ref - take an addition reference on a paged fragment.
- * @frag: the paged fragment
+ * skb_netmem_ref - acquire a fragment reference on a netmem
+ * @netmem: netmem to reference
+ * @recycle: whether page_pool recycling is enabled (e.g. skb->pp_recycle)
*
- * Takes an additional reference on the paged fragment @frag.
+ * Acquires pp_ref_count via napi_pp_get_page() if @recycle is true and
+ * @netmem belongs to a page_pool; otherwise acquires a non-pp backing
+ * reference via get_netmem(). Counterpart to skb_netmem_unref().
*/
-static __always_inline void __skb_frag_ref(skb_frag_t *frag)
+static __always_inline void skb_netmem_ref(netmem_ref netmem, bool recycle)
{
- get_netmem(skb_frag_netmem(frag));
+#ifdef CONFIG_PAGE_POOL
+ if (recycle && napi_pp_get_page(netmem))
+ return;
+#endif
+ get_netmem(netmem);
}
/**
- * skb_frag_ref - take an addition reference on a paged fragment of an skb.
+ * skb_frag_ref - acquire a reference on a paged fragment of an skb
* @skb: the buffer
- * @f: the fragment offset.
+ * @f: the fragment offset
*
- * Takes an additional reference on the @f'th paged fragment of @skb.
+ * Acquires a reference on the @f'th fragment of @skb via skb_netmem_ref()
+ * (pp_ref_count if skb->pp_recycle and page_pool-backed, else non-pp
+ * backing refcount). Counterpart to skb_frag_unref().
*/
static __always_inline void skb_frag_ref(struct sk_buff *skb, int f)
{
- __skb_frag_ref(&skb_shinfo(skb)->frags[f]);
+ skb_netmem_ref(skb_frag_netmem(&skb_shinfo(skb)->frags[f]),
+ skb->pp_recycle);
}
bool napi_pp_put_page(netmem_ref netmem);
-static __always_inline void skb_page_unref(netmem_ref netmem, bool recycle)
+/**
+ * skb_netmem_unref - release a fragment reference on a netmem
+ * @netmem: netmem to unreference
+ * @recycle: whether page_pool recycling is enabled (e.g. skb->pp_recycle)
+ *
+ * Drops pp_ref_count via napi_pp_put_page() if @recycle is true and @netmem
+ * belongs to a page_pool; otherwise drops a non-pp backing reference via
+ * put_netmem(). Counterpart to skb_netmem_ref().
+ */
+static __always_inline void skb_netmem_unref(netmem_ref netmem, bool recycle)
{
#ifdef CONFIG_PAGE_POOL
if (recycle && napi_pp_put_page(netmem))
@@ -44,31 +65,22 @@ static __always_inline void skb_page_unref(netmem_ref netmem, bool recycle)
}
/**
- * __skb_frag_unref - release a reference on a paged fragment.
- * @frag: the paged fragment
- * @recycle: recycle the page if allocated via page_pool
- *
- * Releases a reference on the paged fragment @frag
- * or recycles the page via the page_pool API.
- */
-static __always_inline void __skb_frag_unref(skb_frag_t *frag, bool recycle)
-{
- skb_page_unref(skb_frag_netmem(frag), recycle);
-}
-
-/**
- * skb_frag_unref - release a reference on a paged fragment of an skb.
+ * skb_frag_unref - release a reference on a paged fragment of an skb
* @skb: the buffer
* @f: the fragment offset
*
- * Releases a reference on the @f'th paged fragment of @skb.
+ * Drops a reference on the @f'th fragment of @skb via skb_netmem_unref()
+ * (pp_ref_count if skb->pp_recycle and page_pool-backed, else non-pp
+ * backing refcount), skipping managed zerocopy fragments. Counterpart to
+ * skb_frag_ref().
*/
static __always_inline void skb_frag_unref(struct sk_buff *skb, int f)
{
struct skb_shared_info *shinfo = skb_shinfo(skb);
if (!skb_zcopy_managed(skb))
- __skb_frag_unref(&shinfo->frags[f], skb->pp_recycle);
+ skb_netmem_unref(skb_frag_netmem(&shinfo->frags[f]),
+ skb->pp_recycle);
}
#endif /* _LINUX_SKBUFF_REF_H */
diff --git a/include/net/netmem.h b/include/net/netmem.h
index c4beb6fbc130..193de0d21490 100644
--- a/include/net/netmem.h
+++ b/include/net/netmem.h
@@ -409,21 +409,37 @@ static inline bool net_is_devmem_iov(const struct net_iov *niov)
}
#endif
-void __get_netmem(netmem_ref netmem);
-void __put_netmem(netmem_ref netmem);
+void get_net_iov(struct net_iov *niov);
+void put_net_iov(struct net_iov *niov);
+/**
+ * get_netmem - acquire a non-page_pool reference on a netmem
+ * @netmem: netmem to reference
+ *
+ * Acquires a non-pp backing reference (page._refcount via get_page() or
+ * binding->ref via get_net_iov()) regardless of page_pool membership.
+ * Counterpart to put_netmem().
+ */
static __always_inline void get_netmem(netmem_ref netmem)
{
if (netmem_is_net_iov(netmem))
- __get_netmem(netmem);
+ get_net_iov(netmem_to_net_iov(netmem));
else
get_page(netmem_to_page(netmem));
}
+/**
+ * put_netmem - release a non-page_pool reference on a netmem
+ * @netmem: netmem to unreference
+ *
+ * Drops a non-pp backing reference (page._refcount via put_page() or
+ * binding->ref via put_net_iov()) regardless of page_pool membership.
+ * Counterpart to get_netmem().
+ */
static __always_inline void put_netmem(netmem_ref netmem)
{
if (netmem_is_net_iov(netmem))
- __put_netmem(netmem);
+ put_net_iov(netmem_to_net_iov(netmem));
else
put_page(netmem_to_page(netmem));
}
diff --git a/net/core/net_test.c b/net/core/net_test.c
index bc2ef7612e7d..2bb01430a89f 100644
--- a/net/core/net_test.c
+++ b/net/core/net_test.c
@@ -457,10 +457,1231 @@ static void ip_tunnel_flags_test_run(struct kunit *test)
KUNIT_ASSERT_TRUE(test, __ipt_flag_op(bitmap_equal, exp, out));
}
+/* Netmem & SKB fragment refcounting */
+
+#include <linux/skbuff_ref.h>
+#include <net/netmem.h>
+#include <net/netdev_rx_queue.h>
+#include <net/page_pool/helpers.h>
+#include <net/page_pool/memory_provider.h>
+#include <net/tcp.h>
+#include "devmem.h"
+#include "netmem_priv.h"
+
+static void netmem_ref_test_page(struct kunit *test)
+{
+ struct sk_buff *skb;
+ netmem_ref netmem;
+ struct page *page;
+ int base_ref;
+
+ page = alloc_page(GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, page);
+ netmem = page_to_netmem(page);
+ base_ref = page_ref_count(page);
+
+ /* Layer 1: get_netmem / put_netmem use page non-pp refcount */
+ get_netmem(netmem);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1);
+ put_netmem(netmem);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+
+#ifdef CONFIG_PAGE_POOL
+ /* Layer 2: napi_pp_get/put_page return false on non-PP page */
+ KUNIT_EXPECT_FALSE(test, napi_pp_get_page(netmem));
+ KUNIT_EXPECT_FALSE(test, napi_pp_put_page(netmem));
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+#endif
+
+ /* Layer 3: skb_netmem_ref / unref fall back to non-pp on non-PP page */
+ skb_netmem_ref(netmem, false);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1);
+ skb_netmem_unref(netmem, false);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+
+ skb_netmem_ref(netmem, true);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1);
+ skb_netmem_unref(netmem, true);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+
+ /* Layer 4: skb_frag_ref / skb_frag_unref on non-PP page */
+ skb = alloc_skb(64, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, skb);
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+
+ skb->pp_recycle = 1;
+ skb_frag_ref(skb, 0);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1);
+ skb_frag_unref(skb, 0);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+
+ skb_shinfo(skb)->nr_frags = 0;
+ consume_skb(skb);
+ __free_page(page);
+}
+
+#ifdef CONFIG_PAGE_POOL
+static void netmem_ref_test_pp_page(struct kunit *test)
+{
+ struct page_pool_params pp_params = {
+ .order = 0,
+ .pool_size = 4,
+ .nid = NUMA_NO_NODE,
+ };
+ struct page_pool *pool;
+ struct sk_buff *skb, *copy, *clone, *split;
+ netmem_ref netmem;
+ struct page *page;
+ long base_pp_ref;
+ int base_ref;
+
+ pool = page_pool_create(&pp_params);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool);
+
+ page = page_pool_dev_alloc_pages(pool);
+ KUNIT_ASSERT_NOT_NULL(test, page);
+ netmem = page_to_netmem(page);
+ base_ref = page_ref_count(page);
+ base_pp_ref = atomic_long_read(&page->pp_ref_count);
+
+ /* Layer 1: get_netmem / put_netmem always use non-pp refcount */
+ get_netmem(netmem);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+ put_netmem(netmem);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+
+ /* Layer 2: napi_pp_get/put_page use pp_ref_count */
+ KUNIT_EXPECT_TRUE(test, napi_pp_get_page(netmem));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ KUNIT_EXPECT_TRUE(test, napi_pp_put_page(netmem));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ /* Layer 3: skb_netmem_ref / unref obey recycle flag */
+ skb_netmem_ref(netmem, false);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+ skb_netmem_unref(netmem, false);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+
+ skb_netmem_ref(netmem, true);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ skb_netmem_unref(netmem, true);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ /* Layer 4: skb_frag_ref / skb_frag_unref match on pp_recycle=1 & 0 */
+ skb = alloc_skb(64, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, skb);
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len = 64;
+ skb->data_len = 64;
+
+ skb->pp_recycle = 1;
+ skb_frag_ref(skb, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ skb_frag_unref(skb, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ /* pskb_copy on pp_recycle=1 skb propagates pp_recycle & pp_ref_count */
+ copy = pskb_copy(skb, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, copy);
+ KUNIT_EXPECT_TRUE(test, copy->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ consume_skb(copy);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ /* skb_zerocopy on pp_recycle=1 skb propagates pp_ref_count */
+ copy = alloc_skb(0, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, copy);
+ skb_put(copy, skb_tailroom(copy));
+ skb->head_frag = 1;
+ KUNIT_ASSERT_EQ(test, skb_zerocopy(copy, skb, skb->len, 0), 0);
+ skb->head_frag = 0;
+ KUNIT_EXPECT_TRUE(test, copy->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ consume_skb(copy);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ /* skb_clone + pskb_expand_head preserves pp_recycle and pp_ref_count */
+ clone = skb_clone(skb, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, clone);
+ KUNIT_ASSERT_EQ(test, pskb_expand_head(clone, 32, 0, GFP_KERNEL), 0);
+ KUNIT_EXPECT_TRUE(test, clone->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ consume_skb(clone);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ /* skb_split inside linear header & inside nonlinear page_pool frags */
+ skb_shinfo(skb)->nr_frags = 0;
+ skb->len = 0;
+ skb->data_len = 0;
+ skb_reset_tail_pointer(skb);
+ skb_put(skb, 32);
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len += 64;
+ skb->data_len = 64;
+ split = alloc_skb(32, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, split);
+ skb_split(skb, split, 16);
+ KUNIT_EXPECT_TRUE(test, split->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ skb_shinfo(split)->nr_frags = 0;
+ consume_skb(split);
+ skb->len = 0;
+ skb->data_len = 0;
+ skb_reset_tail_pointer(skb);
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len = 64;
+ skb->data_len = 64;
+
+ split = alloc_skb(0, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, split);
+ skb_split(skb, split, 32);
+ KUNIT_EXPECT_TRUE(test, split->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ consume_skb(split);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ /* skb_segment on pp_recycle=1 skb propagates pp_recycle to segments */
+ skb->protocol = htons(ETH_P_IP);
+ skb_reset_mac_header(skb);
+ skb_reset_network_header(skb);
+ skb_shinfo(skb)->gso_size = 16;
+ split = skb_segment(skb, NETIF_F_SG | NETIF_F_HW_CSUM);
+ KUNIT_ASSERT_FALSE(test, IS_ERR_OR_NULL(split));
+ KUNIT_EXPECT_TRUE(test, split->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 2);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ kfree_skb_list(split);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ /* skb_shift balances pp_ref_count across split, partial coalesce,
+ * multi-frag coalesce + partial next-frag shift, and full merge.
+ */
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 32);
+ skb_fill_netmem_desc(skb, 1, netmem, 64, 32);
+ page_pool_ref_page(page);
+ skb->len = 64;
+ skb->data_len = 64;
+ split = alloc_skb(0, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, split);
+ split->pp_recycle = 1;
+ KUNIT_EXPECT_EQ(test, skb_shift(split, skb, 16), 16);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 2);
+ KUNIT_EXPECT_EQ(test, skb_shift(split, skb, 8), 8);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 2);
+ KUNIT_EXPECT_EQ(test, skb_shift(split, skb, 24), 24);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 2);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ KUNIT_EXPECT_EQ(test, skb_shift(split, skb, 16), 16);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ skb_shinfo(split)->nr_frags = 0;
+ consume_skb(split);
+ page_pool_put_full_page(pool, page, false);
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len = 64;
+ skb->data_len = 64;
+
+ skb->pp_recycle = 0;
+ skb_frag_ref(skb, 0);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref + 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+ skb_frag_unref(skb, 0);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+
+ skb_shinfo(skb)->nr_frags = 0;
+ consume_skb(skb);
+ page_pool_put_full_page(pool, page, false);
+ page_pool_destroy(pool);
+
+ /* Order-1 compound page_pool page: tail netmem resolves to head &
+ * build_skb(head_frag=1, pp_recycle=1) skb_zerocopy uses pp_ref_count.
+ */
+ pp_params.order = 1;
+ pool = page_pool_create(&pp_params);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool);
+ page = page_pool_dev_alloc_pages(pool);
+ KUNIT_ASSERT_NOT_NULL(test, page);
+ base_ref = page_ref_count(page);
+ base_pp_ref = atomic_long_read(&page->pp_ref_count);
+
+ netmem = page_to_netmem(page + 1);
+ KUNIT_EXPECT_TRUE(test, napi_pp_get_page(netmem));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ KUNIT_EXPECT_TRUE(test, napi_pp_put_page(netmem));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ skb = build_skb(page_address(page), PAGE_SIZE);
+ KUNIT_ASSERT_NOT_NULL(test, skb);
+ skb->pp_recycle = 1;
+ skb_put(skb, 64);
+ page_pool_ref_page(page);
+
+ copy = alloc_skb(0, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, copy);
+ skb_put(copy, skb_tailroom(copy));
+ KUNIT_ASSERT_EQ(test, skb_zerocopy(copy, skb, skb->len, 0), 0);
+ KUNIT_EXPECT_TRUE(test, copy->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 2);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ consume_skb(copy);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ consume_skb(skb);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ page_pool_put_full_page(pool, page, false);
+ page_pool_destroy(pool);
+}
+#endif
+
+#if defined(CONFIG_PAGE_POOL) && defined(CONFIG_NET_DEVMEM)
+static void dummy_percpu_ref_release(struct percpu_ref *ref)
+{
+}
+
+static void netmem_ref_test_net_iov(struct kunit *test)
+{
+ struct net_devmem_dmabuf_binding binding = {};
+ struct page_pool_params pp_params = {
+ .order = 0,
+ .pool_size = 4,
+ .nid = NUMA_NO_NODE,
+ };
+ struct sk_buff *skb, *clone, *copy;
+ struct page_pool *pool;
+ struct net_iov niov = {};
+ netmem_ref netmem;
+ long base_ref;
+ int err;
+
+ err = percpu_ref_init(&binding.ref, dummy_percpu_ref_release,
+ PERCPU_REF_INIT_ATOMIC, GFP_KERNEL);
+ KUNIT_ASSERT_EQ(test, err, 0);
+
+ net_iov_init(&niov, &binding.area, NET_IOV_DMABUF);
+ netmem = net_iov_to_netmem(&niov);
+ base_ref = atomic_long_read(&binding.ref.data->count);
+
+ /* Non-PP net_iov: get/put_net_iov & get/put_netmem */
+ get_net_iov(&niov);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref + 1);
+ put_net_iov(&niov);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+
+ KUNIT_EXPECT_FALSE(test, napi_pp_get_page(netmem));
+ KUNIT_EXPECT_FALSE(test, napi_pp_put_page(netmem));
+
+ skb_netmem_ref(netmem, true);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref + 1);
+ skb_netmem_unref(netmem, true);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+
+ /* Attach net_iov to a page_pool to test PP net_iov behavior */
+ pool = page_pool_create(&pp_params);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool);
+ netmem_set_pp(netmem, pool);
+ netmem_or_pp_magic(netmem, PP_SIGNATURE);
+ atomic_long_set(&niov.desc.pp_ref_count, 2);
+
+ /* Layer 0 & 1: get_net_iov / get_netmem grab non-pp ref even on PP */
+ get_net_iov(&niov);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref + 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L);
+ put_net_iov(&niov);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+
+ /* Layer 2: napi_pp_get/put_page use pp_ref_count */
+ KUNIT_EXPECT_TRUE(test, napi_pp_get_page(netmem));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+ KUNIT_EXPECT_TRUE(test, napi_pp_put_page(netmem));
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L);
+
+ /* Layer 4: skb_frag_ref / skb_frag_unref on PP net_iov */
+ skb = alloc_skb(64, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, skb);
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len = 64;
+ skb->data_len = 64;
+
+ skb->pp_recycle = 1;
+ skb_frag_ref(skb, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+ skb_frag_unref(skb, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L);
+
+ /* pskb_copy on pp_recycle=1 net_iov skb keeps pp_ref_count */
+ copy = pskb_copy(skb, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, copy);
+ KUNIT_EXPECT_TRUE(test, copy->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+ consume_skb(copy);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L);
+
+ /* skb_zerocopy on pp_recycle=1 net_iov skb keeps pp_ref_count */
+ copy = alloc_skb(0, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, copy);
+ skb_put(copy, skb_tailroom(copy));
+ skb->head_frag = 1;
+ skb->unreadable = 1;
+ KUNIT_ASSERT_EQ(test, skb_zerocopy(copy, skb, skb->len, 0), 0);
+ skb->head_frag = 0;
+ KUNIT_EXPECT_TRUE(test, copy->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+ consume_skb(copy);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L);
+
+ /* skb_shift balances pp_ref_count on split and merge */
+ copy = alloc_skb(0, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, copy);
+ copy->pp_recycle = 1;
+ KUNIT_EXPECT_EQ(test, skb_shift(copy, skb, 16), 16);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+ KUNIT_EXPECT_EQ(test, skb_shift(copy, skb, 48), 48);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L);
+ skb_shinfo(copy)->nr_frags = 0;
+ consume_skb(copy);
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len = 64;
+ skb->data_len = 64;
+
+ /* Uncloning a cloned pp_recycle=1 net_iov skb keeps pp_ref_count */
+ clone = skb_clone(skb, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, clone);
+ KUNIT_ASSERT_EQ(test, pskb_expand_head(skb, 32, 0, GFP_KERNEL), 0);
+ KUNIT_EXPECT_TRUE(test, skb->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+ consume_skb(clone);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L);
+
+ skb->pp_recycle = 0;
+ skb_frag_ref(skb, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref + 1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L);
+ skb_frag_unref(skb, 0);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&binding.ref.data->count),
+ base_ref);
+
+ skb_shinfo(skb)->nr_frags = 0;
+ consume_skb(skb);
+ netmem_clear_pp_magic(netmem);
+ netmem_set_pp(netmem, NULL);
+ page_pool_destroy(pool);
+ percpu_ref_exit(&binding.ref);
+}
+
+static void netmem_ref_test_release_data(struct kunit *test)
+{
+ struct page_pool_params pp_params = {
+ .order = 0,
+ .pool_size = 4,
+ .nid = NUMA_NO_NODE,
+ };
+ struct sk_buff *skb, *clone1, *clone2, *ext;
+ struct net_devmem_dmabuf_binding binding = {};
+ struct net_iov niov = {};
+ struct page_pool *pool;
+ struct page *page;
+ netmem_ref netmem;
+ long base_pp_ref;
+ int base_ref;
+ int i;
+
+ pool = page_pool_create(&pp_params);
+ KUNIT_ASSERT_FALSE(test, IS_ERR(pool));
+ page = page_pool_dev_alloc_pages(pool);
+ KUNIT_ASSERT_NOT_NULL(test, page);
+ netmem = page_to_netmem(page);
+ page_pool_ref_netmem(netmem);
+
+ /* 3-way clone chain: only last clone drops pp_ref_count */
+ skb = alloc_skb(64, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, skb);
+ skb_put(skb, 32);
+ skb->pp_recycle = 1;
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len += 64;
+ skb->data_len = 64;
+ base_ref = page_ref_count(page);
+ base_pp_ref = atomic_long_read(&page->pp_ref_count);
+
+ page_pool_ref_netmem(netmem);
+ clone1 = skb_clone(skb, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, clone1);
+ clone2 = skb_clone(clone1, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, clone2);
+
+ /* Freeing head skb first leaves clone1/clone2 pp_recycle=1 intact */
+ consume_skb(skb);
+ KUNIT_EXPECT_TRUE(test, clone1->pp_recycle);
+ KUNIT_EXPECT_TRUE(test, clone2->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ consume_skb(clone2);
+ KUNIT_EXPECT_TRUE(test, clone1->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ consume_skb(clone1);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+
+ /* All 4 combinations of (unclone skb vs clone) x (free order) */
+ for (i = 0; i < 4; i++) {
+ skb = alloc_skb(64, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, skb);
+ skb_put(skb, 32);
+ skb->pp_recycle = 1;
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len += 64;
+ skb->data_len = 64;
+ page_pool_ref_netmem(netmem);
+
+ clone1 = skb_clone(skb, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, clone1);
+ KUNIT_ASSERT_EQ(test,
+ pskb_expand_head((i & 1) ? clone1 : skb,
+ 32, 0, GFP_KERNEL), 0);
+ KUNIT_EXPECT_TRUE(test, skb->pp_recycle);
+ KUNIT_EXPECT_TRUE(test, clone1->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 2);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+
+ if (i & 2) {
+ consume_skb(clone1);
+ KUNIT_EXPECT_EQ(test,
+ atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ consume_skb(skb);
+ } else {
+ consume_skb(skb);
+ KUNIT_EXPECT_EQ(test,
+ atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ consume_skb(clone1);
+ }
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ }
+
+ /* pskb_extract carving inside header & inside nonlinear frags */
+ skb = alloc_skb(64, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, skb);
+ skb_put(skb, 32);
+ skb->pp_recycle = 1;
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len += 64;
+ skb->data_len = 64;
+ page_pool_ref_netmem(netmem);
+
+ /* Carve inside header keeping frag -> pp_ref_count +1 until free */
+ ext = pskb_extract(skb, 16, skb->len - 16, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, ext);
+ KUNIT_EXPECT_TRUE(test, ext->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 2);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ consume_skb(ext);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+
+ /* Carve + trim + skb_condense ref/unrefs pp_ref_count cleanly */
+ ext = pskb_extract(skb, 16, 48, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, ext);
+ KUNIT_EXPECT_TRUE(test, ext->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ consume_skb(ext);
+
+ /* Carve inside nonlinear keeping frag -> pp_ref_count +1 until free */
+ ext = pskb_extract(skb, 48, skb->len - 48, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, ext);
+ KUNIT_EXPECT_TRUE(test, ext->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 2);
+ KUNIT_EXPECT_EQ(test, page_ref_count(page), base_ref);
+ consume_skb(ext);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+
+ /* skb_morph drops old skb data via skb_release_data */
+ page_pool_ref_netmem(netmem);
+ clone1 = alloc_skb(64, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, clone1);
+ clone1->pp_recycle = 1;
+ skb_fill_netmem_desc(clone1, 0, netmem, 0, 64);
+ clone1->len = 64;
+ clone1->data_len = 64;
+ skb_morph(clone1, skb);
+ KUNIT_EXPECT_TRUE(test, clone1->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&page->pp_ref_count),
+ base_pp_ref + 1);
+ consume_skb(clone1);
+ kfree_skb(skb);
+ page_pool_put_full_page(pool, page, false);
+ page_pool_put_full_page(pool, page, false);
+
+ /* Also verify pskb_extract on page_pool net_iov */
+ KUNIT_ASSERT_EQ(test,
+ percpu_ref_init(&binding.ref, dummy_percpu_ref_release,
+ PERCPU_REF_INIT_ATOMIC, GFP_KERNEL),
+ 0);
+ net_iov_init(&niov, &binding.area, NET_IOV_DMABUF);
+ netmem = net_iov_to_netmem(&niov);
+ netmem_set_pp(netmem, pool);
+ netmem_or_pp_magic(netmem, PP_SIGNATURE);
+ atomic_long_set(&niov.desc.pp_ref_count, 2);
+
+ skb = alloc_skb(64, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, skb);
+ skb_put(skb, 32);
+ skb->pp_recycle = 1;
+ skb_fill_netmem_desc(skb, 0, netmem, 0, 64);
+ skb->len += 64;
+ skb->data_len = 64;
+
+ ext = pskb_extract(skb, 48, skb->len - 48, GFP_KERNEL);
+ KUNIT_ASSERT_NOT_NULL(test, ext);
+ KUNIT_EXPECT_TRUE(test, ext->pp_recycle);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 3L);
+ consume_skb(ext);
+ KUNIT_EXPECT_EQ(test, atomic_long_read(&niov.desc.pp_ref_count), 2L);
+
+ skb_shinfo(skb)->nr_frags = 0;
+ consume_skb(s