ixgbevf: multi-buffer AF_XDP Tx

Transmitting multi-buffer AF_XDP packets is not very straightforward
given HW limitations in ixgbevf, namely that the first data descriptor
must contain the length of the whole packet.

Use private data of an sqe to store the length of an unfinished packet so
far and the first descriptor index. Once EoP zero-copy descriptor is
processed, write the accumulated length into the saved first descriptor.

Signed-off-by: Larysa Zaremba <larysa.zaremba@intel.com>

Author: walking-machine

drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

@@ -4314,7 +4314,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 			    NETIF_F_HW_VLAN_CTAG_TX;
 
 	netdev->priv_flags |= IFF_UNICAST_FLT;
-	libeth_xdp_set_features_noredir(netdev, NULL, 1, NULL);
+	libeth_xdp_set_features_noredir(netdev, NULL, IXGBEVF_XSK_MAX_ZC_FRAGS);
 
 	/* MTU range: 68 - 1504 or 9710 */
 	netdev->min_mtu = ETH_MIN_MTU;

drivers/net/ethernet/intel/ixgbevf/ixgbevf_txrx_lib.h

@@ -30,6 +30,9 @@ static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush)
 	xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1;
 	xdp_ring->cached_ntu = xdp_ring->next_to_use;
 
+	/* In case the packet was interrupted, discard it */
+	xdp_ring->xdp_sqes[ltu].priv = 0;
+
 	/* Finish descriptor writes before bumping tail */
 	wmb();
 	ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use);

drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c

@@ -133,24 +133,73 @@ void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring)
 	}
 }
 
+struct ixgbevf_zc_sqe_priv {
+	u32 first_desc;
+	u32 len;
+};
+
+static_assert(sizeof(struct ixgbevf_zc_sqe_priv) <=
+	      sizeof(typeof_member(struct libeth_sqe, priv)));
+
 static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i,
 				  const struct libeth_xdpsq *sq, u64 priv)
 {
-	union ixgbe_adv_tx_desc *tx_desc =
-		&((union ixgbe_adv_tx_desc *)sq->descs)[i];
+	union ixgbe_adv_tx_desc *descs = sq->descs, *tx_desc = &descs[i];
+	u32 ltu = (i ? : sq->count) - 1;
 
 	u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA |
 		       IXGBE_ADVTXD_DCMD_DEXT |
 		       IXGBE_ADVTXD_DCMD_IFCS |
-		       IXGBE_TXD_CMD_EOP |
 		       desc.len;
 
-	tx_desc->read.olinfo_status =
-		cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) |
-			    IXGBE_ADVTXD_CC);
-
 	tx_desc->read.buffer_addr = cpu_to_le64(desc.addr);
-	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
+
+	if (likely((desc.flags & LIBETH_XDP_TX_LAST) && !sq->sqes[ltu].priv)) {
+		tx_desc->read.olinfo_status =
+			cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) |
+				    IXGBE_ADVTXD_CC);
+		tx_desc->read.cmd_type_len =
+			cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP);
+		return;
+	}
+
+	/* No previous packet */
+	if (!sq->sqes[ltu].priv) {
+		struct ixgbevf_zc_sqe_priv *sqe_priv =
+						(void *)&sq->sqes[i].priv;
+
+		sqe_priv->first_desc = i;
+		sqe_priv->len = desc.len;
+
+		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
+
+		return;
+	}
+
+	if (sq->sqes[ltu].priv) {
+		struct ixgbevf_zc_sqe_priv *sqe_priv =
+						(void *)&sq->sqes[i].priv;
+
+		sq->sqes[i].priv = sq->sqes[ltu].priv;
+		sq->sqes[ltu].priv = 0;
+		sqe_priv->len += desc.len;
+
+		if (desc.flags & LIBETH_XDP_TX_LAST) {
+			union ixgbe_adv_tx_desc *first_desc =
+						&descs[sqe_priv->first_desc];
+
+			first_desc->read.olinfo_status =
+				cpu_to_le32((sqe_priv->len <<
+					     IXGBE_ADVTXD_PAYLEN_SHIFT) |
+					    IXGBE_ADVTXD_CC);
+			tx_desc->read.cmd_type_len =
+				cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP);
+			cmd_type |= IXGBE_TXD_CMD_EOP;
+			sq->sqes[i].priv = 0;
+		}
+
+		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
+	}
 }
 
 LIBETH_XDP_DEFINE_START();

drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.h

@@ -6,6 +6,7 @@
 
 /* Process compeletions as soon as possible */
 #define IXGBEVF_XSK_TX_CLEAN_THRESH(r)	((r)->count - 1)
+#define IXGBEVF_XSK_MAX_ZC_FRAGS	18
 
 int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter,
 			   struct xsk_buff_pool *pool, u16 qid);