/* SPDX-License-Identifier: BSD-3-Clause */ /* Copyright (c) 2021, Intel Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * 3. Neither the name of the Intel Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /** * @file iavf_txrx_iflib.c * @brief Tx/Rx hotpath implementation for the iflib driver * * Contains functions used to implement the Tx and Rx hotpaths of the iflib * driver implementation. */ #include "iavf_iflib.h" #include "iavf_txrx_common.h" #ifdef RSS #include #endif /* Local Prototypes */ static void iavf_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype); static int iavf_isc_txd_encap(void *arg, if_pkt_info_t pi); static void iavf_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); static int iavf_isc_txd_credits_update_hwb(void *arg, uint16_t txqid, bool clear); static int iavf_isc_txd_credits_update_dwb(void *arg, uint16_t txqid, bool clear); static void iavf_isc_rxd_refill(void *arg, if_rxd_update_t iru); static void iavf_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx); static int iavf_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget); static int iavf_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); /** * @var iavf_txrx_hwb * @brief iflib Tx/Rx operations for head write back * * iflib ops structure for when operating the device in head write back mode. */ struct if_txrx iavf_txrx_hwb = { iavf_isc_txd_encap, iavf_isc_txd_flush, iavf_isc_txd_credits_update_hwb, iavf_isc_rxd_available, iavf_isc_rxd_pkt_get, iavf_isc_rxd_refill, iavf_isc_rxd_flush, NULL }; /** * @var iavf_txrx_dwb * @brief iflib Tx/Rx operations for descriptor write back * * iflib ops structure for when operating the device in descriptor write back * mode. */ struct if_txrx iavf_txrx_dwb = { iavf_isc_txd_encap, iavf_isc_txd_flush, iavf_isc_txd_credits_update_dwb, iavf_isc_rxd_available, iavf_isc_rxd_pkt_get, iavf_isc_rxd_refill, iavf_isc_rxd_flush, NULL }; /** * iavf_is_tx_desc_done - Check if a Tx descriptor is ready * @txr: the Tx ring to check in * @idx: ring index to check * * @returns true if the descriptor has been written back by hardware, and * false otherwise. */ static bool iavf_is_tx_desc_done(struct tx_ring *txr, int idx) { return (((txr->tx_base[idx].cmd_type_offset_bsz >> IAVF_TXD_QW1_DTYPE_SHIFT) & IAVF_TXD_QW1_DTYPE_MASK) == IAVF_TX_DESC_DTYPE_DESC_DONE); } /** * iavf_tso_detect_sparse - detect TSO packets with too many segments * @segs: packet segments array * @nsegs: number of packet segments * @pi: packet information * * Hardware only transmits packets with a maximum of 8 descriptors. For TSO * packets, hardware needs to be able to build the split packets using 8 or * fewer descriptors. Additionally, the header must be contained within at * most 3 descriptors. * * To verify this, we walk the headers to find out how many descriptors the * headers require (usually 1). Then we ensure that, for each TSO segment, its * data plus the headers are contained within 8 or fewer descriptors. * * @returns zero if the packet is valid, one otherwise. */ static int iavf_tso_detect_sparse(bus_dma_segment_t *segs, int nsegs, if_pkt_info_t pi) { int count, curseg, i, hlen, segsz, seglen, tsolen; if (nsegs <= IAVF_MAX_TX_SEGS-2) return (0); segsz = pi->ipi_tso_segsz; curseg = count = 0; hlen = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; tsolen = pi->ipi_len - hlen; i = 0; curseg = segs[0].ds_len; while (hlen > 0) { count++; if (count > IAVF_MAX_TX_SEGS - 2) return (1); if (curseg == 0) { i++; if (__predict_false(i == nsegs)) return (1); curseg = segs[i].ds_len; } seglen = min(curseg, hlen); curseg -= seglen; hlen -= seglen; } while (tsolen > 0) { segsz = pi->ipi_tso_segsz; while (segsz > 0 && tsolen != 0) { count++; if (count > IAVF_MAX_TX_SEGS - 2) { return (1); } if (curseg == 0) { i++; if (__predict_false(i == nsegs)) { return (1); } curseg = segs[i].ds_len; } seglen = min(curseg, segsz); segsz -= seglen; curseg -= seglen; tsolen -= seglen; } count = 0; } return (0); } /** * iavf_tx_setup_offload - Setup Tx offload parameters * @que: pointer to the Tx queue * @pi: Tx packet info * @cmd: pointer to command descriptor value * @off: pointer to offset descriptor value * * Based on packet type and Tx offloads requested, sets up the command and * offset values for a Tx descriptor to enable the requested offloads. */ static void iavf_tx_setup_offload(struct iavf_tx_queue *que __unused, if_pkt_info_t pi, u32 *cmd, u32 *off) { switch (pi->ipi_etype) { #ifdef INET case ETHERTYPE_IP: if (pi->ipi_csum_flags & IAVF_CSUM_IPV4) *cmd |= IAVF_TX_DESC_CMD_IIPT_IPV4_CSUM; else *cmd |= IAVF_TX_DESC_CMD_IIPT_IPV4; break; #endif #ifdef INET6 case ETHERTYPE_IPV6: *cmd |= IAVF_TX_DESC_CMD_IIPT_IPV6; break; #endif default: break; } *off |= (pi->ipi_ehdrlen >> 1) << IAVF_TX_DESC_LENGTH_MACLEN_SHIFT; *off |= (pi->ipi_ip_hlen >> 2) << IAVF_TX_DESC_LENGTH_IPLEN_SHIFT; switch (pi->ipi_ipproto) { case IPPROTO_TCP: if (pi->ipi_csum_flags & IAVF_CSUM_TCP) { *cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_TCP; *off |= (pi->ipi_tcp_hlen >> 2) << IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; /* Check for NO_HEAD MDD event */ MPASS(pi->ipi_tcp_hlen != 0); } break; case IPPROTO_UDP: if (pi->ipi_csum_flags & IAVF_CSUM_UDP) { *cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_UDP; *off |= (sizeof(struct udphdr) >> 2) << IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; } break; case IPPROTO_SCTP: if (pi->ipi_csum_flags & IAVF_CSUM_SCTP) { *cmd |= IAVF_TX_DESC_CMD_L4T_EOFT_SCTP; *off |= (sizeof(struct sctphdr) >> 2) << IAVF_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; } /* Fall Thru */ default: break; } } /** * iavf_tso_setup - Setup TSO context descriptor * @txr: the Tx ring to process * @pi: packet info structure * * Enable hardware segmentation offload (TSO) for a given packet by creating * a context descriptor with the necessary details for offloading. * * @returns the new ring index to use for the data descriptor. */ static int iavf_tso_setup(struct tx_ring *txr, if_pkt_info_t pi) { if_softc_ctx_t scctx; struct iavf_tx_context_desc *TXD; u32 cmd, mss, type, tsolen; int idx, total_hdr_len; u64 type_cmd_tso_mss; idx = pi->ipi_pidx; TXD = (struct iavf_tx_context_desc *) &txr->tx_base[idx]; total_hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; tsolen = pi->ipi_len - total_hdr_len; scctx = txr->que->vsi->shared; type = IAVF_TX_DESC_DTYPE_CONTEXT; cmd = IAVF_TX_CTX_DESC_TSO; /* * TSO MSS must not be less than 64; this prevents a * BAD_LSO_MSS MDD event when the MSS is too small. */ if (pi->ipi_tso_segsz < IAVF_MIN_TSO_MSS) { txr->mss_too_small++; pi->ipi_tso_segsz = IAVF_MIN_TSO_MSS; } mss = pi->ipi_tso_segsz; /* Check for BAD_LS0_MSS MDD event (mss too large) */ MPASS(mss <= IAVF_MAX_TSO_MSS); /* Check for NO_HEAD MDD event (header lengths are 0) */ MPASS(pi->ipi_ehdrlen != 0); MPASS(pi->ipi_ip_hlen != 0); /* Partial check for BAD_LSO_LEN MDD event */ MPASS(tsolen != 0); /* Partial check for WRONG_SIZE MDD event (during TSO) */ MPASS(total_hdr_len + mss <= IAVF_MAX_FRAME); type_cmd_tso_mss = ((u64)type << IAVF_TXD_CTX_QW1_DTYPE_SHIFT) | ((u64)cmd << IAVF_TXD_CTX_QW1_CMD_SHIFT) | ((u64)tsolen << IAVF_TXD_CTX_QW1_TSO_LEN_SHIFT) | ((u64)mss << IAVF_TXD_CTX_QW1_MSS_SHIFT); TXD->type_cmd_tso_mss = htole64(type_cmd_tso_mss); TXD->tunneling_params = htole32(0); txr->que->tso++; return ((idx + 1) & (scctx->isc_ntxd[0]-1)); } #define IAVF_TXD_CMD (IAVF_TX_DESC_CMD_EOP | IAVF_TX_DESC_CMD_RS) /** * iavf_isc_txd_encap - Encapsulate a Tx packet into descriptors * @arg: void pointer to the VSI structure * @pi: packet info to encapsulate * * This routine maps the mbufs to tx descriptors, allowing the * TX engine to transmit the packets. * * @returns 0 on success, positive on failure */ static int iavf_isc_txd_encap(void *arg, if_pkt_info_t pi) { struct iavf_vsi *vsi = arg; if_softc_ctx_t scctx = vsi->shared; struct iavf_tx_queue *que = &vsi->tx_queues[pi->ipi_qsidx]; struct tx_ring *txr = &que->txr; int nsegs = pi->ipi_nsegs; bus_dma_segment_t *segs = pi->ipi_segs; struct iavf_tx_desc *txd = NULL; int i, j, mask, pidx_last; u32 cmd, off, tx_intr; if (__predict_false(pi->ipi_len < IAVF_MIN_FRAME)) { que->pkt_too_small++; return (EINVAL); } cmd = off = 0; i = pi->ipi_pidx; tx_intr = (pi->ipi_flags & IPI_TX_INTR); /* Set up the TSO/CSUM offload */ if (pi->ipi_csum_flags & CSUM_OFFLOAD) { /* Set up the TSO context descriptor if required */ if (pi->ipi_csum_flags & CSUM_TSO) { /* Prevent MAX_BUFF MDD event (for TSO) */ if (iavf_tso_detect_sparse(segs, nsegs, pi)) return (EFBIG); i = iavf_tso_setup(txr, pi); } iavf_tx_setup_offload(que, pi, &cmd, &off); } if (pi->ipi_mflags & M_VLANTAG) cmd |= IAVF_TX_DESC_CMD_IL2TAG1; cmd |= IAVF_TX_DESC_CMD_ICRC; mask = scctx->isc_ntxd[0] - 1; /* Check for WRONG_SIZE MDD event */ MPASS(pi->ipi_len >= IAVF_MIN_FRAME); #ifdef INVARIANTS if (!(pi->ipi_csum_flags & CSUM_TSO)) MPASS(pi->ipi_len <= IAVF_MAX_FRAME); #endif for (j = 0; j < nsegs; j++) { bus_size_t seglen; txd = &txr->tx_base[i]; seglen = segs[j].ds_len; /* Check for ZERO_BSIZE MDD event */ MPASS(seglen != 0); txd->buffer_addr = htole64(segs[j].ds_addr); txd->cmd_type_offset_bsz = htole64(IAVF_TX_DESC_DTYPE_DATA | ((u64)cmd << IAVF_TXD_QW1_CMD_SHIFT) | ((u64)off << IAVF_TXD_QW1_OFFSET_SHIFT) | ((u64)seglen << IAVF_TXD_QW1_TX_BUF_SZ_SHIFT) | ((u64)htole16(pi->ipi_vtag) << IAVF_TXD_QW1_L2TAG1_SHIFT)); txr->tx_bytes += seglen; pidx_last = i; i = (i+1) & mask; } /* Set the last descriptor for report */ txd->cmd_type_offset_bsz |= htole64(((u64)IAVF_TXD_CMD << IAVF_TXD_QW1_CMD_SHIFT)); /* Add to report status array (if using TX interrupts) */ if (!vsi->enable_head_writeback && tx_intr) { txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & mask; MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); } pi->ipi_new_pidx = i; ++txr->tx_packets; return (0); } /** * iavf_isc_txd_flush - Flush Tx ring * @arg: void pointer to the VSI * @txqid: the Tx queue to flush * @pidx: the ring index to flush to * * Advance the Transmit Descriptor Tail (Tdt), this tells the * hardware that this frame is available to transmit. */ static void iavf_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) { struct iavf_vsi *vsi = arg; struct tx_ring *txr = &vsi->tx_queues[txqid].txr; /* Check for ENDLESS_TX MDD event */ MPASS(pidx < vsi->shared->isc_ntxd[0]); wr32(vsi->hw, txr->tail, pidx); } /** * iavf_init_tx_ring - Initialize queue Tx ring * @vsi: pointer to the VSI * @que: pointer to queue to initialize * * (Re)Initialize a queue transmit ring by clearing its memory. */ void iavf_init_tx_ring(struct iavf_vsi *vsi, struct iavf_tx_queue *que) { struct tx_ring *txr = &que->txr; /* Clear the old ring contents */ bzero((void *)txr->tx_base, (sizeof(struct iavf_tx_desc)) * (vsi->shared->isc_ntxd[0] + (vsi->enable_head_writeback ? 1 : 0))); wr32(vsi->hw, txr->tail, 0); } /** * iavf_get_tx_head - Get the index of the head of a ring * @que: queue to read * * Retrieve the value from the location the HW records its HEAD index * * @returns the index of the HW head of the Tx queue */ static inline u32 iavf_get_tx_head(struct iavf_tx_queue *que) { if_softc_ctx_t scctx = que->vsi->shared; struct tx_ring *txr = &que->txr; void *head = &txr->tx_base[scctx->isc_ntxd[0]]; return LE32_TO_CPU(*(volatile __le32 *)head); } /** * iavf_isc_txd_credits_update_hwb - Update Tx ring credits * @arg: void pointer to the VSI * @qid: the queue id to update * @clear: whether to update or only report current status * * Checks the number of packets in the queue that could be cleaned up. * * if clear is true, the iflib stack has cleaned the packets and is * notifying the driver to update its processed ring pointer. * * @returns the number of packets in the ring that can be cleaned. * * @remark this function is intended for the head write back mode. */ static int iavf_isc_txd_credits_update_hwb(void *arg, uint16_t qid, bool clear) { struct iavf_vsi *vsi = arg; if_softc_ctx_t scctx = vsi->shared; struct iavf_tx_queue *que = &vsi->tx_queues[qid]; struct tx_ring *txr = &que->txr; int head, credits; /* Get the Head WB value */ head = iavf_get_tx_head(que); credits = head - txr->tx_cidx_processed; if (credits < 0) credits += scctx->isc_ntxd[0]; if (clear) txr->tx_cidx_processed = head; return (credits); } /** * iavf_isc_txd_credits_update_dwb - Update Tx ring credits * @arg: void pointer to the VSI * @txqid: the queue id to update * @clear: whether to update or only report current status * * Checks the number of packets in the queue that could be cleaned up. * * if clear is true, the iflib stack has cleaned the packets and is * notifying the driver to update its processed ring pointer. * * @returns the number of packets in the ring that can be cleaned. * * @remark this function is intended for the descriptor write back mode. */ static int iavf_isc_txd_credits_update_dwb(void *arg, uint16_t txqid, bool clear) { struct iavf_vsi *vsi = arg; struct iavf_tx_queue *tx_que = &vsi->tx_queues[txqid]; if_softc_ctx_t scctx = vsi->shared; struct tx_ring *txr = &tx_que->txr; qidx_t processed = 0; qidx_t cur, prev, ntxd, rs_cidx; int32_t delta; bool is_done; rs_cidx = txr->tx_rs_cidx; if (rs_cidx == txr->tx_rs_pidx) return (0); cur = txr->tx_rsq[rs_cidx]; MPASS(cur != QIDX_INVALID); is_done = iavf_is_tx_desc_done(txr, cur); if (!is_done) return (0); /* If clear is false just let caller know that there * are descriptors to reclaim */ if (!clear) return (1); prev = txr->tx_cidx_processed; ntxd = scctx->isc_ntxd[0]; do { MPASS(prev != cur); delta = (int32_t)cur - (int32_t)prev; if (delta < 0) delta += ntxd; MPASS(delta > 0); processed += delta; prev = cur; rs_cidx = (rs_cidx + 1) & (ntxd-1); if (rs_cidx == txr->tx_rs_pidx) break; cur = txr->tx_rsq[rs_cidx]; MPASS(cur != QIDX_INVALID); is_done = iavf_is_tx_desc_done(txr, cur); } while (is_done); txr->tx_rs_cidx = rs_cidx; txr->tx_cidx_processed = prev; return (processed); } /** * iavf_isc_rxd_refill - Prepare descriptors for re-use * @arg: void pointer to the VSI * @iru: the Rx descriptor update structure * * Update Rx descriptors for a given queue so that they can be re-used by * hardware for future packets. */ static void iavf_isc_rxd_refill(void *arg, if_rxd_update_t iru) { struct iavf_vsi *vsi = arg; if_softc_ctx_t scctx = vsi->shared; struct rx_ring *rxr = &((vsi->rx_queues[iru->iru_qsidx]).rxr); uint64_t *paddrs; uint16_t next_pidx, pidx; uint16_t count; int i; paddrs = iru->iru_paddrs; pidx = iru->iru_pidx; count = iru->iru_count; for (i = 0, next_pidx = pidx; i < count; i++) { rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]); if (++next_pidx == scctx->isc_nrxd[0]) next_pidx = 0; } } /** * iavf_isc_rxd_flush - Notify hardware of new Rx descriptors * @arg: void pointer to the VSI * @rxqid: Rx queue to update * @flid: unused parameter * @pidx: ring index to update to * * Updates the tail pointer of the Rx ring, notifying hardware of new * descriptors available for receiving packets. */ static void iavf_isc_rxd_flush(void * arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) { struct iavf_vsi *vsi = arg; struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; wr32(vsi->hw, rxr->tail, pidx); } /** * iavf_isc_rxd_available - Calculate number of available Rx descriptors * @arg: void pointer to the VSI * @rxqid: Rx queue to check * @idx: starting index to check from * @budget: maximum Rx budget * * Determines how many packets are ready to be processed in the Rx queue, up * to the specified budget. * * @returns the number of packets ready to be processed, up to the budget. */ static int iavf_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) { struct iavf_vsi *vsi = arg; struct rx_ring *rxr = &vsi->rx_queues[rxqid].rxr; union iavf_rx_desc *rxd; u64 qword; uint32_t status; int cnt, i, nrxd; nrxd = vsi->shared->isc_nrxd[0]; for (cnt = 0, i = idx; cnt < nrxd - 1 && cnt <= budget;) { rxd = &rxr->rx_base[i]; qword = le64toh(rxd->wb.qword1.status_error_len); status = (qword & IAVF_RXD_QW1_STATUS_MASK) >> IAVF_RXD_QW1_STATUS_SHIFT; if ((status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT)) == 0) break; if (++i == nrxd) i = 0; if (status & (1 << IAVF_RX_DESC_STATUS_EOF_SHIFT)) cnt++; } return (cnt); } /** * iavf_isc_rxd_pkt_get - Decapsulate packet from Rx descriptors * @arg: void pointer to the VSI * @ri: packet info structure * * Read packet data from the Rx ring descriptors and fill in the packet info * structure so that the iflib stack can process the packet. * * @remark this routine executes in ithread context. * * @returns zero success, or EBADMSG if the packet is corrupted. */ static int iavf_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) { struct iavf_vsi *vsi = arg; struct iavf_rx_queue *que = &vsi->rx_queues[ri->iri_qsidx]; struct rx_ring *rxr = &que->rxr; union iavf_rx_desc *cur; u32 status, error; u16 plen, vtag; u64 qword; u8 ptype; bool eop; int i, cidx; cidx = ri->iri_cidx; i = 0; do { /* 5 descriptor receive limit */ MPASS(i < IAVF_MAX_RX_SEGS); cur = &rxr->rx_base[cidx]; qword = le64toh(cur->wb.qword1.status_error_len); status = (qword & IAVF_RXD_QW1_STATUS_MASK) >> IAVF_RXD_QW1_STATUS_SHIFT; error = (qword & IAVF_RXD_QW1_ERROR_MASK) >> IAVF_RXD_QW1_ERROR_SHIFT; plen = (qword & IAVF_RXD_QW1_LENGTH_PBUF_MASK) >> IAVF_RXD_QW1_LENGTH_PBUF_SHIFT; ptype = (qword & IAVF_RXD_QW1_PTYPE_MASK) >> IAVF_RXD_QW1_PTYPE_SHIFT; /* we should never be called without a valid descriptor */ MPASS((status & (1 << IAVF_RX_DESC_STATUS_DD_SHIFT)) != 0); ri->iri_len += plen; rxr->rx_bytes += plen; cur->wb.qword1.status_error_len = 0; eop = (status & (1 << IAVF_RX_DESC_STATUS_EOF_SHIFT)); if (status & (1 << IAVF_RX_DESC_STATUS_L2TAG1P_SHIFT)) vtag = le16toh(cur->wb.qword0.lo_dword.l2tag1); else vtag = 0; /* ** Make sure bad packets are discarded, ** note that only EOP descriptor has valid ** error results. */ if (eop && (error & (1 << IAVF_RX_DESC_ERROR_RXE_SHIFT))) { rxr->desc_errs++; return (EBADMSG); } ri->iri_frags[i].irf_flid = 0; ri->iri_frags[i].irf_idx = cidx; ri->iri_frags[i].irf_len = plen; if (++cidx == vsi->shared->isc_nrxd[0]) cidx = 0; i++; } while (!eop); /* capture data for dynamic ITR adjustment */ rxr->packets++; rxr->rx_packets++; if ((if_getcapenable(vsi->ifp) & IFCAP_RXCSUM) != 0) iavf_rx_checksum(ri, status, error, ptype); ri->iri_flowid = le32toh(cur->wb.qword0.hi_dword.rss); ri->iri_rsstype = iavf_ptype_to_hash(ptype); ri->iri_vtag = vtag; ri->iri_nfrags = i; if (vtag) ri->iri_flags |= M_VLANTAG; return (0); } /** * iavf_rx_checksum - Handle Rx hardware checksum indication * @ri: Rx packet info structure * @status: status from Rx descriptor * @error: error from Rx descriptor * @ptype: packet type * * Verify that the hardware indicated that the checksum is valid. * Inform the stack about the status of checksum so that stack * doesn't spend time verifying the checksum. */ static void iavf_rx_checksum(if_rxd_info_t ri, u32 status, u32 error, u8 ptype) { struct iavf_rx_ptype_decoded decoded; ri->iri_csum_flags = 0; /* No L3 or L4 checksum was calculated */ if (!(status & (1 << IAVF_RX_DESC_STATUS_L3L4P_SHIFT))) return; decoded = decode_rx_desc_ptype(ptype); /* IPv6 with extension headers likely have bad csum */ if (decoded.outer_ip == IAVF_RX_PTYPE_OUTER_IP && decoded.outer_ip_ver == IAVF_RX_PTYPE_OUTER_IPV6) { if (status & (1 << IAVF_RX_DESC_STATUS_IPV6EXADD_SHIFT)) { ri->iri_csum_flags = 0; return; } } ri->iri_csum_flags |= CSUM_L3_CALC; /* IPv4 checksum error */ if (error & (1 << IAVF_RX_DESC_ERROR_IPE_SHIFT)) return; ri->iri_csum_flags |= CSUM_L3_VALID; ri->iri_csum_flags |= CSUM_L4_CALC; /* L4 checksum error */ if (error & (1 << IAVF_RX_DESC_ERROR_L4E_SHIFT)) return; ri->iri_csum_flags |= CSUM_L4_VALID; ri->iri_csum_data |= htons(0xffff); }