/*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2001-2024, Intel Corporation * Copyright (c) 2026 Kevin Bowling * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "if_em.h" #include #define IGBV_82576_QUEUES 2 #define IGBV_I350_QUEUES 1 #define IGBV_MAX_MAC_FILTERS 3 #define IGBV_QUEUE_DISABLE_BUSY_RETRIES 10 #define IGBV_QUEUE_DISABLE_DELAY_US 10 #define IGBV_QUEUE_DISABLE_PAUSE (100 * SBT_1US) #define IGBV_QUEUE_DISABLE_RETRIES 20 #define IGBV_QUEUE_SANITIZE_ATTEMPTS 3 #define IGBV_VLAN_RETRY_BATCH 4 #define IGBV_VLAN_RETRY_WINDOW (8 * SBT_1S) static const struct timeval igbv_queue_log_interval = { 2, 0 }; static const sbintime_t igbv_queue_retry_delay[] = { 100 * SBT_1MS, 500 * SBT_1MS, }; _Static_assert(nitems(igbv_queue_retry_delay) + 1 == IGBV_QUEUE_SANITIZE_ATTEMPTS, "missing queue retry delay"); struct igb_vf_uc_addr_list { struct e1000_softc *sc; u8 addrs[IGBV_MAX_MAC_FILTERS][ETHER_ADDR_LEN]; }; static bool igbv_tx_pending(struct e1000_softc *); static bool igbv_vlan_retry_pending(const struct e1000_softc *); static void igbv_vlan_retry_tick(struct e1000_softc *); static void igbv_queue_retry_callout(void *arg) { struct e1000_softc *sc; if_t ifp; sc = arg; if (atomic_readandclear_32(&sc->vf_queue_retry_pending) == 0) return; ifp = iflib_get_ifp(sc->ctx); if ((if_getflags(ifp) & IFF_UP) == 0) { atomic_set_32(&sc->vf_queue_retry_new_epoch, 1); return; } iflib_request_reset(sc->ctx); iflib_admin_intr_deferred(sc->ctx); } void igbv_queue_retry_detach(struct e1000_softc *sc) { if (!sc->vf_queue_retry_initialized) return; atomic_readandclear_32(&sc->vf_queue_retry_pending); callout_drain(&sc->vf_queue_retry); sc->vf_queue_retry_initialized = false; } void igbv_queue_retry_stop(struct e1000_softc *sc) { if (!sc->vf_queue_retry_initialized) return; if (atomic_readandclear_32(&sc->vf_queue_retry_pending) != 0) atomic_set_32(&sc->vf_queue_retry_new_epoch, 1); callout_stop(&sc->vf_queue_retry); } void igbv_queue_retry_prepare(struct e1000_softc *sc) { bool new_epoch; new_epoch = atomic_readandclear_32(&sc->vf_queue_retry_new_epoch) != 0; if (!sc->vf_queue_gave_up && !new_epoch) return; sc->vf_queue_failures = 0; sc->vf_queue_gave_up = false; } static void igbv_queue_retry_succeeded(struct e1000_softc *sc) { atomic_readandclear_32(&sc->vf_queue_retry_pending); atomic_readandclear_32(&sc->vf_queue_retry_new_epoch); if (sc->vf_queue_retry_initialized) callout_stop(&sc->vf_queue_retry); sc->vf_queue_failures = 0; sc->vf_queue_gave_up = false; } void igbv_queue_retry_failed(if_ctx_t ctx) { struct e1000_softc *sc; sbintime_t delay; sc = iflib_get_softc(ctx); KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) sc->vf_queue_failures++; if (sc->vf_queue_failures < IGBV_QUEUE_SANITIZE_ATTEMPTS) { delay = igbv_queue_retry_delay[sc->vf_queue_failures - 1]; atomic_set_32(&sc->vf_queue_retry_pending, 1); callout_reset_sbt(&sc->vf_queue_retry, delay, 0, igbv_queue_retry_callout, sc, C_PREL(1)); } else if (!sc->vf_queue_gave_up) { atomic_readandclear_32(&sc->vf_queue_retry_pending); callout_stop(&sc->vf_queue_retry); sc->vf_queue_gave_up = true; device_printf(sc->dev, "retained VF queues remained active after %u attempts; " "interface left down; toggle it down/up to retry\n", sc->vf_queue_failures); } iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); iflib_admin_intr_deferred(ctx); } void igbv_vlan_retry_add(struct e1000_softc *sc, u16 vid) { bool pending; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); pending = igbv_vlan_retry_pending(sc); sc->vf_vfta_retry[vid >> 5] |= 1U << (vid & 0x1f); /* Bound the whole batch from its first failure, not each new VID. */ if (!pending) sc->vf_vlan_retry_deadline = getsbinuptime() + IGBV_VLAN_RETRY_WINDOW; } void igbv_vlan_retry_clear(struct e1000_softc *sc, u16 vid) { KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); sc->vf_vfta_retry[vid >> 5] &= ~(1U << (vid & 0x1f)); } static bool igbv_vlan_retry_pending(const struct e1000_softc *sc) { int i; for (i = 0; i < EM_VFTA_SIZE; i++) if (sc->vf_vfta_retry[i] != 0) return (true); return (false); } static void igbv_vlan_retry_tick(struct e1000_softc *sc) { u32 bit; u16 vid; int attempts, i, remaining; if (!igbv_vlan_retry_pending(sc)) { sc->vf_vlan_retry_deadline = 0; return; } if (getsbinuptime() >= sc->vf_vlan_retry_deadline) { remaining = 0; for (i = 0; i < EM_VFTA_SIZE; i++) remaining += bitcount32(sc->vf_vfta_retry[i]); memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); sc->vf_vlan_retry_deadline = 0; device_printf(sc->dev, "VF VLAN restore retries exhausted for %d VIDs\n", remaining); return; } /* * The mailbox NACK does not distinguish a transient PF rate limit * from permanent VLVF exhaustion. Retry at the PF's sustained * allowance, but bound the entire recovery window so ENOSPC cannot * create a permanent mailbox poller. */ for (attempts = 0, i = 0; attempts < IGBV_VLAN_RETRY_BATCH && i < 4096; i++) { vid = sc->vf_vlan_retry_cursor; sc->vf_vlan_retry_cursor = (vid + 1) & 0xfff; bit = 1U << (vid & 0x1f); if ((sc->vf_vfta_retry[vid >> 5] & bit) == 0) continue; attempts++; if ((sc->shadow_vfta[vid >> 5] & bit) == 0 || e1000_vfta_set_vf(&sc->hw, vid, true) == E1000_SUCCESS) sc->vf_vfta_retry[vid >> 5] &= ~bit; } if (!igbv_vlan_retry_pending(sc)) sc->vf_vlan_retry_deadline = 0; } int igbv_if_attach_pre(if_ctx_t ctx) { struct e1000_softc *sc; device_t dev; int error; dev = iflib_get_dev(ctx); if (pci_msix_count(dev) < 2) { device_printf(dev, "VF operation requires two MSI-X vectors\n"); return (ENXIO); } error = em_if_attach_pre(ctx); if (error != 0) return (error); sc = iflib_get_softc(ctx); callout_init(&sc->vf_queue_retry, 1); sc->vf_queue_retry_initialized = true; KASSERT(sc->vf_ifp && (iflib_get_sctx(ctx)->isc_flags & IFLIB_IS_VF) != 0, ("%s: igbv attached without VF policy", __func__)); return (0); } int igbv_if_attach_post(if_ctx_t ctx) { struct e1000_softc *sc; int error; sc = iflib_get_softc(ctx); KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); if (sc->intr_type != IFLIB_INTR_MSIX) { device_printf(sc->dev, "VF operation requires MSI-X\n"); return (ENXIO); } error = em_if_attach_post(ctx); if (error != 0) return (error); /* * Attach failures can leave the device sysctl tree registered when * hw.bus.disable_failed_devices is set. Do not publish handlers with * softc arguments until iflib has successfully allocated MSI-X. */ em_add_device_sysctls(sc); return (0); } int igbv_if_media_change(if_ctx_t ctx __unused) { return (EOPNOTSUPP); } void igbv_if_update_admin_status(if_ctx_t ctx) { struct e1000_softc *sc; struct e1000_hw *hw; device_t dev; bool link_check, timer_tick; sc = iflib_get_softc(ctx); hw = &sc->hw; dev = iflib_get_dev(ctx); KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); /* * iflib's init callback cannot report failure and marks the interface * running after it returns. Complete the failed-init transition from * this deferred task, after iflib has set its driver flags. */ if (!sc->vf_queues_sanitized) { igbv_if_intr_disable(ctx); if_setdrvflagbits(iflib_get_ifp(ctx), IFF_DRV_OACTIVE, IFF_DRV_RUNNING); return; } if (!sc->vf_reset_pending && atomic_readandclear_32(&sc->promisc_pending) != 0) (void)em_if_set_promisc_impl(ctx, if_getflags(iflib_get_ifp(ctx))); if (e1000_check_for_link(hw) != E1000_SUCCESS && !sc->vf_reset_pending) { sc->vf_reset_pending = true; iflib_request_reset(ctx); iflib_admin_intr_deferred(ctx); } link_check = !hw->mac.get_link_status; if (link_check && (sc->link_state == EM_LINK_STATE_DOWN || sc->link_state == EM_LINK_STATE_DOWN_RESET_PENDING)) { e1000_get_speed_and_duplex(hw, &sc->link_speed, &sc->link_duplex); if (bootverbose) device_printf(dev, "Link is up %d Mbps %s\n", sc->link_speed, sc->link_duplex == FULL_DUPLEX ? "Full Duplex" : "Half Duplex"); sc->link_state = EM_LINK_STATE_UP; iflib_link_state_change(ctx, LINK_STATE_UP, IF_Mbps(sc->link_speed)); } else if (!link_check && (sc->link_state == EM_LINK_STATE_UP || sc->link_state == EM_LINK_STATE_UP_RESET_PENDING)) { sc->link_speed = 0; sc->link_duplex = 0; sc->link_state = EM_LINK_STATE_DOWN; iflib_link_state_change(ctx, LINK_STATE_DOWN, 0); } /* * A VF stops transmit DMA when its PF reports link down. Reset if * descriptors remain queued so they cannot be sent stale when carrier * returns, matching the periodic check in Linux igbvf. */ if (!link_check && !sc->vf_reset_pending && igbv_tx_pending(sc)) { sc->vf_reset_pending = true; iflib_request_reset(ctx); iflib_admin_intr_deferred(ctx); } /* em_if_init() establishes a new counter baseline after the reset. */ timer_tick = !sc->vf_reset_pending && atomic_readandclear_32(&sc->stats_pending) != 0; if (timer_tick) { em_update_stats_counters(sc); igbv_vlan_retry_tick(sc); } } static bool igbv_tx_pending(struct e1000_softc *sc) { struct tx_ring *txr; u32 head, tail; for (int i = 0; i < sc->tx_num_queues; i++) { txr = &sc->tx_queues[i].txr; head = E1000_READ_REG(&sc->hw, E1000_TDH(txr->me)); tail = E1000_READ_REG(&sc->hw, E1000_TDT(txr->me)); if (head != tail) return (true); } return (false); } static bool igbv_sanitize_queues(struct e1000_softc *sc) { struct e1000_hw *hw; u32 rxdctl, txdctl; int i, nqueues, retry; hw = &sc->hw; switch (hw->mac.type) { case e1000_vfadapt: nqueues = IGBV_82576_QUEUES; break; case e1000_vfadapt_i350: nqueues = IGBV_I350_QUEUES; break; default: return (true); } /* * The 82576 and I350 specification updates, Software Clarification 3, * note that VFLR leaves this queue configuration intact. Clear it * before programming the new rings so igbv does not depend on its PF * to sanitize state left by a previous VF owner. igbv uses only queue * zero, but must also clear the unused second 82576 queue. * * Disable every queue first and wait for outstanding DMA activity to * stop before programming TDWBAL/H. Spin only for the normal fast * transition, then sleep until the bounded deadline. */ for (i = 0; i < nqueues; i++) { E1000_WRITE_REG(hw, E1000_RXDCTL(i), 0); E1000_WRITE_REG(hw, E1000_TXDCTL(i), 0); } E1000_WRITE_FLUSH(hw); for (retry = 0; retry < IGBV_QUEUE_DISABLE_RETRIES; retry++) { for (i = 0; i < nqueues; i++) { rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(i)); txdctl = E1000_READ_REG(hw, E1000_TXDCTL(i)); if ((rxdctl & E1000_RXDCTL_QUEUE_ENABLE) != 0 || (txdctl & E1000_TXDCTL_QUEUE_ENABLE) != 0) break; } if (i == nqueues) break; if (retry + 1 < IGBV_QUEUE_DISABLE_RETRIES) { if (retry < IGBV_QUEUE_DISABLE_BUSY_RETRIES) DELAY(IGBV_QUEUE_DISABLE_DELAY_US); else pause_sbt("igbvqds", IGBV_QUEUE_DISABLE_PAUSE, 0, C_PREL(1)); } } if (retry == IGBV_QUEUE_DISABLE_RETRIES) { if (ratecheck(&sc->vf_last_queue_log, &igbv_queue_log_interval)) device_printf(sc->dev, "could not disable retained VF queues; " "reset deferred\n"); return (false); } for (i = 0; i < nqueues; i++) { E1000_WRITE_REG(hw, E1000_SRRCTL(i), 0); E1000_WRITE_REG(hw, E1000_DCA_RXCTRL(i), 0); E1000_WRITE_REG(hw, E1000_TDWBAL(i), 0); E1000_WRITE_REG(hw, E1000_TDWBAH(i), 0); E1000_WRITE_REG(hw, E1000_DCA_TXCTRL(i), 0); } E1000_WRITE_REG(hw, E1000_VFPSRTYPE, 0); E1000_WRITE_FLUSH(hw); return (true); } bool igbv_reset(if_ctx_t ctx) { struct e1000_softc *sc; struct e1000_hw *hw; s32 error; sc = iflib_get_softc(ctx); hw = &sc->hw; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); /* * Receive-buffer allocation and flow control are port resources owned * by the PF. Zero is an unavailable PBA sentinel, not a per-VF size. */ sc->pba = 0; hw->fc = (struct e1000_fc_info){ .current_mode = e1000_fc_none, .requested_mode = e1000_fc_none, }; error = e1000_reset_hw(hw); sc->vf_queues_sanitized = igbv_sanitize_queues(sc); if (!sc->vf_queues_sanitized) { e1000_check_for_link(hw); return (false); } igbv_queue_retry_succeeded(sc); if (error != E1000_SUCCESS) { e1000_check_for_link(hw); return (false); } memset(sc->vf_vfta_stale, 0, sizeof(sc->vf_vfta_stale)); memset(sc->vf_vfta_retry, 0, sizeof(sc->vf_vfta_retry)); sc->vf_vlan_retry_deadline = 0; sc->vf_vlan_retry_cursor = 0; if (e1000_init_hw(hw) < 0) { device_printf(sc->dev, "Hardware Initialization Failed\n"); return (false); } e1000_check_for_link(hw); return (true); } void igbv_initialize_transmit_unit(if_ctx_t ctx) { KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp, ("%s called for a PF", __func__)); em_initialize_transmit_rings(ctx); } void igbv_initialize_receive_unit(if_ctx_t ctx) { KASSERT(((struct e1000_softc *)iflib_get_softc(ctx))->vf_ifp, ("%s called for a PF", __func__)); igb_initialize_receive_rings(ctx, true); } void igbv_if_intr_enable(if_ctx_t ctx) { struct e1000_softc *sc; struct e1000_hw *hw; u32 mask; sc = iflib_get_softc(ctx); hw = &sc->hw; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); if (!sc->vf_queues_sanitized) return; mask = sc->que_mask | sc->link_mask; E1000_WRITE_REG(hw, E1000_EIAC, mask); E1000_WRITE_REG(hw, E1000_EIAM, mask); E1000_WRITE_REG(hw, E1000_EIMS, mask); E1000_WRITE_FLUSH(hw); } void igbv_if_intr_disable(if_ctx_t ctx) { struct e1000_softc *sc; struct e1000_hw *hw; sc = iflib_get_softc(ctx); hw = &sc->hw; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); E1000_WRITE_REG(hw, E1000_EIMC, 0xffffffff); E1000_WRITE_REG(hw, E1000_EIAC, 0); E1000_WRITE_FLUSH(hw); } int igbv_get_regs(SYSCTL_HANDLER_ARGS) { struct e1000_softc *sc; struct e1000_hw *hw; struct sbuf *sb; int error; sc = (struct e1000_softc *)arg1; hw = &sc->hw; KASSERT(sc->vf_ifp, ("%s called for a PF", __func__)); sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); if (sb == NULL) return (ENOMEM); /* * Limited VF register set: * Don't read EICR here because it is clear-on-read. The VF register * file exposes its queue pair at index zero, so this diagnostic does * not depend on the narrower lifetime of iflib's queue arrays. */ sbuf_printf(sb, "VF Registers\n"); sbuf_printf(sb, "\tVTCTRL\t %08x\n", E1000_READ_REG(hw, E1000_CTRL)); sbuf_printf(sb, "\tSTATUS\t %08x\n", E1000_READ_REG(hw, E1000_STATUS)); sbuf_printf(sb, "\tRDLEN\t %08x\n", E1000_READ_REG(hw, E1000_RDLEN(0))); sbuf_printf(sb, "\tRDH\t %08x\n", E1000_READ_REG(hw, E1000_RDH(0))); sbuf_printf(sb, "\tRDT\t %08x\n", E1000_READ_REG(hw, E1000_RDT(0))); sbuf_printf(sb, "\tTDLEN\t %08x\n", E1000_READ_REG(hw, E1000_TDLEN(0))); sbuf_printf(sb, "\tTDH\t %08x\n", E1000_READ_REG(hw, E1000_TDH(0))); sbuf_printf(sb, "\tTDT\t %08x\n", E1000_READ_REG(hw, E1000_TDT(0))); error = sbuf_finish(sb); sbuf_delete(sb); return (error); } static u_int igbv_copy_uc_addr(void *arg, struct sockaddr_dl *sdl, u_int idx) { struct igb_vf_uc_addr_list *list; const u8 *addr; list = arg; addr = (const u8 *)LLADDR(sdl); if (memcmp(addr, list->sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) return (0); if (idx < IGBV_MAX_MAC_FILTERS) memcpy(list->addrs[idx], addr, ETHER_ADDR_LEN); return (1); } void igbv_update_uc_addr_list(struct e1000_softc *sc, if_t ifp) { struct igb_vf_uc_addr_list list = { .sc = sc, }; u_int count; count = if_foreach_lladdr(ifp, igbv_copy_uc_addr, &list); if (count > IGBV_MAX_MAC_FILTERS) { device_printf(sc->dev, "too many secondary unicast addresses; maximum is %u\n", IGBV_MAX_MAC_FILTERS); } if (count == 0 && !sc->vf_uc_filters_set) return; /* * Linux igb PFs validate the address field before dispatching the CLR * subcommand. Supply the primary address rather than the zero payload * used by igbvf so those PFs actually remove the old filters. FreeBSD * PFs dispatch CLR before inspecting the otherwise-ignored address. */ if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_CLR, sc->hw.mac.addr) != E1000_SUCCESS) { device_printf(sc->dev, "VF secondary unicast filter clear request failed\n"); return; } sc->vf_uc_filters_set = false; if (count > IGBV_MAX_MAC_FILTERS) return; for (u_int i = 0; i < count; i++) { if (e1000_set_uc_addr_vf(&sc->hw, E1000_VF_MAC_FILTER_ADD, list.addrs[i]) != E1000_SUCCESS) { device_printf(sc->dev, "VF secondary unicast filter add request failed " "for %6D\n", list.addrs[i], ":"); } else sc->vf_uc_filters_set = true; usec_delay(200); } } void igbv_reconcile_mac(struct e1000_softc *sc, if_t ifp) { u8 *lladdr; if (!em_is_valid_ether_addr(sc->hw.mac.addr)) return; lladdr = (u8 *)if_getlladdr(ifp); if (memcmp(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN) == 0) return; device_printf(sc->dev, "PF rejected or replaced the requested MAC; using %6D\n", sc->hw.mac.addr, ":"); /* * if_setlladdr() would re-enter the driver's address-change path. * Initialization already holds the context lock, so update the * storage directly and issue the notification it would have sent. */ memcpy(lladdr, sc->hw.mac.addr, ETHER_ADDR_LEN); CURVNET_SET_QUIET(if_getvnet(ifp)); EVENTHANDLER_INVOKE(iflladdr_event, ifp); CURVNET_RESTORE(); }