/*- * Copyright (c) 2016 Chelsio Communications, Inc. * All rights reserved. * Written by: John Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "opt_inet.h" #include "opt_inet6.h" #include #include #include #include #include #include #include #if defined(__i386__) || defined(__amd64__) #include #include #endif #include "common/common.h" #include "common/t4_regs.h" #include "t4_ioctl.h" #include "t4_mp_ring.h" /* * Some notes: * * The Virtual Interfaces are connected to an internal switch on the chip * which allows VIs attached to the same port to talk to each other even when * the port link is down. As a result, we might want to always report a * VF's link as being "up". * * XXX: Add a TUNABLE and possible per-device sysctl for this? */ struct intrs_and_queues { uint16_t intr_type; /* MSI, or MSI-X */ uint16_t nirq; /* Total # of vectors */ uint16_t ntxq; /* # of NIC txq's for each port */ uint16_t nrxq; /* # of NIC rxq's for each port */ }; struct { uint16_t device; char *desc; } t4vf_pciids[] = { {0x4800, "Chelsio T440-dbg VF"}, {0x4801, "Chelsio T420-CR VF"}, {0x4802, "Chelsio T422-CR VF"}, {0x4803, "Chelsio T440-CR VF"}, {0x4804, "Chelsio T420-BCH VF"}, {0x4805, "Chelsio T440-BCH VF"}, {0x4806, "Chelsio T440-CH VF"}, {0x4807, "Chelsio T420-SO VF"}, {0x4808, "Chelsio T420-CX VF"}, {0x4809, "Chelsio T420-BT VF"}, {0x480a, "Chelsio T404-BT VF"}, {0x480e, "Chelsio T440-LP-CR VF"}, }, t5vf_pciids[] = { {0x5800, "Chelsio T580-dbg VF"}, {0x5801, "Chelsio T520-CR VF"}, /* 2 x 10G */ {0x5802, "Chelsio T522-CR VF"}, /* 2 x 10G, 2 X 1G */ {0x5803, "Chelsio T540-CR VF"}, /* 4 x 10G */ {0x5807, "Chelsio T520-SO VF"}, /* 2 x 10G, nomem */ {0x5809, "Chelsio T520-BT VF"}, /* 2 x 10GBaseT */ {0x580a, "Chelsio T504-BT VF"}, /* 4 x 1G */ {0x580d, "Chelsio T580-CR VF"}, /* 2 x 40G */ {0x580e, "Chelsio T540-LP-CR VF"}, /* 4 x 10G */ {0x5810, "Chelsio T580-LP-CR VF"}, /* 2 x 40G */ {0x5811, "Chelsio T520-LL-CR VF"}, /* 2 x 10G */ {0x5812, "Chelsio T560-CR VF"}, /* 1 x 40G, 2 x 10G */ {0x5814, "Chelsio T580-LP-SO-CR VF"}, /* 2 x 40G, nomem */ {0x5815, "Chelsio T502-BT VF"}, /* 2 x 1G */ {0x5818, "Chelsio T540-BT VF"}, /* 4 x 10GBaseT */ {0x5819, "Chelsio T540-LP-BT VF"}, /* 4 x 10GBaseT */ {0x581a, "Chelsio T540-SO-BT VF"}, /* 4 x 10GBaseT, nomem */ {0x581b, "Chelsio T540-SO-CR VF"}, /* 4 x 10G, nomem */ }, t6vf_pciids[] = { {0x6800, "Chelsio T6-DBG-25 VF"}, /* 2 x 10/25G, debug */ {0x6801, "Chelsio T6225-CR VF"}, /* 2 x 10/25G */ {0x6802, "Chelsio T6225-SO-CR VF"}, /* 2 x 10/25G, nomem */ {0x6803, "Chelsio T6425-CR VF"}, /* 4 x 10/25G */ {0x6804, "Chelsio T6425-SO-CR VF"}, /* 4 x 10/25G, nomem */ {0x6805, "Chelsio T6225-SO-OCP3 VF"}, /* 2 x 10/25G, nomem */ {0x6806, "Chelsio T6225-OCP3 VF"}, /* 2 x 10/25G */ {0x6807, "Chelsio T62100-LP-CR VF"}, /* 2 x 40/50/100G */ {0x6808, "Chelsio T62100-SO-CR VF"}, /* 2 x 40/50/100G, nomem */ {0x6809, "Chelsio T6210-BT VF"}, /* 2 x 10GBASE-T */ {0x680d, "Chelsio T62100-CR VF"}, /* 2 x 40/50/100G */ {0x6810, "Chelsio T6-DBG-100 VF"}, /* 2 x 40/50/100G, debug */ {0x6811, "Chelsio T6225-LL-CR VF"}, /* 2 x 10/25G */ {0x6814, "Chelsio T62100-SO-OCP3 VF"}, /* 2 x 40/50/100G, nomem */ {0x6815, "Chelsio T6201-BT VF"}, /* 2 x 1000BASE-T */ /* Custom */ {0x6880, "Chelsio T6225 80 VF"}, {0x6881, "Chelsio T62100 81 VF"}, {0x6882, "Chelsio T6225-CR 82 VF"}, {0x6883, "Chelsio T62100-CR 83 VF"}, {0x6884, "Chelsio T64100-CR 84 VF"}, {0x6885, "Chelsio T6240-SO 85 VF"}, {0x6886, "Chelsio T6225-SO-CR 86 VF"}, {0x6887, "Chelsio T6225-CR 87 VF"}, }; static d_ioctl_t t4vf_ioctl; static struct cdevsw t4vf_cdevsw = { .d_version = D_VERSION, .d_ioctl = t4vf_ioctl, .d_name = "t4vf", }; static int t4vf_probe(device_t dev) { uint16_t d; size_t i; d = pci_get_device(dev); for (i = 0; i < nitems(t4vf_pciids); i++) { if (d == t4vf_pciids[i].device) { device_set_desc(dev, t4vf_pciids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int t5vf_probe(device_t dev) { uint16_t d; size_t i; d = pci_get_device(dev); for (i = 0; i < nitems(t5vf_pciids); i++) { if (d == t5vf_pciids[i].device) { device_set_desc(dev, t5vf_pciids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } static int t6vf_probe(device_t dev) { uint16_t d; size_t i; d = pci_get_device(dev); for (i = 0; i < nitems(t6vf_pciids); i++) { if (d == t6vf_pciids[i].device) { device_set_desc(dev, t6vf_pciids[i].desc); return (BUS_PROBE_DEFAULT); } } return (ENXIO); } #define FW_PARAM_DEV(param) \ (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) #define FW_PARAM_PFVF(param) \ (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) static int get_params__pre_init(struct adapter *sc) { int rc; uint32_t param[3], val[3]; param[0] = FW_PARAM_DEV(FWREV); param[1] = FW_PARAM_DEV(TPREV); param[2] = FW_PARAM_DEV(CCLK); rc = -t4vf_query_params(sc, nitems(param), param, val); if (rc != 0) { device_printf(sc->dev, "failed to query parameters (pre_init): %d.\n", rc); return (rc); } sc->params.fw_vers = val[0]; sc->params.tp_vers = val[1]; sc->params.vpd.cclk = val[2]; snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u", G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); return (0); } static int get_params__post_init(struct adapter *sc) { int rc; uint32_t param, val; rc = -t4vf_get_sge_params(sc); if (rc != 0) { device_printf(sc->dev, "unable to retrieve adapter SGE parameters: %d\n", rc); return (rc); } rc = -t4vf_get_rss_glb_config(sc); if (rc != 0) { device_printf(sc->dev, "unable to retrieve adapter RSS parameters: %d\n", rc); return (rc); } if (sc->params.rss.mode != FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) { device_printf(sc->dev, "unable to operate with global RSS mode %d\n", sc->params.rss.mode); return (EINVAL); } /* * Grab our Virtual Interface resource allocation, extract the * features that we're interested in and do a bit of sanity testing on * what we discover. */ rc = -t4vf_get_vfres(sc); if (rc != 0) { device_printf(sc->dev, "unable to get virtual interface resources: %d\n", rc); return (rc); } /* * Check for various parameter sanity issues. */ if (sc->params.vfres.pmask == 0) { device_printf(sc->dev, "no port access configured/usable!\n"); return (EINVAL); } if (sc->params.vfres.nvi == 0) { device_printf(sc->dev, "no virtual interfaces configured/usable!\n"); return (EINVAL); } sc->params.portvec = sc->params.vfres.pmask; param = FW_PARAM_PFVF(MAX_PKTS_PER_ETH_TX_PKTS_WR); rc = -t4vf_query_params(sc, 1, ¶m, &val); if (rc == 0) sc->params.max_pkts_per_eth_tx_pkts_wr = val; else sc->params.max_pkts_per_eth_tx_pkts_wr = 14; rc = t4_verify_chip_settings(sc); if (rc != 0) return (rc); t4_init_rx_buf_info(sc); return (0); } static int set_params__post_init(struct adapter *sc) { uint32_t param, val; /* ask for encapsulated CPLs */ param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); val = 1; (void)t4vf_set_params(sc, 1, ¶m, &val); /* Enable 32b port caps if the firmware supports it. */ param = FW_PARAM_PFVF(PORT_CAPS32); val = 1; if (t4vf_set_params(sc, 1, ¶m, &val) == 0) sc->params.port_caps32 = 1; return (0); } #undef FW_PARAM_PFVF #undef FW_PARAM_DEV static int cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq) { struct vf_resources *vfres; int nrxq, ntxq, nports; int itype, iq_avail, navail, rc; /* * Figure out the layout of queues across our VIs and ensure * we can allocate enough interrupts for our layout. */ vfres = &sc->params.vfres; nports = sc->params.nports; bzero(iaq, sizeof(*iaq)); for (itype = INTR_MSIX; itype != 0; itype >>= 1) { if (itype == INTR_INTX) continue; if (itype == INTR_MSIX) navail = pci_msix_count(sc->dev); else navail = pci_msi_count(sc->dev); if (navail == 0) continue; iaq->intr_type = itype; /* * XXX: The Linux driver reserves an Ingress Queue for * forwarded interrupts when using MSI (but not MSI-X). * It seems it just always asks for 2 interrupts and * forwards all rxqs to the forwarded interrupt. * * We must reserve one IRQ for the for the firmware * event queue. * * Every rxq requires an ingress queue with a free * list and interrupts and an egress queue. Every txq * requires an ETH egress queue. */ iaq->nirq = T4VF_EXTRA_INTR; /* * First, determine how many queues we can allocate. * Start by finding the upper bound on rxqs from the * limit on ingress queues. */ iq_avail = vfres->niqflint - iaq->nirq; if (iq_avail < nports) { device_printf(sc->dev, "Not enough ingress queues (%d) for %d ports\n", vfres->niqflint, nports); return (ENXIO); } /* * Try to honor the cap on interrupts. If there aren't * enough interrupts for at least one interrupt per * port, then don't bother, we will just forward all * interrupts to one interrupt in that case. */ if (iaq->nirq + nports <= navail) { if (iq_avail > navail - iaq->nirq) iq_avail = navail - iaq->nirq; } nrxq = nports * t4_nrxq; if (nrxq > iq_avail) { /* * Too many ingress queues. Use what we can. */ nrxq = (iq_avail / nports) * nports; } KASSERT(nrxq <= iq_avail, ("too many ingress queues")); /* * Next, determine the upper bound on txqs from the limit * on ETH queues. */ if (vfres->nethctrl < nports) { device_printf(sc->dev, "Not enough ETH queues (%d) for %d ports\n", vfres->nethctrl, nports); return (ENXIO); } ntxq = nports * t4_ntxq; if (ntxq > vfres->nethctrl) { /* * Too many ETH queues. Use what we can. */ ntxq = (vfres->nethctrl / nports) * nports; } KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); /* * Finally, ensure we have enough egress queues. */ if (vfres->neq < nports * 2) { device_printf(sc->dev, "Not enough egress queues (%d) for %d ports\n", vfres->neq, nports); return (ENXIO); } if (nrxq + ntxq > vfres->neq) { /* Just punt and use 1 for everything. */ nrxq = ntxq = nports; } KASSERT(nrxq <= iq_avail, ("too many ingress queues")); KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues")); KASSERT(nrxq + ntxq <= vfres->neq, ("too many egress queues")); /* * Do we have enough interrupts? For MSI the interrupts * have to be a power of 2 as well. */ iaq->nirq += nrxq; iaq->ntxq = ntxq; iaq->nrxq = nrxq; if (iaq->nirq <= navail && (itype != INTR_MSI || powerof2(iaq->nirq))) { navail = iaq->nirq; if (itype == INTR_MSIX) rc = pci_alloc_msix(sc->dev, &navail); else rc = pci_alloc_msi(sc->dev, &navail); if (rc != 0) { device_printf(sc->dev, "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", itype, rc, iaq->nirq, navail); return (rc); } if (navail == iaq->nirq) { return (0); } pci_release_msi(sc->dev); } /* Fall back to a single interrupt. */ iaq->nirq = 1; navail = iaq->nirq; if (itype == INTR_MSIX) rc = pci_alloc_msix(sc->dev, &navail); else rc = pci_alloc_msi(sc->dev, &navail); if (rc != 0) device_printf(sc->dev, "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n", itype, rc, iaq->nirq, navail); return (rc); } device_printf(sc->dev, "failed to find a usable interrupt type. " "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types, pci_msix_count(sc->dev), pci_msi_count(sc->dev)); return (ENXIO); } static int t4vf_attach(device_t dev) { struct adapter *sc; int rc = 0, i, j, rqidx, tqidx, n, p, pmask; struct make_dev_args mda; struct intrs_and_queues iaq; struct sge *s; sc = device_get_softc(dev); sc->dev = dev; sysctl_ctx_init(&sc->ctx); pci_enable_busmaster(dev); pci_set_max_read_req(dev, 4096); sc->params.pci.mps = pci_get_max_payload(dev); sc->flags |= IS_VF; TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags); sc->sge_gts_reg = VF_SGE_REG(A_SGE_VF_GTS); sc->sge_kdoorbell_reg = VF_SGE_REG(A_SGE_VF_KDOORBELL); snprintf(sc->lockname, sizeof(sc->lockname), "%s", device_get_nameunit(dev)); mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF); t4_add_adapter(sc); mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF); TAILQ_INIT(&sc->sfl); callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0); mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF); rc = t4_map_bars_0_and_4(sc); if (rc != 0) goto done; /* error message displayed already */ rc = -t4vf_prep_adapter(sc); if (rc != 0) goto done; t4_init_devnames(sc); if (sc->names == NULL) { rc = ENOTSUP; goto done; /* error message displayed already */ } /* * Leave the 'pf' and 'mbox' values as zero. This ensures * that various firmware messages do not set the fields which * is the correct thing to do for a VF. */ memset(sc->chan_map, 0xff, sizeof(sc->chan_map)); make_dev_args_init(&mda); mda.mda_devsw = &t4vf_cdevsw; mda.mda_uid = UID_ROOT; mda.mda_gid = GID_WHEEL; mda.mda_mode = 0600; mda.mda_si_drv1 = sc; rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev)); if (rc != 0) device_printf(dev, "failed to create nexus char device: %d.\n", rc); #if defined(__i386__) if ((cpu_feature & CPUID_CX8) == 0) { device_printf(dev, "64 bit atomics not available.\n"); rc = ENOTSUP; goto done; } #endif /* * Some environments do not properly handle PCIE FLRs -- e.g. in Linux * 2.6.31 and later we can't call pci_reset_function() in order to * issue an FLR because of a self- deadlock on the device semaphore. * Meanwhile, the OS infrastructure doesn't issue FLRs in all the * cases where they're needed -- for instance, some versions of KVM * fail to reset "Assigned Devices" when the VM reboots. Therefore we * use the firmware based reset in order to reset any per function * state. */ rc = -t4vf_fw_reset(sc); if (rc != 0) { device_printf(dev, "FW reset failed: %d\n", rc); goto done; } sc->flags |= FW_OK; /* * Grab basic operational parameters. These will predominantly have * been set up by the Physical Function Driver or will be hard coded * into the adapter. We just have to live with them ... Note that * we _must_ get our VPD parameters before our SGE parameters because * we need to know the adapter's core clock from the VPD in order to * properly decode the SGE Timer Values. */ rc = get_params__pre_init(sc); if (rc != 0) goto done; /* error message displayed already */ rc = get_params__post_init(sc); if (rc != 0) goto done; /* error message displayed already */ rc = set_params__post_init(sc); if (rc != 0) goto done; /* error message displayed already */ rc = t4_map_bar_2(sc); if (rc != 0) goto done; /* error message displayed already */ rc = t4_adj_doorbells(sc); if (rc != 0) goto done; /* error message displayed already */ rc = t4_create_dma_tag(sc); if (rc != 0) goto done; /* error message displayed already */ /* * The number of "ports" which we support is equal to the number of * Virtual Interfaces with which we've been provisioned. */ sc->params.nports = imin(sc->params.vfres.nvi, MAX_NPORTS); /* * We may have been provisioned with more VIs than the number of * ports we're allowed to access (our Port Access Rights Mask). * Just use a single VI for each port. */ sc->params.nports = imin(sc->params.nports, bitcount32(sc->params.vfres.pmask)); #ifdef notyet /* * XXX: The Linux VF driver will lower nports if it thinks there * are too few resources in vfres (niqflint, nethctrl, neq). */ #endif /* * First pass over all the ports - allocate VIs and initialize some * basic parameters like mac address, port type, etc. */ pmask = sc->params.vfres.pmask; for_each_port(sc, i) { struct port_info *pi; uint8_t mac[ETHER_ADDR_LEN]; pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK); sc->port[i] = pi; /* These must be set before t4_port_init */ pi->adapter = sc; pi->port_id = i; pi->nvi = 1; pi->vi = malloc(sizeof(struct vi_info) * pi->nvi, M_CXGBE, M_ZERO | M_WAITOK); /* * Allocate the "main" VI and initialize parameters * like mac addr. */ rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i); if (rc != 0) { device_printf(dev, "unable to initialize port %d: %d\n", i, rc); free(pi->vi, M_CXGBE); free(pi, M_CXGBE); sc->port[i] = NULL; goto done; } /* Prefer the MAC address set by the PF, if there is one. */ n = 1; p = ffs(pmask) - 1; MPASS(p >= 0); rc = t4vf_get_vf_mac(sc, p, &n, mac); if (rc == 0 && n == 1) t4_os_set_hw_addr(pi, mac); pmask &= ~(1 << p); sc->vlan_id = t4vf_get_vf_vlan(sc); /* No t4_link_start. */ snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d", device_get_nameunit(dev), i); mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF); sc->chan_map[pi->tx_chan] = i; /* All VIs on this port share this media. */ ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change, cxgbe_media_status); pi->dev = device_add_child(dev, sc->names->vf_ifnet_name, -1); if (pi->dev == NULL) { device_printf(dev, "failed to add device for port %d.\n", i); rc = ENXIO; goto done; } pi->vi[0].dev = pi->dev; device_set_softc(pi->dev, pi); } /* * Interrupt type, # of interrupts, # of rx/tx queues, etc. */ rc = cfg_itype_and_nqueues(sc, &iaq); if (rc != 0) goto done; /* error message displayed already */ sc->intr_type = iaq.intr_type; sc->intr_count = iaq.nirq; s = &sc->sge; s->nrxq = sc->params.nports * iaq.nrxq; s->ntxq = sc->params.nports * iaq.ntxq; s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */ s->neq += sc->params.nports; /* ctrl queues: 1 per port */ s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ s->iqmap_sz = s->niq; s->eqmap_sz = s->neq; s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE, M_ZERO | M_WAITOK); s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE, M_ZERO | M_WAITOK); s->iqmap = malloc(s->iqmap_sz * sizeof(struct sge_iq *), M_CXGBE, M_ZERO | M_WAITOK); s->eqmap = malloc(s->eqmap_sz * sizeof(struct sge_eq *), M_CXGBE, M_ZERO | M_WAITOK); sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE, M_ZERO | M_WAITOK); /* * Second pass over the ports. This time we know the number of rx and * tx queues that each port should get. */ rqidx = tqidx = 0; for_each_port(sc, i) { struct port_info *pi = sc->port[i]; struct vi_info *vi; if (pi == NULL) continue; for_each_vi(pi, j, vi) { vi->pi = pi; vi->adapter = sc; vi->qsize_rxq = t4_qsize_rxq; vi->qsize_txq = t4_qsize_txq; vi->first_rxq = rqidx; vi->first_txq = tqidx; vi->tmr_idx = t4_tmr_idx; vi->pktc_idx = t4_pktc_idx; vi->nrxq = j == 0 ? iaq.nrxq: 1; vi->ntxq = j == 0 ? iaq.ntxq: 1; rqidx += vi->nrxq; tqidx += vi->ntxq; vi->rsrv_noflowq = 0; } } rc = t4_setup_intr_handlers(sc); if (rc != 0) { device_printf(dev, "failed to setup interrupt handlers: %d\n", rc); goto done; } rc = bus_generic_attach(dev); if (rc != 0) { device_printf(dev, "failed to attach all child ports: %d\n", rc); goto done; } device_printf(dev, "%d ports, %d %s interrupt%s, %d eq, %d iq\n", sc->params.nports, sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" : "MSI", sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq); done: if (rc != 0) t4_detach_common(dev); else t4_sysctls(sc); return (rc); } static void get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf) { /* 0x3f is used as the revision for VFs. */ regs->version = chip_id(sc) | (0x3f << 10); t4_get_regs(sc, buf, regs->len); } static void t4_clr_vi_stats(struct adapter *sc) { int reg; for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L; reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4) t4_write_reg(sc, VF_MPS_REG(reg), 0); } static int t4vf_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, struct thread *td) { int rc; struct adapter *sc = dev->si_drv1; rc = priv_check(td, PRIV_DRIVER); if (rc != 0) return (rc); switch (cmd) { case CHELSIO_T4_GETREG: { struct t4_reg *edata = (struct t4_reg *)data; if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) return (EFAULT); if (edata->size == 4) edata->val = t4_read_reg(sc, edata->addr); else if (edata->size == 8) edata->val = t4_read_reg64(sc, edata->addr); else return (EINVAL); break; } case CHELSIO_T4_SETREG: { struct t4_reg *edata = (struct t4_reg *)data; if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len) return (EFAULT); if (edata->size == 4) { if (edata->val & 0xffffffff00000000) return (EINVAL); t4_write_reg(sc, edata->addr, (uint32_t) edata->val); } else if (edata->size == 8) t4_write_reg64(sc, edata->addr, edata->val); else return (EINVAL); break; } case CHELSIO_T4_REGDUMP: { struct t4_regdump *regs = (struct t4_regdump *)data; int reglen = t4_get_regs_len(sc); uint8_t *buf; if (regs->len < reglen) { regs->len = reglen; /* hint to the caller */ return (ENOBUFS); } regs->len = reglen; buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO); get_regs(sc, regs, buf); rc = copyout(buf, regs->data, reglen); free(buf, M_CXGBE); break; } case CHELSIO_T4_CLEAR_STATS: { int i, v; u_int port_id = *(uint32_t *)data; struct port_info *pi; struct vi_info *vi; if (port_id >= sc->params.nports) return (EINVAL); pi = sc->port[port_id]; /* MAC stats */ pi->tx_parse_error = 0; t4_clr_vi_stats(sc); /* * Since this command accepts a port, clear stats for * all VIs on this port. */ for_each_vi(pi, v, vi) { if (vi->flags & VI_INIT_DONE) { struct sge_rxq *rxq; struct sge_txq *txq; for_each_rxq(vi, i, rxq) { #if defined(INET) || defined(INET6) rxq->lro.lro_queued = 0; rxq->lro.lro_flushed = 0; #endif rxq->rxcsum = 0; rxq->vlan_extraction = 0; } for_each_txq(vi, i, txq) { txq->txcsum = 0; txq->tso_wrs = 0; txq->vlan_insertion = 0; txq->imm_wrs = 0; txq->sgl_wrs = 0; txq->txpkt_wrs = 0; txq->txpkts0_wrs = 0; txq->txpkts1_wrs = 0; txq->txpkts0_pkts = 0; txq->txpkts1_pkts = 0; txq->txpkts_flush = 0; mp_ring_reset_stats(txq->r); } } } break; } case CHELSIO_T4_SCHED_CLASS: rc = t4_set_sched_class(sc, (struct t4_sched_params *)data); break; case CHELSIO_T4_SCHED_QUEUE: rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data); break; default: rc = ENOTTY; } return (rc); } static device_method_t t4vf_methods[] = { DEVMETHOD(device_probe, t4vf_probe), DEVMETHOD(device_attach, t4vf_attach), DEVMETHOD(device_detach, t4_detach_common), DEVMETHOD_END }; static driver_t t4vf_driver = { "t4vf", t4vf_methods, sizeof(struct adapter) }; static device_method_t t5vf_methods[] = { DEVMETHOD(device_probe, t5vf_probe), DEVMETHOD(device_attach, t4vf_attach), DEVMETHOD(device_detach, t4_detach_common), DEVMETHOD_END }; static driver_t t5vf_driver = { "t5vf", t5vf_methods, sizeof(struct adapter) }; static device_method_t t6vf_methods[] = { DEVMETHOD(device_probe, t6vf_probe), DEVMETHOD(device_attach, t4vf_attach), DEVMETHOD(device_detach, t4_detach_common), DEVMETHOD_END }; static driver_t t6vf_driver = { "t6vf", t6vf_methods, sizeof(struct adapter) }; static driver_t cxgbev_driver = { "cxgbev", cxgbe_methods, sizeof(struct port_info) }; static driver_t cxlv_driver = { "cxlv", cxgbe_methods, sizeof(struct port_info) }; static driver_t ccv_driver = { "ccv", cxgbe_methods, sizeof(struct port_info) }; DRIVER_MODULE(t4vf, pci, t4vf_driver, 0, 0); MODULE_VERSION(t4vf, 1); MODULE_DEPEND(t4vf, t4nex, 1, 1, 1); DRIVER_MODULE(t5vf, pci, t5vf_driver, 0, 0); MODULE_VERSION(t5vf, 1); MODULE_DEPEND(t5vf, t5nex, 1, 1, 1); DRIVER_MODULE(t6vf, pci, t6vf_driver, 0, 0); MODULE_VERSION(t6vf, 1); MODULE_DEPEND(t6vf, t6nex, 1, 1, 1); DRIVER_MODULE(cxgbev, t4vf, cxgbev_driver, 0, 0); MODULE_VERSION(cxgbev, 1); DRIVER_MODULE(cxlv, t5vf, cxlv_driver, 0, 0); MODULE_VERSION(cxlv, 1); DRIVER_MODULE(ccv, t6vf, ccv_driver, 0, 0); MODULE_VERSION(ccv, 1);