/*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright © 2021-2022 Dmitry Salychev * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include /* * QBMan command interface and the DPAA2 I/O (DPIO) driver. * * The DPIO object allows configuration of the QBMan software portal with * optional notification capabilities. * * Software portals are used by the driver to communicate with the QBMan. The * DPIO object’s main purpose is to enable the driver to perform I/O – enqueue * and dequeue operations, as well as buffer release and acquire operations – * using QBMan. */ #include "opt_rss.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef RSS #include #endif #include "pcib_if.h" #include "pci_if.h" #include "dpaa2_mc.h" #include "dpaa2_mcp.h" #include "dpaa2_swp.h" #include "dpaa2_swp_if.h" #include "dpaa2_cmd_if.h" #include "dpaa2_io.h" #include "dpaa2_ni.h" #include "dpaa2_channel.h" #define DPIO_IRQ_INDEX 0 /* index of the only DPIO IRQ */ #define DPIO_POLL_MAX 32 /* * Memory: * 0: cache-enabled part of the QBMan software portal. * 1: cache-inhibited part of the QBMan software portal. * 2: control registers of the QBMan software portal? * * Note that MSI should be allocated separately using pseudo-PCI interface. */ struct resource_spec dpaa2_io_spec[] = { /* * System Memory resources. */ #define MEM_RES_NUM (3u) #define MEM_RID_OFF (0u) #define MEM_RID(rid) ((rid) + MEM_RID_OFF) { SYS_RES_MEMORY, MEM_RID(0), RF_ACTIVE | RF_UNMAPPED }, { SYS_RES_MEMORY, MEM_RID(1), RF_ACTIVE | RF_UNMAPPED }, { SYS_RES_MEMORY, MEM_RID(2), RF_ACTIVE | RF_UNMAPPED | RF_OPTIONAL }, /* * DPMCP resources. * * NOTE: MC command portals (MCPs) are used to send commands to, and * receive responses from, the MC firmware. One portal per DPIO. */ #define MCP_RES_NUM (1u) #define MCP_RID_OFF (MEM_RID_OFF + MEM_RES_NUM) #define MCP_RID(rid) ((rid) + MCP_RID_OFF) /* --- */ { DPAA2_DEV_MCP, MCP_RID(0), RF_ACTIVE | RF_SHAREABLE | RF_OPTIONAL }, /* --- */ RESOURCE_SPEC_END }; /* Configuration routines. */ static int dpaa2_io_setup_irqs(device_t dev); static int dpaa2_io_release_irqs(device_t dev); static int dpaa2_io_setup_msi(struct dpaa2_io_softc *sc); static int dpaa2_io_release_msi(struct dpaa2_io_softc *sc); /* Interrupt handlers */ static void dpaa2_io_intr(void *arg); static int dpaa2_io_probe(device_t dev) { /* DPIO device will be added by a parent resource container itself. */ device_set_desc(dev, "DPAA2 I/O"); return (BUS_PROBE_DEFAULT); } static int dpaa2_io_detach(device_t dev) { device_t pdev = device_get_parent(dev); device_t child = dev; struct dpaa2_io_softc *sc = device_get_softc(dev); struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); struct dpaa2_devinfo *dinfo = device_get_ivars(dev); struct dpaa2_cmd cmd; uint16_t rc_token, io_token; int error; DPAA2_CMD_INIT(&cmd); /* Tear down interrupt handler and release IRQ resources. */ dpaa2_io_release_irqs(dev); /* Free software portal helper object. */ dpaa2_swp_free_portal(sc->swp); error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); if (error) { device_printf(dev, "%s: failed to open DPRC: error=%d\n", __func__, error); goto err_exit; } error = DPAA2_CMD_IO_OPEN(dev, child, &cmd, dinfo->id, &io_token); if (error) { device_printf(dev, "%s: failed to open DPIO: id=%d, error=%d\n", __func__, dinfo->id, error); goto close_rc; } error = DPAA2_CMD_IO_DISABLE(dev, child, &cmd); if (error && bootverbose) { device_printf(dev, "%s: failed to disable DPIO: id=%d, " "error=%d\n", __func__, dinfo->id, error); } (void)DPAA2_CMD_IO_CLOSE(dev, child, &cmd); (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); /* Unmap memory resources of the portal. */ for (int i = 0; i < MEM_RES_NUM; i++) { if (sc->res[MEM_RID(i)] == NULL) { continue; } error = bus_unmap_resource(sc->dev, SYS_RES_MEMORY, sc->res[MEM_RID(i)], &sc->map[MEM_RID(i)]); if (error && bootverbose) { device_printf(dev, "%s: failed to unmap memory " "resource: rid=%d, error=%d\n", __func__, MEM_RID(i), error); } } /* Release allocated resources. */ bus_release_resources(dev, dpaa2_io_spec, sc->res); return (0); close_rc: (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); err_exit: return (error); } static int dpaa2_io_attach(device_t dev) { device_t pdev = device_get_parent(dev); device_t child = dev; device_t mcp_dev; struct dpaa2_io_softc *sc = device_get_softc(dev); struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev); struct dpaa2_devinfo *dinfo = device_get_ivars(dev); struct dpaa2_devinfo *mcp_dinfo; struct dpaa2_cmd cmd; struct resource_map_request req; struct { vm_memattr_t memattr; char *label; } map_args[MEM_RES_NUM] = { { VM_MEMATTR_WRITE_BACK, "cache-enabled part" }, { VM_MEMATTR_DEVICE, "cache-inhibited part" }, { VM_MEMATTR_DEVICE, "control registers" } }; uint16_t rc_token, io_token; int error; sc->dev = dev; sc->swp = NULL; sc->intr = NULL; sc->irq_resource = NULL; /* Allocate resources. */ error = bus_alloc_resources(sc->dev, dpaa2_io_spec, sc->res); if (error) { device_printf(dev, "%s: failed to allocate resources: " "error=%d\n", __func__, error); return (ENXIO); } /* Set allocated MC portal up. */ mcp_dev = (device_t) rman_get_start(sc->res[MCP_RID(0)]); mcp_dinfo = device_get_ivars(mcp_dev); dinfo->portal = mcp_dinfo->portal; /* Map memory resources of the portal. */ for (int i = 0; i < MEM_RES_NUM; i++) { if (sc->res[MEM_RID(i)] == NULL) { continue; } resource_init_map_request(&req); req.memattr = map_args[i].memattr; error = bus_map_resource(sc->dev, SYS_RES_MEMORY, sc->res[MEM_RID(i)], &req, &sc->map[MEM_RID(i)]); if (error) { device_printf(dev, "%s: failed to map %s: error=%d\n", __func__, map_args[i].label, error); goto err_exit; } } DPAA2_CMD_INIT(&cmd); error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rc_token); if (error) { device_printf(dev, "%s: failed to open DPRC: error=%d\n", __func__, error); goto err_exit; } error = DPAA2_CMD_IO_OPEN(dev, child, &cmd, dinfo->id, &io_token); if (error) { device_printf(dev, "%s: failed to open DPIO: id=%d, error=%d\n", __func__, dinfo->id, error); goto close_rc; } error = DPAA2_CMD_IO_RESET(dev, child, &cmd); if (error) { device_printf(dev, "%s: failed to reset DPIO: id=%d, error=%d\n", __func__, dinfo->id, error); goto close_io; } error = DPAA2_CMD_IO_GET_ATTRIBUTES(dev, child, &cmd, &sc->attr); if (error) { device_printf(dev, "%s: failed to get DPIO attributes: id=%d, " "error=%d\n", __func__, dinfo->id, error); goto close_io; } error = DPAA2_CMD_IO_ENABLE(dev, child, &cmd); if (error) { device_printf(dev, "%s: failed to enable DPIO: id=%d, " "error=%d\n", __func__, dinfo->id, error); goto close_io; } /* Prepare descriptor of the QBMan software portal. */ sc->swp_desc.dpio_dev = dev; sc->swp_desc.swp_version = sc->attr.swp_version; sc->swp_desc.swp_clk = sc->attr.swp_clk; sc->swp_desc.swp_id = sc->attr.swp_id; sc->swp_desc.has_notif = sc->attr.priors_num ? true : false; sc->swp_desc.has_8prio = sc->attr.priors_num == 8u ? true : false; sc->swp_desc.cena_res = sc->res[0]; sc->swp_desc.cena_map = &sc->map[0]; sc->swp_desc.cinh_res = sc->res[1]; sc->swp_desc.cinh_map = &sc->map[1]; /* * Compute how many 256 QBMAN cycles fit into one ns. This is because * the interrupt timeout period register needs to be specified in QBMAN * clock cycles in increments of 256. */ sc->swp_desc.swp_cycles_ratio = 256000 / (sc->swp_desc.swp_clk / 1000000); /* Initialize QBMan software portal. */ error = dpaa2_swp_init_portal(&sc->swp, &sc->swp_desc, DPAA2_SWP_DEF); if (error) { device_printf(dev, "%s: failed to initialize dpaa2_swp: " "error=%d\n", __func__, error); goto err_exit; } error = dpaa2_io_setup_irqs(dev); if (error) { device_printf(dev, "%s: failed to setup IRQs: error=%d\n", __func__, error); goto err_exit; } if (bootverbose) { device_printf(dev, "dpio_id=%d, swp_id=%d, chan_mode=%s, " "notif_priors=%d, swp_version=0x%x\n", sc->attr.id, sc->attr.swp_id, sc->attr.chan_mode == DPAA2_IO_LOCAL_CHANNEL ? "local_channel" : "no_channel", sc->attr.priors_num, sc->attr.swp_version); } (void)DPAA2_CMD_IO_CLOSE(dev, child, &cmd); (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); return (0); close_io: (void)DPAA2_CMD_IO_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, io_token)); close_rc: (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rc_token)); err_exit: dpaa2_io_detach(dev); return (ENXIO); } /** * @brief Enqueue multiple frames to a frame queue using one FQID. */ static int dpaa2_io_enq_multiple_fq(device_t iodev, uint32_t fqid, struct dpaa2_fd *fd, int frames_n) { struct dpaa2_io_softc *sc = device_get_softc(iodev); struct dpaa2_swp *swp = sc->swp; struct dpaa2_eq_desc ed; uint32_t flags = 0; memset(&ed, 0, sizeof(ed)); /* Setup enqueue descriptor. */ dpaa2_swp_set_ed_norp(&ed, false); dpaa2_swp_set_ed_fq(&ed, fqid); return (dpaa2_swp_enq_mult(swp, &ed, fd, &flags, frames_n)); } /** * @brief Configure the channel data availability notification (CDAN) * in a particular WQ channel paired with DPIO. */ static int dpaa2_io_conf_wq_channel(device_t iodev, struct dpaa2_io_notif_ctx *ctx) { struct dpaa2_io_softc *sc = device_get_softc(iodev); /* Enable generation of the CDAN notifications. */ if (ctx->cdan_en) { return (dpaa2_swp_conf_wq_channel(sc->swp, ctx->fq_chan_id, DPAA2_WQCHAN_WE_EN | DPAA2_WQCHAN_WE_CTX, ctx->cdan_en, ctx->qman_ctx)); } return (0); } /** * @brief Query current configuration/state of the buffer pool. */ static int dpaa2_io_query_bp(device_t iodev, uint16_t bpid, struct dpaa2_bp_conf *conf) { struct dpaa2_io_softc *sc = device_get_softc(iodev); return (dpaa2_swp_query_bp(sc->swp, bpid, conf)); } /** * @brief Release one or more buffer pointers to the QBMan buffer pool. */ static int dpaa2_io_release_bufs(device_t iodev, uint16_t bpid, bus_addr_t *buf, uint32_t buf_num) { struct dpaa2_io_softc *sc = device_get_softc(iodev); return (dpaa2_swp_release_bufs(sc->swp, bpid, buf, buf_num)); } /** * @brief Configure DPNI object to generate interrupts. */ static int dpaa2_io_setup_irqs(device_t dev) { struct dpaa2_io_softc *sc = device_get_softc(dev); int error; /* * Setup interrupts generated by the software portal. */ dpaa2_swp_set_intr_trigger(sc->swp, DPAA2_SWP_INTR_DQRI); dpaa2_swp_clear_intr_status(sc->swp, 0xFFFFFFFFu); /* Configure IRQs. */ error = dpaa2_io_setup_msi(sc); if (error) { device_printf(dev, "%s: failed to allocate MSI: error=%d\n", __func__, error); return (error); } if ((sc->irq_resource = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irq_rid[0], RF_ACTIVE | RF_SHAREABLE)) == NULL) { device_printf(dev, "%s: failed to allocate IRQ resource\n", __func__); return (ENXIO); } if (bus_setup_intr(dev, sc->irq_resource, INTR_TYPE_NET | INTR_MPSAFE | INTR_ENTROPY, NULL, dpaa2_io_intr, sc, &sc->intr)) { device_printf(dev, "%s: failed to setup IRQ resource\n", __func__); return (ENXIO); } /* Wrap DPIO ID around number of CPUs/RSS buckets */ #ifdef RSS sc->cpu = rss_getcpu(sc->attr.id % rss_getnumbuckets()); #else sc->cpu = sc->attr.id % mp_ncpus; #endif CPU_SETOF(sc->cpu, &sc->cpu_mask); bus_bind_intr(dev, sc->irq_resource, sc->cpu); /* * Setup and enable Static Dequeue Command to receive CDANs from * channel 0. */ if (sc->swp_desc.has_notif) dpaa2_swp_set_push_dequeue(sc->swp, 0, true); return (0); } static int dpaa2_io_release_irqs(device_t dev) { struct dpaa2_io_softc *sc = device_get_softc(dev); /* Disable receiving CDANs from channel 0. */ if (sc->swp_desc.has_notif) dpaa2_swp_set_push_dequeue(sc->swp, 0, false); /* Release IRQ resources. */ if (sc->intr != NULL) bus_teardown_intr(dev, sc->irq_resource, &sc->intr); if (sc->irq_resource != NULL) bus_release_resource(dev, SYS_RES_IRQ, sc->irq_rid[0], sc->irq_resource); (void)dpaa2_io_release_msi(device_get_softc(dev)); /* Configure software portal to stop generating interrupts. */ dpaa2_swp_set_intr_trigger(sc->swp, 0); dpaa2_swp_clear_intr_status(sc->swp, 0xFFFFFFFFu); return (0); } /** * @brief Allocate MSI interrupts for this DPAA2 I/O object. */ static int dpaa2_io_setup_msi(struct dpaa2_io_softc *sc) { int val; val = pci_msi_count(sc->dev); if (val < DPAA2_IO_MSI_COUNT) device_printf(sc->dev, "MSI: actual=%d, expected=%d\n", val, DPAA2_IO_MSI_COUNT); val = MIN(val, DPAA2_IO_MSI_COUNT); if (pci_alloc_msi(sc->dev, &val) != 0) return (EINVAL); for (int i = 0; i < val; i++) sc->irq_rid[i] = i + 1; return (0); } static int dpaa2_io_release_msi(struct dpaa2_io_softc *sc) { int error; error = pci_release_msi(sc->dev); if (error) { device_printf(sc->dev, "%s: failed to release MSI: error=%d/n", __func__, error); return (error); } return (0); } /** * @brief DPAA2 I/O interrupt handler. */ static void dpaa2_io_intr(void *arg) { struct dpaa2_io_softc *sc = (struct dpaa2_io_softc *) arg; /* struct dpaa2_ni_softc *nisc = NULL; */ struct dpaa2_io_notif_ctx *ctx[DPIO_POLL_MAX]; struct dpaa2_channel *chan; struct dpaa2_dq dq; uint32_t idx, status; uint16_t flags; int rc, cdan_n = 0; status = dpaa2_swp_read_intr_status(sc->swp); if (status == 0) { return; } DPAA2_SWP_LOCK(sc->swp, &flags); if (flags & DPAA2_SWP_DESTROYED) { /* Terminate operation if portal is destroyed. */ DPAA2_SWP_UNLOCK(sc->swp); return; } for (int i = 0; i < DPIO_POLL_MAX; i++) { rc = dpaa2_swp_dqrr_next_locked(sc->swp, &dq, &idx); if (rc) { break; } if ((dq.common.verb & DPAA2_DQRR_RESULT_MASK) == DPAA2_DQRR_RESULT_CDAN) { ctx[cdan_n++] = (struct dpaa2_io_notif_ctx *) dq.scn.ctx; } else { /* TODO: Report unknown DQRR entry. */ } dpaa2_swp_write_reg(sc->swp, DPAA2_SWP_CINH_DCAP, idx); } DPAA2_SWP_UNLOCK(sc->swp); for (int i = 0; i < cdan_n; i++) { chan = (struct dpaa2_channel *)ctx[i]->channel; /* nisc = device_get_softc(chan->ni_dev); */ taskqueue_enqueue(chan->cleanup_tq, &chan->cleanup_task); } /* Enable software portal interrupts back */ dpaa2_swp_clear_intr_status(sc->swp, status); dpaa2_swp_write_reg(sc->swp, DPAA2_SWP_CINH_IIR, 0); } static device_method_t dpaa2_io_methods[] = { /* Device interface */ DEVMETHOD(device_probe, dpaa2_io_probe), DEVMETHOD(device_attach, dpaa2_io_attach), DEVMETHOD(device_detach, dpaa2_io_detach), /* QBMan software portal interface */ DEVMETHOD(dpaa2_swp_enq_multiple_fq, dpaa2_io_enq_multiple_fq), DEVMETHOD(dpaa2_swp_conf_wq_channel, dpaa2_io_conf_wq_channel), DEVMETHOD(dpaa2_swp_query_bp, dpaa2_io_query_bp), DEVMETHOD(dpaa2_swp_release_bufs, dpaa2_io_release_bufs), DEVMETHOD_END }; static driver_t dpaa2_io_driver = { "dpaa2_io", dpaa2_io_methods, sizeof(struct dpaa2_io_softc), }; DRIVER_MODULE(dpaa2_io, dpaa2_rc, dpaa2_io_driver, 0, 0);