/*-
 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
 *
 * Copyright (c) 2021 Adrian Chadd <adrian@FreeBSD.org>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>

#include <sys/kernel.h>
#include <sys/module.h>
#include <sys/rman.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/socket.h>
#include <sys/sockio.h>

#include <net/if.h>
#include <net/if_var.h>
#include <net/if_media.h>
#include <net/ethernet.h>

#include <machine/bus.h>
#include <machine/resource.h>

#include <dev/fdt/fdt_common.h>
#include <dev/ofw/ofw_bus.h>
#include <dev/ofw/ofw_bus_subr.h>

#include <dev/qcom_ess_edma/qcom_ess_edma_var.h>
#include <dev/qcom_ess_edma/qcom_ess_edma_reg.h>
#include <dev/qcom_ess_edma/qcom_ess_edma_hw.h>
#include <dev/qcom_ess_edma/qcom_ess_edma_debug.h>

/*
 * Reset the ESS EDMA core.
 *
 * This is ... problematic.  There's only a single clock control
 * for the ESS core - and that includes both the EDMA (ethernet)
 * and switch hardware.
 *
 * AND, it's a placeholder for what the linux ess-edma driver
 * is doing directly to the ess core because in some instances
 * where there's a single PHY hooked up, it's possible that
 * ess-switch won't be initialised.  In that case it defaults
 * to a very minimal switch config.  Now, that's honestly pretty
 * bad, and instead we should be doing that kind of awareness
 * in ar40xx_switch.
 *
 * So, for now this is a big no-op, at least until everything
 * is implemented enough that I can get the switch/phy code and
 * this EDMA driver code to co-exist.
 */
int
qcom_ess_edma_hw_reset(struct qcom_ess_edma_softc *sc)
{

	EDMA_LOCK_ASSERT(sc);

	device_printf(sc->sc_dev, "%s: called, TODO!\n", __func__);

	/*
	 * This is where the linux ess-edma driver would reset the
	 * ESS core.
	 */

	/*
	 * and here's where the linux ess-edma driver would program
	 * in the initial port config, rgmii control, traffic
	 * port forwarding and broadcast/multicast traffic forwarding.
	 *
	 * instead, this should be done by the ar40xx_switch driver!
	 */

	return (0);
}

/*
 * Get the TX interrupt moderation timer.
 *
 * The resolution of this register is 2uS.
 */
int
qcom_ess_edma_hw_get_tx_intr_moderation(struct qcom_ess_edma_softc *sc,
    uint32_t *usec)
{
	uint32_t reg;

	EDMA_LOCK_ASSERT(sc);

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_IRQ_MODRT_TIMER_INIT);
	reg = reg >> EDMA_IRQ_MODRT_TX_TIMER_SHIFT;
	reg &= EDMA_IRQ_MODRT_TIMER_MASK;

	*usec = reg * 2;

	return (0);
}


/*
 * Set the TX interrupt moderation timer.
 *
 * The resolution of this register is 2uS.
 */
int
qcom_ess_edma_hw_set_tx_intr_moderation(struct qcom_ess_edma_softc *sc,
    uint32_t usec)
{
	uint32_t reg;

	usec = usec / 2;

	EDMA_LOCK_ASSERT(sc);

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_IRQ_MODRT_TIMER_INIT);
	reg &= ~(EDMA_IRQ_MODRT_TIMER_MASK << EDMA_IRQ_MODRT_TX_TIMER_SHIFT);
	reg |= (usec & EDMA_IRQ_MODRT_TIMER_MASK)
	    << EDMA_IRQ_MODRT_TX_TIMER_SHIFT;
	EDMA_REG_WRITE(sc, EDMA_REG_IRQ_MODRT_TIMER_INIT, reg);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Set the RX interrupt moderation timer.
 *
 * The resolution of this register is 2uS.
 */
int
qcom_ess_edma_hw_set_rx_intr_moderation(struct qcom_ess_edma_softc *sc,
    uint32_t usec)
{
	uint32_t reg;

	EDMA_LOCK_ASSERT(sc);

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_IRQ_MODRT_TIMER_INIT);
	reg &= ~(EDMA_IRQ_MODRT_TIMER_MASK << EDMA_IRQ_MODRT_RX_TIMER_SHIFT);
	reg |= (usec & EDMA_IRQ_MODRT_TIMER_MASK)
	    << EDMA_IRQ_MODRT_RX_TIMER_SHIFT;
	EDMA_REG_WRITE(sc, EDMA_REG_IRQ_MODRT_TIMER_INIT, reg);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Disable all interrupts.
 */
int
qcom_ess_edma_hw_intr_disable(struct qcom_ess_edma_softc *sc)
{
	int i;

	/* Disable TX interrupts */
	for (i = 0; i < QCOM_ESS_EDMA_NUM_TX_IRQS; i++) {
		EDMA_REG_WRITE(sc, EDMA_REG_TX_INT_MASK_Q(i), 0);
	}

	/* Disable RX interrupts */
	for (i = 0; i < QCOM_ESS_EDMA_NUM_RX_IRQS; i++) {
		EDMA_REG_WRITE(sc, EDMA_REG_RX_INT_MASK_Q(i), 0);
	}

	/* Disable misc/WOL interrupts */
	EDMA_REG_WRITE(sc, EDMA_REG_MISC_IMR, 0);
	EDMA_REG_WRITE(sc, EDMA_REG_WOL_IMR, 0);

	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Enable/disable the given RX ring interrupt.
 */
int
qcom_ess_edma_hw_intr_rx_intr_set_enable(struct qcom_ess_edma_softc *sc,
    int rxq, bool state)
{
	EDMA_REG_WRITE(sc, EDMA_REG_RX_INT_MASK_Q(rxq), state ? 1 : 0);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Enable/disable the given TX ring interrupt.
 */
int
qcom_ess_edma_hw_intr_tx_intr_set_enable(struct qcom_ess_edma_softc *sc,
    int txq, bool state)
{

	EDMA_REG_WRITE(sc, EDMA_REG_TX_INT_MASK_Q(txq), state ? 1 : 0);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Enable interrupts.
 */
int
qcom_ess_edma_hw_intr_enable(struct qcom_ess_edma_softc *sc)
{
	int i;

	/* ACK, then Enable TX interrupts */
	EDMA_REG_WRITE(sc, EDMA_REG_TX_ISR, 0xffff);
	for (i = 0; i < QCOM_ESS_EDMA_NUM_TX_IRQS; i++) {
		EDMA_REG_WRITE(sc, EDMA_REG_TX_INT_MASK_Q(i),
		    sc->sc_config.tx_intr_mask);
	}

	/* ACK, then Enable RX interrupts */
	EDMA_REG_WRITE(sc, EDMA_REG_RX_ISR, 0xff);
	for (i = 0; i < QCOM_ESS_EDMA_NUM_RX_IRQS; i++) {
		EDMA_REG_WRITE(sc, EDMA_REG_RX_INT_MASK_Q(i),
		    sc->sc_config.rx_intr_mask);
	}

	/* Disable misc/WOL interrupts */
	EDMA_REG_WRITE(sc, EDMA_REG_MISC_IMR, 0);
	EDMA_REG_WRITE(sc, EDMA_REG_WOL_IMR, 0);

	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Clear interrupt status.
 */
int
qcom_ess_edma_hw_intr_status_clear(struct qcom_ess_edma_softc *sc)
{

	EDMA_REG_WRITE(sc, EDMA_REG_RX_ISR, 0xff);
	EDMA_REG_WRITE(sc, EDMA_REG_TX_ISR, 0xffff);
	EDMA_REG_WRITE(sc, EDMA_REG_MISC_ISR, 0x1fff);
	EDMA_REG_WRITE(sc, EDMA_REG_WOL_ISR, 0x1);

	return (0);
}

/*
 * ACK the given RX queue ISR.
 *
 * Must be called with the RX ring lock held!
 */
int
qcom_ess_edma_hw_intr_rx_ack(struct qcom_ess_edma_softc *sc, int rx_queue)
{

	EDMA_RING_LOCK_ASSERT(&sc->sc_rx_ring[rx_queue]);
	EDMA_REG_WRITE(sc, EDMA_REG_RX_ISR, (1U << rx_queue));
	(void) EDMA_REG_READ(sc, EDMA_REG_RX_ISR);

	return (0);
}

/*
 * ACK the given TX queue ISR.
 *
 * Must be called with the TX ring lock held!
 */
int
qcom_ess_edma_hw_intr_tx_ack(struct qcom_ess_edma_softc *sc, int tx_queue)
{

	EDMA_RING_LOCK_ASSERT(&sc->sc_tx_ring[tx_queue]);
	EDMA_REG_WRITE(sc, EDMA_REG_TX_ISR, (1U << tx_queue));
	(void) EDMA_REG_READ(sc, EDMA_REG_TX_ISR);

	return (0);
}

/*
 * Configure the default RSS indirection table.
 */
int
qcom_ess_edma_hw_configure_rss_table(struct qcom_ess_edma_softc *sc)
{
	int i;

	/*
	 * The default IDT value configures the hash buckets
	 * to a repeating pattern of q0, q2, q4, q6.
	 */
	for (i = 0; i < EDMA_NUM_IDT; i++) {
		EDMA_REG_WRITE(sc, EDMA_REG_RSS_IDT(i), EDMA_RSS_IDT_VALUE);
	}
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Configure the default load balance mapping table.
 */
int
qcom_ess_edma_hw_configure_load_balance_table(struct qcom_ess_edma_softc *sc)
{

	/*
	 * I think this is mapping things to queues 0,2,4,6.
	 * Linux says it's 0,1,3,4 but that doesn't match the
	 * EDMA_LB_REG_VALUE field.
	 */
	EDMA_REG_WRITE(sc, EDMA_REG_LB_RING, EDMA_LB_REG_VALUE);
	EDMA_REG_BARRIER_WRITE(sc);
	return (0);
}

/*
 * Configure the default virtual tx ring queues.
 */
int
qcom_ess_edma_hw_configure_tx_virtual_queue(struct qcom_ess_edma_softc *sc)
{

	EDMA_REG_WRITE(sc, EDMA_REG_VQ_CTRL0, EDMA_VQ_REG_VALUE);
	EDMA_REG_WRITE(sc, EDMA_REG_VQ_CTRL1, EDMA_VQ_REG_VALUE);

	EDMA_REG_BARRIER_WRITE(sc);
	return (0);
}

/*
 * Configure the default maximum AXI bus transaction size.
 */
int
qcom_ess_edma_hw_configure_default_axi_transaction_size(
    struct qcom_ess_edma_softc *sc)
{

	EDMA_REG_WRITE(sc, EDMA_REG_AXIW_CTRL_MAXWRSIZE,
	    EDMA_AXIW_MAXWRSIZE_VALUE);
	return (0);
}

/*
 * Stop the TX/RX queues.
 */
int
qcom_ess_edma_hw_stop_txrx_queues(struct qcom_ess_edma_softc *sc)
{
	uint32_t reg;

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_RXQ_CTRL);
	reg &= ~EDMA_RXQ_CTRL_EN;
	EDMA_REG_WRITE(sc, EDMA_REG_RXQ_CTRL, reg);
	EDMA_REG_BARRIER_WRITE(sc);

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_TXQ_CTRL);
	reg &= ~EDMA_TXQ_CTRL_TXQ_EN;
	EDMA_REG_WRITE(sc, EDMA_REG_TXQ_CTRL, reg);
	EDMA_REG_BARRIER_WRITE(sc);
	return (0);
}

/*
 * Stop the EDMA block, disable interrupts.
 */
int
qcom_ess_edma_hw_stop(struct qcom_ess_edma_softc *sc)
{
	int ret;

	EDMA_LOCK_ASSERT(sc);

	ret = qcom_ess_edma_hw_intr_disable(sc);
	if (ret != 0) {
		QCOM_ESS_EDMA_DPRINTF(sc, QCOM_ESS_EDMA_DBG_INTERRUPT,
		    "%s: hw_intr_disable failed (%d)\n",
		    __func__,
		    ret);
	}

	ret = qcom_ess_edma_hw_intr_status_clear(sc);
	if (ret != 0) {
		QCOM_ESS_EDMA_DPRINTF(sc, QCOM_ESS_EDMA_DBG_INTERRUPT,
		    "%s: hw_intr_status_clear failed (%d)\n",
		    __func__,
		    ret);
	}

	ret = qcom_ess_edma_hw_stop_txrx_queues(sc);
	if (ret != 0) {
		QCOM_ESS_EDMA_DPRINTF(sc, QCOM_ESS_EDMA_DBG_INTERRUPT,
		    "%s: hw_stop_txrx_queues failed (%d)\n",
		    __func__,
		    ret);
	}

	return (0);
}

/*
 * Update the producer index for the given receive queue.
 *
 * Note: the RX ring lock must be held!
 *
 * Return 0 if OK, an error number if there's an error.
 */
int
qcom_ess_edma_hw_rfd_prod_index_update(struct qcom_ess_edma_softc *sc,
    int queue, int idx)
{
	uint32_t reg;

	EDMA_RING_LOCK_ASSERT(&sc->sc_rx_ring[queue]);

	QCOM_ESS_EDMA_DPRINTF(sc, QCOM_ESS_EDMA_DBG_RX_RING_MGMT,
	    "%s: called; q=%d idx=0x%x\n",
	    __func__, queue, idx);

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_RFD_IDX_Q(queue));
	QCOM_ESS_EDMA_DPRINTF(sc, QCOM_ESS_EDMA_DBG_RX_RING_MGMT,
	    "%s: q=%d reg was 0x%08x\n", __func__, queue, reg);
	reg &= ~EDMA_RFD_PROD_IDX_BITS;
	reg |= idx;
	EDMA_REG_WRITE(sc, EDMA_REG_RFD_IDX_Q(queue), reg);
	QCOM_ESS_EDMA_DPRINTF(sc, QCOM_ESS_EDMA_DBG_RX_RING_MGMT,
	    "%s: q=%d reg now 0x%08x\n", __func__, queue, reg);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Fetch the consumer index for the given receive queue.
 *
 * Returns the current consumer index.
 *
 * Note - since it's used in statistics/debugging it isn't asserting the
 * RX ring lock, so be careful when/how you use this!
 */
int
qcom_ess_edma_hw_rfd_get_cons_index(struct qcom_ess_edma_softc *sc, int queue)
{
	uint32_t reg;

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_RFD_IDX_Q(queue));
	return (reg >> EDMA_RFD_CONS_IDX_SHIFT) & EDMA_RFD_CONS_IDX_MASK;
}

/*
 * Update the software consumed index to the hardware, so
 * it knows what we've read.
 *
 * Note: the RX ring lock must be held when calling this!
 *
 * Returns 0 if OK, error number if error.
 */
int
qcom_ess_edma_hw_rfd_sw_cons_index_update(struct qcom_ess_edma_softc *sc,
    int queue, int idx)
{
	EDMA_RING_LOCK_ASSERT(&sc->sc_rx_ring[queue]);

	EDMA_REG_WRITE(sc, EDMA_REG_RX_SW_CONS_IDX_Q(queue), idx);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Setup initial hardware configuration.
 */
int
qcom_ess_edma_hw_setup(struct qcom_ess_edma_softc *sc)
{
	uint32_t reg;

	EDMA_LOCK_ASSERT(sc);

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_INTR_CTRL);
	reg &= ~(1 << EDMA_INTR_SW_IDX_W_TYP_SHIFT);
	reg |= sc->sc_state.intr_sw_idx_w << EDMA_INTR_SW_IDX_W_TYP_SHIFT;
	EDMA_REG_WRITE(sc, EDMA_REG_INTR_CTRL, reg);


	/* Clear wake-on-lan config */
	EDMA_REG_WRITE(sc, EDMA_REG_WOL_CTRL, 0);

	/* configure initial interrupt moderation config */
	reg = (EDMA_TX_IMT << EDMA_IRQ_MODRT_TX_TIMER_SHIFT);
	reg |= (EDMA_RX_IMT << EDMA_IRQ_MODRT_RX_TIMER_SHIFT);
	EDMA_REG_WRITE(sc, EDMA_REG_IRQ_MODRT_TIMER_INIT, reg);

	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Setup TX DMA burst configuration.
 */
int
qcom_ess_edma_hw_setup_tx(struct qcom_ess_edma_softc *sc)
{
	uint32_t reg;

	EDMA_LOCK_ASSERT(sc);

	reg = (EDMA_TPD_BURST << EDMA_TXQ_NUM_TPD_BURST_SHIFT);
	reg |= EDMA_TXQ_CTRL_TPD_BURST_EN;
	reg |= (EDMA_TXF_BURST << EDMA_TXQ_TXF_BURST_NUM_SHIFT);
	EDMA_REG_WRITE(sc, EDMA_REG_TXQ_CTRL, reg);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Setup default RSS, RX burst/prefetch/interrupt thresholds.
 *
 * Strip VLANs, those are offloaded in the RX descriptor.
 */
int
qcom_ess_edma_hw_setup_rx(struct qcom_ess_edma_softc *sc)
{
	uint32_t reg;

	EDMA_LOCK_ASSERT(sc);

	/* Configure RSS types */
	EDMA_REG_WRITE(sc, EDMA_REG_RSS_TYPE, sc->sc_config.rss_type);

	/* Configure RFD burst */
	reg = (EDMA_RFD_BURST << EDMA_RXQ_RFD_BURST_NUM_SHIFT);
	/* .. and RFD prefetch threshold */
	reg |= (EDMA_RFD_THR << EDMA_RXQ_RFD_PF_THRESH_SHIFT);
	/* ... and threshold to generate RFD interrupt */
	reg |= (EDMA_RFD_LTHR << EDMA_RXQ_RFD_LOW_THRESH_SHIFT);
	EDMA_REG_WRITE(sc, EDMA_REG_RX_DESC1, reg);

	/* Set RX FIFO threshold to begin DMAing data to host */
	reg = EDMA_FIFO_THRESH_128_BYTE;
	/* Remove VLANs (??) */
	reg |= EDMA_RXQ_CTRL_RMV_VLAN;
	EDMA_REG_WRITE(sc, EDMA_REG_RXQ_CTRL, reg);

	EDMA_REG_BARRIER_WRITE(sc);
	return (0);
}

/*
 * XXX TODO: this particular routine is a bit big and likely should be split
 * across main, hw, desc, rx and tx.  But to expedite initial bring-up,
 * let's just commit the sins here and get receive up and going.
 */
int
qcom_ess_edma_hw_setup_txrx_desc_rings(struct qcom_ess_edma_softc *sc)
{
	uint32_t reg, i, idx;
	int len;

	EDMA_LOCK_ASSERT(sc);

	/*
	 * setup base addresses for each transmit ring, and
	 * read in the initial index to use for transmit.
	 */
	for (i = 0; i < QCOM_ESS_EDMA_NUM_TX_RINGS; i++) {
		/* Descriptor ring based address */
		QCOM_ESS_EDMA_DPRINTF(sc, QCOM_ESS_EDMA_DBG_TX_RING_MGMT,
		    "TXQ[%d]: ring paddr=0x%08lx\n",
		    i, sc->sc_tx_ring[i].hw_desc_paddr);
		EDMA_REG_WRITE(sc, EDMA_REG_TPD_BASE_ADDR_Q(i),
		    sc->sc_tx_ring[i].hw_desc_paddr);

		/* And now, grab the consumer index */
		reg = EDMA_REG_READ(sc, EDMA_REG_TPD_IDX_Q(i));
		idx = (reg >> EDMA_TPD_CONS_IDX_SHIFT) & 0xffff;

		sc->sc_tx_ring[i].next_to_fill = idx;
		sc->sc_tx_ring[i].next_to_clean = idx;

		/* Update prod and sw consumer indexes */
		reg &= ~(EDMA_TPD_PROD_IDX_MASK << EDMA_TPD_PROD_IDX_SHIFT);
		reg |= idx;
		EDMA_REG_WRITE(sc, EDMA_REG_TPD_IDX_Q(i), reg);
		EDMA_REG_WRITE(sc, EDMA_REG_TX_SW_CONS_IDX_Q(i), idx);

		/* Set the ring size */
		EDMA_REG_WRITE(sc, EDMA_REG_TPD_RING_SIZE,
		    sc->sc_config.tx_ring_count & EDMA_TPD_RING_SIZE_MASK);

	}

	/* Set base addresses for each RFD ring */
	for (i = 0; i < QCOM_ESS_EDMA_NUM_RX_RINGS; i++) {
		QCOM_ESS_EDMA_DPRINTF(sc, QCOM_ESS_EDMA_DBG_RX_RING_MGMT,
		    "RXQ[%d]: ring paddr=0x%08lx\n",
		    i, sc->sc_rx_ring[i].hw_desc_paddr);
		EDMA_REG_WRITE(sc, EDMA_REG_RFD_BASE_ADDR_Q(i),
		    sc->sc_rx_ring[i].hw_desc_paddr);
	}
	EDMA_REG_BARRIER_WRITE(sc);

	/* Configure RX buffer size */
	len = sc->sc_config.rx_buf_size;
	if (sc->sc_config.rx_buf_ether_align)
		len -= ETHER_ALIGN;
	reg = (len & EDMA_RX_BUF_SIZE_MASK)
	    << EDMA_RX_BUF_SIZE_SHIFT;
	/* .. and RFD ring size */
	reg |= (sc->sc_config.rx_ring_count & EDMA_RFD_RING_SIZE_MASK)
	    << EDMA_RFD_RING_SIZE_SHIFT;
	EDMA_REG_WRITE(sc, EDMA_REG_RX_DESC0, reg);

	/* Disable the TX low/high watermark (for interrupts?) */
	EDMA_REG_WRITE(sc, EDMA_REG_TXF_WATER_MARK, 0);

	EDMA_REG_BARRIER_WRITE(sc);

	/* Load all the ring base addresses into the hardware */
	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_TX_SRAM_PART);
	reg |= 1 << EDMA_LOAD_PTR_SHIFT;
	EDMA_REG_WRITE(sc, EDMA_REG_TX_SRAM_PART, reg);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Enable general MAC TX DMA.
 */
int
qcom_ess_edma_hw_tx_enable(struct qcom_ess_edma_softc *sc)
{
	uint32_t reg;

	EDMA_LOCK_ASSERT(sc);

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_TXQ_CTRL);
	reg |= EDMA_TXQ_CTRL_TXQ_EN;
	EDMA_REG_WRITE(sc, EDMA_REG_TXQ_CTRL, reg);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Enable general MAC RX DMA.
 */
int
qcom_ess_edma_hw_rx_enable(struct qcom_ess_edma_softc *sc)
{
	EDMA_LOCK_ASSERT(sc);
	uint32_t reg;

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_RXQ_CTRL);
	reg |= EDMA_RXQ_CTRL_EN;
	EDMA_REG_WRITE(sc, EDMA_REG_RXQ_CTRL, reg);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Read the TPD consumer index register for the given transmit ring.
 */
int
qcom_ess_edma_hw_tx_read_tpd_cons_idx(struct qcom_ess_edma_softc *sc,
    int queue_id, uint16_t *idx)
{
	uint32_t reg;

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_TPD_IDX_Q(queue_id));
	*idx = (reg >> EDMA_TPD_CONS_IDX_SHIFT) & EDMA_TPD_CONS_IDX_MASK;

	return (0);
}

/*
 * Update the TPD producer index for the given transmit wring.
 */
int
qcom_ess_edma_hw_tx_update_tpd_prod_idx(struct qcom_ess_edma_softc *sc,
    int queue_id, uint16_t idx)
{
	uint32_t reg;

	EDMA_REG_BARRIER_READ(sc);
	reg = EDMA_REG_READ(sc, EDMA_REG_TPD_IDX_Q(queue_id));
	reg &= ~EDMA_TPD_PROD_IDX_BITS;
	reg |= (idx & EDMA_TPD_PROD_IDX_MASK) << EDMA_TPD_PROD_IDX_SHIFT;
	EDMA_REG_WRITE(sc, EDMA_REG_TPD_IDX_Q(queue_id), reg);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}

/*
 * Update the TPD software consumer index register for the given
 * transmit ring - ie, what software has cleaned.
 */
int
qcom_ess_edma_hw_tx_update_cons_idx(struct qcom_ess_edma_softc *sc,
    int queue_id, uint16_t idx)
{

	EDMA_REG_WRITE(sc, EDMA_REG_TX_SW_CONS_IDX_Q(queue_id), idx);
	EDMA_REG_BARRIER_WRITE(sc);

	return (0);
}