cancel
Showing results for 
Show  only  | Search instead for 
Did you mean: 
viggy
Observer
Observer
445 Views
Registered: ‎11-28-2018

DMA between PS and PL doesn't appear to be working...

I can't seem to get DMA between PS and PL to work for the Ultra96 board. I followed the test client from https://xilinx-wiki.atlassian.net/wiki/spaces/A/pages/18842528/Zynqmp+DMA as recommended by Xilinx, and it works great. I can use DMA to transfer data between two allocated buffers in the kernel driver.

However, once I change one of the buffer pointers to a pointer returned from a call to "ioremap_nocache", nothing is copied. The DMA engine says the transaction is complete, but nothing was copied to the PL, or copied from the PL. This is a snippet of the code:

// Write to FPGA...
static ssize_t device_write(struct file *pFile, const char *pBuf, size_t len, loff_t *pOffset)
{
    enum dma_status status;
    void __iomem *pFpga = ioremap_nocache(pFile->f_pos, POOL_SIZE);
    void *pTmpBuf;

    pTmpBuf = kmalloc(len, GFP_KERNEL);
    memcpy(pTmpBuf, pBuf, len);
    status = dmaTransfer(pTmpBuf/*pBuf*/, pFpga, len, true);

    if (status != DMA_COMPLETE) {
        printk(KERN_ERR pr_fmt("Error writing to device"));
        kfree(pTmpBuf);
        iounmap(pFpga);
        return -status;
    }
    // Free resources...
    kfree(pTmpBuf);
    iounmap(pFpga);

    // Increment the file position...
    pFile->f_pos += len;

    return len;
}

And, the DMA code:

static int dmaTransfer(const void *pSrc, void *pDst, u64 len, bool bWrite)
{
    enum dma_status status = DMA_ERROR;

    struct cqzdma_params params;
    strcpy(params._chan, strim(dma_channel));
    strcpy(params._device, strim(dma_device));
    if (getChannel(&params, DMA_MEMCPY)) {
        struct dma_chan *pChan;
        struct dma_device *pDev;
        u8 align = 0;
        struct dmaengine_unmap_data *pUmData;
        unsigned int alignLen = len;

        smp_rmb();  // Set hardware memory barrier

        pChan = params._pChan;
        pDev = pChan->device;
        align = pDev->copy_align;

        alignLen = (alignLen >> align) << align;
        if (alignLen == 0) {
            alignLen = 1 << align;
        }

        pUmData = dmaengine_get_unmap_data(pDev->dev, 2, GFP_KERNEL);
        if (pUmData != NULL) {
            struct page *pPage;
            unsigned long pgOff;
            struct dma_async_tx_descriptor *pTx = NULL;
            dma_addr_t dmaSrcAddr, dmaDstAddr;
            dma_cookie_t dmaCookie;
            pUmData->len = len;

            pPage = virt_to_page(pSrc);
            pgOff = offset_in_page(pSrc);
            pUmData->addr[0] = dma_map_page(pDev->dev, pPage, pgOff, pUmData->len, (bWrite) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
            dmaSrcAddr = pUmData->addr[0];
            if (dma_mapping_error(pDev->dev, pUmData->addr[0])) {
                printk(KERN_ERR pr_fmt("Error DMA mapping source\n"));
                goto cleanup;
            }
            if (bWrite) {
                pUmData->to_cnt++;
            }
            else {
                pUmData->from_cnt++;
            }
            pPage = virt_to_page(pDst);
            pgOff = offset_in_page(pDst);
            pUmData->addr[1] = dma_map_page(pDev->dev, pPage, pgOff, pUmData->len, (bWrite) ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
            dmaDstAddr = pUmData->addr[1];
            if (dma_mapping_error(pDev->dev, pUmData->addr[1])) {
                printk(KERN_ERR pr_fmt("Error DMA mapping destination\n"));
                goto cleanup;
            }
            if (bWrite) {
                pUmData->from_cnt++;
            }
            else {
                pUmData->to_cnt++;
            }
            pTx = pDev->device_prep_dma_memcpy(pChan, dmaDstAddr, dmaSrcAddr, alignLen, DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
            if (pTx == NULL) {
                printk(KERN_ERR pr_fmt("Error prepping DMA transaction\n"));
                msleep(100);
                goto cleanup;
            }
            dmaCookie = pTx->tx_submit(pTx);
            if (dma_submit_error(dmaCookie)) {
                printk(KERN_ERR pr_fmt("Error submitting transaction\n"));
                msleep(100);
                goto cleanup;
            }

            status = dma_sync_wait(pChan, dmaCookie);
            dmaengine_terminate_sync(pChan);
            if (status != DMA_COMPLETE) {
                switch (status) {
                case DMA_IN_PROGRESS:
                    printk(KERN_INFO pr_fmt("DMA in progres..."));
                    break;
                case DMA_PAUSED:
                    printk(KERN_INFO pr_fmt("DMA paused..."));
                    break;
                case DMA_ERROR:
                    printk(KERN_ERR pr_fmt("DMA error"));
                    break;
                default:
                    ;
                }
            }
            else {
#ifdef DEBUG
                printk(KERN_DEBUG pr_fmt("DMA transfer complete..."));
#endif
            }

        cleanup:
            dmaengine_unmap_put(pUmData);
            dmaengine_terminate_sync(pChan);
            dma_release_channel(pChan);
        }
        else {
            printk(KERN_ERR pr_fmt("Unable to get DMA unmap data for channel %s\n"), dma_channel);
            dmaengine_terminate_sync(pChan);
            dma_release_channel(pChan);
        }
    }
    else {
        if (dma_device[0] == '\0') {
            printk(KERN_ERR pr_fmt("Cannot find DMA channel %s\n"), params._chan);
        }
        else {
            printk(KERN_ERR pr_fmt("Cannot find DMA channel %s, device %s\n"),
                params._chan, params._device);
        }
    }
    return status;
}

"DMA transfer complete..." is printed in the logs, leading me to believe that it worked, however the buffer in the PL is garbage.

3 Replies
bkamen
Explorer
Explorer
303 Views
Registered: ‎07-17-2014

Hey Viggy,

 I've been looking at your code -- and mine's a bit different and on older petalinux for a uZed-020 from AVNET.. so it's hard for me to give you an exact answer -- but I can tell you this:

Keeping the pointers straight between userland and kernel space was where I found similar things happening to what you described.

DMA worked -- but my data is all wonky.

Turned out it was pointer issues (and type casting)

I realize that's not very specific but I hope it helps.

I also ended up doing a mmap to userspace to save time with writing my DMA buffer to disk (if I needed to).

Copying from kernel -> userspace was not optimal.

Anyway - not sure if you fixed your problem. Hopefully so. DMA can be an elusive beast.

Cheers,

 -Ben

0 Kudos
viggy
Observer
Observer
269 Views
Registered: ‎11-28-2018

Thanks Ben.

Actually, the issue we were running into was that the clock to the DMA block wasn't enabled! Setting bit 24 in the register at 0xFD1A00B8 fixed the issues.

Viggy

0 Kudos
bkamen
Explorer
Explorer
252 Views
Registered: ‎07-17-2014

That's weird. I never ran into that particular issue.

Sounds like you found something new?

I also ran into a Xilinx driver bug that the driver didn't properly reset a DMA channel with a DMA timeout.

I had to go edit the driver to fix the problem.

If you have any issues with DMA timeouts breaking the DMA channel -- let me know and we can look to see if the DMA driver was fixed.