Writing simple DMA driver

Question

I want to use DMA in a driver that I'm working on. The ultimate goal is to ensure that the data is in physical RAM and not hidden in a cache. For that I was trying to implement a simple test driver before merging it with my current project.

From what I understand I can setup a DMA mask and allocate a coherent buffer and then simply write to the virtual address. My other device can then read from the physical address. It would be nice if someone could confirm if that is indeed how it works.

Unfortunately the allocation fails and I can't decipher the syslogs for why that could be. I'm probably doing something wrong with the struct device. What is the device reference in dma_alloc_coherent used for?

Here's my current attempt. It's a modified dummy character device:

#include <linux/module.h>
#include <linux/fs.h>
#include <linux/device.h>
#include <linux/cdev.h>
#include <linux/uaccess.h>
#include <linux/dma-mapping.h>

static char hello_world[]="Hello World\n";

static dev_t hello_dev_number;
static struct cdev *driver_object;
static struct class *hello_class;
static struct device *hello_dev;
int errorType;

void *virtAddr;
size_t size = 32;
dma_addr_t *physAddr;
int flag = GFP_KERNEL;

static ssize_t driver_read( struct file *instance, char __user *user, size_t count, loff_t *offset )
{
    unsigned long not_copied, to_copy;

    to_copy = min( count, strlen(hello_world)+1 );
    not_copied=copy_to_user(user,hello_world,to_copy);
    *offset += to_copy-not_copied;
    return to_copy-not_copied;
}

static struct file_operations fops = {
    .owner= THIS_MODULE,
    .read= driver_read,
};

static int __init mod_init( void )
{
    int debug;
    printk("starting insertion");
    if (alloc_chrdev_region(&hello_dev_number,0,1,"Hello")<0)
        return -EIO;
    driver_object = cdev_alloc();
    if (driver_object==NULL){
        errorType = EIO;
        goto free_device_number;
    }
    driver_object->owner = THIS_MODULE;
    driver_object->ops = &fops;
    if (cdev_add(driver_object,hello_dev_number,1)){
        errorType=EIO;
        goto free_cdev;
    }
    hello_class = class_create( THIS_MODULE, "Hello" );
    if (IS_ERR( hello_class )) {
        pr_err( "hello: no udev support\n");
        errorType=EIO;
        goto free_cdev;
    }
    hello_dev = device_create( hello_class, NULL, hello_dev_number, NULL, "%s", "hello" );
    if (IS_ERR( hello_dev )) {
        pr_err( "hello: device_create failed\n");
        errorType=EIO;
        goto free_class;
    }
    printk("dma is happening");
    debug = dma_set_mask_and_coherent(hello_dev, DMA_BIT_MASK(32));
    printk("dma mask returns: %d", debug);
    if(debug==0){
        printk("setting mask failed");
        errorType=EIO;
        goto free_dev;
    }
    printk("goodbye crule world!");
    virtAddr = dma_alloc_coherent(hello_dev, size, physAddr, flag);
    printk("I'm actually alive");
    if(virtAddr==NULL){
        printk("virtual address null, dma failed!");
        errorType=ENOMEM;
        goto free_dev;
    }
    printk("success");
    return 0;
free_dev:
    device_destroy( hello_class, hello_dev_number );
free_class:
    class_destroy( hello_class );
free_cdev:
    kobject_put( &driver_object->kobj );
free_device_number:
    unregister_chrdev_region( hello_dev_number, 1 );
    return -errorType;
}

static void __exit mod_exit( void )
{
    dma_free_coherent(hello_dev, size, virtAddr, *physAddr);
    device_destroy( hello_class, hello_dev_number );
    class_destroy( hello_class );
    cdev_del( driver_object );
    unregister_chrdev_region( hello_dev_number, 1 );
    return;
}

module_init( mod_init );
module_exit( mod_exit );

MODULE_AUTHOR("ME");
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Testing DMA.");

Here's the Makefile that I use:

obj-m+=dma_test.o

all:
    make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules
clean:
    make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean

Running this on an ARMv7 AM335x Sitara CPU (Beaglebone Black), Linux Kernel 5.4.106-ti-r27 Here's the kernel log:

Mar 14 02:38:23 beaglebone kernel: [  100.662640] starting insertion
Mar 14 02:38:23 beaglebone kernel: [  100.667068] dma is happening
Mar 14 02:38:23 beaglebone kernel: [  100.667087] dma mask returns: -5
Mar 14 02:38:23 beaglebone kernel: [  100.667092] goodbye crule world!
Mar 14 02:38:23 beaglebone kernel: [  100.667100] ------------[ cut here ]------------
Mar 14 02:38:23 beaglebone kernel: [  100.667128] WARNING: CPU: 0 PID: 2360 at kernel/dma/mapping.c:272 dma_alloc_attrs+0x118/0x128
Mar 14 02:38:23 beaglebone kernel: [  100.667134] Modules linked in: dma_test(O+) c_can_platform c_can can_dev evdev usb_f_acm u_serial usb_f_ecm usb_f_mass_storage usb_f_rndis u_ether libcomposite uio_pdrv_genirq(O) uio iptable_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 iptable_mangle iptable_filter dmatest(O) ip_tables x_tables icss_iep prueth_ecap spidev
Mar 14 02:38:23 beaglebone kernel: [  100.667216] CPU: 0 PID: 2360 Comm: insmod Tainted: G           O      5.4.106-ti-r27 #1buster
Mar 14 02:38:23 beaglebone kernel: [  100.667222] Hardware name: Generic AM33XX (Flattened Device Tree)
Mar 14 02:38:23 beaglebone kernel: [  100.667228] Backtrace: 
Mar 14 02:38:23 beaglebone kernel: [  100.667248] [<c0e37fd8>] (dump_backtrace) from [<c0e38390>] (show_stack+0x20/0x24)
Mar 14 02:38:23 beaglebone kernel: [  100.667259]  r7:600f0113 r6:c14e3154 r5:00000000 r4:c14e3154
Mar 14 02:38:23 beaglebone kernel: [  100.667274] [<c0e38370>] (show_stack) from [<c0e49af0>] (dump_stack+0xb8/0xcc)
Mar 14 02:38:23 beaglebone kernel: [  100.667289] [<c0e49a38>] (dump_stack) from [<c013c6cc>] (__warn+0xe0/0x108)
Mar 14 02:38:23 beaglebone kernel: [  100.667299]  r7:00000110 r6:00000009 r5:c01c7090 r4:c11194a8
Mar 14 02:38:23 beaglebone kernel: [  100.667309] [<c013c5ec>] (__warn) from [<c0e38c00>] (warn_slowpath_fmt+0x70/0xd8)
Mar 14 02:38:23 beaglebone kernel: [  100.667318]  r7:00000110 r6:c11194a8 r5:c1405fc8 r4:00000000
Mar 14 02:38:23 beaglebone kernel: [  100.667328] [<c0e38b94>] (warn_slowpath_fmt) from [<c01c7090>] (dma_alloc_attrs+0x118/0x128)
Mar 14 02:38:23 beaglebone kernel: [  100.667339]  r9:00000cc0 r8:00000000 r7:00000020 r6:dc78d600 r5:c1405fc8 r4:c0f01734
Mar 14 02:38:23 beaglebone kernel: [  100.667368] [<c01c6f78>] (dma_alloc_attrs) from [<bf00c1b4>] (mod_init+0x1b4/0x1000 [dma_test])
Mar 14 02:38:23 beaglebone kernel: [  100.667379]  r9:bf0e00cc r8:d29cff30 r7:dc78d600 r6:fffffffb r5:00000000 r4:bf0e0300
Mar 14 02:38:23 beaglebone kernel: [  100.667399] [<bf00c000>] (mod_init [dma_test]) from [<c0103268>] (do_one_initcall+0x50/0x2d0)
Mar 14 02:38:23 beaglebone kernel: [  100.667408]  r7:00000000 r6:bf0e01f0 r5:bf00c000 r4:c1405fc8
Mar 14 02:38:23 beaglebone kernel: [  100.667423] [<c0103218>] (do_one_initcall) from [<c01f0878>] (do_init_module+0x70/0x274)
Mar 14 02:38:23 beaglebone kernel: [  100.667433]  r8:d29cff30 r7:bf0e00c0 r6:bf0e01f0 r5:db20cf40 r4:bf0e00c0
Mar 14 02:38:23 beaglebone kernel: [  100.667444] [<c01f0808>] (do_init_module) from [<c01f2adc>] (load_module+0x1f64/0x2370)
Mar 14 02:38:23 beaglebone kernel: [  100.667452]  r6:bf0e01f0 r5:00000000 r4:bf0e01c0
Mar 14 02:38:23 beaglebone kernel: [  100.667463] [<c01f0b78>] (load_module) from [<c01f318c>] (sys_finit_module+0xc0/0x110)
Mar 14 02:38:23 beaglebone kernel: [  100.667474]  r10:0000017b r9:d29ce000 r8:c0101204 r7:0043c7e0 r6:00000003 r5:00000000
Mar 14 02:38:23 beaglebone kernel: [  100.667480]  r4:c1405fc8
Mar 14 02:38:23 beaglebone kernel: [  100.667491] [<c01f30cc>] (sys_finit_module) from [<c0101000>] (ret_fast_syscall+0x0/0x54)
Mar 14 02:38:23 beaglebone kernel: [  100.667499] Exception stack(0xd29cffa8 to 0xd29cfff0)
Mar 14 02:38:23 beaglebone kernel: [  100.667511] ffa0:                   3d6d5800 00000000 00000003 0043c7e0 00000000 bece2578
Mar 14 02:38:23 beaglebone kernel: [  100.667522] ffc0: 3d6d5800 00000000 00000000 0000017b 015347e0 00000000 bece26f8 00000000
Mar 14 02:38:23 beaglebone kernel: [  100.667531] ffe0: bece2528 bece2518 00434e41 b6cbbd92
Mar 14 02:38:23 beaglebone kernel: [  100.667540]  r7:0000017b r6:00000000 r5:00000000 r4:3d6d5800
Mar 14 02:38:23 beaglebone kernel: [  100.667547] ---[ end trace 870c2d2ad09e80fa ]---
Mar 14 02:38:23 beaglebone kernel: [  100.667565] Hello hello: coherent DMA mask is unset
Mar 14 02:38:23 beaglebone kernel: [  100.667571] I'm actually alive

Answer 1

You need to allocate space for the dma_addr_t value.

- dma_addr_t *physAddr;
+ dma_addr_t physAddr;

- virtAddr = dma_alloc_coherent(hello_dev, size, physAddr, flag);
+ virtAddr = dma_alloc_coherent(hello_dev, size, &physAddr, flag);

-  dma_free_coherent(hello_dev, size, virtAddr, *physAddr);
+  dma_free_coherent(hello_dev, size, virtAddr, physAddr);

Here is an online example and looking through the source for callers of dma_alloc_coherent() should confirm this.

---

The API is this way as 'C' does not allow multiple return values. In C++, it might have been a reference. To most kernel developers that have a breadth of experience with 'C', this would seem second nature.

---

How did I deduce this? This helps,

[<c013c5ec>] (__warn) from [<c0e38c00>] (warn_slowpath_fmt+0x70/0xd8)
[<c01c7090>] (dma_alloc_attrs+0x118/0x128)
[<bf00c1b4>] (mod_init+0x1b4/0x1000 [dma_test])

A call to warn was issued so some parameter seemed to be wrong.