diff --git a/README_BUILD.md b/README_BUILD.md index 29e92e103..54fbe0378 100644 --- a/README_BUILD.md +++ b/README_BUILD.md @@ -109,6 +109,7 @@ Cmake Options | Default | Description | `WITH_IIOD_NETWORK` | ON | Add network (TCP/IP) support | `WITH_IIOD_SERIAL` | ON | Add serial (UART) support | `WITH_IIOD_USBD` | ON | Add support for USB through FunctionFS within IIOD | +`WITH_IIOD_USB_DMABUF` | OFF | Enable DMABUF support on the USB stack | `WITH_IIOD_V0_COMPAT` | ON | Add support for Libiio v0.x protocol and clients | `WITH_LIBTINYIIOD` | OFF | Build libtinyiiod | `WITH_AIO` | ON | Build IIOD with async. I/O support | diff --git a/block.c b/block.c index f6b0f9bbc..fe19e0608 100644 --- a/block.c +++ b/block.c @@ -21,6 +21,8 @@ struct iio_block { struct iio_task_token *token; size_t bytes_used; + + int dmabuf_fd; }; struct iio_block * @@ -41,13 +43,19 @@ iio_buffer_create_block(struct iio_buffer *buf, size_t size) if (!block) return iio_ptr(-ENOMEM); + block->dmabuf_fd = -EINVAL; + if (ops->create_block) { pdata = ops->create_block(buf->pdata, size, &block->data); ret = iio_err(pdata); - if (!ret) + if (!ret) { block->pdata = pdata; - else if (ret != -ENOSYS) + + if (ops->get_dmabuf_fd) + block->dmabuf_fd = ops->get_dmabuf_fd(pdata); + } else if (ret != -ENOSYS) { goto err_free_block; + } } if (!block->pdata) { @@ -295,3 +303,18 @@ struct iio_buffer * iio_block_get_buffer(const struct iio_block *block) { return block->buffer; } + +int iio_block_get_dmabuf_fd(const struct iio_block *block) +{ + return block->dmabuf_fd; +} + +int iio_block_disable_cpu_access(struct iio_block *block, bool disable) +{ + const struct iio_backend_ops *ops = block->buffer->dev->ctx->ops; + + if (ops->disable_cpu_access) + return ops->disable_cpu_access(block->pdata, disable); + + return -ENOSYS; +} diff --git a/iio-config.h.cmakein b/iio-config.h.cmakein index 3f647eb09..e3ea030a0 100644 --- a/iio-config.h.cmakein +++ b/iio-config.h.cmakein @@ -32,6 +32,7 @@ #cmakedefine01 WITH_IIOD_NETWORK #cmakedefine01 WITH_IIOD_USBD #cmakedefine01 WITH_IIOD_SERIAL +#cmakedefine01 WITH_IIOD_USB_DMABUF #cmakedefine01 WITH_IIOD_V0_COMPAT #cmakedefine01 WITH_LOCAL_CONFIG #cmakedefine01 WITH_LOCAL_DMABUF_API diff --git a/iiod/CMakeLists.txt b/iiod/CMakeLists.txt index b419bbbb7..27e9cfcff 100644 --- a/iiod/CMakeLists.txt +++ b/iiod/CMakeLists.txt @@ -68,6 +68,12 @@ if (WITH_AIO) endif() target_sources(iiod PRIVATE usbd.c) + + + option(WITH_IIOD_USB_DMABUF "Enable DMABUF support on the USB stack" OFF) + if (WITH_IIOD_USB_DMABUF) + target_sources(iiod PRIVATE usb-dmabuf.c) + endif() endif() target_include_directories(iiod PRIVATE ${LIBAIO_INCLUDE_DIR}) diff --git a/iiod/ops.h b/iiod/ops.h index 3eaea7f1c..e4d9681e5 100644 --- a/iiod/ops.h +++ b/iiod/ops.h @@ -47,6 +47,7 @@ struct iio_mutex; struct iio_task; struct iiod_io; struct pollfd; +struct parser_pdata; struct thread_pool; extern struct thread_pool *main_thread_pool; struct DevEntry; @@ -61,6 +62,8 @@ struct block_entry { uint64_t bytes_used; uint16_t idx; bool cyclic; + int dmabuf_fd; + int ep_fd; }; struct buffer_entry { @@ -71,6 +74,7 @@ struct buffer_entry { struct iio_task *enqueue_task, *dequeue_task; uint32_t *words; uint16_t idx; + bool is_tx; SLIST_HEAD(BlockList, block_entry) blocklist; struct iio_mutex *lock; @@ -139,6 +143,10 @@ int start_network_daemon(struct iio_context *ctx, struct thread_pool *pool, const void *xml_zstd, size_t xml_zstd_len, uint16_t port); +int usb_attach_dmabuf(int ep_fd, int fd); +int usb_detach_dmabuf(int ep_fd, int fd); +int usb_transfer_dmabuf(int ep_fd, int fd, uint64_t size); + int binary_parse(struct parser_pdata *pdata); void enable_binary(struct parser_pdata *pdata); diff --git a/iiod/responder.c b/iiod/responder.c index d9d8d8c7c..756ec0cc5 100644 --- a/iiod/responder.c +++ b/iiod/responder.c @@ -325,7 +325,17 @@ static int buffer_dequeue_block(void *priv, void *d) if (ret < 0) goto out_send_response; - if (!iio_buffer_is_tx(buffer->buf)) { + if (!buffer->is_tx) { + if (WITH_IIOD_USB_DMABUF && entry->dmabuf_fd > 0) { + /* We need to send the error code before the data. + * If usb_transfer_dmabuf() fails, we're screwed... */ + iiod_io_send_response_code(entry->io, entry->bytes_used); + + return usb_transfer_dmabuf(buffer->pdata->fd_out, + entry->dmabuf_fd, + entry->bytes_used); + } + data.ptr = iio_block_start(entry->block); data.size = iio_block_end(entry->block) - data.ptr; nb_data++; @@ -364,6 +374,8 @@ static void handle_create_buffer(struct parser_pdata *pdata, goto err_send_response; } + entry->pdata = pdata; + nb_channels = iio_device_get_channels_count(dev); nb_words = (nb_channels + 31) / 32; @@ -434,6 +446,8 @@ static void handle_create_buffer(struct parser_pdata *pdata, entry->words[BIT_WORD(i)] &= ~BIT_MASK(i); } + entry->is_tx = iio_buffer_is_tx(buf); + /* Success, destroy the temporary mask object */ iio_channels_mask_destroy(mask); @@ -628,7 +642,7 @@ static void handle_create_block(struct parser_pdata *pdata, struct iiod_buf data; uint64_t block_size; struct iiod_io *io; - int ret; + int ret, ep_fd; io = iiod_command_create_io(cmd, cmd_data); ret = iio_err(io); @@ -674,6 +688,25 @@ static void handle_create_block(struct parser_pdata *pdata, entry->io = io; entry->idx = cmd->code >> 16; + if (WITH_IIOD_USB_DMABUF && pdata->is_usb) { + entry->dmabuf_fd = iio_block_get_dmabuf_fd(block); + if (entry->dmabuf_fd > 0) { + ep_fd = buf_entry->is_tx ? pdata->fd_in : pdata->fd_out; + entry->ep_fd = ep_fd; + + ret = usb_attach_dmabuf(ep_fd, entry->dmabuf_fd); + if (!ret) { + /* We could attach to functionfs. Disable CPU + * access to the block as we won't need it. */ + iio_block_disable_cpu_access(block, true); + } else { + /* If we can't attach - no problem. The + * data will be transferred the regular way. */ + entry->dmabuf_fd = -ENOSYS; + } + } + } + /* Keep a reference to the iiod_io until the block is freed. */ iiod_io_ref(io); @@ -695,7 +728,7 @@ static void handle_free_block(struct parser_pdata *pdata, struct iio_buffer *buf; struct iio_block *block; struct iiod_io *io; - int ret; + int ret, ep_fd; buf = get_iio_buffer(pdata, cmd, &buf_entry); ret = iio_err(buf); @@ -715,6 +748,9 @@ static void handle_free_block(struct parser_pdata *pdata, if (entry->block != block) continue; + if (WITH_IIOD_USB_DMABUF && entry->dmabuf_fd > 0) + usb_detach_dmabuf(entry->ep_fd, entry->dmabuf_fd); + SLIST_REMOVE(&buf_entry->blocklist, entry, block_entry, entry); free_block_entry(entry); @@ -779,13 +815,21 @@ static void handle_transfer_block(struct parser_pdata *pdata, } /* Read the data into the block if we are dealing with a TX buffer */ - if (iio_buffer_is_tx(buf)) { - readbuf.ptr = iio_block_start(block); - readbuf.size = iio_block_end(block) - readbuf.ptr; - - ret = iiod_command_data_read(cmd_data, &readbuf); - if (ret < 0) - goto out_send_response; + if (entry->is_tx) { + if (WITH_IIOD_USB_DMABUF && block_entry->dmabuf_fd > 0) { + ret = usb_transfer_dmabuf(pdata->fd_in, + block_entry->dmabuf_fd, + bytes_used); + if (ret) + goto out_send_response; + } else { + readbuf.ptr = iio_block_start(block); + readbuf.size = iio_block_end(block) - readbuf.ptr; + + ret = iiod_command_data_read(cmd_data, &readbuf); + if (ret < 0) + goto out_send_response; + } } block_entry->bytes_used = bytes_used; diff --git a/iiod/usb-dmabuf.c b/iiod/usb-dmabuf.c new file mode 100644 index 000000000..36dea651d --- /dev/null +++ b/iiod/usb-dmabuf.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +/* + * libiio - Library for interfacing industrial I/O (IIO) devices + * + * Copyright (C) 2023 Analog Devices, Inc. + * Author: Paul Cercueil + */ + +#include +#include +#include +#include + +#define IIO_FFS_DMABUF_ATTACH _IOW('g', 131, int) +#define IIO_FFS_DMABUF_DETACH _IOW('g', 132, int) +#define IIO_FFS_DMABUF_TRANSFER _IOW('g', 133, struct iio_ffs_dmabuf_transfer) + +struct iio_ffs_dmabuf_transfer { + int fd; + uint32_t flags; + uint64_t length; +}; + +int usb_attach_dmabuf(int ep_fd, int fd) +{ + int ret; + + ret = ioctl(ep_fd, IIO_FFS_DMABUF_ATTACH, &fd); + if (ret == -1) + return -errno; + + return 0; +} + +int usb_detach_dmabuf(int ep_fd, int fd) +{ + int ret; + + ret = ioctl(ep_fd, IIO_FFS_DMABUF_DETACH, &fd); + if (ret == -1) + return -errno; + + return 0; +} + +int usb_transfer_dmabuf(int ep_fd, int fd, uint64_t size) +{ + struct iio_ffs_dmabuf_transfer req; + int ret; + + req.fd = fd; + req.length = size; + req.flags = 0; + + ret = ioctl(ep_fd, IIO_FFS_DMABUF_TRANSFER, &req); + if (ret == -1) + return -errno; + + return 0; +} diff --git a/include/iio/iio-backend.h b/include/iio/iio-backend.h index a97e76605..823da8a9b 100644 --- a/include/iio/iio-backend.h +++ b/include/iio/iio-backend.h @@ -125,6 +125,7 @@ struct iio_backend_ops { int (*dequeue_block)(struct iio_block_pdata *pdata, bool nonblock); int (*get_dmabuf_fd)(struct iio_block_pdata *pdata); + int (*disable_cpu_access)(struct iio_block_pdata *pdata, bool disable); struct iio_event_stream_pdata *(*open_ev)(const struct iio_device *dev); void (*close_ev)(struct iio_event_stream_pdata *pdata); diff --git a/include/iio/iio.h b/include/iio/iio.h index 6a353230e..82395aabf 100644 --- a/include/iio/iio.h +++ b/include/iio/iio.h @@ -1139,6 +1139,25 @@ iio_buffer_create_block(struct iio_buffer *buffer, size_t size); __api void iio_block_destroy(struct iio_block *block); +/** @brief Get the file descriptor of the underlying DMABUF object + * @param block A pointer to an iio_block structure + * @return The file descriptor of the underlying DMABUF object. + * If the iio_block is not backed by a DMABUF object, -EINVAL is returned. + * Otherwise, the file descriptor will be valid until the block is destroyed. */ +__api __check_ret int iio_block_get_dmabuf_fd(const struct iio_block *block); + + +/** @brief Disable CPU access of a given block + * @param block A pointer to an iio_block structure + * @param disable Whether or not to disable CPU access + * + * NOTE:Disabling CPU access is useful when manipulating DMABUF objects. + * If CPU access is disabled, the block's internal buffer of samples should not + * be accessed. Therefore, the following functions should not be called: + * iio_block_start, iio_block_first, iio_block_end, iio_block_foreach_sample. */ +__api int iio_block_disable_cpu_access(struct iio_block *block, bool disable); + + /** @brief Get the start address of the block * @param block A pointer to an iio_block structure * @return A pointer corresponding to the start address of the block */ diff --git a/local-dmabuf.c b/local-dmabuf.c index f49e83293..1b5f1414e 100644 --- a/local-dmabuf.c +++ b/local-dmabuf.c @@ -6,20 +6,35 @@ * Author: Paul Cercueil */ +#define _GNU_SOURCE #include "local.h" #include +#include #include #include +#include #include #include +#include #include #include +#include #include #include -#define IIO_DMABUF_ALLOC_IOCTL _IOW('i', 0x92, struct iio_dmabuf_req) -#define IIO_DMABUF_ENQUEUE_IOCTL _IOW('i', 0x93, struct iio_dmabuf) +struct iio_dmabuf_heap_data { + uint64_t len; + uint32_t fd; + uint32_t fd_flags; + uint64_t heap_flags; +}; + +#define IIO_DMA_HEAP_ALLOC _IOWR('H', 0x0, struct iio_dmabuf_heap_data) + +#define IIO_DMABUF_ATTACH_IOCTL _IOW('i', 0x92, int) +#define IIO_DMABUF_DETACH_IOCTL _IOW('i', 0x93, int) +#define IIO_DMABUF_ENQUEUE_IOCTL _IOW('i', 0x94, struct iio_dmabuf) #define IIO_DMABUF_SYNC_IOCTL _IOW('b', 0, struct dma_buf_sync) #define IIO_DMABUF_FLAG_CYCLIC (1 << 0) @@ -30,11 +45,6 @@ #define DMA_BUF_SYNC_START (0 << 2) #define DMA_BUF_SYNC_END (1 << 2) -struct iio_dmabuf_req { - uint64_t size; - uint64_t resv; -}; - struct iio_dmabuf { int32_t fd; uint32_t flags; @@ -45,30 +55,54 @@ struct dma_buf_sync { uint64_t flags; }; +static int enable_cpu_access(struct iio_block_pdata *pdata, bool enable) +{ + struct dma_buf_sync dbuf_sync = { 0 }; + int fd = (int)(intptr_t) pdata->pdata; + + dbuf_sync.flags = DMA_BUF_SYNC_RW; + + if (enable) + dbuf_sync.flags |= DMA_BUF_SYNC_START; + else + dbuf_sync.flags |= DMA_BUF_SYNC_END; + + return ioctl_nointr(fd, IIO_DMABUF_SYNC_IOCTL, &dbuf_sync); +} + struct iio_block_pdata * local_create_dmabuf(struct iio_buffer_pdata *pdata, size_t size, void **data) { + struct iio_dmabuf_heap_data req = { + .len = size, + .fd_flags = O_CLOEXEC | O_RDWR, + }; struct iio_block_pdata *priv; - struct iio_dmabuf_req req; - int ret, fd; + int ret, fd, devfd; priv = zalloc(sizeof(*priv)); if (!priv) return iio_ptr(-ENOMEM); - req.size = size; - req.resv = 0; + devfd = open("/dev/dma_heap/system", O_RDONLY | O_CLOEXEC | O_NOFOLLOW); /* Flawfinder: ignore */ + if (devfd < 0) { + ret = -errno; - ret = ioctl_nointr(pdata->fd, IIO_DMABUF_ALLOC_IOCTL, &req); + /* If we're running on an old kernel, return -ENOSYS to mark + * the DMABUF interface as unavailable */ + if (ret == -ENOENT) + ret = -ENOSYS; - /* If we get -ENODEV or -EINVAL errors here, the ioctl is wrong and the - * high-speed DMABUF interface is not supported. */ - if (ret == -ENODEV || ret == -EINVAL || ret == -ENOTTY) - ret = -ENOSYS; - if (ret < 0) goto err_free_priv; + } + + ret = ioctl(devfd, IIO_DMA_HEAP_ALLOC, &req); + if (ret < 0) { + ret = -errno; + goto err_close_devfd; + } - fd = ret; + fd = req.fd; *data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (*data == MAP_FAILED) { @@ -81,12 +115,38 @@ local_create_dmabuf(struct iio_buffer_pdata *pdata, size_t size, void **data) priv->size = size; priv->buf = pdata; priv->dequeued = true; + + /* The new block is dequeued by default, so enable CPU access */ + ret = enable_cpu_access(priv, true); + if (ret) + goto err_data_unmap; + + /* Attach DMABUF to the buffer */ + ret = ioctl(pdata->fd, IIO_DMABUF_ATTACH_IOCTL, &fd); + if (ret) { + ret = -errno; + + if (ret == -ENODEV) { + /* If the ioctl is not available, return -ENOSYS to mark + * the DMABUF interface as unavailable */ + ret = -ENOSYS; + } + + goto err_data_unmap; + } + pdata->dmabuf_supported = true; + close(devfd); + return priv; +err_data_unmap: + munmap(priv->data, priv->size); err_close_fd: close(fd); +err_close_devfd: + close(devfd); err_free_priv: free(priv); return iio_ptr(ret); @@ -94,40 +154,53 @@ local_create_dmabuf(struct iio_buffer_pdata *pdata, size_t size, void **data) void local_free_dmabuf(struct iio_block_pdata *pdata) { - int fd = (int)(intptr_t) pdata->pdata; + int ret, fd = (int)(intptr_t) pdata->pdata; + + ret = ioctl(pdata->buf->fd, IIO_DMABUF_DETACH_IOCTL, &fd); + if (ret) + dev_perror(pdata->buf->dev, ret, "Unable to detach DMABUF"); munmap(pdata->data, pdata->size); close(fd); free(pdata); } +int local_dmabuf_get_fd(struct iio_block_pdata *pdata) +{ + return (int)(intptr_t) pdata->pdata; +} + int local_enqueue_dmabuf(struct iio_block_pdata *pdata, size_t bytes_used, bool cyclic) { - struct dma_buf_sync dbuf_sync; struct iio_dmabuf dmabuf; int ret, fd = (int)(intptr_t) pdata->pdata; if (!pdata->dequeued) return -EPERM; + if (bytes_used > pdata->size || bytes_used == 0) + return -EINVAL; + dmabuf.fd = fd; dmabuf.flags = 0; dmabuf.bytes_used = bytes_used; if (cyclic) - dmabuf.flags |= IIO_DMABUF_FLAG_CYCLIC; - - dbuf_sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_RW; + dmabuf.flags |= IIO_DMABUF_FLAG_CYCLIC; - /* Disable CPU access to last block */ - ret = ioctl_nointr(fd, IIO_DMABUF_SYNC_IOCTL, &dbuf_sync); - if (ret) - return ret; + if (!pdata->cpu_access_disabled) { + /* Disable CPU access to last block */ + ret = enable_cpu_access(pdata, false); + if (ret) + return ret; + } ret = ioctl_nointr(pdata->buf->fd, IIO_DMABUF_ENQUEUE_IOCTL, &dmabuf); - if (ret) + if (ret) { + dev_perror(pdata->buf->dev, ret, "Unable to enqueue DMABUF"); return ret; + } pdata->dequeued = false; @@ -137,7 +210,6 @@ int local_enqueue_dmabuf(struct iio_block_pdata *pdata, int local_dequeue_dmabuf(struct iio_block_pdata *pdata, bool nonblock) { struct iio_buffer_pdata *buf_pdata = pdata->buf; - struct dma_buf_sync dbuf_sync; struct timespec start, *time_ptr = NULL; int ret, fd = (int)(intptr_t) pdata->pdata; @@ -153,14 +225,29 @@ int local_dequeue_dmabuf(struct iio_block_pdata *pdata, bool nonblock) if (ret < 0) return ret; - dbuf_sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_RW; - - /* Enable CPU access to new block */ - ret = ioctl_nointr(fd, IIO_DMABUF_SYNC_IOCTL, &dbuf_sync); - if (ret < 0) - return ret; + if (!pdata->cpu_access_disabled) { + /* Enable CPU access to new block */ + ret = enable_cpu_access(pdata, true); + if (ret < 0) + return ret; + } pdata->dequeued = true; return 0; } + +int local_dmabuf_disable_cpu_access(struct iio_block_pdata *pdata, bool disable) +{ + int ret; + + if (pdata->dequeued) { + ret = enable_cpu_access(pdata, !disable); + if (ret) + return ret; + } + + pdata->cpu_access_disabled = disable; + + return 0; +} diff --git a/local.c b/local.c index 77f8f20b6..9322c4b67 100644 --- a/local.c +++ b/local.c @@ -1697,6 +1697,26 @@ static int local_read_event(struct iio_event_stream_pdata *pdata, return 0; } +static int local_get_dmabuf_fd(struct iio_block_pdata *pdata) +{ + if (WITH_LOCAL_DMABUF_API && pdata->buf->dmabuf_supported) + return local_dmabuf_get_fd(pdata); + + return -EINVAL; +} + +static int local_disable_cpu_access(struct iio_block_pdata *pdata, bool disable) +{ + if (WITH_LOCAL_DMABUF_API && pdata->buf->dmabuf_supported) { + if (pdata->cpu_access_disabled == disable) + return 0; + + return local_dmabuf_disable_cpu_access(pdata, disable); + } + + return -EINVAL; +} + static const struct iio_backend_ops local_ops = { .scan = local_context_scan, .create = local_create_context, @@ -1722,6 +1742,9 @@ static const struct iio_backend_ops local_ops = { .open_ev = local_open_events_fd, .close_ev = local_close_events_fd, .read_ev = local_read_event, + + .get_dmabuf_fd = local_get_dmabuf_fd, + .disable_cpu_access = local_disable_cpu_access, }; const struct iio_backend iio_local_backend = { diff --git a/local.h b/local.h index 4d256f210..547eb9e42 100644 --- a/local.h +++ b/local.h @@ -33,6 +33,7 @@ struct iio_block_pdata { size_t size; void *data; bool dequeued; + bool cpu_access_disabled; }; int ioctl_nointr(int fd, unsigned long request, void *data); @@ -48,6 +49,9 @@ int local_enqueue_dmabuf(struct iio_block_pdata *pdata, size_t bytes_used, bool cyclic); int local_dequeue_dmabuf(struct iio_block_pdata *pdata, bool nonblock); +int local_dmabuf_get_fd(struct iio_block_pdata *pdata); +int local_dmabuf_disable_cpu_access(struct iio_block_pdata *pdata, bool disable); + struct iio_block_pdata * local_create_mmap_block(struct iio_buffer_pdata *pdata, size_t size, void **data);