2304 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright 2020 Xillybus Ltd, http://xillybus.com4 *5 * Driver for the XillyUSB FPGA/host framework.6 *7 * This driver interfaces with a special IP core in an FPGA, setting up8 * a pipe between a hardware FIFO in the programmable logic and a device9 * file in the host. The number of such pipes and their attributes are10 * set up on the logic. This driver detects these automatically and11 * creates the device files accordingly.12 */13 14#include <linux/types.h>15#include <linux/slab.h>16#include <linux/list.h>17#include <linux/device.h>18#include <linux/module.h>19#include <asm/byteorder.h>20#include <linux/io.h>21#include <linux/interrupt.h>22#include <linux/sched.h>23#include <linux/fs.h>24#include <linux/spinlock.h>25#include <linux/mutex.h>26#include <linux/workqueue.h>27#include <linux/crc32.h>28#include <linux/poll.h>29#include <linux/delay.h>30#include <linux/usb.h>31 32#include "xillybus_class.h"33 34MODULE_DESCRIPTION("Driver for XillyUSB FPGA IP Core");35MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");36MODULE_ALIAS("xillyusb");37MODULE_LICENSE("GPL v2");38 39#define XILLY_RX_TIMEOUT (10 * HZ / 1000)40#define XILLY_RESPONSE_TIMEOUT (500 * HZ / 1000)41 42#define BUF_SIZE_ORDER 443#define BUFNUM 844#define LOG2_IDT_FIFO_SIZE 1645#define LOG2_INITIAL_FIFO_BUF_SIZE 1646 47#define MSG_EP_NUM 148#define IN_EP_NUM 149 50static const char xillyname[] = "xillyusb";51 52static unsigned int fifo_buf_order;53static struct workqueue_struct *wakeup_wq;54 55#define USB_VENDOR_ID_XILINX 0x03fd56#define USB_VENDOR_ID_ALTERA 0x09fb57 58#define USB_PRODUCT_ID_XILLYUSB 0xebbe59 60static const struct usb_device_id xillyusb_table[] = {61 { USB_DEVICE(USB_VENDOR_ID_XILINX, USB_PRODUCT_ID_XILLYUSB) },62 { USB_DEVICE(USB_VENDOR_ID_ALTERA, USB_PRODUCT_ID_XILLYUSB) },63 { }64};65 66MODULE_DEVICE_TABLE(usb, xillyusb_table);67 68struct xillyusb_dev;69 70struct xillyfifo {71 unsigned int bufsize; /* In bytes, always a power of 2 */72 unsigned int bufnum;73 unsigned int size; /* Lazy: Equals bufsize * bufnum */74 unsigned int buf_order;75 76 int fill; /* Number of bytes in the FIFO */77 spinlock_t lock;78 wait_queue_head_t waitq;79 80 unsigned int readpos;81 unsigned int readbuf;82 unsigned int writepos;83 unsigned int writebuf;84 char **mem;85};86 87struct xillyusb_channel;88 89struct xillyusb_endpoint {90 struct xillyusb_dev *xdev;91 92 struct mutex ep_mutex; /* serialize operations on endpoint */93 94 struct list_head buffers;95 struct list_head filled_buffers;96 spinlock_t buffers_lock; /* protect these two lists */97 98 unsigned int order;99 unsigned int buffer_size;100 101 unsigned int fill_mask;102 103 int outstanding_urbs;104 105 struct usb_anchor anchor;106 107 struct xillyfifo fifo;108 109 struct work_struct workitem;110 111 bool shutting_down;112 bool drained;113 bool wake_on_drain;114 115 u8 ep_num;116};117 118struct xillyusb_channel {119 struct xillyusb_dev *xdev;120 121 struct xillyfifo *in_fifo;122 struct xillyusb_endpoint *out_ep;123 struct mutex lock; /* protect @out_ep, @in_fifo, bit fields below */124 125 struct mutex in_mutex; /* serialize fops on FPGA to host stream */126 struct mutex out_mutex; /* serialize fops on host to FPGA stream */127 wait_queue_head_t flushq;128 129 int chan_idx;130 131 u32 in_consumed_bytes;132 u32 in_current_checkpoint;133 u32 out_bytes;134 135 unsigned int in_log2_element_size;136 unsigned int out_log2_element_size;137 unsigned int in_log2_fifo_size;138 unsigned int out_log2_fifo_size;139 140 unsigned int read_data_ok; /* EOF not arrived (yet) */141 unsigned int poll_used;142 unsigned int flushing;143 unsigned int flushed;144 unsigned int canceled;145 146 /* Bit fields protected by @lock except for initialization */147 unsigned readable:1;148 unsigned writable:1;149 unsigned open_for_read:1;150 unsigned open_for_write:1;151 unsigned in_synchronous:1;152 unsigned out_synchronous:1;153 unsigned in_seekable:1;154 unsigned out_seekable:1;155};156 157struct xillybuffer {158 struct list_head entry;159 struct xillyusb_endpoint *ep;160 void *buf;161 unsigned int len;162};163 164struct xillyusb_dev {165 struct xillyusb_channel *channels;166 167 struct usb_device *udev;168 struct device *dev; /* For dev_err() and such */169 struct kref kref;170 struct workqueue_struct *workq;171 172 int error;173 spinlock_t error_lock; /* protect @error */174 struct work_struct wakeup_workitem;175 176 int num_channels;177 178 struct xillyusb_endpoint *msg_ep;179 struct xillyusb_endpoint *in_ep;180 181 struct mutex msg_mutex; /* serialize opcode transmission */182 int in_bytes_left;183 int leftover_chan_num;184 unsigned int in_counter;185 struct mutex process_in_mutex; /* synchronize wakeup_all() */186};187 188/*189 * kref_mutex is used in xillyusb_open() to prevent the xillyusb_dev190 * struct from being freed during the gap between being found by191 * xillybus_find_inode() and having its reference count incremented.192 */193 194static DEFINE_MUTEX(kref_mutex);195 196/* FPGA to host opcodes */197enum {198 OPCODE_DATA = 0,199 OPCODE_QUIESCE_ACK = 1,200 OPCODE_EOF = 2,201 OPCODE_REACHED_CHECKPOINT = 3,202 OPCODE_CANCELED_CHECKPOINT = 4,203};204 205/* Host to FPGA opcodes */206enum {207 OPCODE_QUIESCE = 0,208 OPCODE_REQ_IDT = 1,209 OPCODE_SET_CHECKPOINT = 2,210 OPCODE_CLOSE = 3,211 OPCODE_SET_PUSH = 4,212 OPCODE_UPDATE_PUSH = 5,213 OPCODE_CANCEL_CHECKPOINT = 6,214 OPCODE_SET_ADDR = 7,215};216 217/*218 * fifo_write() and fifo_read() are NOT reentrant (i.e. concurrent multiple219 * calls to each on the same FIFO is not allowed) however it's OK to have220 * threads calling each of the two functions once on the same FIFO, and221 * at the same time.222 */223 224static int fifo_write(struct xillyfifo *fifo,225 const void *data, unsigned int len,226 int (*copier)(void *, const void *, int))227{228 unsigned int done = 0;229 unsigned int todo = len;230 unsigned int nmax;231 unsigned int writepos = fifo->writepos;232 unsigned int writebuf = fifo->writebuf;233 unsigned long flags;234 int rc;235 236 nmax = fifo->size - READ_ONCE(fifo->fill);237 238 while (1) {239 unsigned int nrail = fifo->bufsize - writepos;240 unsigned int n = min(todo, nmax);241 242 if (n == 0) {243 spin_lock_irqsave(&fifo->lock, flags);244 fifo->fill += done;245 spin_unlock_irqrestore(&fifo->lock, flags);246 247 fifo->writepos = writepos;248 fifo->writebuf = writebuf;249 250 return done;251 }252 253 if (n > nrail)254 n = nrail;255 256 rc = (*copier)(fifo->mem[writebuf] + writepos, data + done, n);257 258 if (rc)259 return rc;260 261 done += n;262 todo -= n;263 264 writepos += n;265 nmax -= n;266 267 if (writepos == fifo->bufsize) {268 writepos = 0;269 writebuf++;270 271 if (writebuf == fifo->bufnum)272 writebuf = 0;273 }274 }275}276 277static int fifo_read(struct xillyfifo *fifo,278 void *data, unsigned int len,279 int (*copier)(void *, const void *, int))280{281 unsigned int done = 0;282 unsigned int todo = len;283 unsigned int fill;284 unsigned int readpos = fifo->readpos;285 unsigned int readbuf = fifo->readbuf;286 unsigned long flags;287 int rc;288 289 /*290 * The spinlock here is necessary, because otherwise fifo->fill291 * could have been increased by fifo_write() after writing data292 * to the buffer, but this data would potentially not have been293 * visible on this thread at the time the updated fifo->fill was.294 * That could lead to reading invalid data.295 */296 297 spin_lock_irqsave(&fifo->lock, flags);298 fill = fifo->fill;299 spin_unlock_irqrestore(&fifo->lock, flags);300 301 while (1) {302 unsigned int nrail = fifo->bufsize - readpos;303 unsigned int n = min(todo, fill);304 305 if (n == 0) {306 spin_lock_irqsave(&fifo->lock, flags);307 fifo->fill -= done;308 spin_unlock_irqrestore(&fifo->lock, flags);309 310 fifo->readpos = readpos;311 fifo->readbuf = readbuf;312 313 return done;314 }315 316 if (n > nrail)317 n = nrail;318 319 rc = (*copier)(data + done, fifo->mem[readbuf] + readpos, n);320 321 if (rc)322 return rc;323 324 done += n;325 todo -= n;326 327 readpos += n;328 fill -= n;329 330 if (readpos == fifo->bufsize) {331 readpos = 0;332 readbuf++;333 334 if (readbuf == fifo->bufnum)335 readbuf = 0;336 }337 }338}339 340/*341 * These three wrapper functions are used as the @copier argument to342 * fifo_write() and fifo_read(), so that they can work directly with343 * user memory as well.344 */345 346static int xilly_copy_from_user(void *dst, const void *src, int n)347{348 if (copy_from_user(dst, (const void __user *)src, n))349 return -EFAULT;350 351 return 0;352}353 354static int xilly_copy_to_user(void *dst, const void *src, int n)355{356 if (copy_to_user((void __user *)dst, src, n))357 return -EFAULT;358 359 return 0;360}361 362static int xilly_memcpy(void *dst, const void *src, int n)363{364 memcpy(dst, src, n);365 366 return 0;367}368 369static int fifo_init(struct xillyfifo *fifo,370 unsigned int log2_size)371{372 unsigned int log2_bufnum;373 unsigned int buf_order;374 int i;375 376 unsigned int log2_fifo_buf_size;377 378retry:379 log2_fifo_buf_size = fifo_buf_order + PAGE_SHIFT;380 381 if (log2_size > log2_fifo_buf_size) {382 log2_bufnum = log2_size - log2_fifo_buf_size;383 buf_order = fifo_buf_order;384 fifo->bufsize = 1 << log2_fifo_buf_size;385 } else {386 log2_bufnum = 0;387 buf_order = (log2_size > PAGE_SHIFT) ?388 log2_size - PAGE_SHIFT : 0;389 fifo->bufsize = 1 << log2_size;390 }391 392 fifo->bufnum = 1 << log2_bufnum;393 fifo->size = fifo->bufnum * fifo->bufsize;394 fifo->buf_order = buf_order;395 396 fifo->mem = kmalloc_array(fifo->bufnum, sizeof(void *), GFP_KERNEL);397 398 if (!fifo->mem)399 return -ENOMEM;400 401 for (i = 0; i < fifo->bufnum; i++) {402 fifo->mem[i] = (void *)403 __get_free_pages(GFP_KERNEL, buf_order);404 405 if (!fifo->mem[i])406 goto memfail;407 }408 409 fifo->fill = 0;410 fifo->readpos = 0;411 fifo->readbuf = 0;412 fifo->writepos = 0;413 fifo->writebuf = 0;414 spin_lock_init(&fifo->lock);415 init_waitqueue_head(&fifo->waitq);416 return 0;417 418memfail:419 for (i--; i >= 0; i--)420 free_pages((unsigned long)fifo->mem[i], buf_order);421 422 kfree(fifo->mem);423 fifo->mem = NULL;424 425 if (fifo_buf_order) {426 fifo_buf_order--;427 goto retry;428 } else {429 return -ENOMEM;430 }431}432 433static void fifo_mem_release(struct xillyfifo *fifo)434{435 int i;436 437 if (!fifo->mem)438 return;439 440 for (i = 0; i < fifo->bufnum; i++)441 free_pages((unsigned long)fifo->mem[i], fifo->buf_order);442 443 kfree(fifo->mem);444}445 446/*447 * When endpoint_quiesce() returns, the endpoint has no URBs submitted,448 * won't accept any new URB submissions, and its related work item doesn't449 * and won't run anymore.450 */451 452static void endpoint_quiesce(struct xillyusb_endpoint *ep)453{454 mutex_lock(&ep->ep_mutex);455 ep->shutting_down = true;456 mutex_unlock(&ep->ep_mutex);457 458 usb_kill_anchored_urbs(&ep->anchor);459 cancel_work_sync(&ep->workitem);460}461 462/*463 * Note that endpoint_dealloc() also frees fifo memory (if allocated), even464 * though endpoint_alloc doesn't allocate that memory.465 */466 467static void endpoint_dealloc(struct xillyusb_endpoint *ep)468{469 struct list_head *this, *next;470 471 fifo_mem_release(&ep->fifo);472 473 /* Join @filled_buffers with @buffers to free these entries too */474 list_splice(&ep->filled_buffers, &ep->buffers);475 476 list_for_each_safe(this, next, &ep->buffers) {477 struct xillybuffer *xb =478 list_entry(this, struct xillybuffer, entry);479 480 free_pages((unsigned long)xb->buf, ep->order);481 kfree(xb);482 }483 484 kfree(ep);485}486 487static struct xillyusb_endpoint488*endpoint_alloc(struct xillyusb_dev *xdev,489 u8 ep_num,490 void (*work)(struct work_struct *),491 unsigned int order,492 int bufnum)493{494 int i;495 496 struct xillyusb_endpoint *ep;497 498 ep = kzalloc(sizeof(*ep), GFP_KERNEL);499 500 if (!ep)501 return NULL;502 503 INIT_LIST_HEAD(&ep->buffers);504 INIT_LIST_HEAD(&ep->filled_buffers);505 506 spin_lock_init(&ep->buffers_lock);507 mutex_init(&ep->ep_mutex);508 509 init_usb_anchor(&ep->anchor);510 INIT_WORK(&ep->workitem, work);511 512 ep->order = order;513 ep->buffer_size = 1 << (PAGE_SHIFT + order);514 ep->outstanding_urbs = 0;515 ep->drained = true;516 ep->wake_on_drain = false;517 ep->xdev = xdev;518 ep->ep_num = ep_num;519 ep->shutting_down = false;520 521 for (i = 0; i < bufnum; i++) {522 struct xillybuffer *xb;523 unsigned long addr;524 525 xb = kzalloc(sizeof(*xb), GFP_KERNEL);526 527 if (!xb) {528 endpoint_dealloc(ep);529 return NULL;530 }531 532 addr = __get_free_pages(GFP_KERNEL, order);533 534 if (!addr) {535 kfree(xb);536 endpoint_dealloc(ep);537 return NULL;538 }539 540 xb->buf = (void *)addr;541 xb->ep = ep;542 list_add_tail(&xb->entry, &ep->buffers);543 }544 return ep;545}546 547static void cleanup_dev(struct kref *kref)548{549 struct xillyusb_dev *xdev =550 container_of(kref, struct xillyusb_dev, kref);551 552 if (xdev->in_ep)553 endpoint_dealloc(xdev->in_ep);554 555 if (xdev->msg_ep)556 endpoint_dealloc(xdev->msg_ep);557 558 if (xdev->workq)559 destroy_workqueue(xdev->workq);560 561 usb_put_dev(xdev->udev);562 kfree(xdev->channels); /* Argument may be NULL, and that's fine */563 kfree(xdev);564}565 566/*567 * @process_in_mutex is taken to ensure that bulk_in_work() won't call568 * process_bulk_in() after wakeup_all()'s execution: The latter zeroes all569 * @read_data_ok entries, which will make process_bulk_in() report false570 * errors if executed. The mechanism relies on that xdev->error is assigned571 * a non-zero value by report_io_error() prior to queueing wakeup_all(),572 * which prevents bulk_in_work() from calling process_bulk_in().573 */574 575static void wakeup_all(struct work_struct *work)576{577 int i;578 struct xillyusb_dev *xdev = container_of(work, struct xillyusb_dev,579 wakeup_workitem);580 581 mutex_lock(&xdev->process_in_mutex);582 583 for (i = 0; i < xdev->num_channels; i++) {584 struct xillyusb_channel *chan = &xdev->channels[i];585 586 mutex_lock(&chan->lock);587 588 if (chan->in_fifo) {589 /*590 * Fake an EOF: Even if such arrives, it won't be591 * processed.592 */593 chan->read_data_ok = 0;594 wake_up_interruptible(&chan->in_fifo->waitq);595 }596 597 if (chan->out_ep)598 wake_up_interruptible(&chan->out_ep->fifo.waitq);599 600 mutex_unlock(&chan->lock);601 602 wake_up_interruptible(&chan->flushq);603 }604 605 mutex_unlock(&xdev->process_in_mutex);606 607 wake_up_interruptible(&xdev->msg_ep->fifo.waitq);608 609 kref_put(&xdev->kref, cleanup_dev);610}611 612static void report_io_error(struct xillyusb_dev *xdev,613 int errcode)614{615 unsigned long flags;616 bool do_once = false;617 618 spin_lock_irqsave(&xdev->error_lock, flags);619 if (!xdev->error) {620 xdev->error = errcode;621 do_once = true;622 }623 spin_unlock_irqrestore(&xdev->error_lock, flags);624 625 if (do_once) {626 kref_get(&xdev->kref); /* xdev is used by work item */627 queue_work(wakeup_wq, &xdev->wakeup_workitem);628 }629}630 631/*632 * safely_assign_in_fifo() changes the value of chan->in_fifo and ensures633 * the previous pointer is never used after its return.634 */635 636static void safely_assign_in_fifo(struct xillyusb_channel *chan,637 struct xillyfifo *fifo)638{639 mutex_lock(&chan->lock);640 chan->in_fifo = fifo;641 mutex_unlock(&chan->lock);642 643 flush_work(&chan->xdev->in_ep->workitem);644}645 646static void bulk_in_completer(struct urb *urb)647{648 struct xillybuffer *xb = urb->context;649 struct xillyusb_endpoint *ep = xb->ep;650 unsigned long flags;651 652 if (urb->status) {653 if (!(urb->status == -ENOENT ||654 urb->status == -ECONNRESET ||655 urb->status == -ESHUTDOWN))656 report_io_error(ep->xdev, -EIO);657 658 spin_lock_irqsave(&ep->buffers_lock, flags);659 list_add_tail(&xb->entry, &ep->buffers);660 ep->outstanding_urbs--;661 spin_unlock_irqrestore(&ep->buffers_lock, flags);662 663 return;664 }665 666 xb->len = urb->actual_length;667 668 spin_lock_irqsave(&ep->buffers_lock, flags);669 list_add_tail(&xb->entry, &ep->filled_buffers);670 spin_unlock_irqrestore(&ep->buffers_lock, flags);671 672 if (!ep->shutting_down)673 queue_work(ep->xdev->workq, &ep->workitem);674}675 676static void bulk_out_completer(struct urb *urb)677{678 struct xillybuffer *xb = urb->context;679 struct xillyusb_endpoint *ep = xb->ep;680 unsigned long flags;681 682 if (urb->status &&683 (!(urb->status == -ENOENT ||684 urb->status == -ECONNRESET ||685 urb->status == -ESHUTDOWN)))686 report_io_error(ep->xdev, -EIO);687 688 spin_lock_irqsave(&ep->buffers_lock, flags);689 list_add_tail(&xb->entry, &ep->buffers);690 ep->outstanding_urbs--;691 spin_unlock_irqrestore(&ep->buffers_lock, flags);692 693 if (!ep->shutting_down)694 queue_work(ep->xdev->workq, &ep->workitem);695}696 697static void try_queue_bulk_in(struct xillyusb_endpoint *ep)698{699 struct xillyusb_dev *xdev = ep->xdev;700 struct xillybuffer *xb;701 struct urb *urb;702 703 int rc;704 unsigned long flags;705 unsigned int bufsize = ep->buffer_size;706 707 mutex_lock(&ep->ep_mutex);708 709 if (ep->shutting_down || xdev->error)710 goto done;711 712 while (1) {713 spin_lock_irqsave(&ep->buffers_lock, flags);714 715 if (list_empty(&ep->buffers)) {716 spin_unlock_irqrestore(&ep->buffers_lock, flags);717 goto done;718 }719 720 xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);721 list_del(&xb->entry);722 ep->outstanding_urbs++;723 724 spin_unlock_irqrestore(&ep->buffers_lock, flags);725 726 urb = usb_alloc_urb(0, GFP_KERNEL);727 if (!urb) {728 report_io_error(xdev, -ENOMEM);729 goto relist;730 }731 732 usb_fill_bulk_urb(urb, xdev->udev,733 usb_rcvbulkpipe(xdev->udev, ep->ep_num),734 xb->buf, bufsize, bulk_in_completer, xb);735 736 usb_anchor_urb(urb, &ep->anchor);737 738 rc = usb_submit_urb(urb, GFP_KERNEL);739 740 if (rc) {741 report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :742 -EIO);743 goto unanchor;744 }745 746 usb_free_urb(urb); /* This just decrements reference count */747 }748 749unanchor:750 usb_unanchor_urb(urb);751 usb_free_urb(urb);752 753relist:754 spin_lock_irqsave(&ep->buffers_lock, flags);755 list_add_tail(&xb->entry, &ep->buffers);756 ep->outstanding_urbs--;757 spin_unlock_irqrestore(&ep->buffers_lock, flags);758 759done:760 mutex_unlock(&ep->ep_mutex);761}762 763static void try_queue_bulk_out(struct xillyusb_endpoint *ep)764{765 struct xillyfifo *fifo = &ep->fifo;766 struct xillyusb_dev *xdev = ep->xdev;767 struct xillybuffer *xb;768 struct urb *urb;769 770 int rc;771 unsigned int fill;772 unsigned long flags;773 bool do_wake = false;774 775 mutex_lock(&ep->ep_mutex);776 777 if (ep->shutting_down || xdev->error)778 goto done;779 780 fill = READ_ONCE(fifo->fill) & ep->fill_mask;781 782 while (1) {783 int count;784 unsigned int max_read;785 786 spin_lock_irqsave(&ep->buffers_lock, flags);787 788 /*789 * Race conditions might have the FIFO filled while the790 * endpoint is marked as drained here. That doesn't matter,791 * because the sole purpose of @drained is to ensure that792 * certain data has been sent on the USB channel before793 * shutting it down. Hence knowing that the FIFO appears794 * to be empty with no outstanding URBs at some moment795 * is good enough.796 */797 798 if (!fill) {799 ep->drained = !ep->outstanding_urbs;800 if (ep->drained && ep->wake_on_drain)801 do_wake = true;802 803 spin_unlock_irqrestore(&ep->buffers_lock, flags);804 goto done;805 }806 807 ep->drained = false;808 809 if ((fill < ep->buffer_size && ep->outstanding_urbs) ||810 list_empty(&ep->buffers)) {811 spin_unlock_irqrestore(&ep->buffers_lock, flags);812 goto done;813 }814 815 xb = list_first_entry(&ep->buffers, struct xillybuffer, entry);816 list_del(&xb->entry);817 ep->outstanding_urbs++;818 819 spin_unlock_irqrestore(&ep->buffers_lock, flags);820 821 max_read = min(fill, ep->buffer_size);822 823 count = fifo_read(&ep->fifo, xb->buf, max_read, xilly_memcpy);824 825 /*826 * xilly_memcpy always returns 0 => fifo_read can't fail =>827 * count > 0828 */829 830 urb = usb_alloc_urb(0, GFP_KERNEL);831 if (!urb) {832 report_io_error(xdev, -ENOMEM);833 goto relist;834 }835 836 usb_fill_bulk_urb(urb, xdev->udev,837 usb_sndbulkpipe(xdev->udev, ep->ep_num),838 xb->buf, count, bulk_out_completer, xb);839 840 usb_anchor_urb(urb, &ep->anchor);841 842 rc = usb_submit_urb(urb, GFP_KERNEL);843 844 if (rc) {845 report_io_error(xdev, (rc == -ENOMEM) ? -ENOMEM :846 -EIO);847 goto unanchor;848 }849 850 usb_free_urb(urb); /* This just decrements reference count */851 852 fill -= count;853 do_wake = true;854 }855 856unanchor:857 usb_unanchor_urb(urb);858 usb_free_urb(urb);859 860relist:861 spin_lock_irqsave(&ep->buffers_lock, flags);862 list_add_tail(&xb->entry, &ep->buffers);863 ep->outstanding_urbs--;864 spin_unlock_irqrestore(&ep->buffers_lock, flags);865 866done:867 mutex_unlock(&ep->ep_mutex);868 869 if (do_wake)870 wake_up_interruptible(&fifo->waitq);871}872 873static void bulk_out_work(struct work_struct *work)874{875 struct xillyusb_endpoint *ep = container_of(work,876 struct xillyusb_endpoint,877 workitem);878 try_queue_bulk_out(ep);879}880 881static int process_in_opcode(struct xillyusb_dev *xdev,882 int opcode,883 int chan_num)884{885 struct xillyusb_channel *chan;886 struct device *dev = xdev->dev;887 int chan_idx = chan_num >> 1;888 889 if (chan_idx >= xdev->num_channels) {890 dev_err(dev, "Received illegal channel ID %d from FPGA\n",891 chan_num);892 return -EIO;893 }894 895 chan = &xdev->channels[chan_idx];896 897 switch (opcode) {898 case OPCODE_EOF:899 if (!chan->read_data_ok) {900 dev_err(dev, "Received unexpected EOF for channel %d\n",901 chan_num);902 return -EIO;903 }904 905 /*906 * A write memory barrier ensures that the FIFO's fill level907 * is visible before read_data_ok turns zero, so the data in908 * the FIFO isn't missed by the consumer.909 */910 smp_wmb();911 WRITE_ONCE(chan->read_data_ok, 0);912 wake_up_interruptible(&chan->in_fifo->waitq);913 break;914 915 case OPCODE_REACHED_CHECKPOINT:916 chan->flushing = 0;917 wake_up_interruptible(&chan->flushq);918 break;919 920 case OPCODE_CANCELED_CHECKPOINT:921 chan->canceled = 1;922 wake_up_interruptible(&chan->flushq);923 break;924 925 default:926 dev_err(dev, "Received illegal opcode %d from FPGA\n",927 opcode);928 return -EIO;929 }930 931 return 0;932}933 934static int process_bulk_in(struct xillybuffer *xb)935{936 struct xillyusb_endpoint *ep = xb->ep;937 struct xillyusb_dev *xdev = ep->xdev;938 struct device *dev = xdev->dev;939 int dws = xb->len >> 2;940 __le32 *p = xb->buf;941 u32 ctrlword;942 struct xillyusb_channel *chan;943 struct xillyfifo *fifo;944 int chan_num = 0, opcode;945 int chan_idx;946 int bytes, count, dwconsume;947 int in_bytes_left = 0;948 int rc;949 950 if ((dws << 2) != xb->len) {951 dev_err(dev, "Received BULK IN transfer with %d bytes, not a multiple of 4\n",952 xb->len);953 return -EIO;954 }955 956 if (xdev->in_bytes_left) {957 bytes = min(xdev->in_bytes_left, dws << 2);958 in_bytes_left = xdev->in_bytes_left - bytes;959 chan_num = xdev->leftover_chan_num;960 goto resume_leftovers;961 }962 963 while (dws) {964 ctrlword = le32_to_cpu(*p++);965 dws--;966 967 chan_num = ctrlword & 0xfff;968 count = (ctrlword >> 12) & 0x3ff;969 opcode = (ctrlword >> 24) & 0xf;970 971 if (opcode != OPCODE_DATA) {972 unsigned int in_counter = xdev->in_counter++ & 0x3ff;973 974 if (count != in_counter) {975 dev_err(dev, "Expected opcode counter %d, got %d\n",976 in_counter, count);977 return -EIO;978 }979 980 rc = process_in_opcode(xdev, opcode, chan_num);981 982 if (rc)983 return rc;984 985 continue;986 }987 988 bytes = min(count + 1, dws << 2);989 in_bytes_left = count + 1 - bytes;990 991resume_leftovers:992 chan_idx = chan_num >> 1;993 994 if (!(chan_num & 1) || chan_idx >= xdev->num_channels ||995 !xdev->channels[chan_idx].read_data_ok) {996 dev_err(dev, "Received illegal channel ID %d from FPGA\n",997 chan_num);998 return -EIO;999 }1000 chan = &xdev->channels[chan_idx];1001 1002 fifo = chan->in_fifo;1003 1004 if (unlikely(!fifo))1005 return -EIO; /* We got really unexpected data */1006 1007 if (bytes != fifo_write(fifo, p, bytes, xilly_memcpy)) {1008 dev_err(dev, "Misbehaving FPGA overflowed an upstream FIFO!\n");1009 return -EIO;1010 }1011 1012 wake_up_interruptible(&fifo->waitq);1013 1014 dwconsume = (bytes + 3) >> 2;1015 dws -= dwconsume;1016 p += dwconsume;1017 }1018 1019 xdev->in_bytes_left = in_bytes_left;1020 xdev->leftover_chan_num = chan_num;1021 return 0;1022}1023 1024static void bulk_in_work(struct work_struct *work)1025{1026 struct xillyusb_endpoint *ep =1027 container_of(work, struct xillyusb_endpoint, workitem);1028 struct xillyusb_dev *xdev = ep->xdev;1029 unsigned long flags;1030 struct xillybuffer *xb;1031 bool consumed = false;1032 int rc = 0;1033 1034 mutex_lock(&xdev->process_in_mutex);1035 1036 spin_lock_irqsave(&ep->buffers_lock, flags);1037 1038 while (1) {1039 if (rc || list_empty(&ep->filled_buffers)) {1040 spin_unlock_irqrestore(&ep->buffers_lock, flags);1041 mutex_unlock(&xdev->process_in_mutex);1042 1043 if (rc)1044 report_io_error(xdev, rc);1045 else if (consumed)1046 try_queue_bulk_in(ep);1047 1048 return;1049 }1050 1051 xb = list_first_entry(&ep->filled_buffers, struct xillybuffer,1052 entry);1053 list_del(&xb->entry);1054 1055 spin_unlock_irqrestore(&ep->buffers_lock, flags);1056 1057 consumed = true;1058 1059 if (!xdev->error)1060 rc = process_bulk_in(xb);1061 1062 spin_lock_irqsave(&ep->buffers_lock, flags);1063 list_add_tail(&xb->entry, &ep->buffers);1064 ep->outstanding_urbs--;1065 }1066}1067 1068static int xillyusb_send_opcode(struct xillyusb_dev *xdev,1069 int chan_num, char opcode, u32 data)1070{1071 struct xillyusb_endpoint *ep = xdev->msg_ep;1072 struct xillyfifo *fifo = &ep->fifo;1073 __le32 msg[2];1074 1075 int rc = 0;1076 1077 msg[0] = cpu_to_le32((chan_num & 0xfff) |1078 ((opcode & 0xf) << 24));1079 msg[1] = cpu_to_le32(data);1080 1081 mutex_lock(&xdev->msg_mutex);1082 1083 /*1084 * The wait queue is woken with the interruptible variant, so the1085 * wait function matches, however returning because of an interrupt1086 * will mess things up considerably, in particular when the caller is1087 * the release method. And the xdev->error part prevents being stuck1088 * forever in the event of a bizarre hardware bug: Pull the USB plug.1089 */1090 1091 while (wait_event_interruptible(fifo->waitq,1092 fifo->fill <= (fifo->size - 8) ||1093 xdev->error))1094 ; /* Empty loop */1095 1096 if (xdev->error) {1097 rc = xdev->error;1098 goto unlock_done;1099 }1100 1101 fifo_write(fifo, (void *)msg, 8, xilly_memcpy);1102 1103 try_queue_bulk_out(ep);1104 1105unlock_done:1106 mutex_unlock(&xdev->msg_mutex);1107 1108 return rc;1109}1110 1111/*1112 * Note that flush_downstream() merely waits for the data to arrive to1113 * the application logic at the FPGA -- unlike PCIe Xillybus' counterpart,1114 * it does nothing to make it happen (and neither is it necessary).1115 *1116 * This function is not reentrant for the same @chan, but this is covered1117 * by the fact that for any given @chan, it's called either by the open,1118 * write, llseek and flush fops methods, which can't run in parallel (and the1119 * write + flush and llseek method handlers are protected with out_mutex).1120 *1121 * chan->flushed is there to avoid multiple flushes at the same position,1122 * in particular as a result of programs that close the file descriptor1123 * e.g. after a dup2() for redirection.1124 */1125 1126static int flush_downstream(struct xillyusb_channel *chan,1127 long timeout,1128 bool interruptible)1129{1130 struct xillyusb_dev *xdev = chan->xdev;1131 int chan_num = chan->chan_idx << 1;1132 long deadline, left_to_sleep;1133 int rc;1134 1135 if (chan->flushed)1136 return 0;1137 1138 deadline = jiffies + 1 + timeout;1139 1140 if (chan->flushing) {1141 long cancel_deadline = jiffies + 1 + XILLY_RESPONSE_TIMEOUT;1142 1143 chan->canceled = 0;1144 rc = xillyusb_send_opcode(xdev, chan_num,1145 OPCODE_CANCEL_CHECKPOINT, 0);1146 1147 if (rc)1148 return rc; /* Only real error, never -EINTR */1149 1150 /* Ignoring interrupts. Cancellation must be handled */1151 while (!chan->canceled) {1152 left_to_sleep = cancel_deadline - ((long)jiffies);1153 1154 if (left_to_sleep <= 0) {1155 report_io_error(xdev, -EIO);1156 return -EIO;1157 }1158 1159 rc = wait_event_interruptible_timeout(chan->flushq,1160 chan->canceled ||1161 xdev->error,1162 left_to_sleep);1163 1164 if (xdev->error)1165 return xdev->error;1166 }1167 }1168 1169 chan->flushing = 1;1170 1171 /*1172 * The checkpoint is given in terms of data elements, not bytes. As1173 * a result, if less than an element's worth of data is stored in the1174 * FIFO, it's not flushed, including the flush before closing, which1175 * means that such data is lost. This is consistent with PCIe Xillybus.1176 */1177 1178 rc = xillyusb_send_opcode(xdev, chan_num,1179 OPCODE_SET_CHECKPOINT,1180 chan->out_bytes >>1181 chan->out_log2_element_size);1182 1183 if (rc)1184 return rc; /* Only real error, never -EINTR */1185 1186 if (!timeout) {1187 while (chan->flushing) {1188 rc = wait_event_interruptible(chan->flushq,1189 !chan->flushing ||1190 xdev->error);1191 if (xdev->error)1192 return xdev->error;1193 1194 if (interruptible && rc)1195 return -EINTR;1196 }1197 1198 goto done;1199 }1200 1201 while (chan->flushing) {1202 left_to_sleep = deadline - ((long)jiffies);1203 1204 if (left_to_sleep <= 0)1205 return -ETIMEDOUT;1206 1207 rc = wait_event_interruptible_timeout(chan->flushq,1208 !chan->flushing ||1209 xdev->error,1210 left_to_sleep);1211 1212 if (xdev->error)1213 return xdev->error;1214 1215 if (interruptible && rc < 0)1216 return -EINTR;1217 }1218 1219done:1220 chan->flushed = 1;1221 return 0;1222}1223 1224/* request_read_anything(): Ask the FPGA for any little amount of data */1225static int request_read_anything(struct xillyusb_channel *chan,1226 char opcode)1227{1228 struct xillyusb_dev *xdev = chan->xdev;1229 unsigned int sh = chan->in_log2_element_size;1230 int chan_num = (chan->chan_idx << 1) | 1;1231 u32 mercy = chan->in_consumed_bytes + (2 << sh) - 1;1232 1233 return xillyusb_send_opcode(xdev, chan_num, opcode, mercy >> sh);1234}1235 1236static int xillyusb_open(struct inode *inode, struct file *filp)1237{1238 struct xillyusb_dev *xdev;1239 struct xillyusb_channel *chan;1240 struct xillyfifo *in_fifo = NULL;1241 struct xillyusb_endpoint *out_ep = NULL;1242 int rc;1243 int index;1244 1245 mutex_lock(&kref_mutex);1246 1247 rc = xillybus_find_inode(inode, (void **)&xdev, &index);1248 if (rc) {1249 mutex_unlock(&kref_mutex);1250 return rc;1251 }1252 1253 kref_get(&xdev->kref);1254 mutex_unlock(&kref_mutex);1255 1256 chan = &xdev->channels[index];1257 filp->private_data = chan;1258 1259 mutex_lock(&chan->lock);1260 1261 rc = -ENODEV;1262 1263 if (xdev->error)1264 goto unmutex_fail;1265 1266 if (((filp->f_mode & FMODE_READ) && !chan->readable) ||1267 ((filp->f_mode & FMODE_WRITE) && !chan->writable))1268 goto unmutex_fail;1269 1270 if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_READ) &&1271 chan->in_synchronous) {1272 dev_err(xdev->dev,1273 "open() failed: O_NONBLOCK not allowed for read on this device\n");1274 goto unmutex_fail;1275 }1276 1277 if ((filp->f_flags & O_NONBLOCK) && (filp->f_mode & FMODE_WRITE) &&1278 chan->out_synchronous) {1279 dev_err(xdev->dev,1280 "open() failed: O_NONBLOCK not allowed for write on this device\n");1281 goto unmutex_fail;1282 }1283 1284 rc = -EBUSY;1285 1286 if (((filp->f_mode & FMODE_READ) && chan->open_for_read) ||1287 ((filp->f_mode & FMODE_WRITE) && chan->open_for_write))1288 goto unmutex_fail;1289 1290 if (filp->f_mode & FMODE_READ)1291 chan->open_for_read = 1;1292 1293 if (filp->f_mode & FMODE_WRITE)1294 chan->open_for_write = 1;1295 1296 mutex_unlock(&chan->lock);1297 1298 if (filp->f_mode & FMODE_WRITE) {1299 out_ep = endpoint_alloc(xdev,1300 (chan->chan_idx + 2) | USB_DIR_OUT,1301 bulk_out_work, BUF_SIZE_ORDER, BUFNUM);1302 1303 if (!out_ep) {1304 rc = -ENOMEM;1305 goto unopen;1306 }1307 1308 rc = fifo_init(&out_ep->fifo, chan->out_log2_fifo_size);1309 1310 if (rc)1311 goto late_unopen;1312 1313 out_ep->fill_mask = -(1 << chan->out_log2_element_size);1314 chan->out_bytes = 0;1315 chan->flushed = 0;1316 1317 /*1318 * Sending a flush request to a previously closed stream1319 * effectively opens it, and also waits until the command is1320 * confirmed by the FPGA. The latter is necessary because the1321 * data is sent through a separate BULK OUT endpoint, and the1322 * xHCI controller is free to reorder transmissions.1323 *1324 * This can't go wrong unless there's a serious hardware error1325 * (or the computer is stuck for 500 ms?)1326 */1327 rc = flush_downstream(chan, XILLY_RESPONSE_TIMEOUT, false);1328 1329 if (rc == -ETIMEDOUT) {1330 rc = -EIO;1331 report_io_error(xdev, rc);1332 }1333 1334 if (rc)1335 goto late_unopen;1336 }1337 1338 if (filp->f_mode & FMODE_READ) {1339 in_fifo = kzalloc(sizeof(*in_fifo), GFP_KERNEL);1340 1341 if (!in_fifo) {1342 rc = -ENOMEM;1343 goto late_unopen;1344 }1345 1346 rc = fifo_init(in_fifo, chan->in_log2_fifo_size);1347 1348 if (rc) {1349 kfree(in_fifo);1350 goto late_unopen;1351 }1352 }1353 1354 mutex_lock(&chan->lock);1355 if (in_fifo) {1356 chan->in_fifo = in_fifo;1357 chan->read_data_ok = 1;1358 }1359 if (out_ep)1360 chan->out_ep = out_ep;1361 mutex_unlock(&chan->lock);1362 1363 if (in_fifo) {1364 u32 in_checkpoint = 0;1365 1366 if (!chan->in_synchronous)1367 in_checkpoint = in_fifo->size >>1368 chan->in_log2_element_size;1369 1370 chan->in_consumed_bytes = 0;1371 chan->poll_used = 0;1372 chan->in_current_checkpoint = in_checkpoint;1373 rc = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,1374 OPCODE_SET_CHECKPOINT,1375 in_checkpoint);1376 1377 if (rc) /* Failure guarantees that opcode wasn't sent */1378 goto unfifo;1379 1380 /*1381 * In non-blocking mode, request the FPGA to send any data it1382 * has right away. Otherwise, the first read() will always1383 * return -EAGAIN, which is OK strictly speaking, but ugly.1384 * Checking and unrolling if this fails isn't worth the1385 * effort -- the error is propagated to the first read()1386 * anyhow.1387 */1388 if (filp->f_flags & O_NONBLOCK)1389 request_read_anything(chan, OPCODE_SET_PUSH);1390 }1391 1392 return 0;1393 1394unfifo:1395 chan->read_data_ok = 0;1396 safely_assign_in_fifo(chan, NULL);1397 fifo_mem_release(in_fifo);1398 kfree(in_fifo);1399 1400 if (out_ep) {1401 mutex_lock(&chan->lock);1402 chan->out_ep = NULL;1403 mutex_unlock(&chan->lock);1404 }1405 1406late_unopen:1407 if (out_ep)1408 endpoint_dealloc(out_ep);1409 1410unopen:1411 mutex_lock(&chan->lock);1412 1413 if (filp->f_mode & FMODE_READ)1414 chan->open_for_read = 0;1415 1416 if (filp->f_mode & FMODE_WRITE)1417 chan->open_for_write = 0;1418 1419 mutex_unlock(&chan->lock);1420 1421 kref_put(&xdev->kref, cleanup_dev);1422 1423 return rc;1424 1425unmutex_fail:1426 kref_put(&xdev->kref, cleanup_dev);1427 mutex_unlock(&chan->lock);1428 return rc;1429}1430 1431static ssize_t xillyusb_read(struct file *filp, char __user *userbuf,1432 size_t count, loff_t *f_pos)1433{1434 struct xillyusb_channel *chan = filp->private_data;1435 struct xillyusb_dev *xdev = chan->xdev;1436 struct xillyfifo *fifo = chan->in_fifo;1437 int chan_num = (chan->chan_idx << 1) | 1;1438 1439 long deadline, left_to_sleep;1440 int bytes_done = 0;1441 bool sent_set_push = false;1442 int rc;1443 1444 deadline = jiffies + 1 + XILLY_RX_TIMEOUT;1445 1446 rc = mutex_lock_interruptible(&chan->in_mutex);1447 1448 if (rc)1449 return rc;1450 1451 while (1) {1452 u32 fifo_checkpoint_bytes, complete_checkpoint_bytes;1453 u32 complete_checkpoint, fifo_checkpoint;1454 u32 checkpoint;1455 s32 diff, leap;1456 unsigned int sh = chan->in_log2_element_size;1457 bool checkpoint_for_complete;1458 1459 rc = fifo_read(fifo, (__force void *)userbuf + bytes_done,1460 count - bytes_done, xilly_copy_to_user);1461 1462 if (rc < 0)1463 break;1464 1465 bytes_done += rc;1466 chan->in_consumed_bytes += rc;1467 1468 left_to_sleep = deadline - ((long)jiffies);1469 1470 /*1471 * Some 32-bit arithmetic that may wrap. Note that1472 * complete_checkpoint is rounded up to the closest element1473 * boundary, because the read() can't be completed otherwise.1474 * fifo_checkpoint_bytes is rounded down, because it protects1475 * in_fifo from overflowing.1476 */1477 1478 fifo_checkpoint_bytes = chan->in_consumed_bytes + fifo->size;1479 complete_checkpoint_bytes =1480 chan->in_consumed_bytes + count - bytes_done;1481 1482 fifo_checkpoint = fifo_checkpoint_bytes >> sh;1483 complete_checkpoint =1484 (complete_checkpoint_bytes + (1 << sh) - 1) >> sh;1485 1486 diff = (fifo_checkpoint - complete_checkpoint) << sh;1487 1488 if (chan->in_synchronous && diff >= 0) {1489 checkpoint = complete_checkpoint;1490 checkpoint_for_complete = true;1491 } else {1492 checkpoint = fifo_checkpoint;1493 checkpoint_for_complete = false;1494 }1495 1496 leap = (checkpoint - chan->in_current_checkpoint) << sh;1497 1498 /*1499 * To prevent flooding of OPCODE_SET_CHECKPOINT commands as1500 * data is consumed, it's issued only if it moves the1501 * checkpoint by at least an 8th of the FIFO's size, or if1502 * it's necessary to complete the number of bytes requested by1503 * the read() call.1504 *1505 * chan->read_data_ok is checked to spare an unnecessary1506 * submission after receiving EOF, however it's harmless if1507 * such slips away.1508 */1509 1510 if (chan->read_data_ok &&1511 (leap > (fifo->size >> 3) ||1512 (checkpoint_for_complete && leap > 0))) {1513 chan->in_current_checkpoint = checkpoint;1514 rc = xillyusb_send_opcode(xdev, chan_num,1515 OPCODE_SET_CHECKPOINT,1516 checkpoint);1517 1518 if (rc)1519 break;1520 }1521 1522 if (bytes_done == count ||1523 (left_to_sleep <= 0 && bytes_done))1524 break;1525 1526 /*1527 * Reaching here means that the FIFO was empty when1528 * fifo_read() returned, but not necessarily right now. Error1529 * and EOF are checked and reported only now, so that no data1530 * that managed its way to the FIFO is lost.1531 */1532 1533 if (!READ_ONCE(chan->read_data_ok)) { /* FPGA has sent EOF */1534 /* Has data slipped into the FIFO since fifo_read()? */1535 smp_rmb();1536 if (READ_ONCE(fifo->fill))1537 continue;1538 1539 rc = 0;1540 break;1541 }1542 1543 if (xdev->error) {1544 rc = xdev->error;1545 break;1546 }1547 1548 if (filp->f_flags & O_NONBLOCK) {1549 rc = -EAGAIN;1550 break;1551 }1552 1553 if (!sent_set_push) {1554 rc = xillyusb_send_opcode(xdev, chan_num,1555 OPCODE_SET_PUSH,1556 complete_checkpoint);1557 1558 if (rc)1559 break;1560 1561 sent_set_push = true;1562 }1563 1564 if (left_to_sleep > 0) {1565 /*1566 * Note that when xdev->error is set (e.g. when the1567 * device is unplugged), read_data_ok turns zero and1568 * fifo->waitq is awaken.1569 * Therefore no special attention to xdev->error.1570 */1571 1572 rc = wait_event_interruptible_timeout1573 (fifo->waitq,1574 fifo->fill || !chan->read_data_ok,1575 left_to_sleep);1576 } else { /* bytes_done == 0 */1577 /* Tell FPGA to send anything it has */1578 rc = request_read_anything(chan, OPCODE_UPDATE_PUSH);1579 1580 if (rc)1581 break;1582 1583 rc = wait_event_interruptible1584 (fifo->waitq,1585 fifo->fill || !chan->read_data_ok);1586 }1587 1588 if (rc < 0) {1589 rc = -EINTR;1590 break;1591 }1592 }1593 1594 if (((filp->f_flags & O_NONBLOCK) || chan->poll_used) &&1595 !READ_ONCE(fifo->fill))1596 request_read_anything(chan, OPCODE_SET_PUSH);1597 1598 mutex_unlock(&chan->in_mutex);1599 1600 if (bytes_done)1601 return bytes_done;1602 1603 return rc;1604}1605 1606static int xillyusb_flush(struct file *filp, fl_owner_t id)1607{1608 struct xillyusb_channel *chan = filp->private_data;1609 int rc;1610 1611 if (!(filp->f_mode & FMODE_WRITE))1612 return 0;1613 1614 rc = mutex_lock_interruptible(&chan->out_mutex);1615 1616 if (rc)1617 return rc;1618 1619 /*1620 * One second's timeout on flushing. Interrupts are ignored, because if1621 * the user pressed CTRL-C, that interrupt will still be in flight by1622 * the time we reach here, and the opportunity to flush is lost.1623 */1624 rc = flush_downstream(chan, HZ, false);1625 1626 mutex_unlock(&chan->out_mutex);1627 1628 if (rc == -ETIMEDOUT) {1629 /* The things you do to use dev_warn() and not pr_warn() */1630 struct xillyusb_dev *xdev = chan->xdev;1631 1632 mutex_lock(&chan->lock);1633 if (!xdev->error)1634 dev_warn(xdev->dev,1635 "Timed out while flushing. Output data may be lost.\n");1636 mutex_unlock(&chan->lock);1637 }1638 1639 return rc;1640}1641 1642static ssize_t xillyusb_write(struct file *filp, const char __user *userbuf,1643 size_t count, loff_t *f_pos)1644{1645 struct xillyusb_channel *chan = filp->private_data;1646 struct xillyusb_dev *xdev = chan->xdev;1647 struct xillyfifo *fifo = &chan->out_ep->fifo;1648 int rc;1649 1650 rc = mutex_lock_interruptible(&chan->out_mutex);1651 1652 if (rc)1653 return rc;1654 1655 while (1) {1656 if (xdev->error) {1657 rc = xdev->error;1658 break;1659 }1660 1661 if (count == 0)1662 break;1663 1664 rc = fifo_write(fifo, (__force void *)userbuf, count,1665 xilly_copy_from_user);1666 1667 if (rc != 0)1668 break;1669 1670 if (filp->f_flags & O_NONBLOCK) {1671 rc = -EAGAIN;1672 break;1673 }1674 1675 if (wait_event_interruptible1676 (fifo->waitq,1677 fifo->fill != fifo->size || xdev->error)) {1678 rc = -EINTR;1679 break;1680 }1681 }1682 1683 if (rc < 0)1684 goto done;1685 1686 chan->out_bytes += rc;1687 1688 if (rc) {1689 try_queue_bulk_out(chan->out_ep);1690 chan->flushed = 0;1691 }1692 1693 if (chan->out_synchronous) {1694 int flush_rc = flush_downstream(chan, 0, true);1695 1696 if (flush_rc && !rc)1697 rc = flush_rc;1698 }1699 1700done:1701 mutex_unlock(&chan->out_mutex);1702 1703 return rc;1704}1705 1706static int xillyusb_release(struct inode *inode, struct file *filp)1707{1708 struct xillyusb_channel *chan = filp->private_data;1709 struct xillyusb_dev *xdev = chan->xdev;1710 int rc_read = 0, rc_write = 0;1711 1712 if (filp->f_mode & FMODE_READ) {1713 struct xillyfifo *in_fifo = chan->in_fifo;1714 1715 rc_read = xillyusb_send_opcode(xdev, (chan->chan_idx << 1) | 1,1716 OPCODE_CLOSE, 0);1717 /*1718 * If rc_read is nonzero, xdev->error indicates a global1719 * device error. The error is reported later, so that1720 * resources are freed.1721 *1722 * Looping on wait_event_interruptible() kinda breaks the idea1723 * of being interruptible, and this should have been1724 * wait_event(). Only it's being waken with1725 * wake_up_interruptible() for the sake of other uses. If1726 * there's a global device error, chan->read_data_ok is1727 * deasserted and the wait queue is awaken, so this is covered.1728 */1729 1730 while (wait_event_interruptible(in_fifo->waitq,1731 !chan->read_data_ok))1732 ; /* Empty loop */1733 1734 safely_assign_in_fifo(chan, NULL);1735 fifo_mem_release(in_fifo);1736 kfree(in_fifo);1737 1738 mutex_lock(&chan->lock);1739 chan->open_for_read = 0;1740 mutex_unlock(&chan->lock);1741 }1742 1743 if (filp->f_mode & FMODE_WRITE) {1744 struct xillyusb_endpoint *ep = chan->out_ep;1745 /*1746 * chan->flushing isn't zeroed. If the pre-release flush timed1747 * out, a cancel request will be sent before the next1748 * OPCODE_SET_CHECKPOINT (i.e. when the file is opened again).1749 * This is despite that the FPGA forgets about the checkpoint1750 * request as the file closes. Still, in an exceptional race1751 * condition, the FPGA could send an OPCODE_REACHED_CHECKPOINT1752 * just before closing that would reach the host after the1753 * file has re-opened.1754 */1755 1756 mutex_lock(&chan->lock);1757 chan->out_ep = NULL;1758 mutex_unlock(&chan->lock);1759 1760 endpoint_quiesce(ep);1761 endpoint_dealloc(ep);1762 1763 /* See comments on rc_read above */1764 rc_write = xillyusb_send_opcode(xdev, chan->chan_idx << 1,1765 OPCODE_CLOSE, 0);1766 1767 mutex_lock(&chan->lock);1768 chan->open_for_write = 0;1769 mutex_unlock(&chan->lock);1770 }1771 1772 kref_put(&xdev->kref, cleanup_dev);1773 1774 return rc_read ? rc_read : rc_write;1775}1776 1777/*1778 * Xillybus' API allows device nodes to be seekable, giving the user1779 * application access to a RAM array on the FPGA (or logic emulating it).1780 */1781 1782static loff_t xillyusb_llseek(struct file *filp, loff_t offset, int whence)1783{1784 struct xillyusb_channel *chan = filp->private_data;1785 struct xillyusb_dev *xdev = chan->xdev;1786 loff_t pos = filp->f_pos;1787 int rc = 0;1788 unsigned int log2_element_size = chan->readable ?1789 chan->in_log2_element_size : chan->out_log2_element_size;1790 1791 /*1792 * Take both mutexes not allowing interrupts, since it seems like1793 * common applications don't expect an -EINTR here. Besides, multiple1794 * access to a single file descriptor on seekable devices is a mess1795 * anyhow.1796 */1797 1798 mutex_lock(&chan->out_mutex);1799 mutex_lock(&chan->in_mutex);1800 1801 switch (whence) {1802 case SEEK_SET:1803 pos = offset;1804 break;1805 case SEEK_CUR:1806 pos += offset;1807 break;1808 case SEEK_END:1809 pos = offset; /* Going to the end => to the beginning */1810 break;1811 default:1812 rc = -EINVAL;1813 goto end;1814 }1815 1816 /* In any case, we must finish on an element boundary */1817 if (pos & ((1 << log2_element_size) - 1)) {1818 rc = -EINVAL;1819 goto end;1820 }1821 1822 rc = xillyusb_send_opcode(xdev, chan->chan_idx << 1,1823 OPCODE_SET_ADDR,1824 pos >> log2_element_size);1825 1826 if (rc)1827 goto end;1828 1829 if (chan->writable) {1830 chan->flushed = 0;1831 rc = flush_downstream(chan, HZ, false);1832 }1833 1834end:1835 mutex_unlock(&chan->out_mutex);1836 mutex_unlock(&chan->in_mutex);1837 1838 if (rc) /* Return error after releasing mutexes */1839 return rc;1840 1841 filp->f_pos = pos;1842 1843 return pos;1844}1845 1846static __poll_t xillyusb_poll(struct file *filp, poll_table *wait)1847{1848 struct xillyusb_channel *chan = filp->private_data;1849 __poll_t mask = 0;1850 1851 if (chan->in_fifo)1852 poll_wait(filp, &chan->in_fifo->waitq, wait);1853 1854 if (chan->out_ep)1855 poll_wait(filp, &chan->out_ep->fifo.waitq, wait);1856 1857 /*1858 * If this is the first time poll() is called, and the file is1859 * readable, set the relevant flag. Also tell the FPGA to send all it1860 * has, to kickstart the mechanism that ensures there's always some1861 * data in in_fifo unless the stream is dry end-to-end. Note that the1862 * first poll() may not return a EPOLLIN, even if there's data on the1863 * FPGA. Rather, the data will arrive soon, and trigger the relevant1864 * wait queue.1865 */1866 1867 if (!chan->poll_used && chan->in_fifo) {1868 chan->poll_used = 1;1869 request_read_anything(chan, OPCODE_SET_PUSH);1870 }1871 1872 /*1873 * poll() won't play ball regarding read() channels which1874 * are synchronous. Allowing that will create situations where data has1875 * been delivered at the FPGA, and users expecting select() to wake up,1876 * which it may not. So make it never work.1877 */1878 1879 if (chan->in_fifo && !chan->in_synchronous &&1880 (READ_ONCE(chan->in_fifo->fill) || !chan->read_data_ok))1881 mask |= EPOLLIN | EPOLLRDNORM;1882 1883 if (chan->out_ep &&1884 (READ_ONCE(chan->out_ep->fifo.fill) != chan->out_ep->fifo.size))1885 mask |= EPOLLOUT | EPOLLWRNORM;1886 1887 if (chan->xdev->error)1888 mask |= EPOLLERR;1889 1890 return mask;1891}1892 1893static const struct file_operations xillyusb_fops = {1894 .owner = THIS_MODULE,1895 .read = xillyusb_read,1896 .write = xillyusb_write,1897 .open = xillyusb_open,1898 .flush = xillyusb_flush,1899 .release = xillyusb_release,1900 .llseek = xillyusb_llseek,1901 .poll = xillyusb_poll,1902};1903 1904static int xillyusb_setup_base_eps(struct xillyusb_dev *xdev)1905{1906 struct usb_device *udev = xdev->udev;1907 1908 /* Verify that device has the two fundamental bulk in/out endpoints */1909 if (usb_pipe_type_check(udev, usb_sndbulkpipe(udev, MSG_EP_NUM)) ||1910 usb_pipe_type_check(udev, usb_rcvbulkpipe(udev, IN_EP_NUM)))1911 return -ENODEV;1912 1913 xdev->msg_ep = endpoint_alloc(xdev, MSG_EP_NUM | USB_DIR_OUT,1914 bulk_out_work, 1, 2);1915 if (!xdev->msg_ep)1916 return -ENOMEM;1917 1918 if (fifo_init(&xdev->msg_ep->fifo, 13)) /* 8 kiB */1919 goto dealloc;1920 1921 xdev->msg_ep->fill_mask = -8; /* 8 bytes granularity */1922 1923 xdev->in_ep = endpoint_alloc(xdev, IN_EP_NUM | USB_DIR_IN,1924 bulk_in_work, BUF_SIZE_ORDER, BUFNUM);1925 if (!xdev->in_ep)1926 goto dealloc;1927 1928 try_queue_bulk_in(xdev->in_ep);1929 1930 return 0;1931 1932dealloc:1933 endpoint_dealloc(xdev->msg_ep); /* Also frees FIFO mem if allocated */1934 xdev->msg_ep = NULL;1935 return -ENOMEM;1936}1937 1938static int setup_channels(struct xillyusb_dev *xdev,1939 __le16 *chandesc,1940 int num_channels)1941{1942 struct usb_device *udev = xdev->udev;1943 struct xillyusb_channel *chan, *new_channels;1944 int i;1945 1946 chan = kcalloc(num_channels, sizeof(*chan), GFP_KERNEL);1947 if (!chan)1948 return -ENOMEM;1949 1950 new_channels = chan;1951 1952 for (i = 0; i < num_channels; i++, chan++) {1953 unsigned int in_desc = le16_to_cpu(*chandesc++);1954 unsigned int out_desc = le16_to_cpu(*chandesc++);1955 1956 chan->xdev = xdev;1957 mutex_init(&chan->in_mutex);1958 mutex_init(&chan->out_mutex);1959 mutex_init(&chan->lock);1960 init_waitqueue_head(&chan->flushq);1961 1962 chan->chan_idx = i;1963 1964 if (in_desc & 0x80) { /* Entry is valid */1965 chan->readable = 1;1966 chan->in_synchronous = !!(in_desc & 0x40);1967 chan->in_seekable = !!(in_desc & 0x20);1968 chan->in_log2_element_size = in_desc & 0x0f;1969 chan->in_log2_fifo_size = ((in_desc >> 8) & 0x1f) + 16;1970 }1971 1972 /*1973 * A downstream channel should never exist above index 13,1974 * as it would request a nonexistent BULK endpoint > 15.1975 * In the peculiar case that it does, it's ignored silently.1976 */1977 1978 if ((out_desc & 0x80) && i < 14) { /* Entry is valid */1979 if (usb_pipe_type_check(udev,1980 usb_sndbulkpipe(udev, i + 2))) {1981 dev_err(xdev->dev,1982 "Missing BULK OUT endpoint %d\n",1983 i + 2);1984 kfree(new_channels);1985 return -ENODEV;1986 }1987 1988 chan->writable = 1;1989 chan->out_synchronous = !!(out_desc & 0x40);1990 chan->out_seekable = !!(out_desc & 0x20);1991 chan->out_log2_element_size = out_desc & 0x0f;1992 chan->out_log2_fifo_size =1993 ((out_desc >> 8) & 0x1f) + 16;1994 }1995 }1996 1997 xdev->channels = new_channels;1998 return 0;1999}2000 2001static int xillyusb_discovery(struct usb_interface *interface)2002{2003 int rc;2004 struct xillyusb_dev *xdev = usb_get_intfdata(interface);2005 __le16 bogus_chandesc[2];2006 struct xillyfifo idt_fifo;2007 struct xillyusb_channel *chan;2008 unsigned int idt_len, names_offset;2009 unsigned char *idt;2010 int num_channels;2011 2012 rc = xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);2013 2014 if (rc) {2015 dev_err(&interface->dev, "Failed to send quiesce request. Aborting.\n");2016 return rc;2017 }2018 2019 /* Phase I: Set up one fake upstream channel and obtain IDT */2020 2021 /* Set up a fake IDT with one async IN stream */2022 bogus_chandesc[0] = cpu_to_le16(0x80);2023 bogus_chandesc[1] = cpu_to_le16(0);2024 2025 rc = setup_channels(xdev, bogus_chandesc, 1);2026 2027 if (rc)2028 return rc;2029 2030 rc = fifo_init(&idt_fifo, LOG2_IDT_FIFO_SIZE);2031 2032 if (rc)2033 return rc;2034 2035 chan = xdev->channels;2036 2037 chan->in_fifo = &idt_fifo;2038 chan->read_data_ok = 1;2039 2040 xdev->num_channels = 1;2041 2042 rc = xillyusb_send_opcode(xdev, ~0, OPCODE_REQ_IDT, 0);2043 2044 if (rc) {2045 dev_err(&interface->dev, "Failed to send IDT request. Aborting.\n");2046 goto unfifo;2047 }2048 2049 rc = wait_event_interruptible_timeout(idt_fifo.waitq,2050 !chan->read_data_ok,2051 XILLY_RESPONSE_TIMEOUT);2052 2053 if (xdev->error) {2054 rc = xdev->error;2055 goto unfifo;2056 }2057 2058 if (rc < 0) {2059 rc = -EINTR; /* Interrupt on probe method? Interesting. */2060 goto unfifo;2061 }2062 2063 if (chan->read_data_ok) {2064 rc = -ETIMEDOUT;2065 dev_err(&interface->dev, "No response from FPGA. Aborting.\n");2066 goto unfifo;2067 }2068 2069 idt_len = READ_ONCE(idt_fifo.fill);2070 idt = kmalloc(idt_len, GFP_KERNEL);2071 2072 if (!idt) {2073 rc = -ENOMEM;2074 goto unfifo;2075 }2076 2077 fifo_read(&idt_fifo, idt, idt_len, xilly_memcpy);2078 2079 if (crc32_le(~0, idt, idt_len) != 0) {2080 dev_err(&interface->dev, "IDT failed CRC check. Aborting.\n");2081 rc = -ENODEV;2082 goto unidt;2083 }2084 2085 if (*idt > 0x90) {2086 dev_err(&interface->dev, "No support for IDT version 0x%02x. Maybe the xillyusb driver needs an upgrade. Aborting.\n",2087 (int)*idt);2088 rc = -ENODEV;2089 goto unidt;2090 }2091 2092 /* Phase II: Set up the streams as defined in IDT */2093 2094 num_channels = le16_to_cpu(*((__le16 *)(idt + 1)));2095 names_offset = 3 + num_channels * 4;2096 idt_len -= 4; /* Exclude CRC */2097 2098 if (idt_len < names_offset) {2099 dev_err(&interface->dev, "IDT too short. This is exceptionally weird, because its CRC is OK\n");2100 rc = -ENODEV;2101 goto unidt;2102 }2103 2104 rc = setup_channels(xdev, (void *)idt + 3, num_channels);2105 2106 if (rc)2107 goto unidt;2108 2109 /*2110 * Except for wildly misbehaving hardware, or if it was disconnected2111 * just after responding with the IDT, there is no reason for any2112 * work item to be running now. To be sure that xdev->channels2113 * is updated on anything that might run in parallel, flush the2114 * device's workqueue and the wakeup work item. This rarely2115 * does anything.2116 */2117 flush_workqueue(xdev->workq);2118 flush_work(&xdev->wakeup_workitem);2119 2120 xdev->num_channels = num_channels;2121 2122 fifo_mem_release(&idt_fifo);2123 kfree(chan);2124 2125 rc = xillybus_init_chrdev(&interface->dev, &xillyusb_fops,2126 THIS_MODULE, xdev,2127 idt + names_offset,2128 idt_len - names_offset,2129 num_channels,2130 xillyname, true);2131 2132 kfree(idt);2133 2134 return rc;2135 2136unidt:2137 kfree(idt);2138 2139unfifo:2140 safely_assign_in_fifo(chan, NULL);2141 fifo_mem_release(&idt_fifo);2142 2143 return rc;2144}2145 2146static int xillyusb_probe(struct usb_interface *interface,2147 const struct usb_device_id *id)2148{2149 struct xillyusb_dev *xdev;2150 int rc;2151 2152 xdev = kzalloc(sizeof(*xdev), GFP_KERNEL);2153 if (!xdev)2154 return -ENOMEM;2155 2156 kref_init(&xdev->kref);2157 mutex_init(&xdev->process_in_mutex);2158 mutex_init(&xdev->msg_mutex);2159 2160 xdev->udev = usb_get_dev(interface_to_usbdev(interface));2161 xdev->dev = &interface->dev;2162 xdev->error = 0;2163 spin_lock_init(&xdev->error_lock);2164 xdev->in_counter = 0;2165 xdev->in_bytes_left = 0;2166 xdev->workq = alloc_workqueue(xillyname, WQ_HIGHPRI, 0);2167 2168 if (!xdev->workq) {2169 dev_err(&interface->dev, "Failed to allocate work queue\n");2170 rc = -ENOMEM;2171 goto fail;2172 }2173 2174 INIT_WORK(&xdev->wakeup_workitem, wakeup_all);2175 2176 usb_set_intfdata(interface, xdev);2177 2178 rc = xillyusb_setup_base_eps(xdev);2179 if (rc)2180 goto fail;2181 2182 rc = xillyusb_discovery(interface);2183 if (rc)2184 goto latefail;2185 2186 return 0;2187 2188latefail:2189 endpoint_quiesce(xdev->in_ep);2190 endpoint_quiesce(xdev->msg_ep);2191 2192fail:2193 usb_set_intfdata(interface, NULL);2194 kref_put(&xdev->kref, cleanup_dev);2195 return rc;2196}2197 2198static void xillyusb_disconnect(struct usb_interface *interface)2199{2200 struct xillyusb_dev *xdev = usb_get_intfdata(interface);2201 struct xillyusb_endpoint *msg_ep = xdev->msg_ep;2202 struct xillyfifo *fifo = &msg_ep->fifo;2203 int rc;2204 int i;2205 2206 xillybus_cleanup_chrdev(xdev, &interface->dev);2207 2208 /*2209 * Try to send OPCODE_QUIESCE, which will fail silently if the device2210 * was disconnected, but makes sense on module unload.2211 */2212 2213 msg_ep->wake_on_drain = true;2214 xillyusb_send_opcode(xdev, ~0, OPCODE_QUIESCE, 0);2215 2216 /*2217 * If the device has been disconnected, sending the opcode causes2218 * a global device error with xdev->error, if such error didn't2219 * occur earlier. Hence timing out means that the USB link is fine,2220 * but somehow the message wasn't sent. Should never happen.2221 */2222 2223 rc = wait_event_interruptible_timeout(fifo->waitq,2224 msg_ep->drained || xdev->error,2225 XILLY_RESPONSE_TIMEOUT);2226 2227 if (!rc)2228 dev_err(&interface->dev,2229 "Weird timeout condition on sending quiesce request.\n");2230 2231 report_io_error(xdev, -ENODEV); /* Discourage further activity */2232 2233 /*2234 * This device driver is declared with soft_unbind set, or else2235 * sending OPCODE_QUIESCE above would always fail. The price is2236 * that the USB framework didn't kill outstanding URBs, so it has2237 * to be done explicitly before returning from this call.2238 */2239 2240 for (i = 0; i < xdev->num_channels; i++) {2241 struct xillyusb_channel *chan = &xdev->channels[i];2242 2243 /*2244 * Lock taken to prevent chan->out_ep from changing. It also2245 * ensures xillyusb_open() and xillyusb_flush() don't access2246 * xdev->dev after being nullified below.2247 */2248 mutex_lock(&chan->lock);2249 if (chan->out_ep)2250 endpoint_quiesce(chan->out_ep);2251 mutex_unlock(&chan->lock);2252 }2253 2254 endpoint_quiesce(xdev->in_ep);2255 endpoint_quiesce(xdev->msg_ep);2256 2257 usb_set_intfdata(interface, NULL);2258 2259 xdev->dev = NULL;2260 2261 mutex_lock(&kref_mutex);2262 kref_put(&xdev->kref, cleanup_dev);2263 mutex_unlock(&kref_mutex);2264}2265 2266static struct usb_driver xillyusb_driver = {2267 .name = xillyname,2268 .id_table = xillyusb_table,2269 .probe = xillyusb_probe,2270 .disconnect = xillyusb_disconnect,2271 .soft_unbind = 1,2272};2273 2274static int __init xillyusb_init(void)2275{2276 int rc = 0;2277 2278 wakeup_wq = alloc_workqueue(xillyname, 0, 0);2279 if (!wakeup_wq)2280 return -ENOMEM;2281 2282 if (LOG2_INITIAL_FIFO_BUF_SIZE > PAGE_SHIFT)2283 fifo_buf_order = LOG2_INITIAL_FIFO_BUF_SIZE - PAGE_SHIFT;2284 else2285 fifo_buf_order = 0;2286 2287 rc = usb_register(&xillyusb_driver);2288 2289 if (rc)2290 destroy_workqueue(wakeup_wq);2291 2292 return rc;2293}2294 2295static void __exit xillyusb_exit(void)2296{2297 usb_deregister(&xillyusb_driver);2298 2299 destroy_workqueue(wakeup_wq);2300}2301 2302module_init(xillyusb_init);2303module_exit(xillyusb_exit);2304