1992 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * linux/drivers/misc/xillybus_core.c4 *5 * Copyright 2011 Xillybus Ltd, http://xillybus.com6 *7 * Driver for the Xillybus FPGA/host framework.8 *9 * This driver interfaces with a special IP core in an FPGA, setting up10 * a pipe between a hardware FIFO in the programmable logic and a device11 * file in the host. The number of such pipes and their attributes are12 * set up on the logic. This driver detects these automatically and13 * creates the device files accordingly.14 */15 16#include <linux/list.h>17#include <linux/device.h>18#include <linux/module.h>19#include <linux/io.h>20#include <linux/dma-mapping.h>21#include <linux/interrupt.h>22#include <linux/sched.h>23#include <linux/fs.h>24#include <linux/spinlock.h>25#include <linux/mutex.h>26#include <linux/crc32.h>27#include <linux/poll.h>28#include <linux/delay.h>29#include <linux/slab.h>30#include <linux/workqueue.h>31#include "xillybus.h"32#include "xillybus_class.h"33 34MODULE_DESCRIPTION("Xillybus core functions");35MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");36MODULE_ALIAS("xillybus_core");37MODULE_LICENSE("GPL v2");38 39/* General timeout is 100 ms, rx timeout is 10 ms */40#define XILLY_RX_TIMEOUT (10*HZ/1000)41#define XILLY_TIMEOUT (100*HZ/1000)42 43#define fpga_msg_ctrl_reg 0x000844#define fpga_dma_control_reg 0x002045#define fpga_dma_bufno_reg 0x002446#define fpga_dma_bufaddr_lowaddr_reg 0x002847#define fpga_dma_bufaddr_highaddr_reg 0x002c48#define fpga_buf_ctrl_reg 0x003049#define fpga_buf_offset_reg 0x003450#define fpga_endian_reg 0x004051 52#define XILLYMSG_OPCODE_RELEASEBUF 153#define XILLYMSG_OPCODE_QUIESCEACK 254#define XILLYMSG_OPCODE_FIFOEOF 355#define XILLYMSG_OPCODE_FATAL_ERROR 456#define XILLYMSG_OPCODE_NONEMPTY 557 58static const char xillyname[] = "xillybus";59 60static struct workqueue_struct *xillybus_wq;61 62/*63 * Locking scheme: Mutexes protect invocations of character device methods.64 * If both locks are taken, wr_mutex is taken first, rd_mutex second.65 *66 * wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the67 * buffers' end_offset fields against changes made by IRQ handler (and in68 * theory, other file request handlers, but the mutex handles that). Nothing69 * else.70 * They are held for short direct memory manipulations. Needless to say,71 * no mutex locking is allowed when a spinlock is held.72 *73 * rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.74 *75 * register_mutex is endpoint-specific, and is held when non-atomic76 * register operations are performed. wr_mutex and rd_mutex may be77 * held when register_mutex is taken, but none of the spinlocks. Note that78 * register_mutex doesn't protect against sporadic buf_ctrl_reg writes79 * which are unrelated to buf_offset_reg, since they are harmless.80 *81 * Blocking on the wait queues is allowed with mutexes held, but not with82 * spinlocks.83 *84 * Only interruptible blocking is allowed on mutexes and wait queues.85 *86 * All in all, the locking order goes (with skips allowed, of course):87 * wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock88 */89 90static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)91{92 int opcode;93 int msg_channel, msg_bufno, msg_data, msg_dir;94 95 opcode = (buf[0] >> 24) & 0xff;96 msg_dir = buf[0] & 1;97 msg_channel = (buf[0] >> 1) & 0x7ff;98 msg_bufno = (buf[0] >> 12) & 0x3ff;99 msg_data = buf[1] & 0xfffffff;100 101 dev_warn(endpoint->dev,102 "Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",103 opcode, msg_channel, msg_dir, msg_bufno, msg_data);104}105 106/*107 * xillybus_isr assumes the interrupt is allocated exclusively to it,108 * which is the natural case MSI and several other hardware-oriented109 * interrupts. Sharing is not allowed.110 */111 112irqreturn_t xillybus_isr(int irq, void *data)113{114 struct xilly_endpoint *ep = data;115 u32 *buf;116 unsigned int buf_size;117 int i;118 int opcode;119 unsigned int msg_channel, msg_bufno, msg_data, msg_dir;120 struct xilly_channel *channel;121 122 buf = ep->msgbuf_addr;123 buf_size = ep->msg_buf_size/sizeof(u32);124 125 dma_sync_single_for_cpu(ep->dev, ep->msgbuf_dma_addr,126 ep->msg_buf_size, DMA_FROM_DEVICE);127 128 for (i = 0; i < buf_size; i += 2) {129 if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {130 malformed_message(ep, &buf[i]);131 dev_warn(ep->dev,132 "Sending a NACK on counter %x (instead of %x) on entry %d\n",133 ((buf[i+1] >> 28) & 0xf),134 ep->msg_counter,135 i/2);136 137 if (++ep->failed_messages > 10) {138 dev_err(ep->dev,139 "Lost sync with interrupt messages. Stopping.\n");140 } else {141 dma_sync_single_for_device(ep->dev,142 ep->msgbuf_dma_addr,143 ep->msg_buf_size,144 DMA_FROM_DEVICE);145 146 iowrite32(0x01, /* Message NACK */147 ep->registers + fpga_msg_ctrl_reg);148 }149 return IRQ_HANDLED;150 } else if (buf[i] & (1 << 22)) /* Last message */151 break;152 }153 154 if (i >= buf_size) {155 dev_err(ep->dev, "Bad interrupt message. Stopping.\n");156 return IRQ_HANDLED;157 }158 159 buf_size = i + 2;160 161 for (i = 0; i < buf_size; i += 2) { /* Scan through messages */162 opcode = (buf[i] >> 24) & 0xff;163 164 msg_dir = buf[i] & 1;165 msg_channel = (buf[i] >> 1) & 0x7ff;166 msg_bufno = (buf[i] >> 12) & 0x3ff;167 msg_data = buf[i+1] & 0xfffffff;168 169 switch (opcode) {170 case XILLYMSG_OPCODE_RELEASEBUF:171 if ((msg_channel > ep->num_channels) ||172 (msg_channel == 0)) {173 malformed_message(ep, &buf[i]);174 break;175 }176 177 channel = ep->channels[msg_channel];178 179 if (msg_dir) { /* Write channel */180 if (msg_bufno >= channel->num_wr_buffers) {181 malformed_message(ep, &buf[i]);182 break;183 }184 spin_lock(&channel->wr_spinlock);185 channel->wr_buffers[msg_bufno]->end_offset =186 msg_data;187 channel->wr_fpga_buf_idx = msg_bufno;188 channel->wr_empty = 0;189 channel->wr_sleepy = 0;190 spin_unlock(&channel->wr_spinlock);191 192 wake_up_interruptible(&channel->wr_wait);193 194 } else {195 /* Read channel */196 197 if (msg_bufno >= channel->num_rd_buffers) {198 malformed_message(ep, &buf[i]);199 break;200 }201 202 spin_lock(&channel->rd_spinlock);203 channel->rd_fpga_buf_idx = msg_bufno;204 channel->rd_full = 0;205 spin_unlock(&channel->rd_spinlock);206 207 wake_up_interruptible(&channel->rd_wait);208 if (!channel->rd_synchronous)209 queue_delayed_work(210 xillybus_wq,211 &channel->rd_workitem,212 XILLY_RX_TIMEOUT);213 }214 215 break;216 case XILLYMSG_OPCODE_NONEMPTY:217 if ((msg_channel > ep->num_channels) ||218 (msg_channel == 0) || (!msg_dir) ||219 !ep->channels[msg_channel]->wr_supports_nonempty) {220 malformed_message(ep, &buf[i]);221 break;222 }223 224 channel = ep->channels[msg_channel];225 226 if (msg_bufno >= channel->num_wr_buffers) {227 malformed_message(ep, &buf[i]);228 break;229 }230 spin_lock(&channel->wr_spinlock);231 if (msg_bufno == channel->wr_host_buf_idx)232 channel->wr_ready = 1;233 spin_unlock(&channel->wr_spinlock);234 235 wake_up_interruptible(&channel->wr_ready_wait);236 237 break;238 case XILLYMSG_OPCODE_QUIESCEACK:239 ep->idtlen = msg_data;240 wake_up_interruptible(&ep->ep_wait);241 242 break;243 case XILLYMSG_OPCODE_FIFOEOF:244 if ((msg_channel > ep->num_channels) ||245 (msg_channel == 0) || (!msg_dir) ||246 !ep->channels[msg_channel]->num_wr_buffers) {247 malformed_message(ep, &buf[i]);248 break;249 }250 channel = ep->channels[msg_channel];251 spin_lock(&channel->wr_spinlock);252 channel->wr_eof = msg_bufno;253 channel->wr_sleepy = 0;254 255 channel->wr_hangup = channel->wr_empty &&256 (channel->wr_host_buf_idx == msg_bufno);257 258 spin_unlock(&channel->wr_spinlock);259 260 wake_up_interruptible(&channel->wr_wait);261 262 break;263 case XILLYMSG_OPCODE_FATAL_ERROR:264 ep->fatal_error = 1;265 wake_up_interruptible(&ep->ep_wait); /* For select() */266 dev_err(ep->dev,267 "FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");268 break;269 default:270 malformed_message(ep, &buf[i]);271 break;272 }273 }274 275 dma_sync_single_for_device(ep->dev, ep->msgbuf_dma_addr,276 ep->msg_buf_size, DMA_FROM_DEVICE);277 278 ep->msg_counter = (ep->msg_counter + 1) & 0xf;279 ep->failed_messages = 0;280 iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */281 282 return IRQ_HANDLED;283}284EXPORT_SYMBOL(xillybus_isr);285 286/*287 * A few trivial memory management functions.288 * NOTE: These functions are used only on probe and remove, and therefore289 * no locks are applied!290 */291 292static void xillybus_autoflush(struct work_struct *work);293 294struct xilly_alloc_state {295 void *salami;296 int left_of_salami;297 int nbuffer;298 enum dma_data_direction direction;299 u32 regdirection;300};301 302static void xilly_unmap(void *ptr)303{304 struct xilly_mapping *data = ptr;305 306 dma_unmap_single(data->device, data->dma_addr,307 data->size, data->direction);308 309 kfree(ptr);310}311 312static int xilly_map_single(struct xilly_endpoint *ep,313 void *ptr,314 size_t size,315 int direction,316 dma_addr_t *ret_dma_handle317 )318{319 dma_addr_t addr;320 struct xilly_mapping *this;321 322 this = kzalloc(sizeof(*this), GFP_KERNEL);323 if (!this)324 return -ENOMEM;325 326 addr = dma_map_single(ep->dev, ptr, size, direction);327 328 if (dma_mapping_error(ep->dev, addr)) {329 kfree(this);330 return -ENODEV;331 }332 333 this->device = ep->dev;334 this->dma_addr = addr;335 this->size = size;336 this->direction = direction;337 338 *ret_dma_handle = addr;339 340 return devm_add_action_or_reset(ep->dev, xilly_unmap, this);341}342 343static int xilly_get_dma_buffers(struct xilly_endpoint *ep,344 struct xilly_alloc_state *s,345 struct xilly_buffer **buffers,346 int bufnum, int bytebufsize)347{348 int i, rc;349 dma_addr_t dma_addr;350 struct device *dev = ep->dev;351 struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */352 353 if (buffers) { /* Not the message buffer */354 this_buffer = devm_kcalloc(dev, bufnum,355 sizeof(struct xilly_buffer),356 GFP_KERNEL);357 if (!this_buffer)358 return -ENOMEM;359 }360 361 for (i = 0; i < bufnum; i++) {362 /*363 * Buffers are expected in descending size order, so there364 * is either enough space for this buffer or none at all.365 */366 367 if ((s->left_of_salami < bytebufsize) &&368 (s->left_of_salami > 0)) {369 dev_err(ep->dev,370 "Corrupt buffer allocation in IDT. Aborting.\n");371 return -ENODEV;372 }373 374 if (s->left_of_salami == 0) {375 int allocorder, allocsize;376 377 allocsize = PAGE_SIZE;378 allocorder = 0;379 while (bytebufsize > allocsize) {380 allocsize *= 2;381 allocorder++;382 }383 384 s->salami = (void *) devm_get_free_pages(385 dev,386 GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,387 allocorder);388 if (!s->salami)389 return -ENOMEM;390 391 s->left_of_salami = allocsize;392 }393 394 rc = xilly_map_single(ep, s->salami,395 bytebufsize, s->direction,396 &dma_addr);397 if (rc)398 return rc;399 400 iowrite32((u32) (dma_addr & 0xffffffff),401 ep->registers + fpga_dma_bufaddr_lowaddr_reg);402 iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),403 ep->registers + fpga_dma_bufaddr_highaddr_reg);404 405 if (buffers) { /* Not the message buffer */406 this_buffer->addr = s->salami;407 this_buffer->dma_addr = dma_addr;408 buffers[i] = this_buffer++;409 410 iowrite32(s->regdirection | s->nbuffer++,411 ep->registers + fpga_dma_bufno_reg);412 } else {413 ep->msgbuf_addr = s->salami;414 ep->msgbuf_dma_addr = dma_addr;415 ep->msg_buf_size = bytebufsize;416 417 iowrite32(s->regdirection,418 ep->registers + fpga_dma_bufno_reg);419 }420 421 s->left_of_salami -= bytebufsize;422 s->salami += bytebufsize;423 }424 return 0;425}426 427static int xilly_setupchannels(struct xilly_endpoint *ep,428 unsigned char *chandesc,429 int entries)430{431 struct device *dev = ep->dev;432 int i, entry, rc;433 struct xilly_channel *channel;434 int channelnum, bufnum, bufsize, format, is_writebuf;435 int bytebufsize;436 int synchronous, allowpartial, exclusive_open, seekable;437 int supports_nonempty;438 int msg_buf_done = 0;439 440 struct xilly_alloc_state rd_alloc = {441 .salami = NULL,442 .left_of_salami = 0,443 .nbuffer = 1,444 .direction = DMA_TO_DEVICE,445 .regdirection = 0,446 };447 448 struct xilly_alloc_state wr_alloc = {449 .salami = NULL,450 .left_of_salami = 0,451 .nbuffer = 1,452 .direction = DMA_FROM_DEVICE,453 .regdirection = 0x80000000,454 };455 456 channel = devm_kcalloc(dev, ep->num_channels,457 sizeof(struct xilly_channel), GFP_KERNEL);458 if (!channel)459 return -ENOMEM;460 461 ep->channels = devm_kcalloc(dev, ep->num_channels + 1,462 sizeof(struct xilly_channel *),463 GFP_KERNEL);464 if (!ep->channels)465 return -ENOMEM;466 467 ep->channels[0] = NULL; /* Channel 0 is message buf. */468 469 /* Initialize all channels with defaults */470 471 for (i = 1; i <= ep->num_channels; i++) {472 channel->wr_buffers = NULL;473 channel->rd_buffers = NULL;474 channel->num_wr_buffers = 0;475 channel->num_rd_buffers = 0;476 channel->wr_fpga_buf_idx = -1;477 channel->wr_host_buf_idx = 0;478 channel->wr_host_buf_pos = 0;479 channel->wr_empty = 1;480 channel->wr_ready = 0;481 channel->wr_sleepy = 1;482 channel->rd_fpga_buf_idx = 0;483 channel->rd_host_buf_idx = 0;484 channel->rd_host_buf_pos = 0;485 channel->rd_full = 0;486 channel->wr_ref_count = 0;487 channel->rd_ref_count = 0;488 489 spin_lock_init(&channel->wr_spinlock);490 spin_lock_init(&channel->rd_spinlock);491 mutex_init(&channel->wr_mutex);492 mutex_init(&channel->rd_mutex);493 init_waitqueue_head(&channel->rd_wait);494 init_waitqueue_head(&channel->wr_wait);495 init_waitqueue_head(&channel->wr_ready_wait);496 497 INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);498 499 channel->endpoint = ep;500 channel->chan_num = i;501 502 channel->log2_element_size = 0;503 504 ep->channels[i] = channel++;505 }506 507 for (entry = 0; entry < entries; entry++, chandesc += 4) {508 struct xilly_buffer **buffers = NULL;509 510 is_writebuf = chandesc[0] & 0x01;511 channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);512 format = (chandesc[1] >> 4) & 0x03;513 allowpartial = (chandesc[1] >> 6) & 0x01;514 synchronous = (chandesc[1] >> 7) & 0x01;515 bufsize = 1 << (chandesc[2] & 0x1f);516 bufnum = 1 << (chandesc[3] & 0x0f);517 exclusive_open = (chandesc[2] >> 7) & 0x01;518 seekable = (chandesc[2] >> 6) & 0x01;519 supports_nonempty = (chandesc[2] >> 5) & 0x01;520 521 if ((channelnum > ep->num_channels) ||522 ((channelnum == 0) && !is_writebuf)) {523 dev_err(ep->dev,524 "IDT requests channel out of range. Aborting.\n");525 return -ENODEV;526 }527 528 channel = ep->channels[channelnum]; /* NULL for msg channel */529 530 if (!is_writebuf || channelnum > 0) {531 channel->log2_element_size = ((format > 2) ?532 2 : format);533 534 bytebufsize = bufsize *535 (1 << channel->log2_element_size);536 537 buffers = devm_kcalloc(dev, bufnum,538 sizeof(struct xilly_buffer *),539 GFP_KERNEL);540 if (!buffers)541 return -ENOMEM;542 } else {543 bytebufsize = bufsize << 2;544 }545 546 if (!is_writebuf) {547 channel->num_rd_buffers = bufnum;548 channel->rd_buf_size = bytebufsize;549 channel->rd_allow_partial = allowpartial;550 channel->rd_synchronous = synchronous;551 channel->rd_exclusive_open = exclusive_open;552 channel->seekable = seekable;553 554 channel->rd_buffers = buffers;555 rc = xilly_get_dma_buffers(ep, &rd_alloc, buffers,556 bufnum, bytebufsize);557 } else if (channelnum > 0) {558 channel->num_wr_buffers = bufnum;559 channel->wr_buf_size = bytebufsize;560 561 channel->seekable = seekable;562 channel->wr_supports_nonempty = supports_nonempty;563 564 channel->wr_allow_partial = allowpartial;565 channel->wr_synchronous = synchronous;566 channel->wr_exclusive_open = exclusive_open;567 568 channel->wr_buffers = buffers;569 rc = xilly_get_dma_buffers(ep, &wr_alloc, buffers,570 bufnum, bytebufsize);571 } else {572 rc = xilly_get_dma_buffers(ep, &wr_alloc, NULL,573 bufnum, bytebufsize);574 msg_buf_done++;575 }576 577 if (rc)578 return -ENOMEM;579 }580 581 if (!msg_buf_done) {582 dev_err(ep->dev,583 "Corrupt IDT: No message buffer. Aborting.\n");584 return -ENODEV;585 }586 return 0;587}588 589static int xilly_scan_idt(struct xilly_endpoint *endpoint,590 struct xilly_idt_handle *idt_handle)591{592 int count = 0;593 unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;594 unsigned char *end_of_idt = idt + endpoint->idtlen - 4;595 unsigned char *scan;596 int len;597 598 scan = idt + 1;599 idt_handle->names = scan;600 601 while ((scan <= end_of_idt) && *scan) {602 while ((scan <= end_of_idt) && *scan++)603 /* Do nothing, just scan thru string */;604 count++;605 }606 607 idt_handle->names_len = scan - idt_handle->names;608 609 scan++;610 611 if (scan > end_of_idt) {612 dev_err(endpoint->dev,613 "IDT device name list overflow. Aborting.\n");614 return -ENODEV;615 }616 idt_handle->chandesc = scan;617 618 len = endpoint->idtlen - (3 + ((int) (scan - idt)));619 620 if (len & 0x03) {621 dev_err(endpoint->dev,622 "Corrupt IDT device name list. Aborting.\n");623 return -ENODEV;624 }625 626 idt_handle->entries = len >> 2;627 endpoint->num_channels = count;628 629 return 0;630}631 632static int xilly_obtain_idt(struct xilly_endpoint *endpoint)633{634 struct xilly_channel *channel;635 unsigned char *version;636 long t;637 638 channel = endpoint->channels[1]; /* This should be generated ad-hoc */639 640 channel->wr_sleepy = 1;641 642 iowrite32(1 |643 (3 << 24), /* Opcode 3 for channel 0 = Send IDT */644 endpoint->registers + fpga_buf_ctrl_reg);645 646 t = wait_event_interruptible_timeout(channel->wr_wait,647 (!channel->wr_sleepy),648 XILLY_TIMEOUT);649 650 if (t <= 0) {651 dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");652 653 if (endpoint->fatal_error)654 return -EIO;655 656 return -ENODEV;657 }658 659 dma_sync_single_for_cpu(channel->endpoint->dev,660 channel->wr_buffers[0]->dma_addr,661 channel->wr_buf_size,662 DMA_FROM_DEVICE);663 664 if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {665 dev_err(endpoint->dev,666 "IDT length mismatch (%d != %d). Aborting.\n",667 channel->wr_buffers[0]->end_offset, endpoint->idtlen);668 return -ENODEV;669 }670 671 if (crc32_le(~0, channel->wr_buffers[0]->addr,672 endpoint->idtlen+1) != 0) {673 dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");674 return -ENODEV;675 }676 677 version = channel->wr_buffers[0]->addr;678 679 /* Check version number. Reject anything above 0x82. */680 if (*version > 0x82) {681 dev_err(endpoint->dev,682 "No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n",683 *version);684 return -ENODEV;685 }686 687 return 0;688}689 690static ssize_t xillybus_read(struct file *filp, char __user *userbuf,691 size_t count, loff_t *f_pos)692{693 ssize_t rc;694 unsigned long flags;695 int bytes_done = 0;696 int no_time_left = 0;697 long deadline, left_to_sleep;698 struct xilly_channel *channel = filp->private_data;699 700 int empty, reached_eof, exhausted, ready;701 /* Initializations are there only to silence warnings */702 703 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;704 int waiting_bufidx;705 706 if (channel->endpoint->fatal_error)707 return -EIO;708 709 deadline = jiffies + 1 + XILLY_RX_TIMEOUT;710 711 rc = mutex_lock_interruptible(&channel->wr_mutex);712 if (rc)713 return rc;714 715 while (1) { /* Note that we may drop mutex within this loop */716 int bytes_to_do = count - bytes_done;717 718 spin_lock_irqsave(&channel->wr_spinlock, flags);719 720 empty = channel->wr_empty;721 ready = !empty || channel->wr_ready;722 723 if (!empty) {724 bufidx = channel->wr_host_buf_idx;725 bufpos = channel->wr_host_buf_pos;726 howmany = ((channel->wr_buffers[bufidx]->end_offset727 + 1) << channel->log2_element_size)728 - bufpos;729 730 /* Update wr_host_* to its post-operation state */731 if (howmany > bytes_to_do) {732 bufferdone = 0;733 734 howmany = bytes_to_do;735 channel->wr_host_buf_pos += howmany;736 } else {737 bufferdone = 1;738 739 channel->wr_host_buf_pos = 0;740 741 if (bufidx == channel->wr_fpga_buf_idx) {742 channel->wr_empty = 1;743 channel->wr_sleepy = 1;744 channel->wr_ready = 0;745 }746 747 if (bufidx >= (channel->num_wr_buffers - 1))748 channel->wr_host_buf_idx = 0;749 else750 channel->wr_host_buf_idx++;751 }752 }753 754 /*755 * Marking our situation after the possible changes above,756 * for use after releasing the spinlock.757 *758 * empty = empty before change759 * exhasted = empty after possible change760 */761 762 reached_eof = channel->wr_empty &&763 (channel->wr_host_buf_idx == channel->wr_eof);764 channel->wr_hangup = reached_eof;765 exhausted = channel->wr_empty;766 waiting_bufidx = channel->wr_host_buf_idx;767 768 spin_unlock_irqrestore(&channel->wr_spinlock, flags);769 770 if (!empty) { /* Go on, now without the spinlock */771 772 if (bufpos == 0) /* Position zero means it's virgin */773 dma_sync_single_for_cpu(channel->endpoint->dev,774 channel->wr_buffers[bufidx]->dma_addr,775 channel->wr_buf_size,776 DMA_FROM_DEVICE);777 778 if (copy_to_user(779 userbuf,780 channel->wr_buffers[bufidx]->addr781 + bufpos, howmany))782 rc = -EFAULT;783 784 userbuf += howmany;785 bytes_done += howmany;786 787 if (bufferdone) {788 dma_sync_single_for_device(channel->endpoint->dev,789 channel->wr_buffers[bufidx]->dma_addr,790 channel->wr_buf_size,791 DMA_FROM_DEVICE);792 793 /*794 * Tell FPGA the buffer is done with. It's an795 * atomic operation to the FPGA, so what796 * happens with other channels doesn't matter,797 * and the certain channel is protected with798 * the channel-specific mutex.799 */800 801 iowrite32(1 | (channel->chan_num << 1) |802 (bufidx << 12),803 channel->endpoint->registers +804 fpga_buf_ctrl_reg);805 }806 807 if (rc) {808 mutex_unlock(&channel->wr_mutex);809 return rc;810 }811 }812 813 /* This includes a zero-count return = EOF */814 if ((bytes_done >= count) || reached_eof)815 break;816 817 if (!exhausted)818 continue; /* More in RAM buffer(s)? Just go on. */819 820 if ((bytes_done > 0) &&821 (no_time_left ||822 (channel->wr_synchronous && channel->wr_allow_partial)))823 break;824 825 /*826 * Nonblocking read: The "ready" flag tells us that the FPGA827 * has data to send. In non-blocking mode, if it isn't on,828 * just return. But if there is, we jump directly to the point829 * where we ask for the FPGA to send all it has, and wait830 * until that data arrives. So in a sense, we *do* block in831 * nonblocking mode, but only for a very short time.832 */833 834 if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {835 if (bytes_done > 0)836 break;837 838 if (ready)839 goto desperate;840 841 rc = -EAGAIN;842 break;843 }844 845 if (!no_time_left || (bytes_done > 0)) {846 /*847 * Note that in case of an element-misaligned read848 * request, offsetlimit will include the last element,849 * which will be partially read from.850 */851 int offsetlimit = ((count - bytes_done) - 1) >>852 channel->log2_element_size;853 int buf_elements = channel->wr_buf_size >>854 channel->log2_element_size;855 856 /*857 * In synchronous mode, always send an offset limit.858 * Just don't send a value too big.859 */860 861 if (channel->wr_synchronous) {862 /* Don't request more than one buffer */863 if (channel->wr_allow_partial &&864 (offsetlimit >= buf_elements))865 offsetlimit = buf_elements - 1;866 867 /* Don't request more than all buffers */868 if (!channel->wr_allow_partial &&869 (offsetlimit >=870 (buf_elements * channel->num_wr_buffers)))871 offsetlimit = buf_elements *872 channel->num_wr_buffers - 1;873 }874 875 /*876 * In asynchronous mode, force early flush of a buffer877 * only if that will allow returning a full count. The878 * "offsetlimit < ( ... )" rather than "<=" excludes879 * requesting a full buffer, which would obviously880 * cause a buffer transmission anyhow881 */882 883 if (channel->wr_synchronous ||884 (offsetlimit < (buf_elements - 1))) {885 mutex_lock(&channel->endpoint->register_mutex);886 887 iowrite32(offsetlimit,888 channel->endpoint->registers +889 fpga_buf_offset_reg);890 891 iowrite32(1 | (channel->chan_num << 1) |892 (2 << 24) | /* 2 = offset limit */893 (waiting_bufidx << 12),894 channel->endpoint->registers +895 fpga_buf_ctrl_reg);896 897 mutex_unlock(&channel->endpoint->898 register_mutex);899 }900 }901 902 /*903 * If partial completion is disallowed, there is no point in904 * timeout sleeping. Neither if no_time_left is set and905 * there's no data.906 */907 908 if (!channel->wr_allow_partial ||909 (no_time_left && (bytes_done == 0))) {910 /*911 * This do-loop will run more than once if another912 * thread reasserted wr_sleepy before we got the mutex913 * back, so we try again.914 */915 916 do {917 mutex_unlock(&channel->wr_mutex);918 919 if (wait_event_interruptible(920 channel->wr_wait,921 (!channel->wr_sleepy)))922 goto interrupted;923 924 if (mutex_lock_interruptible(925 &channel->wr_mutex))926 goto interrupted;927 } while (channel->wr_sleepy);928 929 continue;930 931interrupted: /* Mutex is not held if got here */932 if (channel->endpoint->fatal_error)933 return -EIO;934 if (bytes_done)935 return bytes_done;936 if (filp->f_flags & O_NONBLOCK)937 return -EAGAIN; /* Don't admit snoozing */938 return -EINTR;939 }940 941 left_to_sleep = deadline - ((long) jiffies);942 943 /*944 * If our time is out, skip the waiting. We may miss wr_sleepy945 * being deasserted but hey, almost missing the train is like946 * missing it.947 */948 949 if (left_to_sleep > 0) {950 left_to_sleep =951 wait_event_interruptible_timeout(952 channel->wr_wait,953 (!channel->wr_sleepy),954 left_to_sleep);955 956 if (left_to_sleep > 0) /* wr_sleepy deasserted */957 continue;958 959 if (left_to_sleep < 0) { /* Interrupt */960 mutex_unlock(&channel->wr_mutex);961 if (channel->endpoint->fatal_error)962 return -EIO;963 if (bytes_done)964 return bytes_done;965 return -EINTR;966 }967 }968 969desperate:970 no_time_left = 1; /* We're out of sleeping time. Desperate! */971 972 if (bytes_done == 0) {973 /*974 * Reaching here means that we allow partial return,975 * that we've run out of time, and that we have976 * nothing to return.977 * So tell the FPGA to send anything it has or gets.978 */979 980 iowrite32(1 | (channel->chan_num << 1) |981 (3 << 24) | /* Opcode 3, flush it all! */982 (waiting_bufidx << 12),983 channel->endpoint->registers +984 fpga_buf_ctrl_reg);985 }986 987 /*988 * Reaching here means that we *do* have data in the buffer,989 * but the "partial" flag disallows returning less than990 * required. And we don't have as much. So loop again,991 * which is likely to end up blocking indefinitely until992 * enough data has arrived.993 */994 }995 996 mutex_unlock(&channel->wr_mutex);997 998 if (channel->endpoint->fatal_error)999 return -EIO;1000 1001 if (rc)1002 return rc;1003 1004 return bytes_done;1005}1006 1007/*1008 * The timeout argument takes values as follows:1009 * >0 : Flush with timeout1010 * ==0 : Flush, and wait idefinitely for the flush to complete1011 * <0 : Autoflush: Flush only if there's a single buffer occupied1012 */1013 1014static int xillybus_myflush(struct xilly_channel *channel, long timeout)1015{1016 int rc;1017 unsigned long flags;1018 1019 int end_offset_plus1;1020 int bufidx, bufidx_minus1;1021 int i;1022 int empty;1023 int new_rd_host_buf_pos;1024 1025 if (channel->endpoint->fatal_error)1026 return -EIO;1027 rc = mutex_lock_interruptible(&channel->rd_mutex);1028 if (rc)1029 return rc;1030 1031 /*1032 * Don't flush a closed channel. This can happen when the work queued1033 * autoflush thread fires off after the file has closed. This is not1034 * an error, just something to dismiss.1035 */1036 1037 if (!channel->rd_ref_count)1038 goto done;1039 1040 bufidx = channel->rd_host_buf_idx;1041 1042 bufidx_minus1 = (bufidx == 0) ?1043 channel->num_rd_buffers - 1 :1044 bufidx - 1;1045 1046 end_offset_plus1 = channel->rd_host_buf_pos >>1047 channel->log2_element_size;1048 1049 new_rd_host_buf_pos = channel->rd_host_buf_pos -1050 (end_offset_plus1 << channel->log2_element_size);1051 1052 /* Submit the current buffer if it's nonempty */1053 if (end_offset_plus1) {1054 unsigned char *tail = channel->rd_buffers[bufidx]->addr +1055 (end_offset_plus1 << channel->log2_element_size);1056 1057 /* Copy unflushed data, so we can put it in next buffer */1058 for (i = 0; i < new_rd_host_buf_pos; i++)1059 channel->rd_leftovers[i] = *tail++;1060 1061 spin_lock_irqsave(&channel->rd_spinlock, flags);1062 1063 /* Autoflush only if a single buffer is occupied */1064 1065 if ((timeout < 0) &&1066 (channel->rd_full ||1067 (bufidx_minus1 != channel->rd_fpga_buf_idx))) {1068 spin_unlock_irqrestore(&channel->rd_spinlock, flags);1069 /*1070 * A new work item may be queued by the ISR exactly1071 * now, since the execution of a work item allows the1072 * queuing of a new one while it's running.1073 */1074 goto done;1075 }1076 1077 /* The 4th element is never needed for data, so it's a flag */1078 channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);1079 1080 /* Set up rd_full to reflect a certain moment's state */1081 1082 if (bufidx == channel->rd_fpga_buf_idx)1083 channel->rd_full = 1;1084 spin_unlock_irqrestore(&channel->rd_spinlock, flags);1085 1086 if (bufidx >= (channel->num_rd_buffers - 1))1087 channel->rd_host_buf_idx = 0;1088 else1089 channel->rd_host_buf_idx++;1090 1091 dma_sync_single_for_device(channel->endpoint->dev,1092 channel->rd_buffers[bufidx]->dma_addr,1093 channel->rd_buf_size,1094 DMA_TO_DEVICE);1095 1096 mutex_lock(&channel->endpoint->register_mutex);1097 1098 iowrite32(end_offset_plus1 - 1,1099 channel->endpoint->registers + fpga_buf_offset_reg);1100 1101 iowrite32((channel->chan_num << 1) | /* Channel ID */1102 (2 << 24) | /* Opcode 2, submit buffer */1103 (bufidx << 12),1104 channel->endpoint->registers + fpga_buf_ctrl_reg);1105 1106 mutex_unlock(&channel->endpoint->register_mutex);1107 } else if (bufidx == 0) {1108 bufidx = channel->num_rd_buffers - 1;1109 } else {1110 bufidx--;1111 }1112 1113 channel->rd_host_buf_pos = new_rd_host_buf_pos;1114 1115 if (timeout < 0)1116 goto done; /* Autoflush */1117 1118 /*1119 * bufidx is now the last buffer written to (or equal to1120 * rd_fpga_buf_idx if buffer was never written to), and1121 * channel->rd_host_buf_idx the one after it.1122 *1123 * If bufidx == channel->rd_fpga_buf_idx we're either empty or full.1124 */1125 1126 while (1) { /* Loop waiting for draining of buffers */1127 spin_lock_irqsave(&channel->rd_spinlock, flags);1128 1129 if (bufidx != channel->rd_fpga_buf_idx)1130 channel->rd_full = 1; /*1131 * Not really full,1132 * but needs waiting.1133 */1134 1135 empty = !channel->rd_full;1136 1137 spin_unlock_irqrestore(&channel->rd_spinlock, flags);1138 1139 if (empty)1140 break;1141 1142 /*1143 * Indefinite sleep with mutex taken. With data waiting for1144 * flushing user should not be surprised if open() for write1145 * sleeps.1146 */1147 if (timeout == 0)1148 wait_event_interruptible(channel->rd_wait,1149 (!channel->rd_full));1150 1151 else if (wait_event_interruptible_timeout(1152 channel->rd_wait,1153 (!channel->rd_full),1154 timeout) == 0) {1155 dev_warn(channel->endpoint->dev,1156 "Timed out while flushing. Output data may be lost.\n");1157 1158 rc = -ETIMEDOUT;1159 break;1160 }1161 1162 if (channel->rd_full) {1163 rc = -EINTR;1164 break;1165 }1166 }1167 1168done:1169 mutex_unlock(&channel->rd_mutex);1170 1171 if (channel->endpoint->fatal_error)1172 return -EIO;1173 1174 return rc;1175}1176 1177static int xillybus_flush(struct file *filp, fl_owner_t id)1178{1179 if (!(filp->f_mode & FMODE_WRITE))1180 return 0;1181 1182 return xillybus_myflush(filp->private_data, HZ); /* 1 second timeout */1183}1184 1185static void xillybus_autoflush(struct work_struct *work)1186{1187 struct delayed_work *workitem = container_of(1188 work, struct delayed_work, work);1189 struct xilly_channel *channel = container_of(1190 workitem, struct xilly_channel, rd_workitem);1191 int rc;1192 1193 rc = xillybus_myflush(channel, -1);1194 if (rc == -EINTR)1195 dev_warn(channel->endpoint->dev,1196 "Autoflush failed because work queue thread got a signal.\n");1197 else if (rc)1198 dev_err(channel->endpoint->dev,1199 "Autoflush failed under weird circumstances.\n");1200}1201 1202static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,1203 size_t count, loff_t *f_pos)1204{1205 ssize_t rc;1206 unsigned long flags;1207 int bytes_done = 0;1208 struct xilly_channel *channel = filp->private_data;1209 1210 int full, exhausted;1211 /* Initializations are there only to silence warnings */1212 1213 int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;1214 int end_offset_plus1 = 0;1215 1216 if (channel->endpoint->fatal_error)1217 return -EIO;1218 1219 rc = mutex_lock_interruptible(&channel->rd_mutex);1220 if (rc)1221 return rc;1222 1223 while (1) {1224 int bytes_to_do = count - bytes_done;1225 1226 spin_lock_irqsave(&channel->rd_spinlock, flags);1227 1228 full = channel->rd_full;1229 1230 if (!full) {1231 bufidx = channel->rd_host_buf_idx;1232 bufpos = channel->rd_host_buf_pos;1233 howmany = channel->rd_buf_size - bufpos;1234 1235 /*1236 * Update rd_host_* to its state after this operation.1237 * count=0 means committing the buffer immediately,1238 * which is like flushing, but not necessarily block.1239 */1240 1241 if ((howmany > bytes_to_do) &&1242 (count ||1243 ((bufpos >> channel->log2_element_size) == 0))) {1244 bufferdone = 0;1245 1246 howmany = bytes_to_do;1247 channel->rd_host_buf_pos += howmany;1248 } else {1249 bufferdone = 1;1250 1251 if (count) {1252 end_offset_plus1 =1253 channel->rd_buf_size >>1254 channel->log2_element_size;1255 channel->rd_host_buf_pos = 0;1256 } else {1257 unsigned char *tail;1258 int i;1259 1260 howmany = 0;1261 1262 end_offset_plus1 = bufpos >>1263 channel->log2_element_size;1264 1265 channel->rd_host_buf_pos -=1266 end_offset_plus1 <<1267 channel->log2_element_size;1268 1269 tail = channel->1270 rd_buffers[bufidx]->addr +1271 (end_offset_plus1 <<1272 channel->log2_element_size);1273 1274 for (i = 0;1275 i < channel->rd_host_buf_pos;1276 i++)1277 channel->rd_leftovers[i] =1278 *tail++;1279 }1280 1281 if (bufidx == channel->rd_fpga_buf_idx)1282 channel->rd_full = 1;1283 1284 if (bufidx >= (channel->num_rd_buffers - 1))1285 channel->rd_host_buf_idx = 0;1286 else1287 channel->rd_host_buf_idx++;1288 }1289 }1290 1291 /*1292 * Marking our situation after the possible changes above,1293 * for use after releasing the spinlock.1294 *1295 * full = full before change1296 * exhasted = full after possible change1297 */1298 1299 exhausted = channel->rd_full;1300 1301 spin_unlock_irqrestore(&channel->rd_spinlock, flags);1302 1303 if (!full) { /* Go on, now without the spinlock */1304 unsigned char *head =1305 channel->rd_buffers[bufidx]->addr;1306 int i;1307 1308 if ((bufpos == 0) || /* Zero means it's virgin */1309 (channel->rd_leftovers[3] != 0)) {1310 dma_sync_single_for_cpu(channel->endpoint->dev,1311 channel->rd_buffers[bufidx]->dma_addr,1312 channel->rd_buf_size,1313 DMA_TO_DEVICE);1314 1315 /* Virgin, but leftovers are due */1316 for (i = 0; i < bufpos; i++)1317 *head++ = channel->rd_leftovers[i];1318 1319 channel->rd_leftovers[3] = 0; /* Clear flag */1320 }1321 1322 if (copy_from_user(1323 channel->rd_buffers[bufidx]->addr + bufpos,1324 userbuf, howmany))1325 rc = -EFAULT;1326 1327 userbuf += howmany;1328 bytes_done += howmany;1329 1330 if (bufferdone) {1331 dma_sync_single_for_device(channel->endpoint->dev,1332 channel->rd_buffers[bufidx]->dma_addr,1333 channel->rd_buf_size,1334 DMA_TO_DEVICE);1335 1336 mutex_lock(&channel->endpoint->register_mutex);1337 1338 iowrite32(end_offset_plus1 - 1,1339 channel->endpoint->registers +1340 fpga_buf_offset_reg);1341 1342 iowrite32((channel->chan_num << 1) |1343 (2 << 24) | /* 2 = submit buffer */1344 (bufidx << 12),1345 channel->endpoint->registers +1346 fpga_buf_ctrl_reg);1347 1348 mutex_unlock(&channel->endpoint->1349 register_mutex);1350 1351 channel->rd_leftovers[3] =1352 (channel->rd_host_buf_pos != 0);1353 }1354 1355 if (rc) {1356 mutex_unlock(&channel->rd_mutex);1357 1358 if (channel->endpoint->fatal_error)1359 return -EIO;1360 1361 if (!channel->rd_synchronous)1362 queue_delayed_work(1363 xillybus_wq,1364 &channel->rd_workitem,1365 XILLY_RX_TIMEOUT);1366 1367 return rc;1368 }1369 }1370 1371 if (bytes_done >= count)1372 break;1373 1374 if (!exhausted)1375 continue; /* If there's more space, just go on */1376 1377 if ((bytes_done > 0) && channel->rd_allow_partial)1378 break;1379 1380 /*1381 * Indefinite sleep with mutex taken. With data waiting for1382 * flushing, user should not be surprised if open() for write1383 * sleeps.1384 */1385 1386 if (filp->f_flags & O_NONBLOCK) {1387 rc = -EAGAIN;1388 break;1389 }1390 1391 if (wait_event_interruptible(channel->rd_wait,1392 (!channel->rd_full))) {1393 mutex_unlock(&channel->rd_mutex);1394 1395 if (channel->endpoint->fatal_error)1396 return -EIO;1397 1398 if (bytes_done)1399 return bytes_done;1400 return -EINTR;1401 }1402 }1403 1404 mutex_unlock(&channel->rd_mutex);1405 1406 if (!channel->rd_synchronous)1407 queue_delayed_work(xillybus_wq,1408 &channel->rd_workitem,1409 XILLY_RX_TIMEOUT);1410 1411 if (channel->endpoint->fatal_error)1412 return -EIO;1413 1414 if (rc)1415 return rc;1416 1417 if ((channel->rd_synchronous) && (bytes_done > 0)) {1418 rc = xillybus_myflush(filp->private_data, 0); /* No timeout */1419 1420 if (rc && (rc != -EINTR))1421 return rc;1422 }1423 1424 return bytes_done;1425}1426 1427static int xillybus_open(struct inode *inode, struct file *filp)1428{1429 int rc;1430 unsigned long flags;1431 struct xilly_endpoint *endpoint;1432 struct xilly_channel *channel;1433 int index;1434 1435 rc = xillybus_find_inode(inode, (void **)&endpoint, &index);1436 if (rc)1437 return rc;1438 1439 if (endpoint->fatal_error)1440 return -EIO;1441 1442 channel = endpoint->channels[1 + index];1443 filp->private_data = channel;1444 1445 /*1446 * It gets complicated because:1447 * 1. We don't want to take a mutex we don't have to1448 * 2. We don't want to open one direction if the other will fail.1449 */1450 1451 if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))1452 return -ENODEV;1453 1454 if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))1455 return -ENODEV;1456 1457 if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&1458 (channel->wr_synchronous || !channel->wr_allow_partial ||1459 !channel->wr_supports_nonempty)) {1460 dev_err(endpoint->dev,1461 "open() failed: O_NONBLOCK not allowed for read on this device\n");1462 return -ENODEV;1463 }1464 1465 if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&1466 (channel->rd_synchronous || !channel->rd_allow_partial)) {1467 dev_err(endpoint->dev,1468 "open() failed: O_NONBLOCK not allowed for write on this device\n");1469 return -ENODEV;1470 }1471 1472 /*1473 * Note: open() may block on getting mutexes despite O_NONBLOCK.1474 * This shouldn't occur normally, since multiple open of the same1475 * file descriptor is almost always prohibited anyhow1476 * (*_exclusive_open is normally set in real-life systems).1477 */1478 1479 if (filp->f_mode & FMODE_READ) {1480 rc = mutex_lock_interruptible(&channel->wr_mutex);1481 if (rc)1482 return rc;1483 }1484 1485 if (filp->f_mode & FMODE_WRITE) {1486 rc = mutex_lock_interruptible(&channel->rd_mutex);1487 if (rc)1488 goto unlock_wr;1489 }1490 1491 if ((filp->f_mode & FMODE_READ) &&1492 (channel->wr_ref_count != 0) &&1493 (channel->wr_exclusive_open)) {1494 rc = -EBUSY;1495 goto unlock;1496 }1497 1498 if ((filp->f_mode & FMODE_WRITE) &&1499 (channel->rd_ref_count != 0) &&1500 (channel->rd_exclusive_open)) {1501 rc = -EBUSY;1502 goto unlock;1503 }1504 1505 if (filp->f_mode & FMODE_READ) {1506 if (channel->wr_ref_count == 0) { /* First open of file */1507 /* Move the host to first buffer */1508 spin_lock_irqsave(&channel->wr_spinlock, flags);1509 channel->wr_host_buf_idx = 0;1510 channel->wr_host_buf_pos = 0;1511 channel->wr_fpga_buf_idx = -1;1512 channel->wr_empty = 1;1513 channel->wr_ready = 0;1514 channel->wr_sleepy = 1;1515 channel->wr_eof = -1;1516 channel->wr_hangup = 0;1517 1518 spin_unlock_irqrestore(&channel->wr_spinlock, flags);1519 1520 iowrite32(1 | (channel->chan_num << 1) |1521 (4 << 24) | /* Opcode 4, open channel */1522 ((channel->wr_synchronous & 1) << 23),1523 channel->endpoint->registers +1524 fpga_buf_ctrl_reg);1525 }1526 1527 channel->wr_ref_count++;1528 }1529 1530 if (filp->f_mode & FMODE_WRITE) {1531 if (channel->rd_ref_count == 0) { /* First open of file */1532 /* Move the host to first buffer */1533 spin_lock_irqsave(&channel->rd_spinlock, flags);1534 channel->rd_host_buf_idx = 0;1535 channel->rd_host_buf_pos = 0;1536 channel->rd_leftovers[3] = 0; /* No leftovers. */1537 channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;1538 channel->rd_full = 0;1539 1540 spin_unlock_irqrestore(&channel->rd_spinlock, flags);1541 1542 iowrite32((channel->chan_num << 1) |1543 (4 << 24), /* Opcode 4, open channel */1544 channel->endpoint->registers +1545 fpga_buf_ctrl_reg);1546 }1547 1548 channel->rd_ref_count++;1549 }1550 1551unlock:1552 if (filp->f_mode & FMODE_WRITE)1553 mutex_unlock(&channel->rd_mutex);1554unlock_wr:1555 if (filp->f_mode & FMODE_READ)1556 mutex_unlock(&channel->wr_mutex);1557 1558 if (!rc && (!channel->seekable))1559 return nonseekable_open(inode, filp);1560 1561 return rc;1562}1563 1564static int xillybus_release(struct inode *inode, struct file *filp)1565{1566 unsigned long flags;1567 struct xilly_channel *channel = filp->private_data;1568 1569 int buf_idx;1570 int eof;1571 1572 if (channel->endpoint->fatal_error)1573 return -EIO;1574 1575 if (filp->f_mode & FMODE_WRITE) {1576 mutex_lock(&channel->rd_mutex);1577 1578 channel->rd_ref_count--;1579 1580 if (channel->rd_ref_count == 0) {1581 /*1582 * We rely on the kernel calling flush()1583 * before we get here.1584 */1585 1586 iowrite32((channel->chan_num << 1) | /* Channel ID */1587 (5 << 24), /* Opcode 5, close channel */1588 channel->endpoint->registers +1589 fpga_buf_ctrl_reg);1590 }1591 mutex_unlock(&channel->rd_mutex);1592 }1593 1594 if (filp->f_mode & FMODE_READ) {1595 mutex_lock(&channel->wr_mutex);1596 1597 channel->wr_ref_count--;1598 1599 if (channel->wr_ref_count == 0) {1600 iowrite32(1 | (channel->chan_num << 1) |1601 (5 << 24), /* Opcode 5, close channel */1602 channel->endpoint->registers +1603 fpga_buf_ctrl_reg);1604 1605 /*1606 * This is crazily cautious: We make sure that not1607 * only that we got an EOF (be it because we closed1608 * the channel or because of a user's EOF), but verify1609 * that it's one beyond the last buffer arrived, so1610 * we have no leftover buffers pending before wrapping1611 * up (which can only happen in asynchronous channels,1612 * BTW)1613 */1614 1615 while (1) {1616 spin_lock_irqsave(&channel->wr_spinlock,1617 flags);1618 buf_idx = channel->wr_fpga_buf_idx;1619 eof = channel->wr_eof;1620 channel->wr_sleepy = 1;1621 spin_unlock_irqrestore(&channel->wr_spinlock,1622 flags);1623 1624 /*1625 * Check if eof points at the buffer after1626 * the last one the FPGA submitted. Note that1627 * no EOF is marked by negative eof.1628 */1629 1630 buf_idx++;1631 if (buf_idx == channel->num_wr_buffers)1632 buf_idx = 0;1633 1634 if (buf_idx == eof)1635 break;1636 1637 /*1638 * Steal extra 100 ms if awaken by interrupt.1639 * This is a simple workaround for an1640 * interrupt pending when entering, which would1641 * otherwise result in declaring the hardware1642 * non-responsive.1643 */1644 1645 if (wait_event_interruptible(1646 channel->wr_wait,1647 (!channel->wr_sleepy)))1648 msleep(100);1649 1650 if (channel->wr_sleepy) {1651 mutex_unlock(&channel->wr_mutex);1652 dev_warn(channel->endpoint->dev,1653 "Hardware failed to respond to close command, therefore left in messy state.\n");1654 return -EINTR;1655 }1656 }1657 }1658 1659 mutex_unlock(&channel->wr_mutex);1660 }1661 1662 return 0;1663}1664 1665static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)1666{1667 struct xilly_channel *channel = filp->private_data;1668 loff_t pos = filp->f_pos;1669 int rc = 0;1670 1671 /*1672 * Take both mutexes not allowing interrupts, since it seems like1673 * common applications don't expect an -EINTR here. Besides, multiple1674 * access to a single file descriptor on seekable devices is a mess1675 * anyhow.1676 */1677 1678 if (channel->endpoint->fatal_error)1679 return -EIO;1680 1681 mutex_lock(&channel->wr_mutex);1682 mutex_lock(&channel->rd_mutex);1683 1684 switch (whence) {1685 case SEEK_SET:1686 pos = offset;1687 break;1688 case SEEK_CUR:1689 pos += offset;1690 break;1691 case SEEK_END:1692 pos = offset; /* Going to the end => to the beginning */1693 break;1694 default:1695 rc = -EINVAL;1696 goto end;1697 }1698 1699 /* In any case, we must finish on an element boundary */1700 if (pos & ((1 << channel->log2_element_size) - 1)) {1701 rc = -EINVAL;1702 goto end;1703 }1704 1705 mutex_lock(&channel->endpoint->register_mutex);1706 1707 iowrite32(pos >> channel->log2_element_size,1708 channel->endpoint->registers + fpga_buf_offset_reg);1709 1710 iowrite32((channel->chan_num << 1) |1711 (6 << 24), /* Opcode 6, set address */1712 channel->endpoint->registers + fpga_buf_ctrl_reg);1713 1714 mutex_unlock(&channel->endpoint->register_mutex);1715 1716end:1717 mutex_unlock(&channel->rd_mutex);1718 mutex_unlock(&channel->wr_mutex);1719 1720 if (rc) /* Return error after releasing mutexes */1721 return rc;1722 1723 filp->f_pos = pos;1724 1725 /*1726 * Since seekable devices are allowed only when the channel is1727 * synchronous, we assume that there is no data pending in either1728 * direction (which holds true as long as no concurrent access on the1729 * file descriptor takes place).1730 * The only thing we may need to throw away is leftovers from partial1731 * write() flush.1732 */1733 1734 channel->rd_leftovers[3] = 0;1735 1736 return pos;1737}1738 1739static __poll_t xillybus_poll(struct file *filp, poll_table *wait)1740{1741 struct xilly_channel *channel = filp->private_data;1742 __poll_t mask = 0;1743 unsigned long flags;1744 1745 poll_wait(filp, &channel->endpoint->ep_wait, wait);1746 1747 /*1748 * poll() won't play ball regarding read() channels which1749 * aren't asynchronous and support the nonempty message. Allowing1750 * that will create situations where data has been delivered at1751 * the FPGA, and users expecting select() to wake up, which it may1752 * not.1753 */1754 1755 if (!channel->wr_synchronous && channel->wr_supports_nonempty) {1756 poll_wait(filp, &channel->wr_wait, wait);1757 poll_wait(filp, &channel->wr_ready_wait, wait);1758 1759 spin_lock_irqsave(&channel->wr_spinlock, flags);1760 if (!channel->wr_empty || channel->wr_ready)1761 mask |= EPOLLIN | EPOLLRDNORM;1762 1763 if (channel->wr_hangup)1764 /*1765 * Not EPOLLHUP, because its behavior is in the1766 * mist, and EPOLLIN does what we want: Wake up1767 * the read file descriptor so it sees EOF.1768 */1769 mask |= EPOLLIN | EPOLLRDNORM;1770 spin_unlock_irqrestore(&channel->wr_spinlock, flags);1771 }1772 1773 /*1774 * If partial data write is disallowed on a write() channel,1775 * it's pointless to ever signal OK to write, because is could1776 * block despite some space being available.1777 */1778 1779 if (channel->rd_allow_partial) {1780 poll_wait(filp, &channel->rd_wait, wait);1781 1782 spin_lock_irqsave(&channel->rd_spinlock, flags);1783 if (!channel->rd_full)1784 mask |= EPOLLOUT | EPOLLWRNORM;1785 spin_unlock_irqrestore(&channel->rd_spinlock, flags);1786 }1787 1788 if (channel->endpoint->fatal_error)1789 mask |= EPOLLERR;1790 1791 return mask;1792}1793 1794static const struct file_operations xillybus_fops = {1795 .owner = THIS_MODULE,1796 .read = xillybus_read,1797 .write = xillybus_write,1798 .open = xillybus_open,1799 .flush = xillybus_flush,1800 .release = xillybus_release,1801 .llseek = xillybus_llseek,1802 .poll = xillybus_poll,1803};1804 1805struct xilly_endpoint *xillybus_init_endpoint(struct device *dev)1806{1807 struct xilly_endpoint *endpoint;1808 1809 endpoint = devm_kzalloc(dev, sizeof(*endpoint), GFP_KERNEL);1810 if (!endpoint)1811 return NULL;1812 1813 endpoint->dev = dev;1814 endpoint->msg_counter = 0x0b;1815 endpoint->failed_messages = 0;1816 endpoint->fatal_error = 0;1817 1818 init_waitqueue_head(&endpoint->ep_wait);1819 mutex_init(&endpoint->register_mutex);1820 1821 return endpoint;1822}1823EXPORT_SYMBOL(xillybus_init_endpoint);1824 1825static int xilly_quiesce(struct xilly_endpoint *endpoint)1826{1827 long t;1828 1829 endpoint->idtlen = -1;1830 1831 iowrite32((u32) (endpoint->dma_using_dac & 0x0001),1832 endpoint->registers + fpga_dma_control_reg);1833 1834 t = wait_event_interruptible_timeout(endpoint->ep_wait,1835 (endpoint->idtlen >= 0),1836 XILLY_TIMEOUT);1837 if (t <= 0) {1838 dev_err(endpoint->dev,1839 "Failed to quiesce the device on exit.\n");1840 return -ENODEV;1841 }1842 return 0;1843}1844 1845int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)1846{1847 int rc;1848 long t;1849 1850 void *bootstrap_resources;1851 int idtbuffersize = (1 << PAGE_SHIFT);1852 struct device *dev = endpoint->dev;1853 1854 /*1855 * The bogus IDT is used during bootstrap for allocating the initial1856 * message buffer, and then the message buffer and space for the IDT1857 * itself. The initial message buffer is of a single page's size, but1858 * it's soon replaced with a more modest one (and memory is freed).1859 */1860 1861 unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,1862 3, 192, PAGE_SHIFT, 0 };1863 struct xilly_idt_handle idt_handle;1864 1865 /*1866 * Writing the value 0x00000001 to Endianness register signals which1867 * endianness this processor is using, so the FPGA can swap words as1868 * necessary.1869 */1870 1871 iowrite32(1, endpoint->registers + fpga_endian_reg);1872 1873 /* Bootstrap phase I: Allocate temporary message buffer */1874 1875 bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);1876 if (!bootstrap_resources)1877 return -ENOMEM;1878 1879 endpoint->num_channels = 0;1880 1881 rc = xilly_setupchannels(endpoint, bogus_idt, 1);1882 if (rc)1883 return rc;1884 1885 /* Clear the message subsystem (and counter in particular) */1886 iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);1887 1888 endpoint->idtlen = -1;1889 1890 /*1891 * Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT1892 * buffer size.1893 */1894 iowrite32((u32) (endpoint->dma_using_dac & 0x0001),1895 endpoint->registers + fpga_dma_control_reg);1896 1897 t = wait_event_interruptible_timeout(endpoint->ep_wait,1898 (endpoint->idtlen >= 0),1899 XILLY_TIMEOUT);1900 if (t <= 0) {1901 dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");1902 return -ENODEV;1903 }1904 1905 /* Enable DMA */1906 iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),1907 endpoint->registers + fpga_dma_control_reg);1908 1909 /* Bootstrap phase II: Allocate buffer for IDT and obtain it */1910 while (endpoint->idtlen >= idtbuffersize) {1911 idtbuffersize *= 2;1912 bogus_idt[6]++;1913 }1914 1915 endpoint->num_channels = 1;1916 1917 rc = xilly_setupchannels(endpoint, bogus_idt, 2);1918 if (rc)1919 goto failed_idt;1920 1921 rc = xilly_obtain_idt(endpoint);1922 if (rc)1923 goto failed_idt;1924 1925 rc = xilly_scan_idt(endpoint, &idt_handle);1926 if (rc)1927 goto failed_idt;1928 1929 devres_close_group(dev, bootstrap_resources);1930 1931 /* Bootstrap phase III: Allocate buffers according to IDT */1932 1933 rc = xilly_setupchannels(endpoint,1934 idt_handle.chandesc,1935 idt_handle.entries);1936 if (rc)1937 goto failed_idt;1938 1939 rc = xillybus_init_chrdev(dev, &xillybus_fops,1940 endpoint->owner, endpoint,1941 idt_handle.names,1942 idt_handle.names_len,1943 endpoint->num_channels,1944 xillyname, false);1945 1946 if (rc)1947 goto failed_idt;1948 1949 devres_release_group(dev, bootstrap_resources);1950 1951 return 0;1952 1953failed_idt:1954 xilly_quiesce(endpoint);1955 flush_workqueue(xillybus_wq);1956 1957 return rc;1958}1959EXPORT_SYMBOL(xillybus_endpoint_discovery);1960 1961void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)1962{1963 xillybus_cleanup_chrdev(endpoint, endpoint->dev);1964 1965 xilly_quiesce(endpoint);1966 1967 /*1968 * Flushing is done upon endpoint release to prevent access to memory1969 * just about to be released. This makes the quiesce complete.1970 */1971 flush_workqueue(xillybus_wq);1972}1973EXPORT_SYMBOL(xillybus_endpoint_remove);1974 1975static int __init xillybus_init(void)1976{1977 xillybus_wq = alloc_workqueue(xillyname, 0, 0);1978 if (!xillybus_wq)1979 return -ENOMEM;1980 1981 return 0;1982}1983 1984static void __exit xillybus_exit(void)1985{1986 /* flush_workqueue() was called for each endpoint released */1987 destroy_workqueue(xillybus_wq);1988}1989 1990module_init(xillybus_init);1991module_exit(xillybus_exit);1992