1922 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * Char device for device raw access4 *5 * Copyright (C) 2005-2007 Kristian Hoegsberg <krh@bitplanet.net>6 */7 8#include <linux/bug.h>9#include <linux/compat.h>10#include <linux/delay.h>11#include <linux/device.h>12#include <linux/dma-mapping.h>13#include <linux/err.h>14#include <linux/errno.h>15#include <linux/firewire.h>16#include <linux/firewire-cdev.h>17#include <linux/irqflags.h>18#include <linux/jiffies.h>19#include <linux/kernel.h>20#include <linux/kref.h>21#include <linux/mm.h>22#include <linux/module.h>23#include <linux/mutex.h>24#include <linux/poll.h>25#include <linux/sched.h> /* required for linux/wait.h */26#include <linux/slab.h>27#include <linux/spinlock.h>28#include <linux/string.h>29#include <linux/time.h>30#include <linux/uaccess.h>31#include <linux/vmalloc.h>32#include <linux/wait.h>33#include <linux/workqueue.h>34 35 36#include "core.h"37#include <trace/events/firewire.h>38 39#include "packet-header-definitions.h"40 41/*42 * ABI version history is documented in linux/firewire-cdev.h.43 */44#define FW_CDEV_KERNEL_VERSION 545#define FW_CDEV_VERSION_EVENT_REQUEST2 446#define FW_CDEV_VERSION_ALLOCATE_REGION_END 447#define FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW 548#define FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP 649 50struct client {51 u32 version;52 struct fw_device *device;53 54 spinlock_t lock;55 bool in_shutdown;56 struct xarray resource_xa;57 struct list_head event_list;58 wait_queue_head_t wait;59 wait_queue_head_t tx_flush_wait;60 u64 bus_reset_closure;61 62 struct fw_iso_context *iso_context;63 u64 iso_closure;64 struct fw_iso_buffer buffer;65 unsigned long vm_start;66 bool buffer_is_mapped;67 68 struct list_head phy_receiver_link;69 u64 phy_receiver_closure;70 71 struct list_head link;72 struct kref kref;73};74 75static inline void client_get(struct client *client)76{77 kref_get(&client->kref);78}79 80static void client_release(struct kref *kref)81{82 struct client *client = container_of(kref, struct client, kref);83 84 fw_device_put(client->device);85 kfree(client);86}87 88static void client_put(struct client *client)89{90 kref_put(&client->kref, client_release);91}92 93struct client_resource;94typedef void (*client_resource_release_fn_t)(struct client *,95 struct client_resource *);96struct client_resource {97 client_resource_release_fn_t release;98 int handle;99};100 101struct address_handler_resource {102 struct client_resource resource;103 struct fw_address_handler handler;104 __u64 closure;105 struct client *client;106};107 108struct outbound_transaction_resource {109 struct client_resource resource;110 struct fw_transaction transaction;111};112 113struct inbound_transaction_resource {114 struct client_resource resource;115 struct fw_card *card;116 struct fw_request *request;117 bool is_fcp;118 void *data;119 size_t length;120};121 122struct descriptor_resource {123 struct client_resource resource;124 struct fw_descriptor descriptor;125 u32 data[];126};127 128struct iso_resource {129 struct client_resource resource;130 struct client *client;131 /* Schedule work and access todo only with client->lock held. */132 struct delayed_work work;133 enum {ISO_RES_ALLOC, ISO_RES_REALLOC, ISO_RES_DEALLOC,134 ISO_RES_ALLOC_ONCE, ISO_RES_DEALLOC_ONCE,} todo;135 int generation;136 u64 channels;137 s32 bandwidth;138 struct iso_resource_event *e_alloc, *e_dealloc;139};140 141static struct address_handler_resource *to_address_handler_resource(struct client_resource *resource)142{143 return container_of(resource, struct address_handler_resource, resource);144}145 146static struct inbound_transaction_resource *to_inbound_transaction_resource(struct client_resource *resource)147{148 return container_of(resource, struct inbound_transaction_resource, resource);149}150 151static struct descriptor_resource *to_descriptor_resource(struct client_resource *resource)152{153 return container_of(resource, struct descriptor_resource, resource);154}155 156static struct iso_resource *to_iso_resource(struct client_resource *resource)157{158 return container_of(resource, struct iso_resource, resource);159}160 161static void release_iso_resource(struct client *, struct client_resource *);162 163static int is_iso_resource(const struct client_resource *resource)164{165 return resource->release == release_iso_resource;166}167 168static void release_transaction(struct client *client,169 struct client_resource *resource);170 171static int is_outbound_transaction_resource(const struct client_resource *resource)172{173 return resource->release == release_transaction;174}175 176static void schedule_iso_resource(struct iso_resource *r, unsigned long delay)177{178 client_get(r->client);179 if (!queue_delayed_work(fw_workqueue, &r->work, delay))180 client_put(r->client);181}182 183/*184 * dequeue_event() just kfree()'s the event, so the event has to be185 * the first field in a struct XYZ_event.186 */187struct event {188 struct { void *data; size_t size; } v[2];189 struct list_head link;190};191 192struct bus_reset_event {193 struct event event;194 struct fw_cdev_event_bus_reset reset;195};196 197struct outbound_transaction_event {198 struct event event;199 struct client *client;200 struct outbound_transaction_resource r;201 union {202 struct fw_cdev_event_response without_tstamp;203 struct fw_cdev_event_response2 with_tstamp;204 } rsp;205};206 207struct inbound_transaction_event {208 struct event event;209 union {210 struct fw_cdev_event_request request;211 struct fw_cdev_event_request2 request2;212 struct fw_cdev_event_request3 with_tstamp;213 } req;214};215 216struct iso_interrupt_event {217 struct event event;218 struct fw_cdev_event_iso_interrupt interrupt;219};220 221struct iso_interrupt_mc_event {222 struct event event;223 struct fw_cdev_event_iso_interrupt_mc interrupt;224};225 226struct iso_resource_event {227 struct event event;228 struct fw_cdev_event_iso_resource iso_resource;229};230 231struct outbound_phy_packet_event {232 struct event event;233 struct client *client;234 struct fw_packet p;235 union {236 struct fw_cdev_event_phy_packet without_tstamp;237 struct fw_cdev_event_phy_packet2 with_tstamp;238 } phy_packet;239};240 241struct inbound_phy_packet_event {242 struct event event;243 union {244 struct fw_cdev_event_phy_packet without_tstamp;245 struct fw_cdev_event_phy_packet2 with_tstamp;246 } phy_packet;247};248 249#ifdef CONFIG_COMPAT250static void __user *u64_to_uptr(u64 value)251{252 if (in_compat_syscall())253 return compat_ptr(value);254 else255 return (void __user *)(unsigned long)value;256}257 258static u64 uptr_to_u64(void __user *ptr)259{260 if (in_compat_syscall())261 return ptr_to_compat(ptr);262 else263 return (u64)(unsigned long)ptr;264}265#else266static inline void __user *u64_to_uptr(u64 value)267{268 return (void __user *)(unsigned long)value;269}270 271static inline u64 uptr_to_u64(void __user *ptr)272{273 return (u64)(unsigned long)ptr;274}275#endif /* CONFIG_COMPAT */276 277static int fw_device_op_open(struct inode *inode, struct file *file)278{279 struct fw_device *device;280 struct client *client;281 282 device = fw_device_get_by_devt(inode->i_rdev);283 if (device == NULL)284 return -ENODEV;285 286 if (fw_device_is_shutdown(device)) {287 fw_device_put(device);288 return -ENODEV;289 }290 291 client = kzalloc(sizeof(*client), GFP_KERNEL);292 if (client == NULL) {293 fw_device_put(device);294 return -ENOMEM;295 }296 297 client->device = device;298 spin_lock_init(&client->lock);299 xa_init_flags(&client->resource_xa, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH);300 INIT_LIST_HEAD(&client->event_list);301 init_waitqueue_head(&client->wait);302 init_waitqueue_head(&client->tx_flush_wait);303 INIT_LIST_HEAD(&client->phy_receiver_link);304 INIT_LIST_HEAD(&client->link);305 kref_init(&client->kref);306 307 file->private_data = client;308 309 return nonseekable_open(inode, file);310}311 312static void queue_event(struct client *client, struct event *event,313 void *data0, size_t size0, void *data1, size_t size1)314{315 event->v[0].data = data0;316 event->v[0].size = size0;317 event->v[1].data = data1;318 event->v[1].size = size1;319 320 scoped_guard(spinlock_irqsave, &client->lock) {321 if (client->in_shutdown)322 kfree(event);323 else324 list_add_tail(&event->link, &client->event_list);325 }326 327 wake_up_interruptible(&client->wait);328}329 330static int dequeue_event(struct client *client,331 char __user *buffer, size_t count)332{333 struct event *event;334 size_t size, total;335 int i, ret;336 337 ret = wait_event_interruptible(client->wait,338 !list_empty(&client->event_list) ||339 fw_device_is_shutdown(client->device));340 if (ret < 0)341 return ret;342 343 if (list_empty(&client->event_list) &&344 fw_device_is_shutdown(client->device))345 return -ENODEV;346 347 scoped_guard(spinlock_irq, &client->lock) {348 event = list_first_entry(&client->event_list, struct event, link);349 list_del(&event->link);350 }351 352 total = 0;353 for (i = 0; i < ARRAY_SIZE(event->v) && total < count; i++) {354 size = min(event->v[i].size, count - total);355 if (copy_to_user(buffer + total, event->v[i].data, size)) {356 ret = -EFAULT;357 goto out;358 }359 total += size;360 }361 ret = total;362 363 out:364 kfree(event);365 366 return ret;367}368 369static ssize_t fw_device_op_read(struct file *file, char __user *buffer,370 size_t count, loff_t *offset)371{372 struct client *client = file->private_data;373 374 return dequeue_event(client, buffer, count);375}376 377static void fill_bus_reset_event(struct fw_cdev_event_bus_reset *event,378 struct client *client)379{380 struct fw_card *card = client->device->card;381 382 guard(spinlock_irq)(&card->lock);383 384 event->closure = client->bus_reset_closure;385 event->type = FW_CDEV_EVENT_BUS_RESET;386 event->generation = client->device->generation;387 event->node_id = client->device->node_id;388 event->local_node_id = card->local_node->node_id;389 event->bm_node_id = card->bm_node_id;390 event->irm_node_id = card->irm_node->node_id;391 event->root_node_id = card->root_node->node_id;392}393 394static void for_each_client(struct fw_device *device,395 void (*callback)(struct client *client))396{397 struct client *c;398 399 guard(mutex)(&device->client_list_mutex);400 401 list_for_each_entry(c, &device->client_list, link)402 callback(c);403}404 405static void queue_bus_reset_event(struct client *client)406{407 struct bus_reset_event *e;408 struct client_resource *resource;409 unsigned long index;410 411 e = kzalloc(sizeof(*e), GFP_KERNEL);412 if (e == NULL)413 return;414 415 fill_bus_reset_event(&e->reset, client);416 417 queue_event(client, &e->event,418 &e->reset, sizeof(e->reset), NULL, 0);419 420 guard(spinlock_irq)(&client->lock);421 422 xa_for_each(&client->resource_xa, index, resource) {423 if (is_iso_resource(resource))424 schedule_iso_resource(to_iso_resource(resource), 0);425 }426}427 428void fw_device_cdev_update(struct fw_device *device)429{430 for_each_client(device, queue_bus_reset_event);431}432 433static void wake_up_client(struct client *client)434{435 wake_up_interruptible(&client->wait);436}437 438void fw_device_cdev_remove(struct fw_device *device)439{440 for_each_client(device, wake_up_client);441}442 443union ioctl_arg {444 struct fw_cdev_get_info get_info;445 struct fw_cdev_send_request send_request;446 struct fw_cdev_allocate allocate;447 struct fw_cdev_deallocate deallocate;448 struct fw_cdev_send_response send_response;449 struct fw_cdev_initiate_bus_reset initiate_bus_reset;450 struct fw_cdev_add_descriptor add_descriptor;451 struct fw_cdev_remove_descriptor remove_descriptor;452 struct fw_cdev_create_iso_context create_iso_context;453 struct fw_cdev_queue_iso queue_iso;454 struct fw_cdev_start_iso start_iso;455 struct fw_cdev_stop_iso stop_iso;456 struct fw_cdev_get_cycle_timer get_cycle_timer;457 struct fw_cdev_allocate_iso_resource allocate_iso_resource;458 struct fw_cdev_send_stream_packet send_stream_packet;459 struct fw_cdev_get_cycle_timer2 get_cycle_timer2;460 struct fw_cdev_send_phy_packet send_phy_packet;461 struct fw_cdev_receive_phy_packets receive_phy_packets;462 struct fw_cdev_set_iso_channels set_iso_channels;463 struct fw_cdev_flush_iso flush_iso;464};465 466static int ioctl_get_info(struct client *client, union ioctl_arg *arg)467{468 struct fw_cdev_get_info *a = &arg->get_info;469 struct fw_cdev_event_bus_reset bus_reset;470 unsigned long ret = 0;471 472 client->version = a->version;473 a->version = FW_CDEV_KERNEL_VERSION;474 a->card = client->device->card->index;475 476 scoped_guard(rwsem_read, &fw_device_rwsem) {477 if (a->rom != 0) {478 size_t want = a->rom_length;479 size_t have = client->device->config_rom_length * 4;480 481 ret = copy_to_user(u64_to_uptr(a->rom), client->device->config_rom,482 min(want, have));483 if (ret != 0)484 return -EFAULT;485 }486 a->rom_length = client->device->config_rom_length * 4;487 }488 489 guard(mutex)(&client->device->client_list_mutex);490 491 client->bus_reset_closure = a->bus_reset_closure;492 if (a->bus_reset != 0) {493 fill_bus_reset_event(&bus_reset, client);494 /* unaligned size of bus_reset is 36 bytes */495 ret = copy_to_user(u64_to_uptr(a->bus_reset), &bus_reset, 36);496 }497 if (ret == 0 && list_empty(&client->link))498 list_add_tail(&client->link, &client->device->client_list);499 500 return ret ? -EFAULT : 0;501}502 503static int add_client_resource(struct client *client, struct client_resource *resource,504 gfp_t gfp_mask)505{506 int ret;507 508 scoped_guard(spinlock_irqsave, &client->lock) {509 u32 index;510 511 if (client->in_shutdown) {512 ret = -ECANCELED;513 } else {514 if (gfpflags_allow_blocking(gfp_mask)) {515 ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b,516 GFP_NOWAIT);517 } else {518 ret = xa_alloc_bh(&client->resource_xa, &index, resource,519 xa_limit_32b, GFP_NOWAIT);520 }521 }522 if (ret >= 0) {523 resource->handle = index;524 client_get(client);525 if (is_iso_resource(resource))526 schedule_iso_resource(to_iso_resource(resource), 0);527 }528 }529 530 return ret < 0 ? ret : 0;531}532 533static int release_client_resource(struct client *client, u32 handle,534 client_resource_release_fn_t release,535 struct client_resource **return_resource)536{537 unsigned long index = handle;538 struct client_resource *resource;539 540 scoped_guard(spinlock_irq, &client->lock) {541 if (client->in_shutdown)542 return -EINVAL;543 544 resource = xa_load(&client->resource_xa, index);545 if (!resource || resource->release != release)546 return -EINVAL;547 548 xa_erase(&client->resource_xa, handle);549 }550 551 if (return_resource)552 *return_resource = resource;553 else554 resource->release(client, resource);555 556 client_put(client);557 558 return 0;559}560 561static void release_transaction(struct client *client,562 struct client_resource *resource)563{564}565 566static void complete_transaction(struct fw_card *card, int rcode, u32 request_tstamp,567 u32 response_tstamp, void *payload, size_t length, void *data)568{569 struct outbound_transaction_event *e = data;570 struct client *client = e->client;571 unsigned long index = e->r.resource.handle;572 573 scoped_guard(spinlock_irqsave, &client->lock) {574 xa_erase(&client->resource_xa, index);575 if (client->in_shutdown)576 wake_up(&client->tx_flush_wait);577 }578 579 switch (e->rsp.without_tstamp.type) {580 case FW_CDEV_EVENT_RESPONSE:581 {582 struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;583 584 if (length < rsp->length)585 rsp->length = length;586 if (rcode == RCODE_COMPLETE)587 memcpy(rsp->data, payload, rsp->length);588 589 rsp->rcode = rcode;590 591 // In the case that sizeof(*rsp) doesn't align with the position of the592 // data, and the read is short, preserve an extra copy of the data593 // to stay compatible with a pre-2.6.27 bug. Since the bug is harmless594 // for short reads and some apps depended on it, this is both safe595 // and prudent for compatibility.596 if (rsp->length <= sizeof(*rsp) - offsetof(typeof(*rsp), data))597 queue_event(client, &e->event, rsp, sizeof(*rsp), rsp->data, rsp->length);598 else599 queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);600 601 break;602 }603 case FW_CDEV_EVENT_RESPONSE2:604 {605 struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;606 607 if (length < rsp->length)608 rsp->length = length;609 if (rcode == RCODE_COMPLETE)610 memcpy(rsp->data, payload, rsp->length);611 612 rsp->rcode = rcode;613 rsp->request_tstamp = request_tstamp;614 rsp->response_tstamp = response_tstamp;615 616 queue_event(client, &e->event, rsp, sizeof(*rsp) + rsp->length, NULL, 0);617 618 break;619 }620 default:621 WARN_ON(1);622 break;623 }624 625 // Drop the xarray's reference.626 client_put(client);627}628 629static int init_request(struct client *client,630 struct fw_cdev_send_request *request,631 int destination_id, int speed)632{633 struct outbound_transaction_event *e;634 void *payload;635 int ret;636 637 if (request->tcode != TCODE_STREAM_DATA &&638 (request->length > 4096 || request->length > 512 << speed))639 return -EIO;640 641 if (request->tcode == TCODE_WRITE_QUADLET_REQUEST &&642 request->length < 4)643 return -EINVAL;644 645 e = kmalloc(sizeof(*e) + request->length, GFP_KERNEL);646 if (e == NULL)647 return -ENOMEM;648 e->client = client;649 650 if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {651 struct fw_cdev_event_response *rsp = &e->rsp.without_tstamp;652 653 rsp->type = FW_CDEV_EVENT_RESPONSE;654 rsp->length = request->length;655 rsp->closure = request->closure;656 payload = rsp->data;657 } else {658 struct fw_cdev_event_response2 *rsp = &e->rsp.with_tstamp;659 660 rsp->type = FW_CDEV_EVENT_RESPONSE2;661 rsp->length = request->length;662 rsp->closure = request->closure;663 payload = rsp->data;664 }665 666 if (request->data && copy_from_user(payload, u64_to_uptr(request->data), request->length)) {667 ret = -EFAULT;668 goto failed;669 }670 671 e->r.resource.release = release_transaction;672 ret = add_client_resource(client, &e->r.resource, GFP_KERNEL);673 if (ret < 0)674 goto failed;675 676 fw_send_request_with_tstamp(client->device->card, &e->r.transaction, request->tcode,677 destination_id, request->generation, speed, request->offset,678 payload, request->length, complete_transaction, e);679 return 0;680 681 failed:682 kfree(e);683 684 return ret;685}686 687static int ioctl_send_request(struct client *client, union ioctl_arg *arg)688{689 switch (arg->send_request.tcode) {690 case TCODE_WRITE_QUADLET_REQUEST:691 case TCODE_WRITE_BLOCK_REQUEST:692 case TCODE_READ_QUADLET_REQUEST:693 case TCODE_READ_BLOCK_REQUEST:694 case TCODE_LOCK_MASK_SWAP:695 case TCODE_LOCK_COMPARE_SWAP:696 case TCODE_LOCK_FETCH_ADD:697 case TCODE_LOCK_LITTLE_ADD:698 case TCODE_LOCK_BOUNDED_ADD:699 case TCODE_LOCK_WRAP_ADD:700 case TCODE_LOCK_VENDOR_DEPENDENT:701 break;702 default:703 return -EINVAL;704 }705 706 return init_request(client, &arg->send_request, client->device->node_id,707 client->device->max_speed);708}709 710static void release_request(struct client *client,711 struct client_resource *resource)712{713 struct inbound_transaction_resource *r = to_inbound_transaction_resource(resource);714 715 if (r->is_fcp)716 fw_request_put(r->request);717 else718 fw_send_response(r->card, r->request, RCODE_CONFLICT_ERROR);719 720 fw_card_put(r->card);721 kfree(r);722}723 724static void handle_request(struct fw_card *card, struct fw_request *request,725 int tcode, int destination, int source,726 int generation, unsigned long long offset,727 void *payload, size_t length, void *callback_data)728{729 struct address_handler_resource *handler = callback_data;730 bool is_fcp = is_in_fcp_region(offset, length);731 struct inbound_transaction_resource *r;732 struct inbound_transaction_event *e;733 size_t event_size0;734 int ret;735 736 /* card may be different from handler->client->device->card */737 fw_card_get(card);738 739 // Extend the lifetime of data for request so that its payload is safely accessible in740 // the process context for the client.741 if (is_fcp)742 fw_request_get(request);743 744 r = kmalloc(sizeof(*r), GFP_ATOMIC);745 e = kmalloc(sizeof(*e), GFP_ATOMIC);746 if (r == NULL || e == NULL)747 goto failed;748 749 r->card = card;750 r->request = request;751 r->is_fcp = is_fcp;752 r->data = payload;753 r->length = length;754 755 r->resource.release = release_request;756 ret = add_client_resource(handler->client, &r->resource, GFP_ATOMIC);757 if (ret < 0)758 goto failed;759 760 if (handler->client->version < FW_CDEV_VERSION_EVENT_REQUEST2) {761 struct fw_cdev_event_request *req = &e->req.request;762 763 if (tcode & 0x10)764 tcode = TCODE_LOCK_REQUEST;765 766 req->type = FW_CDEV_EVENT_REQUEST;767 req->tcode = tcode;768 req->offset = offset;769 req->length = length;770 req->handle = r->resource.handle;771 req->closure = handler->closure;772 event_size0 = sizeof(*req);773 } else if (handler->client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {774 struct fw_cdev_event_request2 *req = &e->req.request2;775 776 req->type = FW_CDEV_EVENT_REQUEST2;777 req->tcode = tcode;778 req->offset = offset;779 req->source_node_id = source;780 req->destination_node_id = destination;781 req->card = card->index;782 req->generation = generation;783 req->length = length;784 req->handle = r->resource.handle;785 req->closure = handler->closure;786 event_size0 = sizeof(*req);787 } else {788 struct fw_cdev_event_request3 *req = &e->req.with_tstamp;789 790 req->type = FW_CDEV_EVENT_REQUEST3;791 req->tcode = tcode;792 req->offset = offset;793 req->source_node_id = source;794 req->destination_node_id = destination;795 req->card = card->index;796 req->generation = generation;797 req->length = length;798 req->handle = r->resource.handle;799 req->closure = handler->closure;800 req->tstamp = fw_request_get_timestamp(request);801 event_size0 = sizeof(*req);802 }803 804 queue_event(handler->client, &e->event,805 &e->req, event_size0, r->data, length);806 return;807 808 failed:809 kfree(r);810 kfree(e);811 812 if (!is_fcp)813 fw_send_response(card, request, RCODE_CONFLICT_ERROR);814 else815 fw_request_put(request);816 817 fw_card_put(card);818}819 820static void release_address_handler(struct client *client,821 struct client_resource *resource)822{823 struct address_handler_resource *r = to_address_handler_resource(resource);824 825 fw_core_remove_address_handler(&r->handler);826 kfree(r);827}828 829static int ioctl_allocate(struct client *client, union ioctl_arg *arg)830{831 struct fw_cdev_allocate *a = &arg->allocate;832 struct address_handler_resource *r;833 struct fw_address_region region;834 int ret;835 836 r = kmalloc(sizeof(*r), GFP_KERNEL);837 if (r == NULL)838 return -ENOMEM;839 840 region.start = a->offset;841 if (client->version < FW_CDEV_VERSION_ALLOCATE_REGION_END)842 region.end = a->offset + a->length;843 else844 region.end = a->region_end;845 846 r->handler.length = a->length;847 r->handler.address_callback = handle_request;848 r->handler.callback_data = r;849 r->closure = a->closure;850 r->client = client;851 852 ret = fw_core_add_address_handler(&r->handler, ®ion);853 if (ret < 0) {854 kfree(r);855 return ret;856 }857 a->offset = r->handler.offset;858 859 r->resource.release = release_address_handler;860 ret = add_client_resource(client, &r->resource, GFP_KERNEL);861 if (ret < 0) {862 release_address_handler(client, &r->resource);863 return ret;864 }865 a->handle = r->resource.handle;866 867 return 0;868}869 870static int ioctl_deallocate(struct client *client, union ioctl_arg *arg)871{872 return release_client_resource(client, arg->deallocate.handle,873 release_address_handler, NULL);874}875 876static int ioctl_send_response(struct client *client, union ioctl_arg *arg)877{878 struct fw_cdev_send_response *a = &arg->send_response;879 struct client_resource *resource;880 struct inbound_transaction_resource *r;881 int ret = 0;882 883 if (release_client_resource(client, a->handle,884 release_request, &resource) < 0)885 return -EINVAL;886 887 r = to_inbound_transaction_resource(resource);888 if (r->is_fcp) {889 fw_request_put(r->request);890 goto out;891 }892 893 if (a->length != fw_get_response_length(r->request)) {894 ret = -EINVAL;895 fw_request_put(r->request);896 goto out;897 }898 if (copy_from_user(r->data, u64_to_uptr(a->data), a->length)) {899 ret = -EFAULT;900 fw_request_put(r->request);901 goto out;902 }903 fw_send_response(r->card, r->request, a->rcode);904 out:905 fw_card_put(r->card);906 kfree(r);907 908 return ret;909}910 911static int ioctl_initiate_bus_reset(struct client *client, union ioctl_arg *arg)912{913 fw_schedule_bus_reset(client->device->card, true,914 arg->initiate_bus_reset.type == FW_CDEV_SHORT_RESET);915 return 0;916}917 918static void release_descriptor(struct client *client,919 struct client_resource *resource)920{921 struct descriptor_resource *r = to_descriptor_resource(resource);922 923 fw_core_remove_descriptor(&r->descriptor);924 kfree(r);925}926 927static int ioctl_add_descriptor(struct client *client, union ioctl_arg *arg)928{929 struct fw_cdev_add_descriptor *a = &arg->add_descriptor;930 struct descriptor_resource *r;931 int ret;932 933 /* Access policy: Allow this ioctl only on local nodes' device files. */934 if (!client->device->is_local)935 return -ENOSYS;936 937 if (a->length > 256)938 return -EINVAL;939 940 r = kmalloc(sizeof(*r) + a->length * 4, GFP_KERNEL);941 if (r == NULL)942 return -ENOMEM;943 944 if (copy_from_user(r->data, u64_to_uptr(a->data), a->length * 4)) {945 ret = -EFAULT;946 goto failed;947 }948 949 r->descriptor.length = a->length;950 r->descriptor.immediate = a->immediate;951 r->descriptor.key = a->key;952 r->descriptor.data = r->data;953 954 ret = fw_core_add_descriptor(&r->descriptor);955 if (ret < 0)956 goto failed;957 958 r->resource.release = release_descriptor;959 ret = add_client_resource(client, &r->resource, GFP_KERNEL);960 if (ret < 0) {961 fw_core_remove_descriptor(&r->descriptor);962 goto failed;963 }964 a->handle = r->resource.handle;965 966 return 0;967 failed:968 kfree(r);969 970 return ret;971}972 973static int ioctl_remove_descriptor(struct client *client, union ioctl_arg *arg)974{975 return release_client_resource(client, arg->remove_descriptor.handle,976 release_descriptor, NULL);977}978 979static void iso_callback(struct fw_iso_context *context, u32 cycle,980 size_t header_length, void *header, void *data)981{982 struct client *client = data;983 struct iso_interrupt_event *e;984 985 e = kmalloc(sizeof(*e) + header_length, GFP_KERNEL);986 if (e == NULL)987 return;988 989 e->interrupt.type = FW_CDEV_EVENT_ISO_INTERRUPT;990 e->interrupt.closure = client->iso_closure;991 e->interrupt.cycle = cycle;992 e->interrupt.header_length = header_length;993 memcpy(e->interrupt.header, header, header_length);994 queue_event(client, &e->event, &e->interrupt,995 sizeof(e->interrupt) + header_length, NULL, 0);996}997 998static void iso_mc_callback(struct fw_iso_context *context,999 dma_addr_t completed, void *data)1000{1001 struct client *client = data;1002 struct iso_interrupt_mc_event *e;1003 1004 e = kmalloc(sizeof(*e), GFP_KERNEL);1005 if (e == NULL)1006 return;1007 1008 e->interrupt.type = FW_CDEV_EVENT_ISO_INTERRUPT_MULTICHANNEL;1009 e->interrupt.closure = client->iso_closure;1010 e->interrupt.completed = fw_iso_buffer_lookup(&client->buffer,1011 completed);1012 queue_event(client, &e->event, &e->interrupt,1013 sizeof(e->interrupt), NULL, 0);1014}1015 1016static enum dma_data_direction iso_dma_direction(struct fw_iso_context *context)1017{1018 if (context->type == FW_ISO_CONTEXT_TRANSMIT)1019 return DMA_TO_DEVICE;1020 else1021 return DMA_FROM_DEVICE;1022}1023 1024static struct fw_iso_context *fw_iso_mc_context_create(struct fw_card *card,1025 fw_iso_mc_callback_t callback,1026 void *callback_data)1027{1028 struct fw_iso_context *ctx;1029 1030 ctx = fw_iso_context_create(card, FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL,1031 0, 0, 0, NULL, callback_data);1032 if (!IS_ERR(ctx))1033 ctx->callback.mc = callback;1034 1035 return ctx;1036}1037 1038static int ioctl_create_iso_context(struct client *client, union ioctl_arg *arg)1039{1040 struct fw_cdev_create_iso_context *a = &arg->create_iso_context;1041 struct fw_iso_context *context;1042 union fw_iso_callback cb;1043 int ret;1044 1045 BUILD_BUG_ON(FW_CDEV_ISO_CONTEXT_TRANSMIT != FW_ISO_CONTEXT_TRANSMIT ||1046 FW_CDEV_ISO_CONTEXT_RECEIVE != FW_ISO_CONTEXT_RECEIVE ||1047 FW_CDEV_ISO_CONTEXT_RECEIVE_MULTICHANNEL !=1048 FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL);1049 1050 switch (a->type) {1051 case FW_ISO_CONTEXT_TRANSMIT:1052 if (a->speed > SCODE_3200 || a->channel > 63)1053 return -EINVAL;1054 1055 cb.sc = iso_callback;1056 break;1057 1058 case FW_ISO_CONTEXT_RECEIVE:1059 if (a->header_size < 4 || (a->header_size & 3) ||1060 a->channel > 63)1061 return -EINVAL;1062 1063 cb.sc = iso_callback;1064 break;1065 1066 case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:1067 cb.mc = iso_mc_callback;1068 break;1069 1070 default:1071 return -EINVAL;1072 }1073 1074 if (a->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL)1075 context = fw_iso_mc_context_create(client->device->card, cb.mc,1076 client);1077 else1078 context = fw_iso_context_create(client->device->card, a->type,1079 a->channel, a->speed,1080 a->header_size, cb.sc, client);1081 if (IS_ERR(context))1082 return PTR_ERR(context);1083 if (client->version < FW_CDEV_VERSION_AUTO_FLUSH_ISO_OVERFLOW)1084 context->drop_overflow_headers = true;1085 1086 // We only support one context at this time.1087 guard(spinlock_irq)(&client->lock);1088 1089 if (client->iso_context != NULL) {1090 fw_iso_context_destroy(context);1091 1092 return -EBUSY;1093 }1094 if (!client->buffer_is_mapped) {1095 ret = fw_iso_buffer_map_dma(&client->buffer,1096 client->device->card,1097 iso_dma_direction(context));1098 if (ret < 0) {1099 fw_iso_context_destroy(context);1100 1101 return ret;1102 }1103 client->buffer_is_mapped = true;1104 }1105 client->iso_closure = a->closure;1106 client->iso_context = context;1107 1108 a->handle = 0;1109 1110 return 0;1111}1112 1113static int ioctl_set_iso_channels(struct client *client, union ioctl_arg *arg)1114{1115 struct fw_cdev_set_iso_channels *a = &arg->set_iso_channels;1116 struct fw_iso_context *ctx = client->iso_context;1117 1118 if (ctx == NULL || a->handle != 0)1119 return -EINVAL;1120 1121 return fw_iso_context_set_channels(ctx, &a->channels);1122}1123 1124/* Macros for decoding the iso packet control header. */1125#define GET_PAYLOAD_LENGTH(v) ((v) & 0xffff)1126#define GET_INTERRUPT(v) (((v) >> 16) & 0x01)1127#define GET_SKIP(v) (((v) >> 17) & 0x01)1128#define GET_TAG(v) (((v) >> 18) & 0x03)1129#define GET_SY(v) (((v) >> 20) & 0x0f)1130#define GET_HEADER_LENGTH(v) (((v) >> 24) & 0xff)1131 1132static int ioctl_queue_iso(struct client *client, union ioctl_arg *arg)1133{1134 struct fw_cdev_queue_iso *a = &arg->queue_iso;1135 struct fw_cdev_iso_packet __user *p, *end, *next;1136 struct fw_iso_context *ctx = client->iso_context;1137 unsigned long payload, buffer_end, transmit_header_bytes = 0;1138 u32 control;1139 int count;1140 struct {1141 struct fw_iso_packet packet;1142 u8 header[256];1143 } u;1144 1145 if (ctx == NULL || a->handle != 0)1146 return -EINVAL;1147 1148 /*1149 * If the user passes a non-NULL data pointer, has mmap()'ed1150 * the iso buffer, and the pointer points inside the buffer,1151 * we setup the payload pointers accordingly. Otherwise we1152 * set them both to 0, which will still let packets with1153 * payload_length == 0 through. In other words, if no packets1154 * use the indirect payload, the iso buffer need not be mapped1155 * and the a->data pointer is ignored.1156 */1157 payload = (unsigned long)a->data - client->vm_start;1158 buffer_end = client->buffer.page_count << PAGE_SHIFT;1159 if (a->data == 0 || client->buffer.pages == NULL ||1160 payload >= buffer_end) {1161 payload = 0;1162 buffer_end = 0;1163 }1164 1165 if (ctx->type == FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL && payload & 3)1166 return -EINVAL;1167 1168 p = (struct fw_cdev_iso_packet __user *)u64_to_uptr(a->packets);1169 1170 end = (void __user *)p + a->size;1171 count = 0;1172 while (p < end) {1173 if (get_user(control, &p->control))1174 return -EFAULT;1175 u.packet.payload_length = GET_PAYLOAD_LENGTH(control);1176 u.packet.interrupt = GET_INTERRUPT(control);1177 u.packet.skip = GET_SKIP(control);1178 u.packet.tag = GET_TAG(control);1179 u.packet.sy = GET_SY(control);1180 u.packet.header_length = GET_HEADER_LENGTH(control);1181 1182 switch (ctx->type) {1183 case FW_ISO_CONTEXT_TRANSMIT:1184 if (u.packet.header_length & 3)1185 return -EINVAL;1186 transmit_header_bytes = u.packet.header_length;1187 break;1188 1189 case FW_ISO_CONTEXT_RECEIVE:1190 if (u.packet.header_length == 0 ||1191 u.packet.header_length % ctx->header_size != 0)1192 return -EINVAL;1193 break;1194 1195 case FW_ISO_CONTEXT_RECEIVE_MULTICHANNEL:1196 if (u.packet.payload_length == 0 ||1197 u.packet.payload_length & 3)1198 return -EINVAL;1199 break;1200 }1201 1202 next = (struct fw_cdev_iso_packet __user *)1203 &p->header[transmit_header_bytes / 4];1204 if (next > end)1205 return -EINVAL;1206 if (copy_from_user1207 (u.packet.header, p->header, transmit_header_bytes))1208 return -EFAULT;1209 if (u.packet.skip && ctx->type == FW_ISO_CONTEXT_TRANSMIT &&1210 u.packet.header_length + u.packet.payload_length > 0)1211 return -EINVAL;1212 if (payload + u.packet.payload_length > buffer_end)1213 return -EINVAL;1214 1215 if (fw_iso_context_queue(ctx, &u.packet,1216 &client->buffer, payload))1217 break;1218 1219 p = next;1220 payload += u.packet.payload_length;1221 count++;1222 }1223 fw_iso_context_queue_flush(ctx);1224 1225 a->size -= uptr_to_u64(p) - a->packets;1226 a->packets = uptr_to_u64(p);1227 a->data = client->vm_start + payload;1228 1229 return count;1230}1231 1232static int ioctl_start_iso(struct client *client, union ioctl_arg *arg)1233{1234 struct fw_cdev_start_iso *a = &arg->start_iso;1235 1236 BUILD_BUG_ON(1237 FW_CDEV_ISO_CONTEXT_MATCH_TAG0 != FW_ISO_CONTEXT_MATCH_TAG0 ||1238 FW_CDEV_ISO_CONTEXT_MATCH_TAG1 != FW_ISO_CONTEXT_MATCH_TAG1 ||1239 FW_CDEV_ISO_CONTEXT_MATCH_TAG2 != FW_ISO_CONTEXT_MATCH_TAG2 ||1240 FW_CDEV_ISO_CONTEXT_MATCH_TAG3 != FW_ISO_CONTEXT_MATCH_TAG3 ||1241 FW_CDEV_ISO_CONTEXT_MATCH_ALL_TAGS != FW_ISO_CONTEXT_MATCH_ALL_TAGS);1242 1243 if (client->iso_context == NULL || a->handle != 0)1244 return -EINVAL;1245 1246 if (client->iso_context->type == FW_ISO_CONTEXT_RECEIVE &&1247 (a->tags == 0 || a->tags > 15 || a->sync > 15))1248 return -EINVAL;1249 1250 return fw_iso_context_start(client->iso_context,1251 a->cycle, a->sync, a->tags);1252}1253 1254static int ioctl_stop_iso(struct client *client, union ioctl_arg *arg)1255{1256 struct fw_cdev_stop_iso *a = &arg->stop_iso;1257 1258 if (client->iso_context == NULL || a->handle != 0)1259 return -EINVAL;1260 1261 return fw_iso_context_stop(client->iso_context);1262}1263 1264static int ioctl_flush_iso(struct client *client, union ioctl_arg *arg)1265{1266 struct fw_cdev_flush_iso *a = &arg->flush_iso;1267 1268 if (client->iso_context == NULL || a->handle != 0)1269 return -EINVAL;1270 1271 return fw_iso_context_flush_completions(client->iso_context);1272}1273 1274static int ioctl_get_cycle_timer2(struct client *client, union ioctl_arg *arg)1275{1276 struct fw_cdev_get_cycle_timer2 *a = &arg->get_cycle_timer2;1277 struct fw_card *card = client->device->card;1278 struct timespec64 ts = {0, 0};1279 u32 cycle_time = 0;1280 int ret;1281 1282 guard(irq)();1283 1284 ret = fw_card_read_cycle_time(card, &cycle_time);1285 if (ret < 0)1286 return ret;1287 1288 switch (a->clk_id) {1289 case CLOCK_REALTIME: ktime_get_real_ts64(&ts); break;1290 case CLOCK_MONOTONIC: ktime_get_ts64(&ts); break;1291 case CLOCK_MONOTONIC_RAW: ktime_get_raw_ts64(&ts); break;1292 default:1293 return -EINVAL;1294 }1295 1296 a->tv_sec = ts.tv_sec;1297 a->tv_nsec = ts.tv_nsec;1298 a->cycle_timer = cycle_time;1299 1300 return 0;1301}1302 1303static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg)1304{1305 struct fw_cdev_get_cycle_timer *a = &arg->get_cycle_timer;1306 struct fw_cdev_get_cycle_timer2 ct2;1307 1308 ct2.clk_id = CLOCK_REALTIME;1309 ioctl_get_cycle_timer2(client, (union ioctl_arg *)&ct2);1310 1311 a->local_time = ct2.tv_sec * USEC_PER_SEC + ct2.tv_nsec / NSEC_PER_USEC;1312 a->cycle_timer = ct2.cycle_timer;1313 1314 return 0;1315}1316 1317static void iso_resource_work(struct work_struct *work)1318{1319 struct iso_resource_event *e;1320 struct iso_resource *r =1321 container_of(work, struct iso_resource, work.work);1322 struct client *client = r->client;1323 unsigned long index = r->resource.handle;1324 int generation, channel, bandwidth, todo;1325 bool skip, free, success;1326 1327 scoped_guard(spinlock_irq, &client->lock) {1328 generation = client->device->generation;1329 todo = r->todo;1330 // Allow 1000ms grace period for other reallocations.1331 if (todo == ISO_RES_ALLOC &&1332 time_before64(get_jiffies_64(), client->device->card->reset_jiffies + HZ)) {1333 schedule_iso_resource(r, DIV_ROUND_UP(HZ, 3));1334 skip = true;1335 } else {1336 // We could be called twice within the same generation.1337 skip = todo == ISO_RES_REALLOC &&1338 r->generation == generation;1339 }1340 free = todo == ISO_RES_DEALLOC ||1341 todo == ISO_RES_ALLOC_ONCE ||1342 todo == ISO_RES_DEALLOC_ONCE;1343 r->generation = generation;1344 }1345 1346 if (skip)1347 goto out;1348 1349 bandwidth = r->bandwidth;1350 1351 fw_iso_resource_manage(client->device->card, generation,1352 r->channels, &channel, &bandwidth,1353 todo == ISO_RES_ALLOC ||1354 todo == ISO_RES_REALLOC ||1355 todo == ISO_RES_ALLOC_ONCE);1356 /*1357 * Is this generation outdated already? As long as this resource sticks1358 * in the xarray, it will be scheduled again for a newer generation or at1359 * shutdown.1360 */1361 if (channel == -EAGAIN &&1362 (todo == ISO_RES_ALLOC || todo == ISO_RES_REALLOC))1363 goto out;1364 1365 success = channel >= 0 || bandwidth > 0;1366 1367 scoped_guard(spinlock_irq, &client->lock) {1368 // Transit from allocation to reallocation, except if the client1369 // requested deallocation in the meantime.1370 if (r->todo == ISO_RES_ALLOC)1371 r->todo = ISO_RES_REALLOC;1372 // Allocation or reallocation failure? Pull this resource out of the1373 // xarray and prepare for deletion, unless the client is shutting down.1374 if (r->todo == ISO_RES_REALLOC && !success &&1375 !client->in_shutdown &&1376 xa_erase(&client->resource_xa, index)) {1377 client_put(client);1378 free = true;1379 }1380 }1381 1382 if (todo == ISO_RES_ALLOC && channel >= 0)1383 r->channels = 1ULL << channel;1384 1385 if (todo == ISO_RES_REALLOC && success)1386 goto out;1387 1388 if (todo == ISO_RES_ALLOC || todo == ISO_RES_ALLOC_ONCE) {1389 e = r->e_alloc;1390 r->e_alloc = NULL;1391 } else {1392 e = r->e_dealloc;1393 r->e_dealloc = NULL;1394 }1395 e->iso_resource.handle = r->resource.handle;1396 e->iso_resource.channel = channel;1397 e->iso_resource.bandwidth = bandwidth;1398 1399 queue_event(client, &e->event,1400 &e->iso_resource, sizeof(e->iso_resource), NULL, 0);1401 1402 if (free) {1403 cancel_delayed_work(&r->work);1404 kfree(r->e_alloc);1405 kfree(r->e_dealloc);1406 kfree(r);1407 }1408 out:1409 client_put(client);1410}1411 1412static void release_iso_resource(struct client *client,1413 struct client_resource *resource)1414{1415 struct iso_resource *r = to_iso_resource(resource);1416 1417 guard(spinlock_irq)(&client->lock);1418 1419 r->todo = ISO_RES_DEALLOC;1420 schedule_iso_resource(r, 0);1421}1422 1423static int init_iso_resource(struct client *client,1424 struct fw_cdev_allocate_iso_resource *request, int todo)1425{1426 struct iso_resource_event *e1, *e2;1427 struct iso_resource *r;1428 int ret;1429 1430 if ((request->channels == 0 && request->bandwidth == 0) ||1431 request->bandwidth > BANDWIDTH_AVAILABLE_INITIAL)1432 return -EINVAL;1433 1434 r = kmalloc(sizeof(*r), GFP_KERNEL);1435 e1 = kmalloc(sizeof(*e1), GFP_KERNEL);1436 e2 = kmalloc(sizeof(*e2), GFP_KERNEL);1437 if (r == NULL || e1 == NULL || e2 == NULL) {1438 ret = -ENOMEM;1439 goto fail;1440 }1441 1442 INIT_DELAYED_WORK(&r->work, iso_resource_work);1443 r->client = client;1444 r->todo = todo;1445 r->generation = -1;1446 r->channels = request->channels;1447 r->bandwidth = request->bandwidth;1448 r->e_alloc = e1;1449 r->e_dealloc = e2;1450 1451 e1->iso_resource.closure = request->closure;1452 e1->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED;1453 e2->iso_resource.closure = request->closure;1454 e2->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;1455 1456 if (todo == ISO_RES_ALLOC) {1457 r->resource.release = release_iso_resource;1458 ret = add_client_resource(client, &r->resource, GFP_KERNEL);1459 if (ret < 0)1460 goto fail;1461 } else {1462 r->resource.release = NULL;1463 r->resource.handle = -1;1464 schedule_iso_resource(r, 0);1465 }1466 request->handle = r->resource.handle;1467 1468 return 0;1469 fail:1470 kfree(r);1471 kfree(e1);1472 kfree(e2);1473 1474 return ret;1475}1476 1477static int ioctl_allocate_iso_resource(struct client *client,1478 union ioctl_arg *arg)1479{1480 return init_iso_resource(client,1481 &arg->allocate_iso_resource, ISO_RES_ALLOC);1482}1483 1484static int ioctl_deallocate_iso_resource(struct client *client,1485 union ioctl_arg *arg)1486{1487 return release_client_resource(client,1488 arg->deallocate.handle, release_iso_resource, NULL);1489}1490 1491static int ioctl_allocate_iso_resource_once(struct client *client,1492 union ioctl_arg *arg)1493{1494 return init_iso_resource(client,1495 &arg->allocate_iso_resource, ISO_RES_ALLOC_ONCE);1496}1497 1498static int ioctl_deallocate_iso_resource_once(struct client *client,1499 union ioctl_arg *arg)1500{1501 return init_iso_resource(client,1502 &arg->allocate_iso_resource, ISO_RES_DEALLOC_ONCE);1503}1504 1505/*1506 * Returns a speed code: Maximum speed to or from this device,1507 * limited by the device's link speed, the local node's link speed,1508 * and all PHY port speeds between the two links.1509 */1510static int ioctl_get_speed(struct client *client, union ioctl_arg *arg)1511{1512 return client->device->max_speed;1513}1514 1515static int ioctl_send_broadcast_request(struct client *client,1516 union ioctl_arg *arg)1517{1518 struct fw_cdev_send_request *a = &arg->send_request;1519 1520 switch (a->tcode) {1521 case TCODE_WRITE_QUADLET_REQUEST:1522 case TCODE_WRITE_BLOCK_REQUEST:1523 break;1524 default:1525 return -EINVAL;1526 }1527 1528 /* Security policy: Only allow accesses to Units Space. */1529 if (a->offset < CSR_REGISTER_BASE + CSR_CONFIG_ROM_END)1530 return -EACCES;1531 1532 return init_request(client, a, LOCAL_BUS | 0x3f, SCODE_100);1533}1534 1535static int ioctl_send_stream_packet(struct client *client, union ioctl_arg *arg)1536{1537 struct fw_cdev_send_stream_packet *a = &arg->send_stream_packet;1538 struct fw_cdev_send_request request;1539 int dest;1540 1541 if (a->speed > client->device->card->link_speed ||1542 a->length > 1024 << a->speed)1543 return -EIO;1544 1545 if (a->tag > 3 || a->channel > 63 || a->sy > 15)1546 return -EINVAL;1547 1548 dest = fw_stream_packet_destination_id(a->tag, a->channel, a->sy);1549 request.tcode = TCODE_STREAM_DATA;1550 request.length = a->length;1551 request.closure = a->closure;1552 request.data = a->data;1553 request.generation = a->generation;1554 1555 return init_request(client, &request, dest, a->speed);1556}1557 1558static void outbound_phy_packet_callback(struct fw_packet *packet,1559 struct fw_card *card, int status)1560{1561 struct outbound_phy_packet_event *e =1562 container_of(packet, struct outbound_phy_packet_event, p);1563 struct client *e_client = e->client;1564 u32 rcode;1565 1566 trace_async_phy_outbound_complete((uintptr_t)packet, card->index, status, packet->generation,1567 packet->timestamp);1568 1569 switch (status) {1570 // expected:1571 case ACK_COMPLETE:1572 rcode = RCODE_COMPLETE;1573 break;1574 // should never happen with PHY packets:1575 case ACK_PENDING:1576 rcode = RCODE_COMPLETE;1577 break;1578 case ACK_BUSY_X:1579 case ACK_BUSY_A:1580 case ACK_BUSY_B:1581 rcode = RCODE_BUSY;1582 break;1583 case ACK_DATA_ERROR:1584 rcode = RCODE_DATA_ERROR;1585 break;1586 case ACK_TYPE_ERROR:1587 rcode = RCODE_TYPE_ERROR;1588 break;1589 // stale generation; cancelled; on certain controllers: no ack1590 default:1591 rcode = status;1592 break;1593 }1594 1595 switch (e->phy_packet.without_tstamp.type) {1596 case FW_CDEV_EVENT_PHY_PACKET_SENT:1597 {1598 struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;1599 1600 pp->rcode = rcode;1601 pp->data[0] = packet->timestamp;1602 queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,1603 NULL, 0);1604 break;1605 }1606 case FW_CDEV_EVENT_PHY_PACKET_SENT2:1607 {1608 struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;1609 1610 pp->rcode = rcode;1611 pp->tstamp = packet->timestamp;1612 queue_event(e->client, &e->event, &e->phy_packet, sizeof(*pp) + pp->length,1613 NULL, 0);1614 break;1615 }1616 default:1617 WARN_ON(1);1618 break;1619 }1620 1621 client_put(e_client);1622}1623 1624static int ioctl_send_phy_packet(struct client *client, union ioctl_arg *arg)1625{1626 struct fw_cdev_send_phy_packet *a = &arg->send_phy_packet;1627 struct fw_card *card = client->device->card;1628 struct outbound_phy_packet_event *e;1629 1630 /* Access policy: Allow this ioctl only on local nodes' device files. */1631 if (!client->device->is_local)1632 return -ENOSYS;1633 1634 e = kzalloc(sizeof(*e) + sizeof(a->data), GFP_KERNEL);1635 if (e == NULL)1636 return -ENOMEM;1637 1638 client_get(client);1639 e->client = client;1640 e->p.speed = SCODE_100;1641 e->p.generation = a->generation;1642 async_header_set_tcode(e->p.header, TCODE_LINK_INTERNAL);1643 e->p.header[1] = a->data[0];1644 e->p.header[2] = a->data[1];1645 e->p.header_length = 12;1646 e->p.callback = outbound_phy_packet_callback;1647 1648 if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {1649 struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;1650 1651 pp->closure = a->closure;1652 pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT;1653 if (is_ping_packet(a->data))1654 pp->length = 4;1655 } else {1656 struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;1657 1658 pp->closure = a->closure;1659 pp->type = FW_CDEV_EVENT_PHY_PACKET_SENT2;1660 // Keep the data field so that application can match the response event to the1661 // request.1662 pp->length = sizeof(a->data);1663 memcpy(pp->data, a->data, sizeof(a->data));1664 }1665 1666 trace_async_phy_outbound_initiate((uintptr_t)&e->p, card->index, e->p.generation,1667 e->p.header[1], e->p.header[2]);1668 1669 card->driver->send_request(card, &e->p);1670 1671 return 0;1672}1673 1674static int ioctl_receive_phy_packets(struct client *client, union ioctl_arg *arg)1675{1676 struct fw_cdev_receive_phy_packets *a = &arg->receive_phy_packets;1677 struct fw_card *card = client->device->card;1678 1679 /* Access policy: Allow this ioctl only on local nodes' device files. */1680 if (!client->device->is_local)1681 return -ENOSYS;1682 1683 guard(spinlock_irq)(&card->lock);1684 1685 list_move_tail(&client->phy_receiver_link, &card->phy_receiver_list);1686 client->phy_receiver_closure = a->closure;1687 1688 return 0;1689}1690 1691void fw_cdev_handle_phy_packet(struct fw_card *card, struct fw_packet *p)1692{1693 struct client *client;1694 1695 guard(spinlock_irqsave)(&card->lock);1696 1697 list_for_each_entry(client, &card->phy_receiver_list, phy_receiver_link) {1698 struct inbound_phy_packet_event *e = kmalloc(sizeof(*e) + 8, GFP_ATOMIC);1699 if (e == NULL)1700 break;1701 1702 if (client->version < FW_CDEV_VERSION_EVENT_ASYNC_TSTAMP) {1703 struct fw_cdev_event_phy_packet *pp = &e->phy_packet.without_tstamp;1704 1705 pp->closure = client->phy_receiver_closure;1706 pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED;1707 pp->rcode = RCODE_COMPLETE;1708 pp->length = 8;1709 pp->data[0] = p->header[1];1710 pp->data[1] = p->header[2];1711 queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);1712 } else {1713 struct fw_cdev_event_phy_packet2 *pp = &e->phy_packet.with_tstamp;1714 1715 pp = &e->phy_packet.with_tstamp;1716 pp->closure = client->phy_receiver_closure;1717 pp->type = FW_CDEV_EVENT_PHY_PACKET_RECEIVED2;1718 pp->rcode = RCODE_COMPLETE;1719 pp->length = 8;1720 pp->tstamp = p->timestamp;1721 pp->data[0] = p->header[1];1722 pp->data[1] = p->header[2];1723 queue_event(client, &e->event, &e->phy_packet, sizeof(*pp) + 8, NULL, 0);1724 }1725 }1726}1727 1728static int (* const ioctl_handlers[])(struct client *, union ioctl_arg *) = {1729 [0x00] = ioctl_get_info,1730 [0x01] = ioctl_send_request,1731 [0x02] = ioctl_allocate,1732 [0x03] = ioctl_deallocate,1733 [0x04] = ioctl_send_response,1734 [0x05] = ioctl_initiate_bus_reset,1735 [0x06] = ioctl_add_descriptor,1736 [0x07] = ioctl_remove_descriptor,1737 [0x08] = ioctl_create_iso_context,1738 [0x09] = ioctl_queue_iso,1739 [0x0a] = ioctl_start_iso,1740 [0x0b] = ioctl_stop_iso,1741 [0x0c] = ioctl_get_cycle_timer,1742 [0x0d] = ioctl_allocate_iso_resource,1743 [0x0e] = ioctl_deallocate_iso_resource,1744 [0x0f] = ioctl_allocate_iso_resource_once,1745 [0x10] = ioctl_deallocate_iso_resource_once,1746 [0x11] = ioctl_get_speed,1747 [0x12] = ioctl_send_broadcast_request,1748 [0x13] = ioctl_send_stream_packet,1749 [0x14] = ioctl_get_cycle_timer2,1750 [0x15] = ioctl_send_phy_packet,1751 [0x16] = ioctl_receive_phy_packets,1752 [0x17] = ioctl_set_iso_channels,1753 [0x18] = ioctl_flush_iso,1754};1755 1756static int dispatch_ioctl(struct client *client,1757 unsigned int cmd, void __user *arg)1758{1759 union ioctl_arg buffer;1760 int ret;1761 1762 if (fw_device_is_shutdown(client->device))1763 return -ENODEV;1764 1765 if (_IOC_TYPE(cmd) != '#' ||1766 _IOC_NR(cmd) >= ARRAY_SIZE(ioctl_handlers) ||1767 _IOC_SIZE(cmd) > sizeof(buffer))1768 return -ENOTTY;1769 1770 memset(&buffer, 0, sizeof(buffer));1771 1772 if (_IOC_DIR(cmd) & _IOC_WRITE)1773 if (copy_from_user(&buffer, arg, _IOC_SIZE(cmd)))1774 return -EFAULT;1775 1776 ret = ioctl_handlers[_IOC_NR(cmd)](client, &buffer);1777 if (ret < 0)1778 return ret;1779 1780 if (_IOC_DIR(cmd) & _IOC_READ)1781 if (copy_to_user(arg, &buffer, _IOC_SIZE(cmd)))1782 return -EFAULT;1783 1784 return ret;1785}1786 1787static long fw_device_op_ioctl(struct file *file,1788 unsigned int cmd, unsigned long arg)1789{1790 return dispatch_ioctl(file->private_data, cmd, (void __user *)arg);1791}1792 1793static int fw_device_op_mmap(struct file *file, struct vm_area_struct *vma)1794{1795 struct client *client = file->private_data;1796 unsigned long size;1797 int page_count, ret;1798 1799 if (fw_device_is_shutdown(client->device))1800 return -ENODEV;1801 1802 /* FIXME: We could support multiple buffers, but we don't. */1803 if (client->buffer.pages != NULL)1804 return -EBUSY;1805 1806 if (!(vma->vm_flags & VM_SHARED))1807 return -EINVAL;1808 1809 if (vma->vm_start & ~PAGE_MASK)1810 return -EINVAL;1811 1812 client->vm_start = vma->vm_start;1813 size = vma->vm_end - vma->vm_start;1814 page_count = size >> PAGE_SHIFT;1815 if (size & ~PAGE_MASK)1816 return -EINVAL;1817 1818 ret = fw_iso_buffer_alloc(&client->buffer, page_count);1819 if (ret < 0)1820 return ret;1821 1822 scoped_guard(spinlock_irq, &client->lock) {1823 if (client->iso_context) {1824 ret = fw_iso_buffer_map_dma(&client->buffer, client->device->card,1825 iso_dma_direction(client->iso_context));1826 if (ret < 0)1827 goto fail;1828 client->buffer_is_mapped = true;1829 }1830 }1831 1832 ret = vm_map_pages_zero(vma, client->buffer.pages,1833 client->buffer.page_count);1834 if (ret < 0)1835 goto fail;1836 1837 return 0;1838 fail:1839 fw_iso_buffer_destroy(&client->buffer, client->device->card);1840 return ret;1841}1842 1843static bool has_outbound_transactions(struct client *client)1844{1845 struct client_resource *resource;1846 unsigned long index;1847 1848 guard(spinlock_irq)(&client->lock);1849 1850 xa_for_each(&client->resource_xa, index, resource) {1851 if (is_outbound_transaction_resource(resource))1852 return true;1853 }1854 1855 return false;1856}1857 1858static int fw_device_op_release(struct inode *inode, struct file *file)1859{1860 struct client *client = file->private_data;1861 struct event *event, *next_event;1862 struct client_resource *resource;1863 unsigned long index;1864 1865 scoped_guard(spinlock_irq, &client->device->card->lock)1866 list_del(&client->phy_receiver_link);1867 1868 scoped_guard(mutex, &client->device->client_list_mutex)1869 list_del(&client->link);1870 1871 if (client->iso_context)1872 fw_iso_context_destroy(client->iso_context);1873 1874 if (client->buffer.pages)1875 fw_iso_buffer_destroy(&client->buffer, client->device->card);1876 1877 // Freeze client->resource_xa and client->event_list.1878 scoped_guard(spinlock_irq, &client->lock)1879 client->in_shutdown = true;1880 1881 wait_event(client->tx_flush_wait, !has_outbound_transactions(client));1882 1883 xa_for_each(&client->resource_xa, index, resource) {1884 resource->release(client, resource);1885 client_put(client);1886 }1887 xa_destroy(&client->resource_xa);1888 1889 list_for_each_entry_safe(event, next_event, &client->event_list, link)1890 kfree(event);1891 1892 client_put(client);1893 1894 return 0;1895}1896 1897static __poll_t fw_device_op_poll(struct file *file, poll_table * pt)1898{1899 struct client *client = file->private_data;1900 __poll_t mask = 0;1901 1902 poll_wait(file, &client->wait, pt);1903 1904 if (fw_device_is_shutdown(client->device))1905 mask |= EPOLLHUP | EPOLLERR;1906 if (!list_empty(&client->event_list))1907 mask |= EPOLLIN | EPOLLRDNORM;1908 1909 return mask;1910}1911 1912const struct file_operations fw_device_ops = {1913 .owner = THIS_MODULE,1914 .open = fw_device_op_open,1915 .read = fw_device_op_read,1916 .unlocked_ioctl = fw_device_op_ioctl,1917 .mmap = fw_device_op_mmap,1918 .release = fw_device_op_release,1919 .poll = fw_device_op_poll,1920 .compat_ioctl = compat_ptr_ioctl,1921};1922