1288 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * ISHTP client logic4 *5 * Copyright (c) 2003-2016, Intel Corporation.6 */7 8#include <linux/slab.h>9#include <linux/sched.h>10#include <linux/wait.h>11#include <linux/delay.h>12#include <linux/dma-mapping.h>13#include <asm/cacheflush.h>14#include "hbm.h"15#include "client.h"16 17int ishtp_cl_get_tx_free_buffer_size(struct ishtp_cl *cl)18{19 unsigned long tx_free_flags;20 int size;21 22 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);23 size = cl->tx_ring_free_size * cl->device->fw_client->props.max_msg_length;24 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);25 26 return size;27}28EXPORT_SYMBOL(ishtp_cl_get_tx_free_buffer_size);29 30int ishtp_cl_get_tx_free_rings(struct ishtp_cl *cl)31{32 return cl->tx_ring_free_size;33}34EXPORT_SYMBOL(ishtp_cl_get_tx_free_rings);35 36/**37 * ishtp_read_list_flush() - Flush read queue38 * @cl: ishtp client instance39 *40 * Used to remove all entries from read queue for a client41 */42static void ishtp_read_list_flush(struct ishtp_cl *cl)43{44 struct ishtp_cl_rb *rb;45 struct ishtp_cl_rb *next;46 unsigned long flags;47 48 spin_lock_irqsave(&cl->dev->read_list_spinlock, flags);49 list_for_each_entry_safe(rb, next, &cl->dev->read_list.list, list)50 if (rb->cl && ishtp_cl_cmp_id(cl, rb->cl)) {51 list_del(&rb->list);52 spin_lock(&cl->free_list_spinlock);53 list_add_tail(&rb->list, &cl->free_rb_list.list);54 spin_unlock(&cl->free_list_spinlock);55 }56 spin_unlock_irqrestore(&cl->dev->read_list_spinlock, flags);57}58 59/**60 * ishtp_cl_flush_queues() - Flush all queues for a client61 * @cl: ishtp client instance62 *63 * Used to remove all queues for a client. This is called when a client device64 * needs reset due to error, S3 resume or during module removal65 *66 * Return: 0 on success else -EINVAL if device is NULL67 */68int ishtp_cl_flush_queues(struct ishtp_cl *cl)69{70 if (WARN_ON(!cl || !cl->dev))71 return -EINVAL;72 73 ishtp_read_list_flush(cl);74 75 return 0;76}77EXPORT_SYMBOL(ishtp_cl_flush_queues);78 79/**80 * ishtp_cl_init() - Initialize all fields of a client device81 * @cl: ishtp client instance82 * @dev: ishtp device83 *84 * Initializes a client device fields: Init spinlocks, init queues etc.85 * This function is called during new client creation86 */87static void ishtp_cl_init(struct ishtp_cl *cl, struct ishtp_device *dev)88{89 memset(cl, 0, sizeof(struct ishtp_cl));90 init_waitqueue_head(&cl->wait_ctrl_res);91 spin_lock_init(&cl->free_list_spinlock);92 spin_lock_init(&cl->in_process_spinlock);93 spin_lock_init(&cl->tx_list_spinlock);94 spin_lock_init(&cl->tx_free_list_spinlock);95 spin_lock_init(&cl->fc_spinlock);96 INIT_LIST_HEAD(&cl->link);97 cl->dev = dev;98 99 INIT_LIST_HEAD(&cl->free_rb_list.list);100 INIT_LIST_HEAD(&cl->tx_list.list);101 INIT_LIST_HEAD(&cl->tx_free_list.list);102 INIT_LIST_HEAD(&cl->in_process_list.list);103 104 cl->rx_ring_size = CL_DEF_RX_RING_SIZE;105 cl->tx_ring_size = CL_DEF_TX_RING_SIZE;106 cl->tx_ring_free_size = cl->tx_ring_size;107 108 /* dma */109 cl->last_tx_path = CL_TX_PATH_IPC;110 cl->last_dma_acked = 1;111 cl->last_dma_addr = NULL;112 cl->last_ipc_acked = 1;113}114 115/**116 * ishtp_cl_allocate() - allocates client structure and sets it up.117 * @cl_device: ishtp client device118 *119 * Allocate memory for new client device and call to initialize each field.120 *121 * Return: The allocated client instance or NULL on failure122 */123struct ishtp_cl *ishtp_cl_allocate(struct ishtp_cl_device *cl_device)124{125 struct ishtp_cl *cl;126 127 cl = kmalloc(sizeof(struct ishtp_cl), GFP_KERNEL);128 if (!cl)129 return NULL;130 131 ishtp_cl_init(cl, cl_device->ishtp_dev);132 return cl;133}134EXPORT_SYMBOL(ishtp_cl_allocate);135 136/**137 * ishtp_cl_free() - Frees a client device138 * @cl: client device instance139 *140 * Frees a client device141 */142void ishtp_cl_free(struct ishtp_cl *cl)143{144 struct ishtp_device *dev;145 unsigned long flags;146 147 if (!cl)148 return;149 150 dev = cl->dev;151 if (!dev)152 return;153 154 spin_lock_irqsave(&dev->cl_list_lock, flags);155 ishtp_cl_free_rx_ring(cl);156 ishtp_cl_free_tx_ring(cl);157 kfree(cl);158 spin_unlock_irqrestore(&dev->cl_list_lock, flags);159}160EXPORT_SYMBOL(ishtp_cl_free);161 162/**163 * ishtp_cl_link() - Reserve a host id and link the client instance164 * @cl: client device instance165 *166 * This allocates a single bit in the hostmap. This function will make sure167 * that not many client sessions are opened at the same time. Once allocated168 * the client device instance is added to the ishtp device in the current169 * client list170 *171 * Return: 0 or error code on failure172 */173int ishtp_cl_link(struct ishtp_cl *cl)174{175 struct ishtp_device *dev;176 unsigned long flags, flags_cl;177 int id, ret = 0;178 179 if (WARN_ON(!cl || !cl->dev))180 return -EINVAL;181 182 dev = cl->dev;183 184 spin_lock_irqsave(&dev->device_lock, flags);185 186 if (dev->open_handle_count >= ISHTP_MAX_OPEN_HANDLE_COUNT) {187 ret = -EMFILE;188 goto unlock_dev;189 }190 191 id = find_first_zero_bit(dev->host_clients_map, ISHTP_CLIENTS_MAX);192 193 if (id >= ISHTP_CLIENTS_MAX) {194 spin_unlock_irqrestore(&dev->device_lock, flags);195 dev_err(&cl->device->dev, "id exceeded %d", ISHTP_CLIENTS_MAX);196 return -ENOENT;197 }198 199 dev->open_handle_count++;200 cl->host_client_id = id;201 spin_lock_irqsave(&dev->cl_list_lock, flags_cl);202 if (dev->dev_state != ISHTP_DEV_ENABLED) {203 ret = -ENODEV;204 goto unlock_cl;205 }206 list_add_tail(&cl->link, &dev->cl_list);207 set_bit(id, dev->host_clients_map);208 cl->state = ISHTP_CL_INITIALIZING;209 210unlock_cl:211 spin_unlock_irqrestore(&dev->cl_list_lock, flags_cl);212unlock_dev:213 spin_unlock_irqrestore(&dev->device_lock, flags);214 return ret;215}216EXPORT_SYMBOL(ishtp_cl_link);217 218/**219 * ishtp_cl_unlink() - remove fw_cl from the client device list220 * @cl: client device instance221 *222 * Remove a previously linked device to a ishtp device223 */224void ishtp_cl_unlink(struct ishtp_cl *cl)225{226 struct ishtp_device *dev;227 struct ishtp_cl *pos;228 unsigned long flags;229 230 /* don't shout on error exit path */231 if (!cl || !cl->dev)232 return;233 234 dev = cl->dev;235 236 spin_lock_irqsave(&dev->device_lock, flags);237 if (dev->open_handle_count > 0) {238 clear_bit(cl->host_client_id, dev->host_clients_map);239 dev->open_handle_count--;240 }241 spin_unlock_irqrestore(&dev->device_lock, flags);242 243 /*244 * This checks that 'cl' is actually linked into device's structure,245 * before attempting 'list_del'246 */247 spin_lock_irqsave(&dev->cl_list_lock, flags);248 list_for_each_entry(pos, &dev->cl_list, link)249 if (cl->host_client_id == pos->host_client_id) {250 list_del_init(&pos->link);251 break;252 }253 spin_unlock_irqrestore(&dev->cl_list_lock, flags);254}255EXPORT_SYMBOL(ishtp_cl_unlink);256 257/**258 * ishtp_cl_disconnect() - Send disconnect request to firmware259 * @cl: client device instance260 *261 * Send a disconnect request for a client to firmware.262 *263 * Return: 0 if successful disconnect response from the firmware or error264 * code on failure265 */266int ishtp_cl_disconnect(struct ishtp_cl *cl)267{268 struct ishtp_device *dev;269 270 if (WARN_ON(!cl || !cl->dev))271 return -ENODEV;272 273 dev = cl->dev;274 275 dev->print_log(dev, "%s() state %d\n", __func__, cl->state);276 277 if (cl->state != ISHTP_CL_DISCONNECTING) {278 dev->print_log(dev, "%s() Disconnect in progress\n", __func__);279 return 0;280 }281 282 if (ishtp_hbm_cl_disconnect_req(dev, cl)) {283 dev->print_log(dev, "%s() Failed to disconnect\n", __func__);284 dev_err(&cl->device->dev, "failed to disconnect.\n");285 return -ENODEV;286 }287 288 wait_event_interruptible_timeout(cl->wait_ctrl_res,289 (dev->dev_state != ISHTP_DEV_ENABLED ||290 cl->state == ISHTP_CL_DISCONNECTED),291 ishtp_secs_to_jiffies(ISHTP_CL_CONNECT_TIMEOUT));292 293 /*294 * If FW reset arrived, this will happen. Don't check cl->,295 * as 'cl' may be freed already296 */297 if (dev->dev_state != ISHTP_DEV_ENABLED) {298 dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n",299 __func__);300 return -ENODEV;301 }302 303 if (cl->state == ISHTP_CL_DISCONNECTED) {304 dev->print_log(dev, "%s() successful\n", __func__);305 return 0;306 }307 308 return -ENODEV;309}310EXPORT_SYMBOL(ishtp_cl_disconnect);311 312/**313 * ishtp_cl_is_other_connecting() - Check other client is connecting314 * @cl: client device instance315 *316 * Checks if other client with the same fw client id is connecting317 *318 * Return: true if other client is connected else false319 */320static bool ishtp_cl_is_other_connecting(struct ishtp_cl *cl)321{322 struct ishtp_device *dev;323 struct ishtp_cl *pos;324 unsigned long flags;325 326 if (WARN_ON(!cl || !cl->dev))327 return false;328 329 dev = cl->dev;330 spin_lock_irqsave(&dev->cl_list_lock, flags);331 list_for_each_entry(pos, &dev->cl_list, link) {332 if ((pos->state == ISHTP_CL_CONNECTING) && (pos != cl) &&333 cl->fw_client_id == pos->fw_client_id) {334 spin_unlock_irqrestore(&dev->cl_list_lock, flags);335 return true;336 }337 }338 spin_unlock_irqrestore(&dev->cl_list_lock, flags);339 340 return false;341}342 343/**344 * ishtp_cl_connect_to_fw() - Send connect request to firmware345 * @cl: client device instance346 *347 * Send a connect request to the firmware and wait for firmware response.348 * If there is successful connection response from the firmware, change349 * client state to ISHTP_CL_CONNECTED, and bind client to related350 * firmware client_id.351 *352 * Return: 0 for success and error code on failure353 */354static int ishtp_cl_connect_to_fw(struct ishtp_cl *cl)355{356 struct ishtp_device *dev;357 int rets;358 359 if (WARN_ON(!cl || !cl->dev))360 return -ENODEV;361 362 dev = cl->dev;363 364 if (ishtp_cl_is_other_connecting(cl)) {365 dev->print_log(dev, "%s() Busy\n", __func__);366 return -EBUSY;367 }368 369 if (ishtp_hbm_cl_connect_req(dev, cl)) {370 dev->print_log(dev, "%s() HBM connect req fail\n", __func__);371 return -ENODEV;372 }373 374 rets = wait_event_interruptible_timeout(cl->wait_ctrl_res,375 (dev->dev_state == ISHTP_DEV_ENABLED &&376 (cl->state == ISHTP_CL_CONNECTED ||377 cl->state == ISHTP_CL_DISCONNECTED)),378 ishtp_secs_to_jiffies(379 ISHTP_CL_CONNECT_TIMEOUT));380 /*381 * If FW reset arrived, this will happen. Don't check cl->,382 * as 'cl' may be freed already383 */384 if (dev->dev_state != ISHTP_DEV_ENABLED) {385 dev->print_log(dev, "%s() dev_state != ISHTP_DEV_ENABLED\n",386 __func__);387 return -EFAULT;388 }389 390 if (cl->state != ISHTP_CL_CONNECTED) {391 dev->print_log(dev, "%s() state != ISHTP_CL_CONNECTED\n",392 __func__);393 return -EFAULT;394 }395 396 rets = cl->status;397 if (rets) {398 dev->print_log(dev, "%s() Invalid status\n", __func__);399 return rets;400 }401 402 rets = ishtp_cl_device_bind(cl);403 if (rets) {404 dev->print_log(dev, "%s() Bind error\n", __func__);405 ishtp_cl_disconnect(cl);406 return rets;407 }408 409 return rets;410}411 412/**413 * ishtp_cl_connect() - Build connection with firmware414 * @cl: client device instance415 *416 * Call ishtp_cl_connect_to_fw() to connect and bind to firmware. If successful,417 * allocate RX and TX ring buffers, and start flow control with firmware to418 * start communication.419 *420 * Return: 0 if there is successful connection to the firmware, allocate421 * ring buffers.422 */423int ishtp_cl_connect(struct ishtp_cl *cl)424{425 struct ishtp_device *dev;426 int rets;427 428 if (!cl || !cl->dev)429 return -ENODEV;430 431 dev = cl->dev;432 433 dev->print_log(dev, "%s() current_state = %d\n", __func__, cl->state);434 435 rets = ishtp_cl_connect_to_fw(cl);436 if (rets) {437 dev->print_log(dev, "%s() Connect to fw failed\n", __func__);438 return rets;439 }440 441 rets = ishtp_cl_alloc_rx_ring(cl);442 if (rets) {443 dev->print_log(dev, "%s() Alloc RX ring failed\n", __func__);444 /* if failed allocation, disconnect */445 ishtp_cl_disconnect(cl);446 return rets;447 }448 449 rets = ishtp_cl_alloc_tx_ring(cl);450 if (rets) {451 dev->print_log(dev, "%s() Alloc TX ring failed\n", __func__);452 /* if failed allocation, disconnect */453 ishtp_cl_free_rx_ring(cl);454 ishtp_cl_disconnect(cl);455 return rets;456 }457 458 /*459 * Upon successful connection and allocation, start flow-control.460 */461 rets = ishtp_cl_read_start(cl);462 463 return rets;464}465EXPORT_SYMBOL(ishtp_cl_connect);466 467/**468 * ishtp_cl_establish_connection() - Establish connection with the firmware469 * @cl: client device instance470 * @uuid: uuid of the client to search471 * @tx_size: TX ring buffer size472 * @rx_size: RX ring buffer size473 * @reset: true if called for reset connection, otherwise for first connection474 *475 * This is a helper function for client driver to build connection with firmware.476 * If it's first time connecting to the firmware, set reset to false, this477 * function will link client to bus, find client id and send connect request to478 * the firmware.479 *480 * If it's called for reset handler where client lost connection after481 * firmware reset, set reset to true, this function will reinit client state and482 * establish connection again. In this case, this function reuses current client483 * structure and ring buffers to avoid allocation failure and memory fragments.484 *485 * Return: 0 for successful connection with the firmware,486 * or error code on failure487 */488int ishtp_cl_establish_connection(struct ishtp_cl *cl, const guid_t *uuid,489 int tx_size, int rx_size, bool reset)490{491 struct ishtp_device *dev;492 struct ishtp_fw_client *fw_client;493 int rets;494 495 if (!cl || !cl->dev)496 return -ENODEV;497 498 dev = cl->dev;499 500 ishtp_set_connection_state(cl, ISHTP_CL_INITIALIZING);501 502 /* reinit ishtp_cl structure if call for reset */503 if (reset) {504 cl->host_client_id = 0;505 cl->fw_client_id = 0;506 cl->ishtp_flow_ctrl_creds = 0;507 cl->out_flow_ctrl_creds = 0;508 509 cl->last_tx_path = CL_TX_PATH_IPC;510 cl->last_dma_acked = 1;511 cl->last_dma_addr = NULL;512 cl->last_ipc_acked = 1;513 514 cl->sending = 0;515 cl->err_send_msg = 0;516 cl->err_send_fc = 0;517 518 cl->send_msg_cnt_ipc = 0;519 cl->send_msg_cnt_dma = 0;520 cl->recv_msg_cnt_ipc = 0;521 cl->recv_msg_cnt_dma = 0;522 cl->recv_msg_num_frags = 0;523 cl->ishtp_flow_ctrl_cnt = 0;524 cl->out_flow_ctrl_cnt = 0;525 }526 527 /* link to bus */528 rets = ishtp_cl_link(cl);529 if (rets) {530 dev->print_log(dev, "%s() ishtp_cl_link failed\n", __func__);531 return rets;532 }533 534 /* find firmware client */535 fw_client = ishtp_fw_cl_get_client(dev, uuid);536 if (!fw_client) {537 dev->print_log(dev,538 "%s() ish client uuid not found\n", __func__);539 return -ENOENT;540 }541 542 ishtp_set_tx_ring_size(cl, tx_size);543 ishtp_set_rx_ring_size(cl, rx_size);544 545 ishtp_cl_set_fw_client_id(cl, ishtp_get_fw_client_id(fw_client));546 547 ishtp_set_connection_state(cl, ISHTP_CL_CONNECTING);548 549 /*550 * For reset case, not allocate tx/rx ring buffer which are already551 * done in ishtp_cl_connect() during first connection.552 */553 if (reset) {554 rets = ishtp_cl_connect_to_fw(cl);555 if (!rets)556 rets = ishtp_cl_read_start(cl);557 else558 dev->print_log(dev,559 "%s() connect to fw failed\n", __func__);560 } else {561 rets = ishtp_cl_connect(cl);562 }563 564 return rets;565}566EXPORT_SYMBOL(ishtp_cl_establish_connection);567 568/**569 * ishtp_cl_destroy_connection() - Disconnect with the firmware570 * @cl: client device instance571 * @reset: true if called for firmware reset, false for normal disconnection572 *573 * This is a helper function for client driver to disconnect with firmware,574 * unlink to bus and flush message queue.575 */576void ishtp_cl_destroy_connection(struct ishtp_cl *cl, bool reset)577{578 if (!cl)579 return;580 581 if (reset) {582 /*583 * For reset case, connection is already lost during fw reset.584 * Just set state to DISCONNECTED is enough.585 */586 ishtp_set_connection_state(cl, ISHTP_CL_DISCONNECTED);587 } else {588 if (cl->state != ISHTP_CL_DISCONNECTED) {589 ishtp_set_connection_state(cl, ISHTP_CL_DISCONNECTING);590 ishtp_cl_disconnect(cl);591 }592 }593 594 ishtp_cl_unlink(cl);595 ishtp_cl_flush_queues(cl);596}597EXPORT_SYMBOL(ishtp_cl_destroy_connection);598 599/**600 * ishtp_cl_read_start() - Prepare to read client message601 * @cl: client device instance602 *603 * Get a free buffer from pool of free read buffers and add to read buffer604 * pool to add contents. Send a flow control request to firmware to be able605 * send next message.606 *607 * Return: 0 if successful or error code on failure608 */609int ishtp_cl_read_start(struct ishtp_cl *cl)610{611 struct ishtp_device *dev;612 struct ishtp_cl_rb *rb;613 int rets;614 int i;615 unsigned long flags;616 unsigned long dev_flags;617 618 if (WARN_ON(!cl || !cl->dev))619 return -ENODEV;620 621 dev = cl->dev;622 623 if (cl->state != ISHTP_CL_CONNECTED)624 return -ENODEV;625 626 if (dev->dev_state != ISHTP_DEV_ENABLED)627 return -ENODEV;628 629 i = ishtp_fw_cl_by_id(dev, cl->fw_client_id);630 if (i < 0) {631 dev_err(&cl->device->dev, "no such fw client %d\n",632 cl->fw_client_id);633 return -ENODEV;634 }635 636 /* The current rb is the head of the free rb list */637 spin_lock_irqsave(&cl->free_list_spinlock, flags);638 if (list_empty(&cl->free_rb_list.list)) {639 dev_warn(&cl->device->dev,640 "[ishtp-ish] Rx buffers pool is empty\n");641 rets = -ENOMEM;642 rb = NULL;643 spin_unlock_irqrestore(&cl->free_list_spinlock, flags);644 goto out;645 }646 rb = list_entry(cl->free_rb_list.list.next, struct ishtp_cl_rb, list);647 list_del_init(&rb->list);648 spin_unlock_irqrestore(&cl->free_list_spinlock, flags);649 650 rb->cl = cl;651 rb->buf_idx = 0;652 653 INIT_LIST_HEAD(&rb->list);654 rets = 0;655 656 /*657 * This must be BEFORE sending flow control -658 * response in ISR may come too fast...659 */660 spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);661 list_add_tail(&rb->list, &dev->read_list.list);662 spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);663 if (ishtp_hbm_cl_flow_control_req(dev, cl)) {664 rets = -ENODEV;665 goto out;666 }667out:668 /* if ishtp_hbm_cl_flow_control_req failed, return rb to free list */669 if (rets && rb) {670 spin_lock_irqsave(&dev->read_list_spinlock, dev_flags);671 list_del(&rb->list);672 spin_unlock_irqrestore(&dev->read_list_spinlock, dev_flags);673 674 spin_lock_irqsave(&cl->free_list_spinlock, flags);675 list_add_tail(&rb->list, &cl->free_rb_list.list);676 spin_unlock_irqrestore(&cl->free_list_spinlock, flags);677 }678 return rets;679}680 681/**682 * ishtp_cl_send() - Send a message to firmware683 * @cl: client device instance684 * @buf: message buffer685 * @length: length of message686 *687 * If the client is correct state to send message, this function gets a buffer688 * from tx ring buffers, copy the message data and call to send the message689 * using ishtp_cl_send_msg()690 *691 * Return: 0 if successful or error code on failure692 */693int ishtp_cl_send(struct ishtp_cl *cl, uint8_t *buf, size_t length)694{695 struct ishtp_device *dev;696 int id;697 struct ishtp_cl_tx_ring *cl_msg;698 int have_msg_to_send = 0;699 unsigned long tx_flags, tx_free_flags;700 701 if (WARN_ON(!cl || !cl->dev))702 return -ENODEV;703 704 dev = cl->dev;705 706 if (cl->state != ISHTP_CL_CONNECTED) {707 ++cl->err_send_msg;708 return -EPIPE;709 }710 711 if (dev->dev_state != ISHTP_DEV_ENABLED) {712 ++cl->err_send_msg;713 return -ENODEV;714 }715 716 /* Check if we have fw client device */717 id = ishtp_fw_cl_by_id(dev, cl->fw_client_id);718 if (id < 0) {719 ++cl->err_send_msg;720 return -ENOENT;721 }722 723 if (length > dev->fw_clients[id].props.max_msg_length) {724 ++cl->err_send_msg;725 return -EMSGSIZE;726 }727 728 /* No free bufs */729 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);730 if (list_empty(&cl->tx_free_list.list)) {731 spin_unlock_irqrestore(&cl->tx_free_list_spinlock,732 tx_free_flags);733 ++cl->err_send_msg;734 return -ENOMEM;735 }736 737 cl_msg = list_first_entry(&cl->tx_free_list.list,738 struct ishtp_cl_tx_ring, list);739 if (!cl_msg->send_buf.data) {740 spin_unlock_irqrestore(&cl->tx_free_list_spinlock,741 tx_free_flags);742 return -EIO;743 /* Should not happen, as free list is pre-allocated */744 }745 /*746 * This is safe, as 'length' is already checked for not exceeding747 * max ISHTP message size per client748 */749 list_del_init(&cl_msg->list);750 --cl->tx_ring_free_size;751 752 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);753 memcpy(cl_msg->send_buf.data, buf, length);754 cl_msg->send_buf.size = length;755 spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);756 have_msg_to_send = !list_empty(&cl->tx_list.list);757 list_add_tail(&cl_msg->list, &cl->tx_list.list);758 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);759 760 if (!have_msg_to_send && cl->ishtp_flow_ctrl_creds > 0)761 ishtp_cl_send_msg(dev, cl);762 763 return 0;764}765EXPORT_SYMBOL(ishtp_cl_send);766 767/**768 * ishtp_cl_read_complete() - read complete769 * @rb: Pointer to client request block770 *771 * If the message is completely received call ishtp_cl_bus_rx_event()772 * to process message773 */774static void ishtp_cl_read_complete(struct ishtp_cl_rb *rb)775{776 unsigned long flags;777 int schedule_work_flag = 0;778 struct ishtp_cl *cl = rb->cl;779 780 spin_lock_irqsave(&cl->in_process_spinlock, flags);781 /*782 * if in-process list is empty, then need to schedule783 * the processing thread784 */785 schedule_work_flag = list_empty(&cl->in_process_list.list);786 list_add_tail(&rb->list, &cl->in_process_list.list);787 spin_unlock_irqrestore(&cl->in_process_spinlock, flags);788 789 if (schedule_work_flag)790 ishtp_cl_bus_rx_event(cl->device);791}792 793/**794 * ipc_tx_send() - IPC tx send function795 * @prm: Pointer to client device instance796 *797 * Send message over IPC. Message will be split into fragments798 * if message size is bigger than IPC FIFO size, and all799 * fragments will be sent one by one.800 */801static void ipc_tx_send(void *prm)802{803 struct ishtp_cl *cl = prm;804 struct ishtp_cl_tx_ring *cl_msg;805 size_t rem;806 struct ishtp_device *dev = (cl ? cl->dev : NULL);807 struct ishtp_msg_hdr ishtp_hdr;808 unsigned long tx_flags, tx_free_flags;809 unsigned char *pmsg;810 811 if (!dev)812 return;813 814 /*815 * Other conditions if some critical error has816 * occurred before this callback is called817 */818 if (dev->dev_state != ISHTP_DEV_ENABLED)819 return;820 821 if (cl->state != ISHTP_CL_CONNECTED)822 return;823 824 spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);825 if (list_empty(&cl->tx_list.list)) {826 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);827 return;828 }829 830 if (cl->ishtp_flow_ctrl_creds != 1 && !cl->sending) {831 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);832 return;833 }834 835 if (!cl->sending) {836 --cl->ishtp_flow_ctrl_creds;837 cl->last_ipc_acked = 0;838 cl->last_tx_path = CL_TX_PATH_IPC;839 cl->sending = 1;840 }841 842 cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring,843 list);844 rem = cl_msg->send_buf.size - cl->tx_offs;845 846 while (rem > 0) {847 ishtp_hdr.host_addr = cl->host_client_id;848 ishtp_hdr.fw_addr = cl->fw_client_id;849 ishtp_hdr.reserved = 0;850 pmsg = cl_msg->send_buf.data + cl->tx_offs;851 852 if (rem <= dev->mtu) {853 /* Last fragment or only one packet */854 ishtp_hdr.length = rem;855 ishtp_hdr.msg_complete = 1;856 /* Submit to IPC queue with no callback */857 ishtp_write_message(dev, &ishtp_hdr, pmsg);858 cl->tx_offs = 0;859 cl->sending = 0;860 861 break;862 } else {863 /* Send ipc fragment */864 ishtp_hdr.length = dev->mtu;865 ishtp_hdr.msg_complete = 0;866 /* All fregments submitted to IPC queue with no callback */867 ishtp_write_message(dev, &ishtp_hdr, pmsg);868 cl->tx_offs += dev->mtu;869 rem = cl_msg->send_buf.size - cl->tx_offs;870 }871 }872 873 list_del_init(&cl_msg->list);874 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);875 876 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);877 list_add_tail(&cl_msg->list, &cl->tx_free_list.list);878 ++cl->tx_ring_free_size;879 spin_unlock_irqrestore(&cl->tx_free_list_spinlock,880 tx_free_flags);881}882 883/**884 * ishtp_cl_send_msg_ipc() -Send message using IPC885 * @dev: ISHTP device instance886 * @cl: Pointer to client device instance887 *888 * Send message over IPC not using DMA889 */890static void ishtp_cl_send_msg_ipc(struct ishtp_device *dev,891 struct ishtp_cl *cl)892{893 /* If last DMA message wasn't acked yet, leave this one in Tx queue */894 if (cl->last_tx_path == CL_TX_PATH_DMA && cl->last_dma_acked == 0)895 return;896 897 cl->tx_offs = 0;898 ipc_tx_send(cl);899 ++cl->send_msg_cnt_ipc;900}901 902/**903 * ishtp_cl_send_msg_dma() -Send message using DMA904 * @dev: ISHTP device instance905 * @cl: Pointer to client device instance906 *907 * Send message using DMA908 */909static void ishtp_cl_send_msg_dma(struct ishtp_device *dev,910 struct ishtp_cl *cl)911{912 struct ishtp_msg_hdr hdr;913 struct dma_xfer_hbm dma_xfer;914 unsigned char *msg_addr;915 int off;916 struct ishtp_cl_tx_ring *cl_msg;917 unsigned long tx_flags, tx_free_flags;918 919 /* If last IPC message wasn't acked yet, leave this one in Tx queue */920 if (cl->last_tx_path == CL_TX_PATH_IPC && cl->last_ipc_acked == 0)921 return;922 923 spin_lock_irqsave(&cl->tx_list_spinlock, tx_flags);924 if (list_empty(&cl->tx_list.list)) {925 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);926 return;927 }928 929 cl_msg = list_entry(cl->tx_list.list.next, struct ishtp_cl_tx_ring,930 list);931 932 msg_addr = ishtp_cl_get_dma_send_buf(dev, cl_msg->send_buf.size);933 if (!msg_addr) {934 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);935 if (dev->transfer_path == CL_TX_PATH_DEFAULT)936 ishtp_cl_send_msg_ipc(dev, cl);937 return;938 }939 940 list_del_init(&cl_msg->list); /* Must be before write */941 spin_unlock_irqrestore(&cl->tx_list_spinlock, tx_flags);942 943 --cl->ishtp_flow_ctrl_creds;944 cl->last_dma_acked = 0;945 cl->last_dma_addr = msg_addr;946 cl->last_tx_path = CL_TX_PATH_DMA;947 948 /* write msg to dma buf */949 memcpy(msg_addr, cl_msg->send_buf.data, cl_msg->send_buf.size);950 951 /*952 * if current fw don't support cache snooping, driver have to953 * flush the cache manually.954 */955 if (dev->ops->dma_no_cache_snooping &&956 dev->ops->dma_no_cache_snooping(dev))957 clflush_cache_range(msg_addr, cl_msg->send_buf.size);958 959 /* send dma_xfer hbm msg */960 off = msg_addr - (unsigned char *)dev->ishtp_host_dma_tx_buf;961 ishtp_hbm_hdr(&hdr, sizeof(struct dma_xfer_hbm));962 dma_xfer.hbm = DMA_XFER;963 dma_xfer.fw_client_id = cl->fw_client_id;964 dma_xfer.host_client_id = cl->host_client_id;965 dma_xfer.reserved = 0;966 dma_xfer.msg_addr = dev->ishtp_host_dma_tx_buf_phys + off;967 dma_xfer.msg_length = cl_msg->send_buf.size;968 dma_xfer.reserved2 = 0;969 ishtp_write_message(dev, &hdr, (unsigned char *)&dma_xfer);970 spin_lock_irqsave(&cl->tx_free_list_spinlock, tx_free_flags);971 list_add_tail(&cl_msg->list, &cl->tx_free_list.list);972 ++cl->tx_ring_free_size;973 spin_unlock_irqrestore(&cl->tx_free_list_spinlock, tx_free_flags);974 ++cl->send_msg_cnt_dma;975}976 977/**978 * ishtp_cl_send_msg() -Send message using DMA or IPC979 * @dev: ISHTP device instance980 * @cl: Pointer to client device instance981 *982 * Send message using DMA or IPC based on transfer_path983 */984void ishtp_cl_send_msg(struct ishtp_device *dev, struct ishtp_cl *cl)985{986 if (dev->transfer_path == CL_TX_PATH_DMA)987 ishtp_cl_send_msg_dma(dev, cl);988 else989 ishtp_cl_send_msg_ipc(dev, cl);990}991 992/**993 * recv_ishtp_cl_msg() -Receive client message994 * @dev: ISHTP device instance995 * @ishtp_hdr: Pointer to message header996 *997 * Receive and dispatch ISHTP client messages. This function executes in ISR998 * or work queue context999 */1000void recv_ishtp_cl_msg(struct ishtp_device *dev,1001 struct ishtp_msg_hdr *ishtp_hdr)1002{1003 struct ishtp_cl *cl;1004 struct ishtp_cl_rb *rb;1005 struct ishtp_cl_rb *new_rb;1006 unsigned char *buffer = NULL;1007 struct ishtp_cl_rb *complete_rb = NULL;1008 unsigned long flags;1009 1010 if (ishtp_hdr->reserved) {1011 dev_err(dev->devc, "corrupted message header.\n");1012 goto eoi;1013 }1014 1015 if (ishtp_hdr->length > IPC_PAYLOAD_SIZE) {1016 dev_err(dev->devc,1017 "ISHTP message length in hdr exceeds IPC MTU\n");1018 goto eoi;1019 }1020 1021 spin_lock_irqsave(&dev->read_list_spinlock, flags);1022 list_for_each_entry(rb, &dev->read_list.list, list) {1023 cl = rb->cl;1024 if (!cl || !(cl->host_client_id == ishtp_hdr->host_addr &&1025 cl->fw_client_id == ishtp_hdr->fw_addr) ||1026 !(cl->state == ISHTP_CL_CONNECTED))1027 continue;1028 1029 /* If no Rx buffer is allocated, disband the rb */1030 if (rb->buffer.size == 0 || rb->buffer.data == NULL) {1031 spin_unlock_irqrestore(&dev->read_list_spinlock, flags);1032 dev_err(&cl->device->dev,1033 "Rx buffer is not allocated.\n");1034 list_del(&rb->list);1035 ishtp_io_rb_free(rb);1036 cl->status = -ENOMEM;1037 goto eoi;1038 }1039 1040 /*1041 * If message buffer overflown (exceeds max. client msg1042 * size, drop message and return to free buffer.1043 * Do we need to disconnect such a client? (We don't send1044 * back FC, so communication will be stuck anyway)1045 */1046 if (rb->buffer.size < ishtp_hdr->length + rb->buf_idx) {1047 spin_unlock_irqrestore(&dev->read_list_spinlock, flags);1048 dev_err(&cl->device->dev,1049 "message overflow. size %d len %d idx %ld\n",1050 rb->buffer.size, ishtp_hdr->length,1051 rb->buf_idx);1052 list_del(&rb->list);1053 ishtp_cl_io_rb_recycle(rb);1054 cl->status = -EIO;1055 goto eoi;1056 }1057 1058 buffer = rb->buffer.data + rb->buf_idx;1059 dev->ops->ishtp_read(dev, buffer, ishtp_hdr->length);1060 1061 rb->buf_idx += ishtp_hdr->length;1062 if (ishtp_hdr->msg_complete) {1063 /* Last fragment in message - it's complete */1064 cl->status = 0;1065 list_del(&rb->list);1066 complete_rb = rb;1067 1068 --cl->out_flow_ctrl_creds;1069 /*1070 * the whole msg arrived, send a new FC, and add a new1071 * rb buffer for the next coming msg1072 */1073 spin_lock(&cl->free_list_spinlock);1074 1075 if (!list_empty(&cl->free_rb_list.list)) {1076 new_rb = list_entry(cl->free_rb_list.list.next,1077 struct ishtp_cl_rb, list);1078 list_del_init(&new_rb->list);1079 spin_unlock(&cl->free_list_spinlock);1080 new_rb->cl = cl;1081 new_rb->buf_idx = 0;1082 INIT_LIST_HEAD(&new_rb->list);1083 list_add_tail(&new_rb->list,1084 &dev->read_list.list);1085 1086 ishtp_hbm_cl_flow_control_req(dev, cl);1087 } else {1088 spin_unlock(&cl->free_list_spinlock);1089 }1090 }1091 /* One more fragment in message (even if this was last) */1092 ++cl->recv_msg_num_frags;1093 1094 /*1095 * We can safely break here (and in BH too),1096 * a single input message can go only to a single request!1097 */1098 break;1099 }1100 1101 spin_unlock_irqrestore(&dev->read_list_spinlock, flags);1102 /* If it's nobody's message, just read and discard it */1103 if (!buffer) {1104 uint8_t rd_msg_buf[ISHTP_RD_MSG_BUF_SIZE];1105 1106 dev_err(dev->devc, "Dropped Rx msg - no request\n");1107 dev->ops->ishtp_read(dev, rd_msg_buf, ishtp_hdr->length);1108 goto eoi;1109 }1110 1111 if (complete_rb) {1112 cl = complete_rb->cl;1113 cl->ts_rx = ktime_get();1114 ++cl->recv_msg_cnt_ipc;1115 ishtp_cl_read_complete(complete_rb);1116 }1117eoi:1118 return;1119}1120 1121/**1122 * recv_ishtp_cl_msg_dma() -Receive client message1123 * @dev: ISHTP device instance1124 * @msg: message pointer1125 * @hbm: hbm buffer1126 *1127 * Receive and dispatch ISHTP client messages using DMA. This function executes1128 * in ISR or work queue context1129 */1130void recv_ishtp_cl_msg_dma(struct ishtp_device *dev, void *msg,1131 struct dma_xfer_hbm *hbm)1132{1133 struct ishtp_cl *cl;1134 struct ishtp_cl_rb *rb;1135 struct ishtp_cl_rb *new_rb;1136 unsigned char *buffer = NULL;1137 struct ishtp_cl_rb *complete_rb = NULL;1138 unsigned long flags;1139 1140 spin_lock_irqsave(&dev->read_list_spinlock, flags);1141 1142 list_for_each_entry(rb, &dev->read_list.list, list) {1143 cl = rb->cl;1144 if (!cl || !(cl->host_client_id == hbm->host_client_id &&1145 cl->fw_client_id == hbm->fw_client_id) ||1146 !(cl->state == ISHTP_CL_CONNECTED))1147 continue;1148 1149 /*1150 * If no Rx buffer is allocated, disband the rb1151 */1152 if (rb->buffer.size == 0 || rb->buffer.data == NULL) {1153 spin_unlock_irqrestore(&dev->read_list_spinlock, flags);1154 dev_err(&cl->device->dev,1155 "response buffer is not allocated.\n");1156 list_del(&rb->list);1157 ishtp_io_rb_free(rb);1158 cl->status = -ENOMEM;1159 goto eoi;1160 }1161 1162 /*1163 * If message buffer overflown (exceeds max. client msg1164 * size, drop message and return to free buffer.1165 * Do we need to disconnect such a client? (We don't send1166 * back FC, so communication will be stuck anyway)1167 */1168 if (rb->buffer.size < hbm->msg_length) {1169 spin_unlock_irqrestore(&dev->read_list_spinlock, flags);1170 dev_err(&cl->device->dev,1171 "message overflow. size %d len %d idx %ld\n",1172 rb->buffer.size, hbm->msg_length, rb->buf_idx);1173 list_del(&rb->list);1174 ishtp_cl_io_rb_recycle(rb);1175 cl->status = -EIO;1176 goto eoi;1177 }1178 1179 buffer = rb->buffer.data;1180 1181 /*1182 * if current fw don't support cache snooping, driver have to1183 * flush the cache manually.1184 */1185 if (dev->ops->dma_no_cache_snooping &&1186 dev->ops->dma_no_cache_snooping(dev))1187 clflush_cache_range(msg, hbm->msg_length);1188 1189 memcpy(buffer, msg, hbm->msg_length);1190 rb->buf_idx = hbm->msg_length;1191 1192 /* Last fragment in message - it's complete */1193 cl->status = 0;1194 list_del(&rb->list);1195 complete_rb = rb;1196 1197 --cl->out_flow_ctrl_creds;1198 /*1199 * the whole msg arrived, send a new FC, and add a new1200 * rb buffer for the next coming msg1201 */1202 spin_lock(&cl->free_list_spinlock);1203 1204 if (!list_empty(&cl->free_rb_list.list)) {1205 new_rb = list_entry(cl->free_rb_list.list.next,1206 struct ishtp_cl_rb, list);1207 list_del_init(&new_rb->list);1208 spin_unlock(&cl->free_list_spinlock);1209 new_rb->cl = cl;1210 new_rb->buf_idx = 0;1211 INIT_LIST_HEAD(&new_rb->list);1212 list_add_tail(&new_rb->list,1213 &dev->read_list.list);1214 1215 ishtp_hbm_cl_flow_control_req(dev, cl);1216 } else {1217 spin_unlock(&cl->free_list_spinlock);1218 }1219 1220 /* One more fragment in message (this is always last) */1221 ++cl->recv_msg_num_frags;1222 1223 /*1224 * We can safely break here (and in BH too),1225 * a single input message can go only to a single request!1226 */1227 break;1228 }1229 1230 spin_unlock_irqrestore(&dev->read_list_spinlock, flags);1231 /* If it's nobody's message, just read and discard it */1232 if (!buffer) {1233 dev_err(dev->devc, "Dropped Rx (DMA) msg - no request\n");1234 goto eoi;1235 }1236 1237 if (complete_rb) {1238 cl = complete_rb->cl;1239 cl->ts_rx = ktime_get();1240 ++cl->recv_msg_cnt_dma;1241 ishtp_cl_read_complete(complete_rb);1242 }1243eoi:1244 return;1245}1246 1247void *ishtp_get_client_data(struct ishtp_cl *cl)1248{1249 return cl->client_data;1250}1251EXPORT_SYMBOL(ishtp_get_client_data);1252 1253void ishtp_set_client_data(struct ishtp_cl *cl, void *data)1254{1255 cl->client_data = data;1256}1257EXPORT_SYMBOL(ishtp_set_client_data);1258 1259struct ishtp_device *ishtp_get_ishtp_device(struct ishtp_cl *cl)1260{1261 return cl->dev;1262}1263EXPORT_SYMBOL(ishtp_get_ishtp_device);1264 1265void ishtp_set_tx_ring_size(struct ishtp_cl *cl, int size)1266{1267 cl->tx_ring_size = size;1268}1269EXPORT_SYMBOL(ishtp_set_tx_ring_size);1270 1271void ishtp_set_rx_ring_size(struct ishtp_cl *cl, int size)1272{1273 cl->rx_ring_size = size;1274}1275EXPORT_SYMBOL(ishtp_set_rx_ring_size);1276 1277void ishtp_set_connection_state(struct ishtp_cl *cl, int state)1278{1279 cl->state = state;1280}1281EXPORT_SYMBOL(ishtp_set_connection_state);1282 1283void ishtp_cl_set_fw_client_id(struct ishtp_cl *cl, int fw_client_id)1284{1285 cl->fw_client_id = fw_client_id;1286}1287EXPORT_SYMBOL(ishtp_cl_set_fw_client_id);1288