1558 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2 3/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */4/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */5 6#include <asm/byteorder.h>7#include <linux/completion.h>8#include <linux/crc32.h>9#include <linux/delay.h>10#include <linux/dma-mapping.h>11#include <linux/kref.h>12#include <linux/list.h>13#include <linux/mhi.h>14#include <linux/mm.h>15#include <linux/moduleparam.h>16#include <linux/mutex.h>17#include <linux/overflow.h>18#include <linux/pci.h>19#include <linux/scatterlist.h>20#include <linux/types.h>21#include <linux/uaccess.h>22#include <linux/workqueue.h>23#include <linux/wait.h>24#include <drm/drm_device.h>25#include <drm/drm_file.h>26#include <uapi/drm/qaic_accel.h>27 28#include "qaic.h"29 30#define MANAGE_MAGIC_NUMBER ((__force __le32)0x43494151) /* "QAIC" in little endian */31#define QAIC_DBC_Q_GAP SZ_25632#define QAIC_DBC_Q_BUF_ALIGN SZ_4K33#define QAIC_MANAGE_EXT_MSG_LENGTH SZ_64K /* Max DMA message length */34#define QAIC_WRAPPER_MAX_SIZE SZ_4K35#define QAIC_MHI_RETRY_WAIT_MS 10036#define QAIC_MHI_RETRY_MAX 2037 38static unsigned int control_resp_timeout_s = 60; /* 60 sec default */39module_param(control_resp_timeout_s, uint, 0600);40MODULE_PARM_DESC(control_resp_timeout_s, "Timeout for NNC responses from QSM");41 42struct manage_msg {43 u32 len;44 u32 count;45 u8 data[];46};47 48/*49 * wire encoding structures for the manage protocol.50 * All fields are little endian on the wire51 */52struct wire_msg_hdr {53 __le32 crc32; /* crc of everything following this field in the message */54 __le32 magic_number;55 __le32 sequence_number;56 __le32 len; /* length of this message */57 __le32 count; /* number of transactions in this message */58 __le32 handle; /* unique id to track the resources consumed */59 __le32 partition_id; /* partition id for the request (signed) */60 __le32 padding; /* must be 0 */61} __packed;62 63struct wire_msg {64 struct wire_msg_hdr hdr;65 u8 data[];66} __packed;67 68struct wire_trans_hdr {69 __le32 type;70 __le32 len;71} __packed;72 73/* Each message sent from driver to device are organized in a list of wrapper_msg */74struct wrapper_msg {75 struct list_head list;76 struct kref ref_count;77 u32 len; /* length of data to transfer */78 struct wrapper_list *head;79 union {80 struct wire_msg msg;81 struct wire_trans_hdr trans;82 };83};84 85struct wrapper_list {86 struct list_head list;87 spinlock_t lock; /* Protects the list state during additions and removals */88};89 90struct wire_trans_passthrough {91 struct wire_trans_hdr hdr;92 u8 data[];93} __packed;94 95struct wire_addr_size_pair {96 __le64 addr;97 __le64 size;98} __packed;99 100struct wire_trans_dma_xfer {101 struct wire_trans_hdr hdr;102 __le32 tag;103 __le32 count;104 __le32 dma_chunk_id;105 __le32 padding;106 struct wire_addr_size_pair data[];107} __packed;108 109/* Initiated by device to continue the DMA xfer of a large piece of data */110struct wire_trans_dma_xfer_cont {111 struct wire_trans_hdr hdr;112 __le32 dma_chunk_id;113 __le32 padding;114 __le64 xferred_size;115} __packed;116 117struct wire_trans_activate_to_dev {118 struct wire_trans_hdr hdr;119 __le64 req_q_addr;120 __le64 rsp_q_addr;121 __le32 req_q_size;122 __le32 rsp_q_size;123 __le32 buf_len;124 __le32 options; /* unused, but BIT(16) has meaning to the device */125} __packed;126 127struct wire_trans_activate_from_dev {128 struct wire_trans_hdr hdr;129 __le32 status;130 __le32 dbc_id;131 __le64 options; /* unused */132} __packed;133 134struct wire_trans_deactivate_from_dev {135 struct wire_trans_hdr hdr;136 __le32 status;137 __le32 dbc_id;138} __packed;139 140struct wire_trans_terminate_to_dev {141 struct wire_trans_hdr hdr;142 __le32 handle;143 __le32 padding;144} __packed;145 146struct wire_trans_terminate_from_dev {147 struct wire_trans_hdr hdr;148 __le32 status;149 __le32 padding;150} __packed;151 152struct wire_trans_status_to_dev {153 struct wire_trans_hdr hdr;154} __packed;155 156struct wire_trans_status_from_dev {157 struct wire_trans_hdr hdr;158 __le16 major;159 __le16 minor;160 __le32 status;161 __le64 status_flags;162} __packed;163 164struct wire_trans_validate_part_to_dev {165 struct wire_trans_hdr hdr;166 __le32 part_id;167 __le32 padding;168} __packed;169 170struct wire_trans_validate_part_from_dev {171 struct wire_trans_hdr hdr;172 __le32 status;173 __le32 padding;174} __packed;175 176struct xfer_queue_elem {177 /*178 * Node in list of ongoing transfer request on control channel.179 * Maintained by root device struct.180 */181 struct list_head list;182 /* Sequence number of this transfer request */183 u32 seq_num;184 /* This is used to wait on until completion of transfer request */185 struct completion xfer_done;186 /* Received data from device */187 void *buf;188};189 190struct dma_xfer {191 /* Node in list of DMA transfers which is used for cleanup */192 struct list_head list;193 /* SG table of memory used for DMA */194 struct sg_table *sgt;195 /* Array pages used for DMA */196 struct page **page_list;197 /* Number of pages used for DMA */198 unsigned long nr_pages;199};200 201struct ioctl_resources {202 /* List of all DMA transfers which is used later for cleanup */203 struct list_head dma_xfers;204 /* Base address of request queue which belongs to a DBC */205 void *buf;206 /*207 * Base bus address of request queue which belongs to a DBC. Response208 * queue base bus address can be calculated by adding size of request209 * queue to base bus address of request queue.210 */211 dma_addr_t dma_addr;212 /* Total size of request queue and response queue in byte */213 u32 total_size;214 /* Total number of elements that can be queued in each of request and response queue */215 u32 nelem;216 /* Base address of response queue which belongs to a DBC */217 void *rsp_q_base;218 /* Status of the NNC message received */219 u32 status;220 /* DBC id of the DBC received from device */221 u32 dbc_id;222 /*223 * DMA transfer request messages can be big in size and it may not be224 * possible to send them in one shot. In such cases the messages are225 * broken into chunks, this field stores ID of such chunks.226 */227 u32 dma_chunk_id;228 /* Total number of bytes transferred for a DMA xfer request */229 u64 xferred_dma_size;230 /* Header of transaction message received from user. Used during DMA xfer request. */231 void *trans_hdr;232};233 234struct resp_work {235 struct work_struct work;236 struct qaic_device *qdev;237 void *buf;238};239 240/*241 * Since we're working with little endian messages, its useful to be able to242 * increment without filling a whole line with conversions back and forth just243 * to add one(1) to a message count.244 */245static __le32 incr_le32(__le32 val)246{247 return cpu_to_le32(le32_to_cpu(val) + 1);248}249 250static u32 gen_crc(void *msg)251{252 struct wrapper_list *wrappers = msg;253 struct wrapper_msg *w;254 u32 crc = ~0;255 256 list_for_each_entry(w, &wrappers->list, list)257 crc = crc32(crc, &w->msg, w->len);258 259 return crc ^ ~0;260}261 262static u32 gen_crc_stub(void *msg)263{264 return 0;265}266 267static bool valid_crc(void *msg)268{269 struct wire_msg_hdr *hdr = msg;270 bool ret;271 u32 crc;272 273 /*274 * The output of this algorithm is always converted to the native275 * endianness.276 */277 crc = le32_to_cpu(hdr->crc32);278 hdr->crc32 = 0;279 ret = (crc32(~0, msg, le32_to_cpu(hdr->len)) ^ ~0) == crc;280 hdr->crc32 = cpu_to_le32(crc);281 return ret;282}283 284static bool valid_crc_stub(void *msg)285{286 return true;287}288 289static void free_wrapper(struct kref *ref)290{291 struct wrapper_msg *wrapper = container_of(ref, struct wrapper_msg, ref_count);292 293 list_del(&wrapper->list);294 kfree(wrapper);295}296 297static void save_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources,298 struct qaic_user *usr)299{300 u32 dbc_id = resources->dbc_id;301 302 if (resources->buf) {303 wait_event_interruptible(qdev->dbc[dbc_id].dbc_release, !qdev->dbc[dbc_id].in_use);304 qdev->dbc[dbc_id].req_q_base = resources->buf;305 qdev->dbc[dbc_id].rsp_q_base = resources->rsp_q_base;306 qdev->dbc[dbc_id].dma_addr = resources->dma_addr;307 qdev->dbc[dbc_id].total_size = resources->total_size;308 qdev->dbc[dbc_id].nelem = resources->nelem;309 enable_dbc(qdev, dbc_id, usr);310 qdev->dbc[dbc_id].in_use = true;311 resources->buf = NULL;312 }313}314 315static void free_dbc_buf(struct qaic_device *qdev, struct ioctl_resources *resources)316{317 if (resources->buf)318 dma_free_coherent(&qdev->pdev->dev, resources->total_size, resources->buf,319 resources->dma_addr);320 resources->buf = NULL;321}322 323static void free_dma_xfers(struct qaic_device *qdev, struct ioctl_resources *resources)324{325 struct dma_xfer *xfer;326 struct dma_xfer *x;327 int i;328 329 list_for_each_entry_safe(xfer, x, &resources->dma_xfers, list) {330 dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);331 sg_free_table(xfer->sgt);332 kfree(xfer->sgt);333 for (i = 0; i < xfer->nr_pages; ++i)334 put_page(xfer->page_list[i]);335 kfree(xfer->page_list);336 list_del(&xfer->list);337 kfree(xfer);338 }339}340 341static struct wrapper_msg *add_wrapper(struct wrapper_list *wrappers, u32 size)342{343 struct wrapper_msg *w = kzalloc(size, GFP_KERNEL);344 345 if (!w)346 return NULL;347 list_add_tail(&w->list, &wrappers->list);348 kref_init(&w->ref_count);349 w->head = wrappers;350 return w;351}352 353static int encode_passthrough(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,354 u32 *user_len)355{356 struct qaic_manage_trans_passthrough *in_trans = trans;357 struct wire_trans_passthrough *out_trans;358 struct wrapper_msg *trans_wrapper;359 struct wrapper_msg *wrapper;360 struct wire_msg *msg;361 u32 msg_hdr_len;362 363 wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);364 msg = &wrapper->msg;365 msg_hdr_len = le32_to_cpu(msg->hdr.len);366 367 if (in_trans->hdr.len % 8 != 0)368 return -EINVAL;369 370 if (size_add(msg_hdr_len, in_trans->hdr.len) > QAIC_MANAGE_EXT_MSG_LENGTH)371 return -ENOSPC;372 373 trans_wrapper = add_wrapper(wrappers,374 offsetof(struct wrapper_msg, trans) + in_trans->hdr.len);375 if (!trans_wrapper)376 return -ENOMEM;377 trans_wrapper->len = in_trans->hdr.len;378 out_trans = (struct wire_trans_passthrough *)&trans_wrapper->trans;379 380 memcpy(out_trans->data, in_trans->data, in_trans->hdr.len - sizeof(in_trans->hdr));381 msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);382 msg->hdr.count = incr_le32(msg->hdr.count);383 *user_len += in_trans->hdr.len;384 out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_PASSTHROUGH_TO_DEV);385 out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);386 387 return 0;388}389 390/* returns error code for failure, 0 if enough pages alloc'd, 1 if dma_cont is needed */391static int find_and_map_user_pages(struct qaic_device *qdev,392 struct qaic_manage_trans_dma_xfer *in_trans,393 struct ioctl_resources *resources, struct dma_xfer *xfer)394{395 u64 xfer_start_addr, remaining, end, total;396 unsigned long need_pages;397 struct page **page_list;398 unsigned long nr_pages;399 struct sg_table *sgt;400 int ret;401 int i;402 403 if (check_add_overflow(in_trans->addr, resources->xferred_dma_size, &xfer_start_addr))404 return -EINVAL;405 406 if (in_trans->size < resources->xferred_dma_size)407 return -EINVAL;408 remaining = in_trans->size - resources->xferred_dma_size;409 if (remaining == 0)410 return 0;411 412 if (check_add_overflow(xfer_start_addr, remaining, &end))413 return -EINVAL;414 415 total = remaining + offset_in_page(xfer_start_addr);416 if (total >= SIZE_MAX)417 return -EINVAL;418 419 need_pages = DIV_ROUND_UP(total, PAGE_SIZE);420 421 nr_pages = need_pages;422 423 while (1) {424 page_list = kmalloc_array(nr_pages, sizeof(*page_list), GFP_KERNEL | __GFP_NOWARN);425 if (!page_list) {426 nr_pages = nr_pages / 2;427 if (!nr_pages)428 return -ENOMEM;429 } else {430 break;431 }432 }433 434 ret = get_user_pages_fast(xfer_start_addr, nr_pages, 0, page_list);435 if (ret < 0)436 goto free_page_list;437 if (ret != nr_pages) {438 nr_pages = ret;439 ret = -EFAULT;440 goto put_pages;441 }442 443 sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);444 if (!sgt) {445 ret = -ENOMEM;446 goto put_pages;447 }448 449 ret = sg_alloc_table_from_pages(sgt, page_list, nr_pages,450 offset_in_page(xfer_start_addr),451 remaining, GFP_KERNEL);452 if (ret) {453 ret = -ENOMEM;454 goto free_sgt;455 }456 457 ret = dma_map_sgtable(&qdev->pdev->dev, sgt, DMA_TO_DEVICE, 0);458 if (ret)459 goto free_table;460 461 xfer->sgt = sgt;462 xfer->page_list = page_list;463 xfer->nr_pages = nr_pages;464 465 return need_pages > nr_pages ? 1 : 0;466 467free_table:468 sg_free_table(sgt);469free_sgt:470 kfree(sgt);471put_pages:472 for (i = 0; i < nr_pages; ++i)473 put_page(page_list[i]);474free_page_list:475 kfree(page_list);476 return ret;477}478 479/* returns error code for failure, 0 if everything was encoded, 1 if dma_cont is needed */480static int encode_addr_size_pairs(struct dma_xfer *xfer, struct wrapper_list *wrappers,481 struct ioctl_resources *resources, u32 msg_hdr_len, u32 *size,482 struct wire_trans_dma_xfer **out_trans)483{484 struct wrapper_msg *trans_wrapper;485 struct sg_table *sgt = xfer->sgt;486 struct wire_addr_size_pair *asp;487 struct scatterlist *sg;488 struct wrapper_msg *w;489 unsigned int dma_len;490 u64 dma_chunk_len;491 void *boundary;492 int nents_dma;493 int nents;494 int i;495 496 nents = sgt->nents;497 nents_dma = nents;498 *size = QAIC_MANAGE_EXT_MSG_LENGTH - msg_hdr_len - sizeof(**out_trans);499 for_each_sgtable_dma_sg(sgt, sg, i) {500 *size -= sizeof(*asp);501 /* Save 1K for possible follow-up transactions. */502 if (*size < SZ_1K) {503 nents_dma = i;504 break;505 }506 }507 508 trans_wrapper = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);509 if (!trans_wrapper)510 return -ENOMEM;511 *out_trans = (struct wire_trans_dma_xfer *)&trans_wrapper->trans;512 513 asp = (*out_trans)->data;514 boundary = (void *)trans_wrapper + QAIC_WRAPPER_MAX_SIZE;515 *size = 0;516 517 dma_len = 0;518 w = trans_wrapper;519 dma_chunk_len = 0;520 for_each_sg(sgt->sgl, sg, nents_dma, i) {521 asp->size = cpu_to_le64(dma_len);522 dma_chunk_len += dma_len;523 if (dma_len) {524 asp++;525 if ((void *)asp + sizeof(*asp) > boundary) {526 w->len = (void *)asp - (void *)&w->msg;527 *size += w->len;528 w = add_wrapper(wrappers, QAIC_WRAPPER_MAX_SIZE);529 if (!w)530 return -ENOMEM;531 boundary = (void *)w + QAIC_WRAPPER_MAX_SIZE;532 asp = (struct wire_addr_size_pair *)&w->msg;533 }534 }535 asp->addr = cpu_to_le64(sg_dma_address(sg));536 dma_len = sg_dma_len(sg);537 }538 /* finalize the last segment */539 asp->size = cpu_to_le64(dma_len);540 w->len = (void *)asp + sizeof(*asp) - (void *)&w->msg;541 *size += w->len;542 dma_chunk_len += dma_len;543 resources->xferred_dma_size += dma_chunk_len;544 545 return nents_dma < nents ? 1 : 0;546}547 548static void cleanup_xfer(struct qaic_device *qdev, struct dma_xfer *xfer)549{550 int i;551 552 dma_unmap_sgtable(&qdev->pdev->dev, xfer->sgt, DMA_TO_DEVICE, 0);553 sg_free_table(xfer->sgt);554 kfree(xfer->sgt);555 for (i = 0; i < xfer->nr_pages; ++i)556 put_page(xfer->page_list[i]);557 kfree(xfer->page_list);558}559 560static int encode_dma(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,561 u32 *user_len, struct ioctl_resources *resources, struct qaic_user *usr)562{563 struct qaic_manage_trans_dma_xfer *in_trans = trans;564 struct wire_trans_dma_xfer *out_trans;565 struct wrapper_msg *wrapper;566 struct dma_xfer *xfer;567 struct wire_msg *msg;568 bool need_cont_dma;569 u32 msg_hdr_len;570 u32 size;571 int ret;572 573 wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);574 msg = &wrapper->msg;575 msg_hdr_len = le32_to_cpu(msg->hdr.len);576 577 /* There should be enough space to hold at least one ASP entry. */578 if (size_add(msg_hdr_len, sizeof(*out_trans) + sizeof(struct wire_addr_size_pair)) >579 QAIC_MANAGE_EXT_MSG_LENGTH)580 return -ENOMEM;581 582 xfer = kmalloc(sizeof(*xfer), GFP_KERNEL);583 if (!xfer)584 return -ENOMEM;585 586 ret = find_and_map_user_pages(qdev, in_trans, resources, xfer);587 if (ret < 0)588 goto free_xfer;589 590 need_cont_dma = (bool)ret;591 592 ret = encode_addr_size_pairs(xfer, wrappers, resources, msg_hdr_len, &size, &out_trans);593 if (ret < 0)594 goto cleanup_xfer;595 596 need_cont_dma = need_cont_dma || (bool)ret;597 598 msg->hdr.len = cpu_to_le32(msg_hdr_len + size);599 msg->hdr.count = incr_le32(msg->hdr.count);600 601 out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);602 out_trans->hdr.len = cpu_to_le32(size);603 out_trans->tag = cpu_to_le32(in_trans->tag);604 out_trans->count = cpu_to_le32((size - sizeof(*out_trans)) /605 sizeof(struct wire_addr_size_pair));606 607 *user_len += in_trans->hdr.len;608 609 if (resources->dma_chunk_id) {610 out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);611 } else if (need_cont_dma) {612 while (resources->dma_chunk_id == 0)613 resources->dma_chunk_id = atomic_inc_return(&usr->chunk_id);614 615 out_trans->dma_chunk_id = cpu_to_le32(resources->dma_chunk_id);616 }617 resources->trans_hdr = trans;618 619 list_add(&xfer->list, &resources->dma_xfers);620 return 0;621 622cleanup_xfer:623 cleanup_xfer(qdev, xfer);624free_xfer:625 kfree(xfer);626 return ret;627}628 629static int encode_activate(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,630 u32 *user_len, struct ioctl_resources *resources)631{632 struct qaic_manage_trans_activate_to_dev *in_trans = trans;633 struct wire_trans_activate_to_dev *out_trans;634 struct wrapper_msg *trans_wrapper;635 struct wrapper_msg *wrapper;636 struct wire_msg *msg;637 dma_addr_t dma_addr;638 u32 msg_hdr_len;639 void *buf;640 u32 nelem;641 u32 size;642 int ret;643 644 wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);645 msg = &wrapper->msg;646 msg_hdr_len = le32_to_cpu(msg->hdr.len);647 648 if (size_add(msg_hdr_len, sizeof(*out_trans)) > QAIC_MANAGE_MAX_MSG_LENGTH)649 return -ENOSPC;650 651 if (!in_trans->queue_size)652 return -EINVAL;653 654 if (in_trans->pad)655 return -EINVAL;656 657 nelem = in_trans->queue_size;658 size = (get_dbc_req_elem_size() + get_dbc_rsp_elem_size()) * nelem;659 if (size / nelem != get_dbc_req_elem_size() + get_dbc_rsp_elem_size())660 return -EINVAL;661 662 if (size + QAIC_DBC_Q_GAP + QAIC_DBC_Q_BUF_ALIGN < size)663 return -EINVAL;664 665 size = ALIGN((size + QAIC_DBC_Q_GAP), QAIC_DBC_Q_BUF_ALIGN);666 667 buf = dma_alloc_coherent(&qdev->pdev->dev, size, &dma_addr, GFP_KERNEL);668 if (!buf)669 return -ENOMEM;670 671 trans_wrapper = add_wrapper(wrappers,672 offsetof(struct wrapper_msg, trans) + sizeof(*out_trans));673 if (!trans_wrapper) {674 ret = -ENOMEM;675 goto free_dma;676 }677 trans_wrapper->len = sizeof(*out_trans);678 out_trans = (struct wire_trans_activate_to_dev *)&trans_wrapper->trans;679 680 out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_ACTIVATE_TO_DEV);681 out_trans->hdr.len = cpu_to_le32(sizeof(*out_trans));682 out_trans->buf_len = cpu_to_le32(size);683 out_trans->req_q_addr = cpu_to_le64(dma_addr);684 out_trans->req_q_size = cpu_to_le32(nelem);685 out_trans->rsp_q_addr = cpu_to_le64(dma_addr + size - nelem * get_dbc_rsp_elem_size());686 out_trans->rsp_q_size = cpu_to_le32(nelem);687 out_trans->options = cpu_to_le32(in_trans->options);688 689 *user_len += in_trans->hdr.len;690 msg->hdr.len = cpu_to_le32(msg_hdr_len + sizeof(*out_trans));691 msg->hdr.count = incr_le32(msg->hdr.count);692 693 resources->buf = buf;694 resources->dma_addr = dma_addr;695 resources->total_size = size;696 resources->nelem = nelem;697 resources->rsp_q_base = buf + size - nelem * get_dbc_rsp_elem_size();698 return 0;699 700free_dma:701 dma_free_coherent(&qdev->pdev->dev, size, buf, dma_addr);702 return ret;703}704 705static int encode_deactivate(struct qaic_device *qdev, void *trans,706 u32 *user_len, struct qaic_user *usr)707{708 struct qaic_manage_trans_deactivate *in_trans = trans;709 710 if (in_trans->dbc_id >= qdev->num_dbc || in_trans->pad)711 return -EINVAL;712 713 *user_len += in_trans->hdr.len;714 715 return disable_dbc(qdev, in_trans->dbc_id, usr);716}717 718static int encode_status(struct qaic_device *qdev, void *trans, struct wrapper_list *wrappers,719 u32 *user_len)720{721 struct qaic_manage_trans_status_to_dev *in_trans = trans;722 struct wire_trans_status_to_dev *out_trans;723 struct wrapper_msg *trans_wrapper;724 struct wrapper_msg *wrapper;725 struct wire_msg *msg;726 u32 msg_hdr_len;727 728 wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);729 msg = &wrapper->msg;730 msg_hdr_len = le32_to_cpu(msg->hdr.len);731 732 if (size_add(msg_hdr_len, in_trans->hdr.len) > QAIC_MANAGE_MAX_MSG_LENGTH)733 return -ENOSPC;734 735 trans_wrapper = add_wrapper(wrappers, sizeof(*trans_wrapper));736 if (!trans_wrapper)737 return -ENOMEM;738 739 trans_wrapper->len = sizeof(*out_trans);740 out_trans = (struct wire_trans_status_to_dev *)&trans_wrapper->trans;741 742 out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_STATUS_TO_DEV);743 out_trans->hdr.len = cpu_to_le32(in_trans->hdr.len);744 msg->hdr.len = cpu_to_le32(msg_hdr_len + in_trans->hdr.len);745 msg->hdr.count = incr_le32(msg->hdr.count);746 *user_len += in_trans->hdr.len;747 748 return 0;749}750 751static int encode_message(struct qaic_device *qdev, struct manage_msg *user_msg,752 struct wrapper_list *wrappers, struct ioctl_resources *resources,753 struct qaic_user *usr)754{755 struct qaic_manage_trans_hdr *trans_hdr;756 struct wrapper_msg *wrapper;757 struct wire_msg *msg;758 u32 user_len = 0;759 int ret;760 int i;761 762 if (!user_msg->count ||763 user_msg->len < sizeof(*trans_hdr)) {764 ret = -EINVAL;765 goto out;766 }767 768 wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);769 msg = &wrapper->msg;770 771 msg->hdr.len = cpu_to_le32(sizeof(msg->hdr));772 773 if (resources->dma_chunk_id) {774 ret = encode_dma(qdev, resources->trans_hdr, wrappers, &user_len, resources, usr);775 msg->hdr.count = cpu_to_le32(1);776 goto out;777 }778 779 for (i = 0; i < user_msg->count; ++i) {780 if (user_len > user_msg->len - sizeof(*trans_hdr)) {781 ret = -EINVAL;782 break;783 }784 trans_hdr = (struct qaic_manage_trans_hdr *)(user_msg->data + user_len);785 if (trans_hdr->len < sizeof(trans_hdr) ||786 size_add(user_len, trans_hdr->len) > user_msg->len) {787 ret = -EINVAL;788 break;789 }790 791 switch (trans_hdr->type) {792 case QAIC_TRANS_PASSTHROUGH_FROM_USR:793 ret = encode_passthrough(qdev, trans_hdr, wrappers, &user_len);794 break;795 case QAIC_TRANS_DMA_XFER_FROM_USR:796 ret = encode_dma(qdev, trans_hdr, wrappers, &user_len, resources, usr);797 break;798 case QAIC_TRANS_ACTIVATE_FROM_USR:799 ret = encode_activate(qdev, trans_hdr, wrappers, &user_len, resources);800 break;801 case QAIC_TRANS_DEACTIVATE_FROM_USR:802 ret = encode_deactivate(qdev, trans_hdr, &user_len, usr);803 break;804 case QAIC_TRANS_STATUS_FROM_USR:805 ret = encode_status(qdev, trans_hdr, wrappers, &user_len);806 break;807 default:808 ret = -EINVAL;809 break;810 }811 812 if (ret)813 break;814 }815 816 if (user_len != user_msg->len)817 ret = -EINVAL;818out:819 if (ret) {820 free_dma_xfers(qdev, resources);821 free_dbc_buf(qdev, resources);822 return ret;823 }824 825 return 0;826}827 828static int decode_passthrough(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,829 u32 *msg_len)830{831 struct qaic_manage_trans_passthrough *out_trans;832 struct wire_trans_passthrough *in_trans = trans;833 u32 len;834 835 out_trans = (void *)user_msg->data + user_msg->len;836 837 len = le32_to_cpu(in_trans->hdr.len);838 if (len % 8 != 0)839 return -EINVAL;840 841 if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)842 return -ENOSPC;843 844 memcpy(out_trans->data, in_trans->data, len - sizeof(in_trans->hdr));845 user_msg->len += len;846 *msg_len += len;847 out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);848 out_trans->hdr.len = len;849 850 return 0;851}852 853static int decode_activate(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,854 u32 *msg_len, struct ioctl_resources *resources, struct qaic_user *usr)855{856 struct qaic_manage_trans_activate_from_dev *out_trans;857 struct wire_trans_activate_from_dev *in_trans = trans;858 u32 len;859 860 out_trans = (void *)user_msg->data + user_msg->len;861 862 len = le32_to_cpu(in_trans->hdr.len);863 if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)864 return -ENOSPC;865 866 user_msg->len += len;867 *msg_len += len;868 out_trans->hdr.type = le32_to_cpu(in_trans->hdr.type);869 out_trans->hdr.len = len;870 out_trans->status = le32_to_cpu(in_trans->status);871 out_trans->dbc_id = le32_to_cpu(in_trans->dbc_id);872 out_trans->options = le64_to_cpu(in_trans->options);873 874 if (!resources->buf)875 /* how did we get an activate response without a request? */876 return -EINVAL;877 878 if (out_trans->dbc_id >= qdev->num_dbc)879 /*880 * The device assigned an invalid resource, which should never881 * happen. Return an error so the user can try to recover.882 */883 return -ENODEV;884 885 if (out_trans->status)886 /*887 * Allocating resources failed on device side. This is not an888 * expected behaviour, user is expected to handle this situation.889 */890 return -ECANCELED;891 892 resources->status = out_trans->status;893 resources->dbc_id = out_trans->dbc_id;894 save_dbc_buf(qdev, resources, usr);895 896 return 0;897}898 899static int decode_deactivate(struct qaic_device *qdev, void *trans, u32 *msg_len,900 struct qaic_user *usr)901{902 struct wire_trans_deactivate_from_dev *in_trans = trans;903 u32 dbc_id = le32_to_cpu(in_trans->dbc_id);904 u32 status = le32_to_cpu(in_trans->status);905 906 if (dbc_id >= qdev->num_dbc)907 /*908 * The device assigned an invalid resource, which should never909 * happen. Inject an error so the user can try to recover.910 */911 return -ENODEV;912 913 if (status) {914 /*915 * Releasing resources failed on the device side, which puts916 * us in a bind since they may still be in use, so enable the917 * dbc. User is expected to retry deactivation.918 */919 enable_dbc(qdev, dbc_id, usr);920 return -ECANCELED;921 }922 923 release_dbc(qdev, dbc_id);924 *msg_len += sizeof(*in_trans);925 926 return 0;927}928 929static int decode_status(struct qaic_device *qdev, void *trans, struct manage_msg *user_msg,930 u32 *user_len, struct wire_msg *msg)931{932 struct qaic_manage_trans_status_from_dev *out_trans;933 struct wire_trans_status_from_dev *in_trans = trans;934 u32 len;935 936 out_trans = (void *)user_msg->data + user_msg->len;937 938 len = le32_to_cpu(in_trans->hdr.len);939 if (user_msg->len + len > QAIC_MANAGE_MAX_MSG_LENGTH)940 return -ENOSPC;941 942 out_trans->hdr.type = QAIC_TRANS_STATUS_FROM_DEV;943 out_trans->hdr.len = len;944 out_trans->major = le16_to_cpu(in_trans->major);945 out_trans->minor = le16_to_cpu(in_trans->minor);946 out_trans->status_flags = le64_to_cpu(in_trans->status_flags);947 out_trans->status = le32_to_cpu(in_trans->status);948 *user_len += le32_to_cpu(in_trans->hdr.len);949 user_msg->len += len;950 951 if (out_trans->status)952 return -ECANCELED;953 if (out_trans->status_flags & BIT(0) && !valid_crc(msg))954 return -EPIPE;955 956 return 0;957}958 959static int decode_message(struct qaic_device *qdev, struct manage_msg *user_msg,960 struct wire_msg *msg, struct ioctl_resources *resources,961 struct qaic_user *usr)962{963 u32 msg_hdr_len = le32_to_cpu(msg->hdr.len);964 struct wire_trans_hdr *trans_hdr;965 u32 msg_len = 0;966 int ret;967 int i;968 969 if (msg_hdr_len < sizeof(*trans_hdr) ||970 msg_hdr_len > QAIC_MANAGE_MAX_MSG_LENGTH)971 return -EINVAL;972 973 user_msg->len = 0;974 user_msg->count = le32_to_cpu(msg->hdr.count);975 976 for (i = 0; i < user_msg->count; ++i) {977 u32 hdr_len;978 979 if (msg_len > msg_hdr_len - sizeof(*trans_hdr))980 return -EINVAL;981 982 trans_hdr = (struct wire_trans_hdr *)(msg->data + msg_len);983 hdr_len = le32_to_cpu(trans_hdr->len);984 if (hdr_len < sizeof(*trans_hdr) ||985 size_add(msg_len, hdr_len) > msg_hdr_len)986 return -EINVAL;987 988 switch (le32_to_cpu(trans_hdr->type)) {989 case QAIC_TRANS_PASSTHROUGH_FROM_DEV:990 ret = decode_passthrough(qdev, trans_hdr, user_msg, &msg_len);991 break;992 case QAIC_TRANS_ACTIVATE_FROM_DEV:993 ret = decode_activate(qdev, trans_hdr, user_msg, &msg_len, resources, usr);994 break;995 case QAIC_TRANS_DEACTIVATE_FROM_DEV:996 ret = decode_deactivate(qdev, trans_hdr, &msg_len, usr);997 break;998 case QAIC_TRANS_STATUS_FROM_DEV:999 ret = decode_status(qdev, trans_hdr, user_msg, &msg_len, msg);1000 break;1001 default:1002 return -EINVAL;1003 }1004 1005 if (ret)1006 return ret;1007 }1008 1009 if (msg_len != (msg_hdr_len - sizeof(msg->hdr)))1010 return -EINVAL;1011 1012 return 0;1013}1014 1015static void *msg_xfer(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 seq_num,1016 bool ignore_signal)1017{1018 struct xfer_queue_elem elem;1019 struct wire_msg *out_buf;1020 struct wrapper_msg *w;1021 long ret = -EAGAIN;1022 int xfer_count = 0;1023 int retry_count;1024 1025 /* Allow QAIC_BOOT state since we need to check control protocol version */1026 if (qdev->dev_state == QAIC_OFFLINE) {1027 mutex_unlock(&qdev->cntl_mutex);1028 return ERR_PTR(-ENODEV);1029 }1030 1031 /* Attempt to avoid a partial commit of a message */1032 list_for_each_entry(w, &wrappers->list, list)1033 xfer_count++;1034 1035 for (retry_count = 0; retry_count < QAIC_MHI_RETRY_MAX; retry_count++) {1036 if (xfer_count <= mhi_get_free_desc_count(qdev->cntl_ch, DMA_TO_DEVICE)) {1037 ret = 0;1038 break;1039 }1040 msleep_interruptible(QAIC_MHI_RETRY_WAIT_MS);1041 if (signal_pending(current))1042 break;1043 }1044 1045 if (ret) {1046 mutex_unlock(&qdev->cntl_mutex);1047 return ERR_PTR(ret);1048 }1049 1050 elem.seq_num = seq_num;1051 elem.buf = NULL;1052 init_completion(&elem.xfer_done);1053 if (likely(!qdev->cntl_lost_buf)) {1054 /*1055 * The max size of request to device is QAIC_MANAGE_EXT_MSG_LENGTH.1056 * The max size of response from device is QAIC_MANAGE_MAX_MSG_LENGTH.1057 */1058 out_buf = kmalloc(QAIC_MANAGE_MAX_MSG_LENGTH, GFP_KERNEL);1059 if (!out_buf) {1060 mutex_unlock(&qdev->cntl_mutex);1061 return ERR_PTR(-ENOMEM);1062 }1063 1064 ret = mhi_queue_buf(qdev->cntl_ch, DMA_FROM_DEVICE, out_buf,1065 QAIC_MANAGE_MAX_MSG_LENGTH, MHI_EOT);1066 if (ret) {1067 mutex_unlock(&qdev->cntl_mutex);1068 return ERR_PTR(ret);1069 }1070 } else {1071 /*1072 * we lost a buffer because we queued a recv buf, but then1073 * queuing the corresponding tx buf failed. To try to avoid1074 * a memory leak, lets reclaim it and use it for this1075 * transaction.1076 */1077 qdev->cntl_lost_buf = false;1078 }1079 1080 list_for_each_entry(w, &wrappers->list, list) {1081 kref_get(&w->ref_count);1082 retry_count = 0;1083 ret = mhi_queue_buf(qdev->cntl_ch, DMA_TO_DEVICE, &w->msg, w->len,1084 list_is_last(&w->list, &wrappers->list) ? MHI_EOT : MHI_CHAIN);1085 if (ret) {1086 qdev->cntl_lost_buf = true;1087 kref_put(&w->ref_count, free_wrapper);1088 mutex_unlock(&qdev->cntl_mutex);1089 return ERR_PTR(ret);1090 }1091 }1092 1093 list_add_tail(&elem.list, &qdev->cntl_xfer_list);1094 mutex_unlock(&qdev->cntl_mutex);1095 1096 if (ignore_signal)1097 ret = wait_for_completion_timeout(&elem.xfer_done, control_resp_timeout_s * HZ);1098 else1099 ret = wait_for_completion_interruptible_timeout(&elem.xfer_done,1100 control_resp_timeout_s * HZ);1101 /*1102 * not using _interruptable because we have to cleanup or we'll1103 * likely cause memory corruption1104 */1105 mutex_lock(&qdev->cntl_mutex);1106 if (!list_empty(&elem.list))1107 list_del(&elem.list);1108 if (!ret && !elem.buf)1109 ret = -ETIMEDOUT;1110 else if (ret > 0 && !elem.buf)1111 ret = -EIO;1112 mutex_unlock(&qdev->cntl_mutex);1113 1114 if (ret < 0) {1115 kfree(elem.buf);1116 return ERR_PTR(ret);1117 } else if (!qdev->valid_crc(elem.buf)) {1118 kfree(elem.buf);1119 return ERR_PTR(-EPIPE);1120 }1121 1122 return elem.buf;1123}1124 1125/* Add a transaction to abort the outstanding DMA continuation */1126static int abort_dma_cont(struct qaic_device *qdev, struct wrapper_list *wrappers, u32 dma_chunk_id)1127{1128 struct wire_trans_dma_xfer *out_trans;1129 u32 size = sizeof(*out_trans);1130 struct wrapper_msg *wrapper;1131 struct wrapper_msg *w;1132 struct wire_msg *msg;1133 1134 wrapper = list_first_entry(&wrappers->list, struct wrapper_msg, list);1135 msg = &wrapper->msg;1136 1137 /* Remove all but the first wrapper which has the msg header */1138 list_for_each_entry_safe(wrapper, w, &wrappers->list, list)1139 if (!list_is_first(&wrapper->list, &wrappers->list))1140 kref_put(&wrapper->ref_count, free_wrapper);1141 1142 wrapper = add_wrapper(wrappers, sizeof(*wrapper));1143 1144 if (!wrapper)1145 return -ENOMEM;1146 1147 out_trans = (struct wire_trans_dma_xfer *)&wrapper->trans;1148 out_trans->hdr.type = cpu_to_le32(QAIC_TRANS_DMA_XFER_TO_DEV);1149 out_trans->hdr.len = cpu_to_le32(size);1150 out_trans->tag = cpu_to_le32(0);1151 out_trans->count = cpu_to_le32(0);1152 out_trans->dma_chunk_id = cpu_to_le32(dma_chunk_id);1153 1154 msg->hdr.len = cpu_to_le32(size + sizeof(*msg));1155 msg->hdr.count = cpu_to_le32(1);1156 wrapper->len = size;1157 1158 return 0;1159}1160 1161static struct wrapper_list *alloc_wrapper_list(void)1162{1163 struct wrapper_list *wrappers;1164 1165 wrappers = kmalloc(sizeof(*wrappers), GFP_KERNEL);1166 if (!wrappers)1167 return NULL;1168 INIT_LIST_HEAD(&wrappers->list);1169 spin_lock_init(&wrappers->lock);1170 1171 return wrappers;1172}1173 1174static int qaic_manage_msg_xfer(struct qaic_device *qdev, struct qaic_user *usr,1175 struct manage_msg *user_msg, struct ioctl_resources *resources,1176 struct wire_msg **rsp)1177{1178 struct wrapper_list *wrappers;1179 struct wrapper_msg *wrapper;1180 struct wrapper_msg *w;1181 bool all_done = false;1182 struct wire_msg *msg;1183 int ret;1184 1185 wrappers = alloc_wrapper_list();1186 if (!wrappers)1187 return -ENOMEM;1188 1189 wrapper = add_wrapper(wrappers, sizeof(*wrapper));1190 if (!wrapper) {1191 kfree(wrappers);1192 return -ENOMEM;1193 }1194 1195 msg = &wrapper->msg;1196 wrapper->len = sizeof(*msg);1197 1198 ret = encode_message(qdev, user_msg, wrappers, resources, usr);1199 if (ret && resources->dma_chunk_id)1200 ret = abort_dma_cont(qdev, wrappers, resources->dma_chunk_id);1201 if (ret)1202 goto encode_failed;1203 1204 ret = mutex_lock_interruptible(&qdev->cntl_mutex);1205 if (ret)1206 goto lock_failed;1207 1208 msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;1209 msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);1210 1211 if (usr) {1212 msg->hdr.handle = cpu_to_le32(usr->handle);1213 msg->hdr.partition_id = cpu_to_le32(usr->qddev->partition_id);1214 } else {1215 msg->hdr.handle = 0;1216 msg->hdr.partition_id = cpu_to_le32(QAIC_NO_PARTITION);1217 }1218 1219 msg->hdr.padding = cpu_to_le32(0);1220 msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));1221 1222 /* msg_xfer releases the mutex */1223 *rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, false);1224 if (IS_ERR(*rsp))1225 ret = PTR_ERR(*rsp);1226 1227lock_failed:1228 free_dma_xfers(qdev, resources);1229encode_failed:1230 spin_lock(&wrappers->lock);1231 list_for_each_entry_safe(wrapper, w, &wrappers->list, list)1232 kref_put(&wrapper->ref_count, free_wrapper);1233 all_done = list_empty(&wrappers->list);1234 spin_unlock(&wrappers->lock);1235 if (all_done)1236 kfree(wrappers);1237 1238 return ret;1239}1240 1241static int qaic_manage(struct qaic_device *qdev, struct qaic_user *usr, struct manage_msg *user_msg)1242{1243 struct wire_trans_dma_xfer_cont *dma_cont = NULL;1244 struct ioctl_resources resources;1245 struct wire_msg *rsp = NULL;1246 int ret;1247 1248 memset(&resources, 0, sizeof(struct ioctl_resources));1249 1250 INIT_LIST_HEAD(&resources.dma_xfers);1251 1252 if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH ||1253 user_msg->count > QAIC_MANAGE_MAX_MSG_LENGTH / sizeof(struct qaic_manage_trans_hdr))1254 return -EINVAL;1255 1256dma_xfer_continue:1257 ret = qaic_manage_msg_xfer(qdev, usr, user_msg, &resources, &rsp);1258 if (ret)1259 return ret;1260 /* dma_cont should be the only transaction if present */1261 if (le32_to_cpu(rsp->hdr.count) == 1) {1262 dma_cont = (struct wire_trans_dma_xfer_cont *)rsp->data;1263 if (le32_to_cpu(dma_cont->hdr.type) != QAIC_TRANS_DMA_XFER_CONT)1264 dma_cont = NULL;1265 }1266 if (dma_cont) {1267 if (le32_to_cpu(dma_cont->dma_chunk_id) == resources.dma_chunk_id &&1268 le64_to_cpu(dma_cont->xferred_size) == resources.xferred_dma_size) {1269 kfree(rsp);1270 goto dma_xfer_continue;1271 }1272 1273 ret = -EINVAL;1274 goto dma_cont_failed;1275 }1276 1277 ret = decode_message(qdev, user_msg, rsp, &resources, usr);1278 1279dma_cont_failed:1280 free_dbc_buf(qdev, &resources);1281 kfree(rsp);1282 return ret;1283}1284 1285int qaic_manage_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)1286{1287 struct qaic_manage_msg *user_msg = data;1288 struct qaic_device *qdev;1289 struct manage_msg *msg;1290 struct qaic_user *usr;1291 u8 __user *user_data;1292 int qdev_rcu_id;1293 int usr_rcu_id;1294 int ret;1295 1296 if (user_msg->len > QAIC_MANAGE_MAX_MSG_LENGTH)1297 return -EINVAL;1298 1299 usr = file_priv->driver_priv;1300 1301 usr_rcu_id = srcu_read_lock(&usr->qddev_lock);1302 if (!usr->qddev) {1303 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);1304 return -ENODEV;1305 }1306 1307 qdev = usr->qddev->qdev;1308 1309 qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);1310 if (qdev->dev_state != QAIC_ONLINE) {1311 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);1312 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);1313 return -ENODEV;1314 }1315 1316 msg = kzalloc(QAIC_MANAGE_MAX_MSG_LENGTH + sizeof(*msg), GFP_KERNEL);1317 if (!msg) {1318 ret = -ENOMEM;1319 goto out;1320 }1321 1322 msg->len = user_msg->len;1323 msg->count = user_msg->count;1324 1325 user_data = u64_to_user_ptr(user_msg->data);1326 1327 if (copy_from_user(msg->data, user_data, user_msg->len)) {1328 ret = -EFAULT;1329 goto free_msg;1330 }1331 1332 ret = qaic_manage(qdev, usr, msg);1333 1334 /*1335 * If the qaic_manage() is successful then we copy the message onto1336 * userspace memory but we have an exception for -ECANCELED.1337 * For -ECANCELED, it means that device has NACKed the message with a1338 * status error code which userspace would like to know.1339 */1340 if (ret == -ECANCELED || !ret) {1341 if (copy_to_user(user_data, msg->data, msg->len)) {1342 ret = -EFAULT;1343 } else {1344 user_msg->len = msg->len;1345 user_msg->count = msg->count;1346 }1347 }1348 1349free_msg:1350 kfree(msg);1351out:1352 srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);1353 srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);1354 return ret;1355}1356 1357int get_cntl_version(struct qaic_device *qdev, struct qaic_user *usr, u16 *major, u16 *minor)1358{1359 struct qaic_manage_trans_status_from_dev *status_result;1360 struct qaic_manage_trans_status_to_dev *status_query;1361 struct manage_msg *user_msg;1362 int ret;1363 1364 user_msg = kmalloc(sizeof(*user_msg) + sizeof(*status_result), GFP_KERNEL);1365 if (!user_msg) {1366 ret = -ENOMEM;1367 goto out;1368 }1369 user_msg->len = sizeof(*status_query);1370 user_msg->count = 1;1371 1372 status_query = (struct qaic_manage_trans_status_to_dev *)user_msg->data;1373 status_query->hdr.type = QAIC_TRANS_STATUS_FROM_USR;1374 status_query->hdr.len = sizeof(status_query->hdr);1375 1376 ret = qaic_manage(qdev, usr, user_msg);1377 if (ret)1378 goto kfree_user_msg;1379 status_result = (struct qaic_manage_trans_status_from_dev *)user_msg->data;1380 *major = status_result->major;1381 *minor = status_result->minor;1382 1383 if (status_result->status_flags & BIT(0)) { /* device is using CRC */1384 /* By default qdev->gen_crc is programmed to generate CRC */1385 qdev->valid_crc = valid_crc;1386 } else {1387 /* By default qdev->valid_crc is programmed to bypass CRC */1388 qdev->gen_crc = gen_crc_stub;1389 }1390 1391kfree_user_msg:1392 kfree(user_msg);1393out:1394 return ret;1395}1396 1397static void resp_worker(struct work_struct *work)1398{1399 struct resp_work *resp = container_of(work, struct resp_work, work);1400 struct qaic_device *qdev = resp->qdev;1401 struct wire_msg *msg = resp->buf;1402 struct xfer_queue_elem *elem;1403 struct xfer_queue_elem *i;1404 bool found = false;1405 1406 mutex_lock(&qdev->cntl_mutex);1407 list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {1408 if (elem->seq_num == le32_to_cpu(msg->hdr.sequence_number)) {1409 found = true;1410 list_del_init(&elem->list);1411 elem->buf = msg;1412 complete_all(&elem->xfer_done);1413 break;1414 }1415 }1416 mutex_unlock(&qdev->cntl_mutex);1417 1418 if (!found)1419 /* request must have timed out, drop packet */1420 kfree(msg);1421 1422 kfree(resp);1423}1424 1425static void free_wrapper_from_list(struct wrapper_list *wrappers, struct wrapper_msg *wrapper)1426{1427 bool all_done = false;1428 1429 spin_lock(&wrappers->lock);1430 kref_put(&wrapper->ref_count, free_wrapper);1431 all_done = list_empty(&wrappers->list);1432 spin_unlock(&wrappers->lock);1433 1434 if (all_done)1435 kfree(wrappers);1436}1437 1438void qaic_mhi_ul_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)1439{1440 struct wire_msg *msg = mhi_result->buf_addr;1441 struct wrapper_msg *wrapper = container_of(msg, struct wrapper_msg, msg);1442 1443 free_wrapper_from_list(wrapper->head, wrapper);1444}1445 1446void qaic_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_result *mhi_result)1447{1448 struct qaic_device *qdev = dev_get_drvdata(&mhi_dev->dev);1449 struct wire_msg *msg = mhi_result->buf_addr;1450 struct resp_work *resp;1451 1452 if (mhi_result->transaction_status || msg->hdr.magic_number != MANAGE_MAGIC_NUMBER) {1453 kfree(msg);1454 return;1455 }1456 1457 resp = kmalloc(sizeof(*resp), GFP_ATOMIC);1458 if (!resp) {1459 kfree(msg);1460 return;1461 }1462 1463 INIT_WORK(&resp->work, resp_worker);1464 resp->qdev = qdev;1465 resp->buf = msg;1466 queue_work(qdev->cntl_wq, &resp->work);1467}1468 1469int qaic_control_open(struct qaic_device *qdev)1470{1471 if (!qdev->cntl_ch)1472 return -ENODEV;1473 1474 qdev->cntl_lost_buf = false;1475 /*1476 * By default qaic should assume that device has CRC enabled.1477 * Qaic comes to know if device has CRC enabled or disabled during the1478 * device status transaction, which is the first transaction performed1479 * on control channel.1480 *1481 * So CRC validation of first device status transaction response is1482 * ignored (by calling valid_crc_stub) and is done later during decoding1483 * if device has CRC enabled.1484 * Now that qaic knows whether device has CRC enabled or not it acts1485 * accordingly.1486 */1487 qdev->gen_crc = gen_crc;1488 qdev->valid_crc = valid_crc_stub;1489 1490 return mhi_prepare_for_transfer(qdev->cntl_ch);1491}1492 1493void qaic_control_close(struct qaic_device *qdev)1494{1495 mhi_unprepare_from_transfer(qdev->cntl_ch);1496}1497 1498void qaic_release_usr(struct qaic_device *qdev, struct qaic_user *usr)1499{1500 struct wire_trans_terminate_to_dev *trans;1501 struct wrapper_list *wrappers;1502 struct wrapper_msg *wrapper;1503 struct wire_msg *msg;1504 struct wire_msg *rsp;1505 1506 wrappers = alloc_wrapper_list();1507 if (!wrappers)1508 return;1509 1510 wrapper = add_wrapper(wrappers, sizeof(*wrapper) + sizeof(*msg) + sizeof(*trans));1511 if (!wrapper)1512 return;1513 1514 msg = &wrapper->msg;1515 1516 trans = (struct wire_trans_terminate_to_dev *)msg->data;1517 1518 trans->hdr.type = cpu_to_le32(QAIC_TRANS_TERMINATE_TO_DEV);1519 trans->hdr.len = cpu_to_le32(sizeof(*trans));1520 trans->handle = cpu_to_le32(usr->handle);1521 1522 mutex_lock(&qdev->cntl_mutex);1523 wrapper->len = sizeof(msg->hdr) + sizeof(*trans);1524 msg->hdr.magic_number = MANAGE_MAGIC_NUMBER;1525 msg->hdr.sequence_number = cpu_to_le32(qdev->next_seq_num++);1526 msg->hdr.len = cpu_to_le32(wrapper->len);1527 msg->hdr.count = cpu_to_le32(1);1528 msg->hdr.handle = cpu_to_le32(usr->handle);1529 msg->hdr.padding = cpu_to_le32(0);1530 msg->hdr.crc32 = cpu_to_le32(qdev->gen_crc(wrappers));1531 1532 /*1533 * msg_xfer releases the mutex1534 * We don't care about the return of msg_xfer since we will not do1535 * anything different based on what happens.1536 * We ignore pending signals since one will be set if the user is1537 * killed, and we need give the device a chance to cleanup, otherwise1538 * DMA may still be in progress when we return.1539 */1540 rsp = msg_xfer(qdev, wrappers, qdev->next_seq_num - 1, true);1541 if (!IS_ERR(rsp))1542 kfree(rsp);1543 free_wrapper_from_list(wrappers, wrapper);1544}1545 1546void wake_all_cntl(struct qaic_device *qdev)1547{1548 struct xfer_queue_elem *elem;1549 struct xfer_queue_elem *i;1550 1551 mutex_lock(&qdev->cntl_mutex);1552 list_for_each_entry_safe(elem, i, &qdev->cntl_xfer_list, list) {1553 list_del_init(&elem->list);1554 complete_all(&elem->xfer_done);1555 }1556 mutex_unlock(&qdev->cntl_mutex);1557}1558