1353 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright 2018-2020 Broadcom.4 */5 6#include <linux/delay.h>7#include <linux/fs.h>8#include <linux/hash.h>9#include <linux/interrupt.h>10#include <linux/list.h>11#include <linux/module.h>12#include <linux/poll.h>13#include <linux/sizes.h>14#include <linux/spinlock.h>15#include <linux/timer.h>16 17#include "bcm_vk.h"18#include "bcm_vk_msg.h"19#include "bcm_vk_sg.h"20 21/* functions to manipulate the transport id in msg block */22#define BCM_VK_MSG_Q_SHIFT 423#define BCM_VK_MSG_Q_MASK 0xF24#define BCM_VK_MSG_ID_MASK 0xFFF25 26#define BCM_VK_DMA_DRAIN_MAX_MS 200027 28/* number x q_size will be the max number of msg processed per loop */29#define BCM_VK_MSG_PROC_MAX_LOOP 230 31/* module parameter */32static bool hb_mon = true;33module_param(hb_mon, bool, 0444);34MODULE_PARM_DESC(hb_mon, "Monitoring heartbeat continuously.\n");35static int batch_log = 1;36module_param(batch_log, int, 0444);37MODULE_PARM_DESC(batch_log, "Max num of logs per batch operation.\n");38 39static bool hb_mon_is_on(void)40{41 return hb_mon;42}43 44static u32 get_q_num(const struct vk_msg_blk *msg)45{46 u32 q_num = msg->trans_id & BCM_VK_MSG_Q_MASK;47 48 if (q_num >= VK_MSGQ_PER_CHAN_MAX)49 q_num = VK_MSGQ_NUM_DEFAULT;50 return q_num;51}52 53static void set_q_num(struct vk_msg_blk *msg, u32 q_num)54{55 u32 trans_q;56 57 if (q_num >= VK_MSGQ_PER_CHAN_MAX)58 trans_q = VK_MSGQ_NUM_DEFAULT;59 else60 trans_q = q_num;61 62 msg->trans_id = (msg->trans_id & ~BCM_VK_MSG_Q_MASK) | trans_q;63}64 65static u32 get_msg_id(const struct vk_msg_blk *msg)66{67 return ((msg->trans_id >> BCM_VK_MSG_Q_SHIFT) & BCM_VK_MSG_ID_MASK);68}69 70static void set_msg_id(struct vk_msg_blk *msg, u32 val)71{72 msg->trans_id = (val << BCM_VK_MSG_Q_SHIFT) | get_q_num(msg);73}74 75static u32 msgq_inc(const struct bcm_vk_sync_qinfo *qinfo, u32 idx, u32 inc)76{77 return ((idx + inc) & qinfo->q_mask);78}79 80static81struct vk_msg_blk __iomem *msgq_blk_addr(const struct bcm_vk_sync_qinfo *qinfo,82 u32 idx)83{84 return qinfo->q_start + (VK_MSGQ_BLK_SIZE * idx);85}86 87static u32 msgq_occupied(const struct bcm_vk_msgq __iomem *msgq,88 const struct bcm_vk_sync_qinfo *qinfo)89{90 u32 wr_idx, rd_idx;91 92 wr_idx = readl_relaxed(&msgq->wr_idx);93 rd_idx = readl_relaxed(&msgq->rd_idx);94 95 return ((wr_idx - rd_idx) & qinfo->q_mask);96}97 98static99u32 msgq_avail_space(const struct bcm_vk_msgq __iomem *msgq,100 const struct bcm_vk_sync_qinfo *qinfo)101{102 return (qinfo->q_size - msgq_occupied(msgq, qinfo) - 1);103}104 105/* number of retries when enqueue message fails before returning EAGAIN */106#define BCM_VK_H2VK_ENQ_RETRY 10107#define BCM_VK_H2VK_ENQ_RETRY_DELAY_MS 50108 109bool bcm_vk_drv_access_ok(struct bcm_vk *vk)110{111 return (!!atomic_read(&vk->msgq_inited));112}113 114void bcm_vk_set_host_alert(struct bcm_vk *vk, u32 bit_mask)115{116 struct bcm_vk_alert *alert = &vk->host_alert;117 unsigned long flags;118 119 /* use irqsave version as this maybe called inside timer interrupt */120 spin_lock_irqsave(&vk->host_alert_lock, flags);121 alert->notfs |= bit_mask;122 spin_unlock_irqrestore(&vk->host_alert_lock, flags);123 124 if (test_and_set_bit(BCM_VK_WQ_NOTF_PEND, vk->wq_offload) == 0)125 queue_work(vk->wq_thread, &vk->wq_work);126}127 128/*129 * Heartbeat related defines130 * The heartbeat from host is a last resort. If stuck condition happens131 * on the card, firmware is supposed to detect it. Therefore, the heartbeat132 * values used will be more relaxed on the driver, which need to be bigger133 * than the watchdog timeout on the card. The watchdog timeout on the card134 * is 20s, with a jitter of 2s => 22s. We use a value of 27s here.135 */136#define BCM_VK_HB_TIMER_S 3137#define BCM_VK_HB_TIMER_VALUE (BCM_VK_HB_TIMER_S * HZ)138#define BCM_VK_HB_LOST_MAX (27 / BCM_VK_HB_TIMER_S)139 140static void bcm_vk_hb_poll(struct work_struct *work)141{142 u32 uptime_s;143 struct bcm_vk_hb_ctrl *hb = container_of(to_delayed_work(work), struct bcm_vk_hb_ctrl,144 work);145 struct bcm_vk *vk = container_of(hb, struct bcm_vk, hb_ctrl);146 147 if (bcm_vk_drv_access_ok(vk) && hb_mon_is_on()) {148 /* read uptime from register and compare */149 uptime_s = vkread32(vk, BAR_0, BAR_OS_UPTIME);150 151 if (uptime_s == hb->last_uptime)152 hb->lost_cnt++;153 else /* reset to avoid accumulation */154 hb->lost_cnt = 0;155 156 dev_dbg(&vk->pdev->dev, "Last uptime %d current %d, lost %d\n",157 hb->last_uptime, uptime_s, hb->lost_cnt);158 159 /*160 * if the interface goes down without any activity, a value161 * of 0xFFFFFFFF will be continuously read, and the detection162 * will be happened eventually.163 */164 hb->last_uptime = uptime_s;165 } else {166 /* reset heart beat lost cnt */167 hb->lost_cnt = 0;168 }169 170 /* next, check if heartbeat exceeds limit */171 if (hb->lost_cnt > BCM_VK_HB_LOST_MAX) {172 dev_err(&vk->pdev->dev, "Heartbeat Misses %d times, %d s!\n",173 BCM_VK_HB_LOST_MAX,174 BCM_VK_HB_LOST_MAX * BCM_VK_HB_TIMER_S);175 176 bcm_vk_blk_drv_access(vk);177 bcm_vk_set_host_alert(vk, ERR_LOG_HOST_HB_FAIL);178 }179 /* re-arm timer */180 schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE);181}182 183void bcm_vk_hb_init(struct bcm_vk *vk)184{185 struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl;186 187 INIT_DELAYED_WORK(&hb->work, bcm_vk_hb_poll);188 schedule_delayed_work(&hb->work, BCM_VK_HB_TIMER_VALUE);189}190 191void bcm_vk_hb_deinit(struct bcm_vk *vk)192{193 struct bcm_vk_hb_ctrl *hb = &vk->hb_ctrl;194 195 cancel_delayed_work_sync(&hb->work);196}197 198static void bcm_vk_msgid_bitmap_clear(struct bcm_vk *vk,199 unsigned int start,200 unsigned int nbits)201{202 spin_lock(&vk->msg_id_lock);203 bitmap_clear(vk->bmap, start, nbits);204 spin_unlock(&vk->msg_id_lock);205}206 207/*208 * allocate a ctx per file struct209 */210static struct bcm_vk_ctx *bcm_vk_get_ctx(struct bcm_vk *vk, const pid_t pid)211{212 u32 i;213 struct bcm_vk_ctx *ctx = NULL;214 u32 hash_idx = hash_32(pid, VK_PID_HT_SHIFT_BIT);215 216 spin_lock(&vk->ctx_lock);217 218 /* check if it is in reset, if so, don't allow */219 if (vk->reset_pid) {220 dev_err(&vk->pdev->dev,221 "No context allowed during reset by pid %d\n",222 vk->reset_pid);223 224 goto in_reset_exit;225 }226 227 for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) {228 if (!vk->ctx[i].in_use) {229 vk->ctx[i].in_use = true;230 ctx = &vk->ctx[i];231 break;232 }233 }234 235 if (!ctx) {236 dev_err(&vk->pdev->dev, "All context in use\n");237 238 goto all_in_use_exit;239 }240 241 /* set the pid and insert it to hash table */242 ctx->pid = pid;243 ctx->hash_idx = hash_idx;244 list_add_tail(&ctx->node, &vk->pid_ht[hash_idx].head);245 246 /* increase kref */247 kref_get(&vk->kref);248 249 /* clear counter */250 atomic_set(&ctx->pend_cnt, 0);251 atomic_set(&ctx->dma_cnt, 0);252 init_waitqueue_head(&ctx->rd_wq);253 254all_in_use_exit:255in_reset_exit:256 spin_unlock(&vk->ctx_lock);257 258 return ctx;259}260 261static u16 bcm_vk_get_msg_id(struct bcm_vk *vk)262{263 u16 rc = VK_MSG_ID_OVERFLOW;264 u16 test_bit_count = 0;265 266 spin_lock(&vk->msg_id_lock);267 while (test_bit_count < (VK_MSG_ID_BITMAP_SIZE - 1)) {268 /*269 * first time come in this loop, msg_id will be 0270 * and the first one tested will be 1. We skip271 * VK_SIMPLEX_MSG_ID (0) for one way host2vk272 * communication273 */274 vk->msg_id++;275 if (vk->msg_id == VK_MSG_ID_BITMAP_SIZE)276 vk->msg_id = 1;277 278 if (test_bit(vk->msg_id, vk->bmap)) {279 test_bit_count++;280 continue;281 }282 rc = vk->msg_id;283 bitmap_set(vk->bmap, vk->msg_id, 1);284 break;285 }286 spin_unlock(&vk->msg_id_lock);287 288 return rc;289}290 291static int bcm_vk_free_ctx(struct bcm_vk *vk, struct bcm_vk_ctx *ctx)292{293 u32 idx;294 u32 hash_idx;295 pid_t pid;296 struct bcm_vk_ctx *entry;297 int count = 0;298 299 if (!ctx) {300 dev_err(&vk->pdev->dev, "NULL context detected\n");301 return -EINVAL;302 }303 idx = ctx->idx;304 pid = ctx->pid;305 306 spin_lock(&vk->ctx_lock);307 308 if (!vk->ctx[idx].in_use) {309 dev_err(&vk->pdev->dev, "context[%d] not in use!\n", idx);310 } else {311 vk->ctx[idx].in_use = false;312 vk->ctx[idx].miscdev = NULL;313 314 /* Remove it from hash list and see if it is the last one. */315 list_del(&ctx->node);316 hash_idx = ctx->hash_idx;317 list_for_each_entry(entry, &vk->pid_ht[hash_idx].head, node) {318 if (entry->pid == pid)319 count++;320 }321 }322 323 spin_unlock(&vk->ctx_lock);324 325 return count;326}327 328static void bcm_vk_free_wkent(struct device *dev, struct bcm_vk_wkent *entry)329{330 int proc_cnt;331 332 bcm_vk_sg_free(dev, entry->dma, VK_DMA_MAX_ADDRS, &proc_cnt);333 if (proc_cnt)334 atomic_dec(&entry->ctx->dma_cnt);335 336 kfree(entry->to_h_msg);337 kfree(entry);338}339 340static void bcm_vk_drain_all_pend(struct device *dev,341 struct bcm_vk_msg_chan *chan,342 struct bcm_vk_ctx *ctx)343{344 u32 num;345 struct bcm_vk_wkent *entry, *tmp;346 struct bcm_vk *vk;347 struct list_head del_q;348 349 if (ctx)350 vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);351 352 INIT_LIST_HEAD(&del_q);353 spin_lock(&chan->pendq_lock);354 for (num = 0; num < chan->q_nr; num++) {355 list_for_each_entry_safe(entry, tmp, &chan->pendq[num], node) {356 if ((!ctx) || (entry->ctx->idx == ctx->idx)) {357 list_move_tail(&entry->node, &del_q);358 }359 }360 }361 spin_unlock(&chan->pendq_lock);362 363 /* batch clean up */364 num = 0;365 list_for_each_entry_safe(entry, tmp, &del_q, node) {366 list_del(&entry->node);367 num++;368 if (ctx) {369 struct vk_msg_blk *msg;370 int bit_set;371 bool responded;372 u32 msg_id;373 374 /* if it is specific ctx, log for any stuck */375 msg = entry->to_v_msg;376 msg_id = get_msg_id(msg);377 bit_set = test_bit(msg_id, vk->bmap);378 responded = entry->to_h_msg ? true : false;379 if (num <= batch_log)380 dev_info(dev,381 "Drained: fid %u size %u msg 0x%x(seq-%x) ctx 0x%x[fd-%d] args:[0x%x 0x%x] resp %s, bmap %d\n",382 msg->function_id, msg->size,383 msg_id, entry->seq_num,384 msg->context_id, entry->ctx->idx,385 msg->cmd, msg->arg,386 responded ? "T" : "F", bit_set);387 if (responded)388 atomic_dec(&ctx->pend_cnt);389 else if (bit_set)390 bcm_vk_msgid_bitmap_clear(vk, msg_id, 1);391 }392 bcm_vk_free_wkent(dev, entry);393 }394 if (num && ctx)395 dev_info(dev, "Total drained items %d [fd-%d]\n",396 num, ctx->idx);397}398 399void bcm_vk_drain_msg_on_reset(struct bcm_vk *vk)400{401 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL);402 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL);403}404 405/*406 * Function to sync up the messages queue info that is provided by BAR1407 */408int bcm_vk_sync_msgq(struct bcm_vk *vk, bool force_sync)409{410 struct bcm_vk_msgq __iomem *msgq;411 struct device *dev = &vk->pdev->dev;412 u32 msgq_off;413 u32 num_q;414 struct bcm_vk_msg_chan *chan_list[] = {&vk->to_v_msg_chan,415 &vk->to_h_msg_chan};416 struct bcm_vk_msg_chan *chan;417 int i, j;418 int ret = 0;419 420 /*421 * If the driver is loaded at startup where vk OS is not up yet,422 * the msgq-info may not be available until a later time. In423 * this case, we skip and the sync function is supposed to be424 * called again.425 */426 if (!bcm_vk_msgq_marker_valid(vk)) {427 dev_info(dev, "BAR1 msgq marker not initialized.\n");428 return -EAGAIN;429 }430 431 msgq_off = vkread32(vk, BAR_1, VK_BAR1_MSGQ_CTRL_OFF);432 433 /* each side is always half the total */434 num_q = vkread32(vk, BAR_1, VK_BAR1_MSGQ_NR) / 2;435 if (!num_q || (num_q > VK_MSGQ_PER_CHAN_MAX)) {436 dev_err(dev,437 "Advertised msgq %d error - max %d allowed\n",438 num_q, VK_MSGQ_PER_CHAN_MAX);439 return -EINVAL;440 }441 442 vk->to_v_msg_chan.q_nr = num_q;443 vk->to_h_msg_chan.q_nr = num_q;444 445 /* first msgq location */446 msgq = vk->bar[BAR_1] + msgq_off;447 448 /*449 * if this function is called when it is already inited,450 * something is wrong451 */452 if (bcm_vk_drv_access_ok(vk) && !force_sync) {453 dev_err(dev, "Msgq info already in sync\n");454 return -EPERM;455 }456 457 for (i = 0; i < ARRAY_SIZE(chan_list); i++) {458 chan = chan_list[i];459 memset(chan->sync_qinfo, 0, sizeof(chan->sync_qinfo));460 461 for (j = 0; j < num_q; j++) {462 struct bcm_vk_sync_qinfo *qinfo;463 u32 msgq_start;464 u32 msgq_size;465 u32 msgq_nxt;466 u32 msgq_db_offset, q_db_offset;467 468 chan->msgq[j] = msgq;469 msgq_start = readl_relaxed(&msgq->start);470 msgq_size = readl_relaxed(&msgq->size);471 msgq_nxt = readl_relaxed(&msgq->nxt);472 msgq_db_offset = readl_relaxed(&msgq->db_offset);473 q_db_offset = (msgq_db_offset & ((1 << DB_SHIFT) - 1));474 if (q_db_offset == (~msgq_db_offset >> DB_SHIFT))475 msgq_db_offset = q_db_offset;476 else477 /* fall back to default */478 msgq_db_offset = VK_BAR0_Q_DB_BASE(j);479 480 dev_info(dev,481 "MsgQ[%d] type %d num %d, @ 0x%x, db_offset 0x%x rd_idx %d wr_idx %d, size %d, nxt 0x%x\n",482 j,483 readw_relaxed(&msgq->type),484 readw_relaxed(&msgq->num),485 msgq_start,486 msgq_db_offset,487 readl_relaxed(&msgq->rd_idx),488 readl_relaxed(&msgq->wr_idx),489 msgq_size,490 msgq_nxt);491 492 qinfo = &chan->sync_qinfo[j];493 /* formulate and record static info */494 qinfo->q_start = vk->bar[BAR_1] + msgq_start;495 qinfo->q_size = msgq_size;496 /* set low threshold as 50% or 1/2 */497 qinfo->q_low = qinfo->q_size >> 1;498 qinfo->q_mask = qinfo->q_size - 1;499 qinfo->q_db_offset = msgq_db_offset;500 501 msgq++;502 }503 }504 atomic_set(&vk->msgq_inited, 1);505 506 return ret;507}508 509static int bcm_vk_msg_chan_init(struct bcm_vk_msg_chan *chan)510{511 u32 i;512 513 mutex_init(&chan->msgq_mutex);514 spin_lock_init(&chan->pendq_lock);515 for (i = 0; i < VK_MSGQ_MAX_NR; i++)516 INIT_LIST_HEAD(&chan->pendq[i]);517 518 return 0;519}520 521static void bcm_vk_append_pendq(struct bcm_vk_msg_chan *chan, u16 q_num,522 struct bcm_vk_wkent *entry)523{524 struct bcm_vk_ctx *ctx;525 526 spin_lock(&chan->pendq_lock);527 list_add_tail(&entry->node, &chan->pendq[q_num]);528 if (entry->to_h_msg) {529 ctx = entry->ctx;530 atomic_inc(&ctx->pend_cnt);531 wake_up_interruptible(&ctx->rd_wq);532 }533 spin_unlock(&chan->pendq_lock);534}535 536static u32 bcm_vk_append_ib_sgl(struct bcm_vk *vk,537 struct bcm_vk_wkent *entry,538 struct _vk_data *data,539 unsigned int num_planes)540{541 unsigned int i;542 unsigned int item_cnt = 0;543 struct device *dev = &vk->pdev->dev;544 struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;545 struct vk_msg_blk *msg = &entry->to_v_msg[0];546 struct bcm_vk_msgq __iomem *msgq;547 struct bcm_vk_sync_qinfo *qinfo;548 u32 ib_sgl_size = 0;549 u8 *buf = (u8 *)&entry->to_v_msg[entry->to_v_blks];550 u32 avail;551 u32 q_num;552 553 /* check if high watermark is hit, and if so, skip */554 q_num = get_q_num(msg);555 msgq = chan->msgq[q_num];556 qinfo = &chan->sync_qinfo[q_num];557 avail = msgq_avail_space(msgq, qinfo);558 if (avail < qinfo->q_low) {559 dev_dbg(dev, "Skip inserting inband SGL, [0x%x/0x%x]\n",560 avail, qinfo->q_size);561 return 0;562 }563 564 for (i = 0; i < num_planes; i++) {565 if (data[i].address &&566 (ib_sgl_size + data[i].size) <= vk->ib_sgl_size) {567 item_cnt++;568 memcpy(buf, entry->dma[i].sglist, data[i].size);569 ib_sgl_size += data[i].size;570 buf += data[i].size;571 }572 }573 574 dev_dbg(dev, "Num %u sgl items appended, size 0x%x, room 0x%x\n",575 item_cnt, ib_sgl_size, vk->ib_sgl_size);576 577 /* round up size */578 ib_sgl_size = (ib_sgl_size + VK_MSGQ_BLK_SIZE - 1)579 >> VK_MSGQ_BLK_SZ_SHIFT;580 581 return ib_sgl_size;582}583 584void bcm_to_v_q_doorbell(struct bcm_vk *vk, u32 q_num, u32 db_val)585{586 struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;587 struct bcm_vk_sync_qinfo *qinfo = &chan->sync_qinfo[q_num];588 589 vkwrite32(vk, db_val, BAR_0, qinfo->q_db_offset);590}591 592static int bcm_to_v_msg_enqueue(struct bcm_vk *vk, struct bcm_vk_wkent *entry)593{594 static u32 seq_num;595 struct bcm_vk_msg_chan *chan = &vk->to_v_msg_chan;596 struct device *dev = &vk->pdev->dev;597 struct vk_msg_blk *src = &entry->to_v_msg[0];598 599 struct vk_msg_blk __iomem *dst;600 struct bcm_vk_msgq __iomem *msgq;601 struct bcm_vk_sync_qinfo *qinfo;602 u32 q_num = get_q_num(src);603 u32 wr_idx; /* local copy */604 u32 i;605 u32 avail;606 u32 retry;607 608 if (entry->to_v_blks != src->size + 1) {609 dev_err(dev, "number of blks %d not matching %d MsgId[0x%x]: func %d ctx 0x%x\n",610 entry->to_v_blks,611 src->size + 1,612 get_msg_id(src),613 src->function_id,614 src->context_id);615 return -EMSGSIZE;616 }617 618 msgq = chan->msgq[q_num];619 qinfo = &chan->sync_qinfo[q_num];620 621 mutex_lock(&chan->msgq_mutex);622 623 avail = msgq_avail_space(msgq, qinfo);624 625 /* if not enough space, return EAGAIN and let app handles it */626 retry = 0;627 while ((avail < entry->to_v_blks) &&628 (retry++ < BCM_VK_H2VK_ENQ_RETRY)) {629 mutex_unlock(&chan->msgq_mutex);630 631 msleep(BCM_VK_H2VK_ENQ_RETRY_DELAY_MS);632 mutex_lock(&chan->msgq_mutex);633 avail = msgq_avail_space(msgq, qinfo);634 }635 if (retry > BCM_VK_H2VK_ENQ_RETRY) {636 mutex_unlock(&chan->msgq_mutex);637 return -EAGAIN;638 }639 640 /* at this point, mutex is taken and there is enough space */641 entry->seq_num = seq_num++; /* update debug seq number */642 wr_idx = readl_relaxed(&msgq->wr_idx);643 644 if (wr_idx >= qinfo->q_size) {645 dev_crit(dev, "Invalid wr_idx 0x%x => max 0x%x!",646 wr_idx, qinfo->q_size);647 bcm_vk_blk_drv_access(vk);648 bcm_vk_set_host_alert(vk, ERR_LOG_HOST_PCIE_DWN);649 goto idx_err;650 }651 652 dst = msgq_blk_addr(qinfo, wr_idx);653 for (i = 0; i < entry->to_v_blks; i++) {654 memcpy_toio(dst, src, sizeof(*dst));655 656 src++;657 wr_idx = msgq_inc(qinfo, wr_idx, 1);658 dst = msgq_blk_addr(qinfo, wr_idx);659 }660 661 /* flush the write pointer */662 writel(wr_idx, &msgq->wr_idx);663 664 /* log new info for debugging */665 dev_dbg(dev,666 "MsgQ[%d] [Rd Wr] = [%d %d] blks inserted %d - Q = [u-%d a-%d]/%d\n",667 readl_relaxed(&msgq->num),668 readl_relaxed(&msgq->rd_idx),669 wr_idx,670 entry->to_v_blks,671 msgq_occupied(msgq, qinfo),672 msgq_avail_space(msgq, qinfo),673 readl_relaxed(&msgq->size));674 /*675 * press door bell based on queue number. 1 is added to the wr_idx676 * to avoid the value of 0 appearing on the VK side to distinguish677 * from initial value.678 */679 bcm_to_v_q_doorbell(vk, q_num, wr_idx + 1);680idx_err:681 mutex_unlock(&chan->msgq_mutex);682 return 0;683}684 685int bcm_vk_send_shutdown_msg(struct bcm_vk *vk, u32 shut_type,686 const pid_t pid, const u32 q_num)687{688 int rc = 0;689 struct bcm_vk_wkent *entry;690 struct device *dev = &vk->pdev->dev;691 692 /*693 * check if the marker is still good. Sometimes, the PCIe interface may694 * have gone done, and if so and we ship down thing based on broken695 * values, kernel may panic.696 */697 if (!bcm_vk_msgq_marker_valid(vk)) {698 dev_info(dev, "PCIe comm chan - invalid marker (0x%x)!\n",699 vkread32(vk, BAR_1, VK_BAR1_MSGQ_DEF_RDY));700 return -EINVAL;701 }702 703 entry = kzalloc(struct_size(entry, to_v_msg, 1), GFP_KERNEL);704 if (!entry)705 return -ENOMEM;706 entry->to_v_blks = 1; /* always 1 block */707 708 /* fill up necessary data */709 entry->to_v_msg[0].function_id = VK_FID_SHUTDOWN;710 set_q_num(&entry->to_v_msg[0], q_num);711 set_msg_id(&entry->to_v_msg[0], VK_SIMPLEX_MSG_ID);712 713 entry->to_v_msg[0].cmd = shut_type;714 entry->to_v_msg[0].arg = pid;715 716 rc = bcm_to_v_msg_enqueue(vk, entry);717 if (rc)718 dev_err(dev,719 "Sending shutdown message to q %d for pid %d fails.\n",720 get_q_num(&entry->to_v_msg[0]), pid);721 722 kfree(entry);723 724 return rc;725}726 727static int bcm_vk_handle_last_sess(struct bcm_vk *vk, const pid_t pid,728 const u32 q_num)729{730 int rc = 0;731 struct device *dev = &vk->pdev->dev;732 733 /*734 * don't send down or do anything if message queue is not initialized735 * and if it is the reset session, clear it.736 */737 if (!bcm_vk_drv_access_ok(vk)) {738 if (vk->reset_pid == pid)739 vk->reset_pid = 0;740 return -EPERM;741 }742 743 dev_dbg(dev, "No more sessions, shut down pid %d\n", pid);744 745 /* only need to do it if it is not the reset process */746 if (vk->reset_pid != pid)747 rc = bcm_vk_send_shutdown_msg(vk, VK_SHUTDOWN_PID, pid, q_num);748 else749 /* put reset_pid to 0 if it is exiting last session */750 vk->reset_pid = 0;751 752 return rc;753}754 755static struct bcm_vk_wkent *bcm_vk_dequeue_pending(struct bcm_vk *vk,756 struct bcm_vk_msg_chan *chan,757 u16 q_num,758 u16 msg_id)759{760 struct bcm_vk_wkent *entry = NULL, *iter;761 762 spin_lock(&chan->pendq_lock);763 list_for_each_entry(iter, &chan->pendq[q_num], node) {764 if (get_msg_id(&iter->to_v_msg[0]) == msg_id) {765 list_del(&iter->node);766 entry = iter;767 bcm_vk_msgid_bitmap_clear(vk, msg_id, 1);768 break;769 }770 }771 spin_unlock(&chan->pendq_lock);772 return entry;773}774 775s32 bcm_to_h_msg_dequeue(struct bcm_vk *vk)776{777 struct device *dev = &vk->pdev->dev;778 struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan;779 struct vk_msg_blk *data;780 struct vk_msg_blk __iomem *src;781 struct vk_msg_blk *dst;782 struct bcm_vk_msgq __iomem *msgq;783 struct bcm_vk_sync_qinfo *qinfo;784 struct bcm_vk_wkent *entry;785 u32 rd_idx, wr_idx;786 u32 q_num, msg_id, j;787 u32 num_blks;788 s32 total = 0;789 int cnt = 0;790 int msg_processed = 0;791 int max_msg_to_process;792 bool exit_loop;793 794 /*795 * drain all the messages from the queues, and find its pending796 * entry in the to_v queue, based on msg_id & q_num, and move the797 * entry to the to_h pending queue, waiting for user space798 * program to extract799 */800 mutex_lock(&chan->msgq_mutex);801 802 for (q_num = 0; q_num < chan->q_nr; q_num++) {803 msgq = chan->msgq[q_num];804 qinfo = &chan->sync_qinfo[q_num];805 max_msg_to_process = BCM_VK_MSG_PROC_MAX_LOOP * qinfo->q_size;806 807 rd_idx = readl_relaxed(&msgq->rd_idx);808 wr_idx = readl_relaxed(&msgq->wr_idx);809 msg_processed = 0;810 exit_loop = false;811 while ((rd_idx != wr_idx) && !exit_loop) {812 u8 src_size;813 814 /*815 * Make a local copy and get pointer to src blk816 * The rd_idx is masked before getting the pointer to817 * avoid out of bound access in case the interface goes818 * down. It will end up pointing to the last block in819 * the buffer, but subsequent src->size check would be820 * able to catch this.821 */822 src = msgq_blk_addr(qinfo, rd_idx & qinfo->q_mask);823 src_size = readb(&src->size);824 825 if ((rd_idx >= qinfo->q_size) ||826 (src_size > (qinfo->q_size - 1))) {827 dev_crit(dev,828 "Invalid rd_idx 0x%x or size 0x%x => max 0x%x!",829 rd_idx, src_size, qinfo->q_size);830 bcm_vk_blk_drv_access(vk);831 bcm_vk_set_host_alert(vk,832 ERR_LOG_HOST_PCIE_DWN);833 goto idx_err;834 }835 836 num_blks = src_size + 1;837 data = kzalloc(num_blks * VK_MSGQ_BLK_SIZE, GFP_KERNEL);838 if (data) {839 /* copy messages and linearize it */840 dst = data;841 for (j = 0; j < num_blks; j++) {842 memcpy_fromio(dst, src, sizeof(*dst));843 844 dst++;845 rd_idx = msgq_inc(qinfo, rd_idx, 1);846 src = msgq_blk_addr(qinfo, rd_idx);847 }848 total++;849 } else {850 /*851 * if we could not allocate memory in kernel,852 * that is fatal.853 */854 dev_crit(dev, "Kernel mem allocation failure.\n");855 total = -ENOMEM;856 goto idx_err;857 }858 859 /* flush rd pointer after a message is dequeued */860 writel(rd_idx, &msgq->rd_idx);861 862 /* log new info for debugging */863 dev_dbg(dev,864 "MsgQ[%d] [Rd Wr] = [%d %d] blks extracted %d - Q = [u-%d a-%d]/%d\n",865 readl_relaxed(&msgq->num),866 rd_idx,867 wr_idx,868 num_blks,869 msgq_occupied(msgq, qinfo),870 msgq_avail_space(msgq, qinfo),871 readl_relaxed(&msgq->size));872 873 /*874 * No need to search if it is an autonomous one-way875 * message from driver, as these messages do not bear876 * a to_v pending item. Currently, only the shutdown877 * message falls into this category.878 */879 if (data->function_id == VK_FID_SHUTDOWN) {880 kfree(data);881 continue;882 }883 884 msg_id = get_msg_id(data);885 /* lookup original message in to_v direction */886 entry = bcm_vk_dequeue_pending(vk,887 &vk->to_v_msg_chan,888 q_num,889 msg_id);890 891 /*892 * if there is message to does not have prior send,893 * this is the location to add here894 */895 if (entry) {896 entry->to_h_blks = num_blks;897 entry->to_h_msg = data;898 bcm_vk_append_pendq(&vk->to_h_msg_chan,899 q_num, entry);900 901 } else {902 if (cnt++ < batch_log)903 dev_info(dev,904 "Could not find MsgId[0x%x] for resp func %d bmap %d\n",905 msg_id, data->function_id,906 test_bit(msg_id, vk->bmap));907 kfree(data);908 }909 /* Fetch wr_idx to handle more back-to-back events */910 wr_idx = readl(&msgq->wr_idx);911 912 /*913 * cap the max so that even we try to handle more back-to-back events,914 * so that it won't hold CPU too long or in case rd/wr idexes are915 * corrupted which triggers infinite looping.916 */917 if (++msg_processed >= max_msg_to_process) {918 dev_warn(dev, "Q[%d] Per loop processing exceeds %d\n",919 q_num, max_msg_to_process);920 exit_loop = true;921 }922 }923 }924idx_err:925 mutex_unlock(&chan->msgq_mutex);926 dev_dbg(dev, "total %d drained from queues\n", total);927 928 return total;929}930 931/*932 * init routine for all required data structures933 */934static int bcm_vk_data_init(struct bcm_vk *vk)935{936 int i;937 938 spin_lock_init(&vk->ctx_lock);939 for (i = 0; i < ARRAY_SIZE(vk->ctx); i++) {940 vk->ctx[i].in_use = false;941 vk->ctx[i].idx = i; /* self identity */942 vk->ctx[i].miscdev = NULL;943 }944 spin_lock_init(&vk->msg_id_lock);945 spin_lock_init(&vk->host_alert_lock);946 vk->msg_id = 0;947 948 /* initialize hash table */949 for (i = 0; i < VK_PID_HT_SZ; i++)950 INIT_LIST_HEAD(&vk->pid_ht[i].head);951 952 return 0;953}954 955irqreturn_t bcm_vk_msgq_irqhandler(int irq, void *dev_id)956{957 struct bcm_vk *vk = dev_id;958 959 if (!bcm_vk_drv_access_ok(vk)) {960 dev_err(&vk->pdev->dev,961 "Interrupt %d received when msgq not inited\n", irq);962 goto skip_schedule_work;963 }964 965 queue_work(vk->wq_thread, &vk->wq_work);966 967skip_schedule_work:968 return IRQ_HANDLED;969}970 971int bcm_vk_open(struct inode *inode, struct file *p_file)972{973 struct bcm_vk_ctx *ctx;974 struct miscdevice *miscdev = (struct miscdevice *)p_file->private_data;975 struct bcm_vk *vk = container_of(miscdev, struct bcm_vk, miscdev);976 struct device *dev = &vk->pdev->dev;977 int rc = 0;978 979 /* get a context and set it up for file */980 ctx = bcm_vk_get_ctx(vk, task_tgid_nr(current));981 if (!ctx) {982 dev_err(dev, "Error allocating context\n");983 rc = -ENOMEM;984 } else {985 /*986 * set up context and replace private data with context for987 * other methods to use. Reason for the context is because988 * it is allowed for multiple sessions to open the sysfs, and989 * for each file open, when upper layer query the response,990 * only those that are tied to a specific open should be991 * returned. The context->idx will be used for such binding992 */993 ctx->miscdev = miscdev;994 p_file->private_data = ctx;995 dev_dbg(dev, "ctx_returned with idx %d, pid %d\n",996 ctx->idx, ctx->pid);997 }998 return rc;999}1000 1001ssize_t bcm_vk_read(struct file *p_file,1002 char __user *buf,1003 size_t count,1004 loff_t *f_pos)1005{1006 ssize_t rc = -ENOMSG;1007 struct bcm_vk_ctx *ctx = p_file->private_data;1008 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk,1009 miscdev);1010 struct device *dev = &vk->pdev->dev;1011 struct bcm_vk_msg_chan *chan = &vk->to_h_msg_chan;1012 struct bcm_vk_wkent *entry = NULL, *iter;1013 u32 q_num;1014 u32 rsp_length;1015 1016 if (!bcm_vk_drv_access_ok(vk))1017 return -EPERM;1018 1019 dev_dbg(dev, "Buf count %zu\n", count);1020 1021 /*1022 * search through the pendq on the to_h chan, and return only those1023 * that belongs to the same context. Search is always from the high to1024 * the low priority queues1025 */1026 spin_lock(&chan->pendq_lock);1027 for (q_num = 0; q_num < chan->q_nr; q_num++) {1028 list_for_each_entry(iter, &chan->pendq[q_num], node) {1029 if (iter->ctx->idx == ctx->idx) {1030 if (count >=1031 (iter->to_h_blks * VK_MSGQ_BLK_SIZE)) {1032 list_del(&iter->node);1033 atomic_dec(&ctx->pend_cnt);1034 entry = iter;1035 } else {1036 /* buffer not big enough */1037 rc = -EMSGSIZE;1038 }1039 goto read_loop_exit;1040 }1041 }1042 }1043read_loop_exit:1044 spin_unlock(&chan->pendq_lock);1045 1046 if (entry) {1047 /* retrieve the passed down msg_id */1048 set_msg_id(&entry->to_h_msg[0], entry->usr_msg_id);1049 rsp_length = entry->to_h_blks * VK_MSGQ_BLK_SIZE;1050 if (copy_to_user(buf, entry->to_h_msg, rsp_length) == 0)1051 rc = rsp_length;1052 1053 bcm_vk_free_wkent(dev, entry);1054 } else if (rc == -EMSGSIZE) {1055 struct vk_msg_blk tmp_msg = entry->to_h_msg[0];1056 1057 /*1058 * in this case, return just the first block, so1059 * that app knows what size it is looking for.1060 */1061 set_msg_id(&tmp_msg, entry->usr_msg_id);1062 tmp_msg.size = entry->to_h_blks - 1;1063 if (copy_to_user(buf, &tmp_msg, VK_MSGQ_BLK_SIZE) != 0) {1064 dev_err(dev, "Error return 1st block in -EMSGSIZE\n");1065 rc = -EFAULT;1066 }1067 }1068 return rc;1069}1070 1071ssize_t bcm_vk_write(struct file *p_file,1072 const char __user *buf,1073 size_t count,1074 loff_t *f_pos)1075{1076 ssize_t rc;1077 struct bcm_vk_ctx *ctx = p_file->private_data;1078 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk,1079 miscdev);1080 struct bcm_vk_msgq __iomem *msgq;1081 struct device *dev = &vk->pdev->dev;1082 struct bcm_vk_wkent *entry;1083 u32 sgl_extra_blks;1084 u32 q_num;1085 u32 msg_size;1086 u32 msgq_size;1087 1088 if (!bcm_vk_drv_access_ok(vk))1089 return -EPERM;1090 1091 dev_dbg(dev, "Msg count %zu\n", count);1092 1093 /* first, do sanity check where count should be multiple of basic blk */1094 if (count & (VK_MSGQ_BLK_SIZE - 1)) {1095 dev_err(dev, "Failure with size %zu not multiple of %zu\n",1096 count, VK_MSGQ_BLK_SIZE);1097 rc = -EINVAL;1098 goto write_err;1099 }1100 1101 /* allocate the work entry + buffer for size count and inband sgl */1102 entry = kzalloc(sizeof(*entry) + count + vk->ib_sgl_size,1103 GFP_KERNEL);1104 if (!entry) {1105 rc = -ENOMEM;1106 goto write_err;1107 }1108 1109 /* now copy msg from user space, and then formulate the work entry */1110 if (copy_from_user(&entry->to_v_msg[0], buf, count)) {1111 rc = -EFAULT;1112 goto write_free_ent;1113 }1114 1115 entry->to_v_blks = count >> VK_MSGQ_BLK_SZ_SHIFT;1116 entry->ctx = ctx;1117 1118 /* do a check on the blk size which could not exceed queue space */1119 q_num = get_q_num(&entry->to_v_msg[0]);1120 msgq = vk->to_v_msg_chan.msgq[q_num];1121 msgq_size = readl_relaxed(&msgq->size);1122 if (entry->to_v_blks + (vk->ib_sgl_size >> VK_MSGQ_BLK_SZ_SHIFT)1123 > (msgq_size - 1)) {1124 dev_err(dev, "Blk size %d exceed max queue size allowed %d\n",1125 entry->to_v_blks, msgq_size - 1);1126 rc = -EINVAL;1127 goto write_free_ent;1128 }1129 1130 /* Use internal message id */1131 entry->usr_msg_id = get_msg_id(&entry->to_v_msg[0]);1132 rc = bcm_vk_get_msg_id(vk);1133 if (rc == VK_MSG_ID_OVERFLOW) {1134 dev_err(dev, "msg_id overflow\n");1135 rc = -EOVERFLOW;1136 goto write_free_ent;1137 }1138 set_msg_id(&entry->to_v_msg[0], rc);1139 ctx->q_num = q_num;1140 1141 dev_dbg(dev,1142 "[Q-%d]Message ctx id %d, usr_msg_id 0x%x sent msg_id 0x%x\n",1143 ctx->q_num, ctx->idx, entry->usr_msg_id,1144 get_msg_id(&entry->to_v_msg[0]));1145 1146 if (entry->to_v_msg[0].function_id == VK_FID_TRANS_BUF) {1147 /* Convert any pointers to sg list */1148 unsigned int num_planes;1149 int dir;1150 struct _vk_data *data;1151 1152 /*1153 * check if we are in reset, if so, no buffer transfer is1154 * allowed and return error.1155 */1156 if (vk->reset_pid) {1157 dev_dbg(dev, "No Transfer allowed during reset, pid %d.\n",1158 ctx->pid);1159 rc = -EACCES;1160 goto write_free_msgid;1161 }1162 1163 num_planes = entry->to_v_msg[0].cmd & VK_CMD_PLANES_MASK;1164 if ((entry->to_v_msg[0].cmd & VK_CMD_MASK) == VK_CMD_DOWNLOAD)1165 dir = DMA_FROM_DEVICE;1166 else1167 dir = DMA_TO_DEVICE;1168 1169 /* Calculate vk_data location */1170 /* Go to end of the message */1171 msg_size = entry->to_v_msg[0].size;1172 if (msg_size > entry->to_v_blks) {1173 rc = -EMSGSIZE;1174 goto write_free_msgid;1175 }1176 1177 data = (struct _vk_data *)&entry->to_v_msg[msg_size + 1];1178 1179 /* Now back up to the start of the pointers */1180 data -= num_planes;1181 1182 /* Convert user addresses to DMA SG List */1183 rc = bcm_vk_sg_alloc(dev, entry->dma, dir, data, num_planes);1184 if (rc)1185 goto write_free_msgid;1186 1187 atomic_inc(&ctx->dma_cnt);1188 /* try to embed inband sgl */1189 sgl_extra_blks = bcm_vk_append_ib_sgl(vk, entry, data,1190 num_planes);1191 entry->to_v_blks += sgl_extra_blks;1192 entry->to_v_msg[0].size += sgl_extra_blks;1193 } else if (entry->to_v_msg[0].function_id == VK_FID_INIT &&1194 entry->to_v_msg[0].context_id == VK_NEW_CTX) {1195 /*1196 * Init happens in 2 stages, only the first stage contains the1197 * pid that needs translating.1198 */1199 pid_t org_pid, pid;1200 1201 /*1202 * translate the pid into the unique host space as user1203 * may run sessions inside containers or process1204 * namespaces.1205 */1206#define VK_MSG_PID_MASK 0xffffff001207#define VK_MSG_PID_SH 81208 org_pid = (entry->to_v_msg[0].arg & VK_MSG_PID_MASK)1209 >> VK_MSG_PID_SH;1210 1211 pid = task_tgid_nr(current);1212 entry->to_v_msg[0].arg =1213 (entry->to_v_msg[0].arg & ~VK_MSG_PID_MASK) |1214 (pid << VK_MSG_PID_SH);1215 if (org_pid != pid)1216 dev_dbg(dev, "In PID 0x%x(%d), converted PID 0x%x(%d)\n",1217 org_pid, org_pid, pid, pid);1218 }1219 1220 /*1221 * store work entry to pending queue until a response is received.1222 * This needs to be done before enqueuing the message1223 */1224 bcm_vk_append_pendq(&vk->to_v_msg_chan, q_num, entry);1225 1226 rc = bcm_to_v_msg_enqueue(vk, entry);1227 if (rc) {1228 dev_err(dev, "Fail to enqueue msg to to_v queue\n");1229 1230 /* remove message from pending list */1231 entry = bcm_vk_dequeue_pending1232 (vk,1233 &vk->to_v_msg_chan,1234 q_num,1235 get_msg_id(&entry->to_v_msg[0]));1236 goto write_free_ent;1237 }1238 1239 return count;1240 1241write_free_msgid:1242 bcm_vk_msgid_bitmap_clear(vk, get_msg_id(&entry->to_v_msg[0]), 1);1243write_free_ent:1244 kfree(entry);1245write_err:1246 return rc;1247}1248 1249__poll_t bcm_vk_poll(struct file *p_file, struct poll_table_struct *wait)1250{1251 __poll_t ret = 0;1252 int cnt;1253 struct bcm_vk_ctx *ctx = p_file->private_data;1254 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);1255 struct device *dev = &vk->pdev->dev;1256 1257 poll_wait(p_file, &ctx->rd_wq, wait);1258 1259 cnt = atomic_read(&ctx->pend_cnt);1260 if (cnt) {1261 ret = (__force __poll_t)(POLLIN | POLLRDNORM);1262 if (cnt < 0) {1263 dev_err(dev, "Error cnt %d, setting back to 0", cnt);1264 atomic_set(&ctx->pend_cnt, 0);1265 }1266 }1267 1268 return ret;1269}1270 1271int bcm_vk_release(struct inode *inode, struct file *p_file)1272{1273 int ret;1274 struct bcm_vk_ctx *ctx = p_file->private_data;1275 struct bcm_vk *vk = container_of(ctx->miscdev, struct bcm_vk, miscdev);1276 struct device *dev = &vk->pdev->dev;1277 pid_t pid = ctx->pid;1278 int dma_cnt;1279 unsigned long timeout, start_time;1280 1281 /*1282 * if there are outstanding DMA transactions, need to delay long enough1283 * to ensure that the card side would have stopped touching the host buffer1284 * and its SGL list. A race condition could happen if the host app is killed1285 * abruptly, eg kill -9, while some DMA transfer orders are still inflight.1286 * Nothing could be done except for a delay as host side is running in a1287 * completely async fashion.1288 */1289 start_time = jiffies;1290 timeout = start_time + msecs_to_jiffies(BCM_VK_DMA_DRAIN_MAX_MS);1291 do {1292 if (time_after(jiffies, timeout)) {1293 dev_warn(dev, "%d dma still pending for [fd-%d] pid %d\n",1294 dma_cnt, ctx->idx, pid);1295 break;1296 }1297 dma_cnt = atomic_read(&ctx->dma_cnt);1298 cpu_relax();1299 cond_resched();1300 } while (dma_cnt);1301 dev_dbg(dev, "Draining for [fd-%d] pid %d - delay %d ms\n",1302 ctx->idx, pid, jiffies_to_msecs(jiffies - start_time));1303 1304 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, ctx);1305 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, ctx);1306 1307 ret = bcm_vk_free_ctx(vk, ctx);1308 if (ret == 0)1309 ret = bcm_vk_handle_last_sess(vk, pid, ctx->q_num);1310 else1311 ret = 0;1312 1313 kref_put(&vk->kref, bcm_vk_release_data);1314 1315 return ret;1316}1317 1318int bcm_vk_msg_init(struct bcm_vk *vk)1319{1320 struct device *dev = &vk->pdev->dev;1321 int ret;1322 1323 if (bcm_vk_data_init(vk)) {1324 dev_err(dev, "Error initializing internal data structures\n");1325 return -EINVAL;1326 }1327 1328 if (bcm_vk_msg_chan_init(&vk->to_v_msg_chan) ||1329 bcm_vk_msg_chan_init(&vk->to_h_msg_chan)) {1330 dev_err(dev, "Error initializing communication channel\n");1331 return -EIO;1332 }1333 1334 /* read msgq info if ready */1335 ret = bcm_vk_sync_msgq(vk, false);1336 if (ret && (ret != -EAGAIN)) {1337 dev_err(dev, "Error reading comm msg Q info\n");1338 return -EIO;1339 }1340 1341 return 0;1342}1343 1344void bcm_vk_msg_remove(struct bcm_vk *vk)1345{1346 bcm_vk_blk_drv_access(vk);1347 1348 /* drain all pending items */1349 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_v_msg_chan, NULL);1350 bcm_vk_drain_all_pend(&vk->pdev->dev, &vk->to_h_msg_chan, NULL);1351}1352 1353