1760 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.4 * Copyright (C) 2017 Linaro Ltd.5 */6 7#include <linux/delay.h>8#include <linux/device.h>9#include <linux/dma-mapping.h>10#include <linux/interrupt.h>11#include <linux/iopoll.h>12#include <linux/kernel.h>13#include <linux/slab.h>14 15#include "core.h"16#include "hfi_cmds.h"17#include "hfi_msgs.h"18#include "hfi_venus.h"19#include "hfi_venus_io.h"20#include "firmware.h"21 22#define HFI_MASK_QHDR_TX_TYPE 0xff00000023#define HFI_MASK_QHDR_RX_TYPE 0x00ff000024#define HFI_MASK_QHDR_PRI_TYPE 0x0000ff0025#define HFI_MASK_QHDR_ID_TYPE 0x000000ff26 27#define HFI_HOST_TO_CTRL_CMD_Q 028#define HFI_CTRL_TO_HOST_MSG_Q 129#define HFI_CTRL_TO_HOST_DBG_Q 230#define HFI_MASK_QHDR_STATUS 0x000000ff31 32#define IFACEQ_NUM 333#define IFACEQ_CMD_IDX 034#define IFACEQ_MSG_IDX 135#define IFACEQ_DBG_IDX 236#define IFACEQ_MAX_BUF_COUNT 5037#define IFACEQ_MAX_PARALLEL_CLNTS 1638#define IFACEQ_DFLT_QHDR 0x0101000039 40#define POLL_INTERVAL_US 5041 42#define IFACEQ_MAX_PKT_SIZE 102443#define IFACEQ_MED_PKT_SIZE 76844#define IFACEQ_MIN_PKT_SIZE 845#define IFACEQ_VAR_SMALL_PKT_SIZE 10046#define IFACEQ_VAR_LARGE_PKT_SIZE 51247#define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12)48 49struct hfi_queue_table_header {50 u32 version;51 u32 size;52 u32 qhdr0_offset;53 u32 qhdr_size;54 u32 num_q;55 u32 num_active_q;56};57 58struct hfi_queue_header {59 u32 status;60 u32 start_addr;61 u32 type;62 u32 q_size;63 u32 pkt_size;64 u32 pkt_drop_cnt;65 u32 rx_wm;66 u32 tx_wm;67 u32 rx_req;68 u32 tx_req;69 u32 rx_irq_status;70 u32 tx_irq_status;71 u32 read_idx;72 u32 write_idx;73};74 75#define IFACEQ_TABLE_SIZE \76 (sizeof(struct hfi_queue_table_header) + \77 sizeof(struct hfi_queue_header) * IFACEQ_NUM)78 79#define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \80 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)81 82#define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \83 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \84 ((i) * sizeof(struct hfi_queue_header)))85 86#define QDSS_SIZE SZ_4K87#define SFR_SIZE SZ_4K88#define QUEUE_SIZE \89 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))90 91#define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K)92#define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K)93#define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K)94#define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \95 ALIGNED_QDSS_SIZE, SZ_1M)96 97struct mem_desc {98 dma_addr_t da; /* device address */99 void *kva; /* kernel virtual address */100 u32 size;101 unsigned long attrs;102};103 104struct iface_queue {105 struct hfi_queue_header *qhdr;106 struct mem_desc qmem;107};108 109enum venus_state {110 VENUS_STATE_DEINIT = 1,111 VENUS_STATE_INIT,112};113 114struct venus_hfi_device {115 struct venus_core *core;116 u32 irq_status;117 u32 last_packet_type;118 bool power_enabled;119 bool suspended;120 enum venus_state state;121 /* serialize read / write to the shared memory */122 struct mutex lock;123 struct completion pwr_collapse_prep;124 struct completion release_resource;125 struct mem_desc ifaceq_table;126 struct mem_desc sfr;127 struct iface_queue queues[IFACEQ_NUM];128 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];129 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];130};131 132static bool venus_pkt_debug;133int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;134static bool venus_fw_low_power_mode = true;135static int venus_hw_rsp_timeout = 1000;136static bool venus_fw_coverage;137 138static void venus_set_state(struct venus_hfi_device *hdev,139 enum venus_state state)140{141 mutex_lock(&hdev->lock);142 hdev->state = state;143 mutex_unlock(&hdev->lock);144}145 146static bool venus_is_valid_state(struct venus_hfi_device *hdev)147{148 return hdev->state != VENUS_STATE_DEINIT;149}150 151static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)152{153 size_t pkt_size = *(u32 *)packet;154 155 if (!venus_pkt_debug)156 return;157 158 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,159 pkt_size, true);160}161 162static int venus_write_queue(struct venus_hfi_device *hdev,163 struct iface_queue *queue,164 void *packet, u32 *rx_req)165{166 struct hfi_queue_header *qhdr;167 u32 dwords, new_wr_idx;168 u32 empty_space, rd_idx, wr_idx, qsize;169 u32 *wr_ptr;170 171 if (!queue->qmem.kva)172 return -EINVAL;173 174 qhdr = queue->qhdr;175 if (!qhdr)176 return -EINVAL;177 178 venus_dump_packet(hdev, packet);179 180 dwords = (*(u32 *)packet) >> 2;181 if (!dwords)182 return -EINVAL;183 184 rd_idx = qhdr->read_idx;185 wr_idx = qhdr->write_idx;186 qsize = qhdr->q_size;187 /* ensure rd/wr indices's are read from memory */188 rmb();189 190 if (wr_idx >= rd_idx)191 empty_space = qsize - (wr_idx - rd_idx);192 else193 empty_space = rd_idx - wr_idx;194 195 if (empty_space <= dwords) {196 qhdr->tx_req = 1;197 /* ensure tx_req is updated in memory */198 wmb();199 return -ENOSPC;200 }201 202 qhdr->tx_req = 0;203 /* ensure tx_req is updated in memory */204 wmb();205 206 new_wr_idx = wr_idx + dwords;207 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));208 209 if (wr_ptr < (u32 *)queue->qmem.kva ||210 wr_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*wr_ptr)))211 return -EINVAL;212 213 if (new_wr_idx < qsize) {214 memcpy(wr_ptr, packet, dwords << 2);215 } else {216 size_t len;217 218 new_wr_idx -= qsize;219 len = (dwords - new_wr_idx) << 2;220 memcpy(wr_ptr, packet, len);221 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);222 }223 224 /* make sure packet is written before updating the write index */225 wmb();226 227 qhdr->write_idx = new_wr_idx;228 *rx_req = qhdr->rx_req ? 1 : 0;229 230 /* make sure write index is updated before an interrupt is raised */231 mb();232 233 return 0;234}235 236static int venus_read_queue(struct venus_hfi_device *hdev,237 struct iface_queue *queue, void *pkt, u32 *tx_req)238{239 struct hfi_queue_header *qhdr;240 u32 dwords, new_rd_idx;241 u32 rd_idx, wr_idx, type, qsize;242 u32 *rd_ptr;243 u32 recv_request = 0;244 int ret = 0;245 246 if (!queue->qmem.kva)247 return -EINVAL;248 249 qhdr = queue->qhdr;250 if (!qhdr)251 return -EINVAL;252 253 type = qhdr->type;254 rd_idx = qhdr->read_idx;255 wr_idx = qhdr->write_idx;256 qsize = qhdr->q_size;257 258 /* make sure data is valid before using it */259 rmb();260 261 /*262 * Do not set receive request for debug queue, if set, Venus generates263 * interrupt for debug messages even when there is no response message264 * available. In general debug queue will not become full as it is being265 * emptied out for every interrupt from Venus. Venus will anyway266 * generates interrupt if it is full.267 */268 if (type & HFI_CTRL_TO_HOST_MSG_Q)269 recv_request = 1;270 271 if (rd_idx == wr_idx) {272 qhdr->rx_req = recv_request;273 *tx_req = 0;274 /* update rx_req field in memory */275 wmb();276 return -ENODATA;277 }278 279 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));280 281 if (rd_ptr < (u32 *)queue->qmem.kva ||282 rd_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*rd_ptr)))283 return -EINVAL;284 285 dwords = *rd_ptr >> 2;286 if (!dwords)287 return -EINVAL;288 289 new_rd_idx = rd_idx + dwords;290 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {291 if (new_rd_idx < qsize) {292 memcpy(pkt, rd_ptr, dwords << 2);293 } else {294 size_t len;295 296 new_rd_idx -= qsize;297 len = (dwords - new_rd_idx) << 2;298 memcpy(pkt, rd_ptr, len);299 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);300 }301 } else {302 /* bad packet received, dropping */303 new_rd_idx = qhdr->write_idx;304 ret = -EBADMSG;305 }306 307 /* ensure the packet is read before updating read index */308 rmb();309 310 qhdr->read_idx = new_rd_idx;311 /* ensure updating read index */312 wmb();313 314 rd_idx = qhdr->read_idx;315 wr_idx = qhdr->write_idx;316 /* ensure rd/wr indices are read from memory */317 rmb();318 319 if (rd_idx != wr_idx)320 qhdr->rx_req = 0;321 else322 qhdr->rx_req = recv_request;323 324 *tx_req = qhdr->tx_req ? 1 : 0;325 326 /* ensure rx_req is stored to memory and tx_req is loaded from memory */327 mb();328 329 venus_dump_packet(hdev, pkt);330 331 return ret;332}333 334static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,335 u32 size)336{337 struct device *dev = hdev->core->dev;338 339 desc->attrs = DMA_ATTR_WRITE_COMBINE;340 desc->size = ALIGN(size, SZ_4K);341 342 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,343 desc->attrs);344 if (!desc->kva)345 return -ENOMEM;346 347 return 0;348}349 350static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)351{352 struct device *dev = hdev->core->dev;353 354 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);355}356 357static void venus_set_registers(struct venus_hfi_device *hdev)358{359 const struct venus_resources *res = hdev->core->res;360 const struct reg_val *tbl = res->reg_tbl;361 unsigned int count = res->reg_tbl_size;362 unsigned int i;363 364 for (i = 0; i < count; i++)365 writel(tbl[i].value, hdev->core->base + tbl[i].reg);366}367 368static void venus_soft_int(struct venus_hfi_device *hdev)369{370 void __iomem *cpu_ic_base = hdev->core->cpu_ic_base;371 u32 clear_bit;372 373 if (IS_V6(hdev->core))374 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6);375 else376 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT);377 378 writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT);379}380 381static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,382 void *pkt, bool sync)383{384 struct device *dev = hdev->core->dev;385 struct hfi_pkt_hdr *cmd_packet;386 struct iface_queue *queue;387 u32 rx_req;388 int ret;389 390 if (!venus_is_valid_state(hdev))391 return -EINVAL;392 393 cmd_packet = (struct hfi_pkt_hdr *)pkt;394 hdev->last_packet_type = cmd_packet->pkt_type;395 396 queue = &hdev->queues[IFACEQ_CMD_IDX];397 398 ret = venus_write_queue(hdev, queue, pkt, &rx_req);399 if (ret) {400 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);401 return ret;402 }403 404 if (sync) {405 /*406 * Inform video hardware to raise interrupt for synchronous407 * commands408 */409 queue = &hdev->queues[IFACEQ_MSG_IDX];410 queue->qhdr->rx_req = 1;411 /* ensure rx_req is updated in memory */412 wmb();413 }414 415 if (rx_req)416 venus_soft_int(hdev);417 418 return 0;419}420 421static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync)422{423 int ret;424 425 mutex_lock(&hdev->lock);426 ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync);427 mutex_unlock(&hdev->lock);428 429 return ret;430}431 432static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,433 u32 size, u32 addr, void *cookie)434{435 struct venus_hfi_device *hdev = to_hfi_priv(core);436 struct hfi_sys_set_resource_pkt *pkt;437 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];438 int ret;439 440 if (id == VIDC_RESOURCE_NONE)441 return 0;442 443 pkt = (struct hfi_sys_set_resource_pkt *)packet;444 445 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);446 if (ret)447 return ret;448 449 ret = venus_iface_cmdq_write(hdev, pkt, false);450 if (ret)451 return ret;452 453 return 0;454}455 456static int venus_boot_core(struct venus_hfi_device *hdev)457{458 struct device *dev = hdev->core->dev;459 static const unsigned int max_tries = 100;460 u32 ctrl_status = 0, mask_val = 0;461 unsigned int count = 0;462 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;463 void __iomem *wrapper_base = hdev->core->wrapper_base;464 int ret = 0;465 466 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {467 mask_val = readl(wrapper_base + WRAPPER_INTR_MASK);468 mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 |469 WRAPPER_INTR_MASK_A2HCPU_MASK);470 } else {471 mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK;472 }473 474 writel(mask_val, wrapper_base + WRAPPER_INTR_MASK);475 if (IS_V1(hdev->core))476 writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3);477 478 writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT);479 while (!ctrl_status && count < max_tries) {480 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);481 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {482 dev_err(dev, "invalid setting for UC_REGION\n");483 ret = -EINVAL;484 break;485 }486 487 usleep_range(500, 1000);488 count++;489 }490 491 if (count >= max_tries)492 ret = -ETIMEDOUT;493 494 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {495 writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6);496 writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6);497 }498 499 return ret;500}501 502static u32 venus_hwversion(struct venus_hfi_device *hdev)503{504 struct device *dev = hdev->core->dev;505 void __iomem *wrapper_base = hdev->core->wrapper_base;506 u32 ver;507 u32 major, minor, step;508 509 ver = readl(wrapper_base + WRAPPER_HW_VERSION);510 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;511 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;512 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;513 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;514 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;515 516 dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);517 518 return major;519}520 521static int venus_run(struct venus_hfi_device *hdev)522{523 struct device *dev = hdev->core->dev;524 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;525 int ret;526 527 /*528 * Re-program all of the registers that get reset as a result of529 * regulator_disable() and _enable()530 */531 venus_set_registers(hdev);532 533 writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR);534 writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE);535 writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2);536 writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1);537 if (hdev->sfr.da)538 writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR);539 540 ret = venus_boot_core(hdev);541 if (ret) {542 dev_err(dev, "failed to reset venus core\n");543 return ret;544 }545 546 venus_hwversion(hdev);547 548 return 0;549}550 551static int venus_halt_axi(struct venus_hfi_device *hdev)552{553 void __iomem *wrapper_base = hdev->core->wrapper_base;554 void __iomem *vbif_base = hdev->core->vbif_base;555 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;556 void __iomem *aon_base = hdev->core->aon_base;557 struct device *dev = hdev->core->dev;558 u32 val;559 u32 mask_val;560 int ret;561 562 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {563 writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6);564 565 if (IS_IRIS2_1(hdev->core))566 goto skip_aon_mvp_noc;567 568 writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL);569 ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS,570 val,571 val & BIT(0),572 POLL_INTERVAL_US,573 VBIF_AXI_HALT_ACK_TIMEOUT_US);574 if (ret)575 return -ETIMEDOUT;576 577skip_aon_mvp_noc:578 mask_val = (BIT(2) | BIT(1) | BIT(0));579 writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);580 581 writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);582 ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6,583 val,584 val == 0,585 POLL_INTERVAL_US,586 VBIF_AXI_HALT_ACK_TIMEOUT_US);587 588 if (ret) {589 dev_err(dev, "DBLP Release: lpi_status %x\n", val);590 return -ETIMEDOUT;591 }592 return 0;593 }594 595 if (IS_V4(hdev->core)) {596 val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT);597 val |= WRAPPER_CPU_AXI_HALT_HALT;598 writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT);599 600 ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS,601 val,602 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,603 POLL_INTERVAL_US,604 VBIF_AXI_HALT_ACK_TIMEOUT_US);605 if (ret) {606 dev_err(dev, "AXI bus port halt timeout\n");607 return ret;608 }609 610 return 0;611 }612 613 /* Halt AXI and AXI IMEM VBIF Access */614 val = readl(vbif_base + VBIF_AXI_HALT_CTRL0);615 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;616 writel(val, vbif_base + VBIF_AXI_HALT_CTRL0);617 618 /* Request for AXI bus port halt */619 ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val,620 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,621 POLL_INTERVAL_US,622 VBIF_AXI_HALT_ACK_TIMEOUT_US);623 if (ret) {624 dev_err(dev, "AXI bus port halt timeout\n");625 return ret;626 }627 628 return 0;629}630 631static int venus_power_off(struct venus_hfi_device *hdev)632{633 int ret;634 635 if (!hdev->power_enabled)636 return 0;637 638 ret = venus_set_hw_state_suspend(hdev->core);639 if (ret)640 return ret;641 642 ret = venus_halt_axi(hdev);643 if (ret)644 return ret;645 646 hdev->power_enabled = false;647 648 return 0;649}650 651static int venus_power_on(struct venus_hfi_device *hdev)652{653 int ret;654 655 if (hdev->power_enabled)656 return 0;657 658 ret = venus_set_hw_state_resume(hdev->core);659 if (ret)660 goto err;661 662 ret = venus_run(hdev);663 if (ret)664 goto err_suspend;665 666 hdev->power_enabled = true;667 668 return 0;669 670err_suspend:671 venus_set_hw_state_suspend(hdev->core);672err:673 hdev->power_enabled = false;674 return ret;675}676 677static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,678 void *pkt)679{680 struct iface_queue *queue;681 u32 tx_req;682 int ret;683 684 if (!venus_is_valid_state(hdev))685 return -EINVAL;686 687 queue = &hdev->queues[IFACEQ_MSG_IDX];688 689 ret = venus_read_queue(hdev, queue, pkt, &tx_req);690 if (ret)691 return ret;692 693 if (tx_req)694 venus_soft_int(hdev);695 696 return 0;697}698 699static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)700{701 int ret;702 703 mutex_lock(&hdev->lock);704 ret = venus_iface_msgq_read_nolock(hdev, pkt);705 mutex_unlock(&hdev->lock);706 707 return ret;708}709 710static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,711 void *pkt)712{713 struct iface_queue *queue;714 u32 tx_req;715 int ret;716 717 ret = venus_is_valid_state(hdev);718 if (!ret)719 return -EINVAL;720 721 queue = &hdev->queues[IFACEQ_DBG_IDX];722 723 ret = venus_read_queue(hdev, queue, pkt, &tx_req);724 if (ret)725 return ret;726 727 if (tx_req)728 venus_soft_int(hdev);729 730 return 0;731}732 733static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)734{735 int ret;736 737 if (!pkt)738 return -EINVAL;739 740 mutex_lock(&hdev->lock);741 ret = venus_iface_dbgq_read_nolock(hdev, pkt);742 mutex_unlock(&hdev->lock);743 744 return ret;745}746 747static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)748{749 qhdr->status = 1;750 qhdr->type = IFACEQ_DFLT_QHDR;751 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;752 qhdr->pkt_size = 0;753 qhdr->rx_wm = 1;754 qhdr->tx_wm = 1;755 qhdr->rx_req = 1;756 qhdr->tx_req = 0;757 qhdr->rx_irq_status = 0;758 qhdr->tx_irq_status = 0;759 qhdr->read_idx = 0;760 qhdr->write_idx = 0;761}762 763static void venus_interface_queues_release(struct venus_hfi_device *hdev)764{765 mutex_lock(&hdev->lock);766 767 venus_free(hdev, &hdev->ifaceq_table);768 venus_free(hdev, &hdev->sfr);769 770 memset(hdev->queues, 0, sizeof(hdev->queues));771 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));772 memset(&hdev->sfr, 0, sizeof(hdev->sfr));773 774 mutex_unlock(&hdev->lock);775}776 777static int venus_interface_queues_init(struct venus_hfi_device *hdev)778{779 struct hfi_queue_table_header *tbl_hdr;780 struct iface_queue *queue;781 struct hfi_sfr *sfr;782 struct mem_desc desc = {0};783 unsigned int offset;784 unsigned int i;785 int ret;786 787 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);788 if (ret)789 return ret;790 791 hdev->ifaceq_table = desc;792 offset = IFACEQ_TABLE_SIZE;793 794 for (i = 0; i < IFACEQ_NUM; i++) {795 queue = &hdev->queues[i];796 queue->qmem.da = desc.da + offset;797 queue->qmem.kva = desc.kva + offset;798 queue->qmem.size = IFACEQ_QUEUE_SIZE;799 offset += queue->qmem.size;800 queue->qhdr =801 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);802 803 venus_set_qhdr_defaults(queue->qhdr);804 805 queue->qhdr->start_addr = queue->qmem.da;806 807 if (i == IFACEQ_CMD_IDX)808 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;809 else if (i == IFACEQ_MSG_IDX)810 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;811 else if (i == IFACEQ_DBG_IDX)812 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;813 }814 815 tbl_hdr = hdev->ifaceq_table.kva;816 tbl_hdr->version = 0;817 tbl_hdr->size = IFACEQ_TABLE_SIZE;818 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);819 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);820 tbl_hdr->num_q = IFACEQ_NUM;821 tbl_hdr->num_active_q = IFACEQ_NUM;822 823 /*824 * Set receive request to zero on debug queue as there is no825 * need of interrupt from video hardware for debug messages826 */827 queue = &hdev->queues[IFACEQ_DBG_IDX];828 queue->qhdr->rx_req = 0;829 830 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);831 if (ret) {832 hdev->sfr.da = 0;833 } else {834 hdev->sfr = desc;835 sfr = hdev->sfr.kva;836 sfr->buf_size = ALIGNED_SFR_SIZE;837 }838 839 /* ensure table and queue header structs are settled in memory */840 wmb();841 842 return 0;843}844 845static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)846{847 struct hfi_sys_set_property_pkt *pkt;848 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];849 850 pkt = (struct hfi_sys_set_property_pkt *)packet;851 852 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);853 854 return venus_iface_cmdq_write(hdev, pkt, false);855}856 857static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)858{859 struct hfi_sys_set_property_pkt *pkt;860 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];861 862 pkt = (struct hfi_sys_set_property_pkt *)packet;863 864 pkt_sys_coverage_config(pkt, mode);865 866 return venus_iface_cmdq_write(hdev, pkt, false);867}868 869static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,870 bool enable)871{872 struct hfi_sys_set_property_pkt *pkt;873 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];874 875 if (!enable)876 return 0;877 878 pkt = (struct hfi_sys_set_property_pkt *)packet;879 880 pkt_sys_idle_indicator(pkt, enable);881 882 return venus_iface_cmdq_write(hdev, pkt, false);883}884 885static int venus_sys_set_power_control(struct venus_hfi_device *hdev,886 bool enable)887{888 struct hfi_sys_set_property_pkt *pkt;889 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];890 891 pkt = (struct hfi_sys_set_property_pkt *)packet;892 893 pkt_sys_power_control(pkt, enable);894 895 return venus_iface_cmdq_write(hdev, pkt, false);896}897 898static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev)899{900 struct hfi_sys_set_property_pkt *pkt;901 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];902 const struct venus_resources *res = hdev->core->res;903 int ret;904 905 pkt = (struct hfi_sys_set_property_pkt *)packet;906 907 pkt_sys_ubwc_config(pkt, res->ubwc_conf);908 909 ret = venus_iface_cmdq_write(hdev, pkt, false);910 if (ret)911 return ret;912 913 return 0;914}915 916static int venus_get_queue_size(struct venus_hfi_device *hdev,917 unsigned int index)918{919 struct hfi_queue_header *qhdr;920 921 if (index >= IFACEQ_NUM)922 return -EINVAL;923 924 qhdr = hdev->queues[index].qhdr;925 if (!qhdr)926 return -EINVAL;927 928 return abs(qhdr->read_idx - qhdr->write_idx);929}930 931static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)932{933 struct device *dev = hdev->core->dev;934 const struct venus_resources *res = hdev->core->res;935 int ret;936 937 ret = venus_sys_set_debug(hdev, venus_fw_debug);938 if (ret)939 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);940 941 /* HFI_PROPERTY_SYS_IDLE_INDICATOR is not supported beyond 8916 (HFI V1) */942 if (IS_V1(hdev->core)) {943 ret = venus_sys_set_idle_message(hdev, false);944 if (ret)945 dev_warn(dev, "setting idle response ON failed (%d)\n", ret);946 }947 948 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);949 if (ret)950 dev_warn(dev, "setting hw power collapse ON failed (%d)\n",951 ret);952 953 /* For specific venus core, it is mandatory to set the UBWC configuration */954 if (res->ubwc_conf) {955 ret = venus_sys_set_ubwc_config(hdev);956 if (ret)957 dev_warn(dev, "setting ubwc config failed (%d)\n", ret);958 }959 960 return ret;961}962 963static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync)964{965 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);966 struct hfi_session_pkt pkt;967 968 pkt_session_cmd(&pkt, pkt_type, inst);969 970 return venus_iface_cmdq_write(hdev, &pkt, sync);971}972 973static void venus_flush_debug_queue(struct venus_hfi_device *hdev)974{975 struct device *dev = hdev->core->dev;976 void *packet = hdev->dbg_buf;977 978 while (!venus_iface_dbgq_read(hdev, packet)) {979 struct hfi_msg_sys_coverage_pkt *pkt = packet;980 981 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {982 struct hfi_msg_sys_debug_pkt *pkt = packet;983 984 dev_dbg(dev, VDBGFW "%s", pkt->msg_data);985 }986 }987}988 989static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,990 bool wait)991{992 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);993 struct hfi_sys_pc_prep_pkt pkt;994 int ret;995 996 init_completion(&hdev->pwr_collapse_prep);997 998 pkt_sys_pc_prep(&pkt);999 1000 ret = venus_iface_cmdq_write(hdev, &pkt, false);1001 if (ret)1002 return ret;1003 1004 if (!wait)1005 return 0;1006 1007 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);1008 if (!ret) {1009 venus_flush_debug_queue(hdev);1010 return -ETIMEDOUT;1011 }1012 1013 return 0;1014}1015 1016static int venus_are_queues_empty(struct venus_hfi_device *hdev)1017{1018 int ret1, ret2;1019 1020 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);1021 if (ret1 < 0)1022 return ret1;1023 1024 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);1025 if (ret2 < 0)1026 return ret2;1027 1028 if (!ret1 && !ret2)1029 return 1;1030 1031 return 0;1032}1033 1034static void venus_sfr_print(struct venus_hfi_device *hdev)1035{1036 struct device *dev = hdev->core->dev;1037 struct hfi_sfr *sfr = hdev->sfr.kva;1038 void *p;1039 1040 if (!sfr)1041 return;1042 1043 p = memchr(sfr->data, '\0', sfr->buf_size);1044 /*1045 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates1046 * that Venus is in the process of crashing.1047 */1048 if (!p)1049 sfr->data[sfr->buf_size - 1] = '\0';1050 1051 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);1052}1053 1054static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,1055 void *packet)1056{1057 struct hfi_msg_event_notify_pkt *event_pkt = packet;1058 1059 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)1060 return;1061 1062 venus_set_state(hdev, VENUS_STATE_DEINIT);1063 1064 venus_sfr_print(hdev);1065}1066 1067static irqreturn_t venus_isr_thread(struct venus_core *core)1068{1069 struct venus_hfi_device *hdev = to_hfi_priv(core);1070 const struct venus_resources *res;1071 void *pkt;1072 u32 msg_ret;1073 1074 if (!hdev)1075 return IRQ_NONE;1076 1077 res = hdev->core->res;1078 pkt = hdev->pkt_buf;1079 1080 1081 while (!venus_iface_msgq_read(hdev, pkt)) {1082 msg_ret = hfi_process_msg_packet(core, pkt);1083 switch (msg_ret) {1084 case HFI_MSG_EVENT_NOTIFY:1085 venus_process_msg_sys_error(hdev, pkt);1086 break;1087 case HFI_MSG_SYS_INIT:1088 venus_hfi_core_set_resource(core, res->vmem_id,1089 res->vmem_size,1090 res->vmem_addr,1091 hdev);1092 break;1093 case HFI_MSG_SYS_RELEASE_RESOURCE:1094 complete(&hdev->release_resource);1095 break;1096 case HFI_MSG_SYS_PC_PREP:1097 complete(&hdev->pwr_collapse_prep);1098 break;1099 default:1100 break;1101 }1102 }1103 1104 venus_flush_debug_queue(hdev);1105 1106 return IRQ_HANDLED;1107}1108 1109static irqreturn_t venus_isr(struct venus_core *core)1110{1111 struct venus_hfi_device *hdev = to_hfi_priv(core);1112 u32 status;1113 void __iomem *cpu_cs_base;1114 void __iomem *wrapper_base;1115 1116 if (!hdev)1117 return IRQ_NONE;1118 1119 cpu_cs_base = hdev->core->cpu_cs_base;1120 wrapper_base = hdev->core->wrapper_base;1121 1122 status = readl(wrapper_base + WRAPPER_INTR_STATUS);1123 if (IS_IRIS2(core) || IS_IRIS2_1(core)) {1124 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||1125 status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 ||1126 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)1127 hdev->irq_status = status;1128 } else {1129 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||1130 status & WRAPPER_INTR_STATUS_A2HWD_MASK ||1131 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)1132 hdev->irq_status = status;1133 }1134 writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR);1135 if (!(IS_IRIS2(core) || IS_IRIS2_1(core)))1136 writel(status, wrapper_base + WRAPPER_INTR_CLEAR);1137 1138 return IRQ_WAKE_THREAD;1139}1140 1141static int venus_core_init(struct venus_core *core)1142{1143 struct venus_hfi_device *hdev = to_hfi_priv(core);1144 struct device *dev = core->dev;1145 struct hfi_sys_get_property_pkt version_pkt;1146 struct hfi_sys_init_pkt pkt;1147 int ret;1148 1149 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);1150 1151 venus_set_state(hdev, VENUS_STATE_INIT);1152 1153 ret = venus_iface_cmdq_write(hdev, &pkt, false);1154 if (ret)1155 return ret;1156 1157 pkt_sys_image_version(&version_pkt);1158 1159 ret = venus_iface_cmdq_write(hdev, &version_pkt, false);1160 if (ret)1161 dev_warn(dev, "failed to send image version pkt to fw\n");1162 1163 ret = venus_sys_set_default_properties(hdev);1164 if (ret)1165 return ret;1166 1167 return 0;1168}1169 1170static int venus_core_deinit(struct venus_core *core)1171{1172 struct venus_hfi_device *hdev = to_hfi_priv(core);1173 1174 venus_set_state(hdev, VENUS_STATE_DEINIT);1175 hdev->suspended = true;1176 hdev->power_enabled = false;1177 1178 return 0;1179}1180 1181static int venus_core_ping(struct venus_core *core, u32 cookie)1182{1183 struct venus_hfi_device *hdev = to_hfi_priv(core);1184 struct hfi_sys_ping_pkt pkt;1185 1186 pkt_sys_ping(&pkt, cookie);1187 1188 return venus_iface_cmdq_write(hdev, &pkt, false);1189}1190 1191static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)1192{1193 struct venus_hfi_device *hdev = to_hfi_priv(core);1194 struct hfi_sys_test_ssr_pkt pkt;1195 int ret;1196 1197 ret = pkt_sys_ssr_cmd(&pkt, trigger_type);1198 if (ret)1199 return ret;1200 1201 return venus_iface_cmdq_write(hdev, &pkt, false);1202}1203 1204static int venus_session_init(struct venus_inst *inst, u32 session_type,1205 u32 codec)1206{1207 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1208 struct hfi_session_init_pkt pkt;1209 int ret;1210 1211 ret = venus_sys_set_debug(hdev, venus_fw_debug);1212 if (ret)1213 goto err;1214 1215 ret = pkt_session_init(&pkt, inst, session_type, codec);1216 if (ret)1217 goto err;1218 1219 ret = venus_iface_cmdq_write(hdev, &pkt, true);1220 if (ret)1221 goto err;1222 1223 return 0;1224 1225err:1226 venus_flush_debug_queue(hdev);1227 return ret;1228}1229 1230static int venus_session_end(struct venus_inst *inst)1231{1232 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1233 struct device *dev = hdev->core->dev;1234 1235 if (venus_fw_coverage) {1236 if (venus_sys_set_coverage(hdev, venus_fw_coverage))1237 dev_warn(dev, "fw coverage msg ON failed\n");1238 }1239 1240 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true);1241}1242 1243static int venus_session_abort(struct venus_inst *inst)1244{1245 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1246 1247 venus_flush_debug_queue(hdev);1248 1249 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true);1250}1251 1252static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)1253{1254 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1255 struct hfi_session_flush_pkt pkt;1256 int ret;1257 1258 ret = pkt_session_flush(&pkt, inst, flush_mode);1259 if (ret)1260 return ret;1261 1262 return venus_iface_cmdq_write(hdev, &pkt, true);1263}1264 1265static int venus_session_start(struct venus_inst *inst)1266{1267 return venus_session_cmd(inst, HFI_CMD_SESSION_START, true);1268}1269 1270static int venus_session_stop(struct venus_inst *inst)1271{1272 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true);1273}1274 1275static int venus_session_continue(struct venus_inst *inst)1276{1277 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false);1278}1279 1280static int venus_session_etb(struct venus_inst *inst,1281 struct hfi_frame_data *in_frame)1282{1283 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1284 u32 session_type = inst->session_type;1285 int ret;1286 1287 if (session_type == VIDC_SESSION_TYPE_DEC) {1288 struct hfi_session_empty_buffer_compressed_pkt pkt;1289 1290 ret = pkt_session_etb_decoder(&pkt, inst, in_frame);1291 if (ret)1292 return ret;1293 1294 ret = venus_iface_cmdq_write(hdev, &pkt, false);1295 } else if (session_type == VIDC_SESSION_TYPE_ENC) {1296 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;1297 1298 ret = pkt_session_etb_encoder(&pkt, inst, in_frame);1299 if (ret)1300 return ret;1301 1302 ret = venus_iface_cmdq_write(hdev, &pkt, false);1303 } else {1304 ret = -EINVAL;1305 }1306 1307 return ret;1308}1309 1310static int venus_session_ftb(struct venus_inst *inst,1311 struct hfi_frame_data *out_frame)1312{1313 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1314 struct hfi_session_fill_buffer_pkt pkt;1315 int ret;1316 1317 ret = pkt_session_ftb(&pkt, inst, out_frame);1318 if (ret)1319 return ret;1320 1321 return venus_iface_cmdq_write(hdev, &pkt, false);1322}1323 1324static int venus_session_set_buffers(struct venus_inst *inst,1325 struct hfi_buffer_desc *bd)1326{1327 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1328 struct hfi_session_set_buffers_pkt *pkt;1329 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];1330 int ret;1331 1332 if (bd->buffer_type == HFI_BUFFER_INPUT)1333 return 0;1334 1335 pkt = (struct hfi_session_set_buffers_pkt *)packet;1336 1337 ret = pkt_session_set_buffers(pkt, inst, bd);1338 if (ret)1339 return ret;1340 1341 return venus_iface_cmdq_write(hdev, pkt, false);1342}1343 1344static int venus_session_unset_buffers(struct venus_inst *inst,1345 struct hfi_buffer_desc *bd)1346{1347 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1348 struct hfi_session_release_buffer_pkt *pkt;1349 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];1350 int ret;1351 1352 if (bd->buffer_type == HFI_BUFFER_INPUT)1353 return 0;1354 1355 pkt = (struct hfi_session_release_buffer_pkt *)packet;1356 1357 ret = pkt_session_unset_buffers(pkt, inst, bd);1358 if (ret)1359 return ret;1360 1361 return venus_iface_cmdq_write(hdev, pkt, true);1362}1363 1364static int venus_session_load_res(struct venus_inst *inst)1365{1366 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true);1367}1368 1369static int venus_session_release_res(struct venus_inst *inst)1370{1371 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true);1372}1373 1374static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,1375 u32 seq_hdr_len)1376{1377 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1378 struct hfi_session_parse_sequence_header_pkt *pkt;1379 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];1380 int ret;1381 1382 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;1383 1384 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);1385 if (ret)1386 return ret;1387 1388 ret = venus_iface_cmdq_write(hdev, pkt, false);1389 if (ret)1390 return ret;1391 1392 return 0;1393}1394 1395static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,1396 u32 seq_hdr_len)1397{1398 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1399 struct hfi_session_get_sequence_header_pkt *pkt;1400 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];1401 int ret;1402 1403 pkt = (struct hfi_session_get_sequence_header_pkt *)packet;1404 1405 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);1406 if (ret)1407 return ret;1408 1409 return venus_iface_cmdq_write(hdev, pkt, false);1410}1411 1412static int venus_session_set_property(struct venus_inst *inst, u32 ptype,1413 void *pdata)1414{1415 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1416 struct hfi_session_set_property_pkt *pkt;1417 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];1418 int ret;1419 1420 pkt = (struct hfi_session_set_property_pkt *)packet;1421 1422 ret = pkt_session_set_property(pkt, inst, ptype, pdata);1423 if (ret == -ENOTSUPP)1424 return 0;1425 if (ret)1426 return ret;1427 1428 return venus_iface_cmdq_write(hdev, pkt, false);1429}1430 1431static int venus_session_get_property(struct venus_inst *inst, u32 ptype)1432{1433 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);1434 struct hfi_session_get_property_pkt pkt;1435 int ret;1436 1437 ret = pkt_session_get_property(&pkt, inst, ptype);1438 if (ret)1439 return ret;1440 1441 return venus_iface_cmdq_write(hdev, &pkt, true);1442}1443 1444static int venus_resume(struct venus_core *core)1445{1446 struct venus_hfi_device *hdev = to_hfi_priv(core);1447 int ret = 0;1448 1449 mutex_lock(&hdev->lock);1450 1451 if (!hdev->suspended)1452 goto unlock;1453 1454 ret = venus_power_on(hdev);1455 1456unlock:1457 if (!ret)1458 hdev->suspended = false;1459 1460 mutex_unlock(&hdev->lock);1461 1462 return ret;1463}1464 1465static int venus_suspend_1xx(struct venus_core *core)1466{1467 struct venus_hfi_device *hdev = to_hfi_priv(core);1468 struct device *dev = core->dev;1469 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;1470 u32 ctrl_status;1471 int ret;1472 1473 if (!hdev->power_enabled || hdev->suspended)1474 return 0;1475 1476 mutex_lock(&hdev->lock);1477 ret = venus_is_valid_state(hdev);1478 mutex_unlock(&hdev->lock);1479 1480 if (!ret) {1481 dev_err(dev, "bad state, cannot suspend\n");1482 return -EINVAL;1483 }1484 1485 ret = venus_prepare_power_collapse(hdev, true);1486 if (ret) {1487 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);1488 return ret;1489 }1490 1491 mutex_lock(&hdev->lock);1492 1493 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {1494 mutex_unlock(&hdev->lock);1495 return -EINVAL;1496 }1497 1498 ret = venus_are_queues_empty(hdev);1499 if (ret < 0 || !ret) {1500 mutex_unlock(&hdev->lock);1501 return -EINVAL;1502 }1503 1504 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);1505 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {1506 mutex_unlock(&hdev->lock);1507 return -EINVAL;1508 }1509 1510 ret = venus_power_off(hdev);1511 if (ret) {1512 mutex_unlock(&hdev->lock);1513 return ret;1514 }1515 1516 hdev->suspended = true;1517 1518 mutex_unlock(&hdev->lock);1519 1520 return 0;1521}1522 1523static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)1524{1525 void __iomem *wrapper_base = hdev->core->wrapper_base;1526 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;1527 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;1528 u32 ctrl_status, cpu_status;1529 1530 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))1531 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);1532 else1533 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);1534 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);1535 1536 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&1537 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)1538 return true;1539 1540 return false;1541}1542 1543static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)1544{1545 void __iomem *wrapper_base = hdev->core->wrapper_base;1546 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;1547 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;1548 u32 ctrl_status, cpu_status;1549 1550 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))1551 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);1552 else1553 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);1554 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);1555 1556 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&1557 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)1558 return true;1559 1560 return false;1561}1562 1563static int venus_suspend_3xx(struct venus_core *core)1564{1565 struct venus_hfi_device *hdev = to_hfi_priv(core);1566 struct device *dev = core->dev;1567 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;1568 u32 ctrl_status;1569 bool val;1570 int ret;1571 1572 if (!hdev->power_enabled || hdev->suspended)1573 return 0;1574 1575 mutex_lock(&hdev->lock);1576 ret = venus_is_valid_state(hdev);1577 mutex_unlock(&hdev->lock);1578 1579 if (!ret) {1580 dev_err(dev, "bad state, cannot suspend\n");1581 return -EINVAL;1582 }1583 1584 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);1585 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)1586 goto power_off;1587 1588 /*1589 * Power collapse sequence for Venus 3xx and 4xx versions:1590 * 1. Check for ARM9 and video core to be idle by checking WFI bit1591 * (bit 0) in CPU status register and by checking Idle (bit 30) in1592 * Control status register for video core.1593 * 2. Send a command to prepare for power collapse.1594 * 3. Check for WFI and PC_READY bits.1595 */1596 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,1597 1500, 100 * 1500);1598 if (ret) {1599 dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret);1600 return ret;1601 }1602 1603 ret = venus_prepare_power_collapse(hdev, false);1604 if (ret) {1605 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);1606 return ret;1607 }1608 1609 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,1610 1500, 100 * 1500);1611 if (ret)1612 return ret;1613 1614power_off:1615 mutex_lock(&hdev->lock);1616 1617 ret = venus_power_off(hdev);1618 if (ret) {1619 dev_err(dev, "venus_power_off (%d)\n", ret);1620 mutex_unlock(&hdev->lock);1621 return ret;1622 }1623 1624 hdev->suspended = true;1625 1626 mutex_unlock(&hdev->lock);1627 1628 return 0;1629}1630 1631static int venus_suspend(struct venus_core *core)1632{1633 if (IS_V3(core) || IS_V4(core) || IS_V6(core))1634 return venus_suspend_3xx(core);1635 1636 return venus_suspend_1xx(core);1637}1638 1639static const struct hfi_ops venus_hfi_ops = {1640 .core_init = venus_core_init,1641 .core_deinit = venus_core_deinit,1642 .core_ping = venus_core_ping,1643 .core_trigger_ssr = venus_core_trigger_ssr,1644 1645 .session_init = venus_session_init,1646 .session_end = venus_session_end,1647 .session_abort = venus_session_abort,1648 .session_flush = venus_session_flush,1649 .session_start = venus_session_start,1650 .session_stop = venus_session_stop,1651 .session_continue = venus_session_continue,1652 .session_etb = venus_session_etb,1653 .session_ftb = venus_session_ftb,1654 .session_set_buffers = venus_session_set_buffers,1655 .session_unset_buffers = venus_session_unset_buffers,1656 .session_load_res = venus_session_load_res,1657 .session_release_res = venus_session_release_res,1658 .session_parse_seq_hdr = venus_session_parse_seq_hdr,1659 .session_get_seq_hdr = venus_session_get_seq_hdr,1660 .session_set_property = venus_session_set_property,1661 .session_get_property = venus_session_get_property,1662 1663 .resume = venus_resume,1664 .suspend = venus_suspend,1665 1666 .isr = venus_isr,1667 .isr_thread = venus_isr_thread,1668};1669 1670void venus_hfi_destroy(struct venus_core *core)1671{1672 struct venus_hfi_device *hdev = to_hfi_priv(core);1673 1674 core->priv = NULL;1675 venus_interface_queues_release(hdev);1676 mutex_destroy(&hdev->lock);1677 kfree(hdev);1678 core->ops = NULL;1679}1680 1681int venus_hfi_create(struct venus_core *core)1682{1683 struct venus_hfi_device *hdev;1684 int ret;1685 1686 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);1687 if (!hdev)1688 return -ENOMEM;1689 1690 mutex_init(&hdev->lock);1691 1692 hdev->core = core;1693 hdev->suspended = true;1694 core->priv = hdev;1695 core->ops = &venus_hfi_ops;1696 1697 ret = venus_interface_queues_init(hdev);1698 if (ret)1699 goto err_kfree;1700 1701 return 0;1702 1703err_kfree:1704 kfree(hdev);1705 core->priv = NULL;1706 core->ops = NULL;1707 return ret;1708}1709 1710void venus_hfi_queues_reinit(struct venus_core *core)1711{1712 struct venus_hfi_device *hdev = to_hfi_priv(core);1713 struct hfi_queue_table_header *tbl_hdr;1714 struct iface_queue *queue;1715 struct hfi_sfr *sfr;1716 unsigned int i;1717 1718 mutex_lock(&hdev->lock);1719 1720 for (i = 0; i < IFACEQ_NUM; i++) {1721 queue = &hdev->queues[i];1722 queue->qhdr =1723 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);1724 1725 venus_set_qhdr_defaults(queue->qhdr);1726 1727 queue->qhdr->start_addr = queue->qmem.da;1728 1729 if (i == IFACEQ_CMD_IDX)1730 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;1731 else if (i == IFACEQ_MSG_IDX)1732 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;1733 else if (i == IFACEQ_DBG_IDX)1734 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;1735 }1736 1737 tbl_hdr = hdev->ifaceq_table.kva;1738 tbl_hdr->version = 0;1739 tbl_hdr->size = IFACEQ_TABLE_SIZE;1740 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);1741 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);1742 tbl_hdr->num_q = IFACEQ_NUM;1743 tbl_hdr->num_active_q = IFACEQ_NUM;1744 1745 /*1746 * Set receive request to zero on debug queue as there is no1747 * need of interrupt from video hardware for debug messages1748 */1749 queue = &hdev->queues[IFACEQ_DBG_IDX];1750 queue->qhdr->rx_req = 0;1751 1752 sfr = hdev->sfr.kva;1753 sfr->buf_size = ALIGNED_SFR_SIZE;1754 1755 /* ensure table and queue header structs are settled in memory */1756 wmb();1757 1758 mutex_unlock(&hdev->lock);1759}1760