792 lines · c
1// SPDX-License-Identifier: BSD-3-Clause2/*3 * Copyright (c) 2020, MIPI Alliance, Inc.4 *5 * Author: Nicolas Pitre <npitre@baylibre.com>6 *7 * Note: The I3C HCI v2.0 spec is still in flux. The IBI support is based on8 * v1.x of the spec and v2.0 will likely be split out.9 */10 11#include <linux/bitfield.h>12#include <linux/device.h>13#include <linux/dma-mapping.h>14#include <linux/errno.h>15#include <linux/i3c/master.h>16#include <linux/io.h>17 18#include "hci.h"19#include "cmd.h"20#include "ibi.h"21 22 23/*24 * Software Parameter Values (somewhat arb itrary for now).25 * Some of them could be determined at run time eventually.26 */27 28#define XFER_RINGS 1 /* max: 8 */29#define XFER_RING_ENTRIES 16 /* max: 255 */30 31#define IBI_RINGS 1 /* max: 8 */32#define IBI_STATUS_RING_ENTRIES 32 /* max: 255 */33#define IBI_CHUNK_CACHELINES 1 /* max: 256 bytes equivalent */34#define IBI_CHUNK_POOL_SIZE 128 /* max: 1023 */35 36/*37 * Ring Header Preamble38 */39 40#define rhs_reg_read(r) readl(hci->RHS_regs + (RHS_##r))41#define rhs_reg_write(r, v) writel(v, hci->RHS_regs + (RHS_##r))42 43#define RHS_CONTROL 0x0044#define PREAMBLE_SIZE GENMASK(31, 24) /* Preamble Section Size */45#define HEADER_SIZE GENMASK(23, 16) /* Ring Header Size */46#define MAX_HEADER_COUNT_CAP GENMASK(7, 4) /* HC Max Header Count */47#define MAX_HEADER_COUNT GENMASK(3, 0) /* Driver Max Header Count */48 49#define RHS_RHn_OFFSET(n) (0x04 + (n)*4)50 51/*52 * Ring Header (Per-Ring Bundle)53 */54 55#define rh_reg_read(r) readl(rh->regs + (RH_##r))56#define rh_reg_write(r, v) writel(v, rh->regs + (RH_##r))57 58#define RH_CR_SETUP 0x00 /* Command/Response Ring */59#define CR_XFER_STRUCT_SIZE GENMASK(31, 24)60#define CR_RESP_STRUCT_SIZE GENMASK(23, 16)61#define CR_RING_SIZE GENMASK(8, 0)62 63#define RH_IBI_SETUP 0x0464#define IBI_STATUS_STRUCT_SIZE GENMASK(31, 24)65#define IBI_STATUS_RING_SIZE GENMASK(23, 16)66#define IBI_DATA_CHUNK_SIZE GENMASK(12, 10)67#define IBI_DATA_CHUNK_COUNT GENMASK(9, 0)68 69#define RH_CHUNK_CONTROL 0x0870 71#define RH_INTR_STATUS 0x1072#define RH_INTR_STATUS_ENABLE 0x1473#define RH_INTR_SIGNAL_ENABLE 0x1874#define RH_INTR_FORCE 0x1c75#define INTR_IBI_READY BIT(12)76#define INTR_TRANSFER_COMPLETION BIT(11)77#define INTR_RING_OP BIT(10)78#define INTR_TRANSFER_ERR BIT(9)79#define INTR_WARN_INS_STOP_MODE BIT(7)80#define INTR_IBI_RING_FULL BIT(6)81#define INTR_TRANSFER_ABORT BIT(5)82 83#define RH_RING_STATUS 0x2084#define RING_STATUS_LOCKED BIT(3)85#define RING_STATUS_ABORTED BIT(2)86#define RING_STATUS_RUNNING BIT(1)87#define RING_STATUS_ENABLED BIT(0)88 89#define RH_RING_CONTROL 0x2490#define RING_CTRL_ABORT BIT(2)91#define RING_CTRL_RUN_STOP BIT(1)92#define RING_CTRL_ENABLE BIT(0)93 94#define RH_RING_OPERATION1 0x2895#define RING_OP1_IBI_DEQ_PTR GENMASK(23, 16)96#define RING_OP1_CR_SW_DEQ_PTR GENMASK(15, 8)97#define RING_OP1_CR_ENQ_PTR GENMASK(7, 0)98 99#define RH_RING_OPERATION2 0x2c100#define RING_OP2_IBI_ENQ_PTR GENMASK(23, 16)101#define RING_OP2_CR_DEQ_PTR GENMASK(7, 0)102 103#define RH_CMD_RING_BASE_LO 0x30104#define RH_CMD_RING_BASE_HI 0x34105#define RH_RESP_RING_BASE_LO 0x38106#define RH_RESP_RING_BASE_HI 0x3c107#define RH_IBI_STATUS_RING_BASE_LO 0x40108#define RH_IBI_STATUS_RING_BASE_HI 0x44109#define RH_IBI_DATA_RING_BASE_LO 0x48110#define RH_IBI_DATA_RING_BASE_HI 0x4c111 112#define RH_CMD_RING_SG 0x50 /* Ring Scatter Gather Support */113#define RH_RESP_RING_SG 0x54114#define RH_IBI_STATUS_RING_SG 0x58115#define RH_IBI_DATA_RING_SG 0x5c116#define RING_SG_BLP BIT(31) /* Buffer Vs. List Pointer */117#define RING_SG_LIST_SIZE GENMASK(15, 0)118 119/*120 * Data Buffer Descriptor (in memory)121 */122 123#define DATA_BUF_BLP BIT(31) /* Buffer Vs. List Pointer */124#define DATA_BUF_IOC BIT(30) /* Interrupt on Completion */125#define DATA_BUF_BLOCK_SIZE GENMASK(15, 0)126 127 128struct hci_rh_data {129 void __iomem *regs;130 void *xfer, *resp, *ibi_status, *ibi_data;131 dma_addr_t xfer_dma, resp_dma, ibi_status_dma, ibi_data_dma;132 unsigned int xfer_entries, ibi_status_entries, ibi_chunks_total;133 unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz;134 unsigned int done_ptr, ibi_chunk_ptr;135 struct hci_xfer **src_xfers;136 spinlock_t lock;137 struct completion op_done;138};139 140struct hci_rings_data {141 unsigned int total;142 struct hci_rh_data headers[] __counted_by(total);143};144 145struct hci_dma_dev_ibi_data {146 struct i3c_generic_ibi_pool *pool;147 unsigned int max_len;148};149 150static void hci_dma_cleanup(struct i3c_hci *hci)151{152 struct hci_rings_data *rings = hci->io_data;153 struct hci_rh_data *rh;154 unsigned int i;155 156 if (!rings)157 return;158 159 for (i = 0; i < rings->total; i++) {160 rh = &rings->headers[i];161 162 rh_reg_write(RING_CONTROL, 0);163 rh_reg_write(CR_SETUP, 0);164 rh_reg_write(IBI_SETUP, 0);165 rh_reg_write(INTR_SIGNAL_ENABLE, 0);166 167 if (rh->xfer)168 dma_free_coherent(&hci->master.dev,169 rh->xfer_struct_sz * rh->xfer_entries,170 rh->xfer, rh->xfer_dma);171 if (rh->resp)172 dma_free_coherent(&hci->master.dev,173 rh->resp_struct_sz * rh->xfer_entries,174 rh->resp, rh->resp_dma);175 kfree(rh->src_xfers);176 if (rh->ibi_status)177 dma_free_coherent(&hci->master.dev,178 rh->ibi_status_sz * rh->ibi_status_entries,179 rh->ibi_status, rh->ibi_status_dma);180 if (rh->ibi_data_dma)181 dma_unmap_single(&hci->master.dev, rh->ibi_data_dma,182 rh->ibi_chunk_sz * rh->ibi_chunks_total,183 DMA_FROM_DEVICE);184 kfree(rh->ibi_data);185 }186 187 rhs_reg_write(CONTROL, 0);188 189 kfree(rings);190 hci->io_data = NULL;191}192 193static int hci_dma_init(struct i3c_hci *hci)194{195 struct hci_rings_data *rings;196 struct hci_rh_data *rh;197 u32 regval;198 unsigned int i, nr_rings, xfers_sz, resps_sz;199 unsigned int ibi_status_ring_sz, ibi_data_ring_sz;200 int ret;201 202 regval = rhs_reg_read(CONTROL);203 nr_rings = FIELD_GET(MAX_HEADER_COUNT_CAP, regval);204 dev_info(&hci->master.dev, "%d DMA rings available\n", nr_rings);205 if (unlikely(nr_rings > 8)) {206 dev_err(&hci->master.dev, "number of rings should be <= 8\n");207 nr_rings = 8;208 }209 if (nr_rings > XFER_RINGS)210 nr_rings = XFER_RINGS;211 rings = kzalloc(struct_size(rings, headers, nr_rings), GFP_KERNEL);212 if (!rings)213 return -ENOMEM;214 hci->io_data = rings;215 rings->total = nr_rings;216 217 regval = FIELD_PREP(MAX_HEADER_COUNT, rings->total);218 rhs_reg_write(CONTROL, regval);219 220 for (i = 0; i < rings->total; i++) {221 u32 offset = rhs_reg_read(RHn_OFFSET(i));222 223 dev_info(&hci->master.dev, "Ring %d at offset %#x\n", i, offset);224 ret = -EINVAL;225 if (!offset)226 goto err_out;227 rh = &rings->headers[i];228 rh->regs = hci->base_regs + offset;229 spin_lock_init(&rh->lock);230 init_completion(&rh->op_done);231 232 rh->xfer_entries = XFER_RING_ENTRIES;233 234 regval = rh_reg_read(CR_SETUP);235 rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval);236 rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval);237 DBG("xfer_struct_sz = %d, resp_struct_sz = %d",238 rh->xfer_struct_sz, rh->resp_struct_sz);239 xfers_sz = rh->xfer_struct_sz * rh->xfer_entries;240 resps_sz = rh->resp_struct_sz * rh->xfer_entries;241 242 rh->xfer = dma_alloc_coherent(&hci->master.dev, xfers_sz,243 &rh->xfer_dma, GFP_KERNEL);244 rh->resp = dma_alloc_coherent(&hci->master.dev, resps_sz,245 &rh->resp_dma, GFP_KERNEL);246 rh->src_xfers =247 kmalloc_array(rh->xfer_entries, sizeof(*rh->src_xfers),248 GFP_KERNEL);249 ret = -ENOMEM;250 if (!rh->xfer || !rh->resp || !rh->src_xfers)251 goto err_out;252 253 rh_reg_write(CMD_RING_BASE_LO, lower_32_bits(rh->xfer_dma));254 rh_reg_write(CMD_RING_BASE_HI, upper_32_bits(rh->xfer_dma));255 rh_reg_write(RESP_RING_BASE_LO, lower_32_bits(rh->resp_dma));256 rh_reg_write(RESP_RING_BASE_HI, upper_32_bits(rh->resp_dma));257 258 regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries);259 rh_reg_write(CR_SETUP, regval);260 261 rh_reg_write(INTR_STATUS_ENABLE, 0xffffffff);262 rh_reg_write(INTR_SIGNAL_ENABLE, INTR_IBI_READY |263 INTR_TRANSFER_COMPLETION |264 INTR_RING_OP |265 INTR_TRANSFER_ERR |266 INTR_WARN_INS_STOP_MODE |267 INTR_IBI_RING_FULL |268 INTR_TRANSFER_ABORT);269 270 /* IBIs */271 272 if (i >= IBI_RINGS)273 goto ring_ready;274 275 regval = rh_reg_read(IBI_SETUP);276 rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval);277 rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES;278 rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE;279 280 rh->ibi_chunk_sz = dma_get_cache_alignment();281 rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES;282 /*283 * Round IBI data chunk size to number of bytes supported by284 * the HW. Chunk size can be 2^n number of DWORDs which is the285 * same as 2^(n+2) bytes, where n is 0..6.286 */287 rh->ibi_chunk_sz = umax(4, rh->ibi_chunk_sz);288 rh->ibi_chunk_sz = roundup_pow_of_two(rh->ibi_chunk_sz);289 if (rh->ibi_chunk_sz > 256) {290 ret = -EINVAL;291 goto err_out;292 }293 294 ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries;295 ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total;296 297 rh->ibi_status =298 dma_alloc_coherent(&hci->master.dev, ibi_status_ring_sz,299 &rh->ibi_status_dma, GFP_KERNEL);300 rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL);301 ret = -ENOMEM;302 if (!rh->ibi_status || !rh->ibi_data)303 goto err_out;304 rh->ibi_data_dma =305 dma_map_single(&hci->master.dev, rh->ibi_data,306 ibi_data_ring_sz, DMA_FROM_DEVICE);307 if (dma_mapping_error(&hci->master.dev, rh->ibi_data_dma)) {308 rh->ibi_data_dma = 0;309 ret = -ENOMEM;310 goto err_out;311 }312 313 rh_reg_write(IBI_STATUS_RING_BASE_LO, lower_32_bits(rh->ibi_status_dma));314 rh_reg_write(IBI_STATUS_RING_BASE_HI, upper_32_bits(rh->ibi_status_dma));315 rh_reg_write(IBI_DATA_RING_BASE_LO, lower_32_bits(rh->ibi_data_dma));316 rh_reg_write(IBI_DATA_RING_BASE_HI, upper_32_bits(rh->ibi_data_dma));317 318 regval = FIELD_PREP(IBI_STATUS_RING_SIZE,319 rh->ibi_status_entries) |320 FIELD_PREP(IBI_DATA_CHUNK_SIZE,321 ilog2(rh->ibi_chunk_sz) - 2) |322 FIELD_PREP(IBI_DATA_CHUNK_COUNT,323 rh->ibi_chunks_total);324 rh_reg_write(IBI_SETUP, regval);325 326 regval = rh_reg_read(INTR_SIGNAL_ENABLE);327 regval |= INTR_IBI_READY;328 rh_reg_write(INTR_SIGNAL_ENABLE, regval);329 330ring_ready:331 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE |332 RING_CTRL_RUN_STOP);333 }334 335 return 0;336 337err_out:338 hci_dma_cleanup(hci);339 return ret;340}341 342static void hci_dma_unmap_xfer(struct i3c_hci *hci,343 struct hci_xfer *xfer_list, unsigned int n)344{345 struct hci_xfer *xfer;346 unsigned int i;347 348 for (i = 0; i < n; i++) {349 xfer = xfer_list + i;350 if (!xfer->data)351 continue;352 dma_unmap_single(&hci->master.dev,353 xfer->data_dma, xfer->data_len,354 xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE);355 }356}357 358static int hci_dma_queue_xfer(struct i3c_hci *hci,359 struct hci_xfer *xfer_list, int n)360{361 struct hci_rings_data *rings = hci->io_data;362 struct hci_rh_data *rh;363 unsigned int i, ring, enqueue_ptr;364 u32 op1_val, op2_val;365 void *buf;366 367 /* For now we only use ring 0 */368 ring = 0;369 rh = &rings->headers[ring];370 371 op1_val = rh_reg_read(RING_OPERATION1);372 enqueue_ptr = FIELD_GET(RING_OP1_CR_ENQ_PTR, op1_val);373 for (i = 0; i < n; i++) {374 struct hci_xfer *xfer = xfer_list + i;375 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr;376 377 /* store cmd descriptor */378 *ring_data++ = xfer->cmd_desc[0];379 *ring_data++ = xfer->cmd_desc[1];380 if (hci->cmd == &mipi_i3c_hci_cmd_v2) {381 *ring_data++ = xfer->cmd_desc[2];382 *ring_data++ = xfer->cmd_desc[3];383 }384 385 /* first word of Data Buffer Descriptor Structure */386 if (!xfer->data)387 xfer->data_len = 0;388 *ring_data++ =389 FIELD_PREP(DATA_BUF_BLOCK_SIZE, xfer->data_len) |390 ((i == n - 1) ? DATA_BUF_IOC : 0);391 392 /* 2nd and 3rd words of Data Buffer Descriptor Structure */393 if (xfer->data) {394 buf = xfer->bounce_buf ? xfer->bounce_buf : xfer->data;395 xfer->data_dma =396 dma_map_single(&hci->master.dev,397 buf,398 xfer->data_len,399 xfer->rnw ?400 DMA_FROM_DEVICE :401 DMA_TO_DEVICE);402 if (dma_mapping_error(&hci->master.dev,403 xfer->data_dma)) {404 hci_dma_unmap_xfer(hci, xfer_list, i);405 return -ENOMEM;406 }407 *ring_data++ = lower_32_bits(xfer->data_dma);408 *ring_data++ = upper_32_bits(xfer->data_dma);409 } else {410 *ring_data++ = 0;411 *ring_data++ = 0;412 }413 414 /* remember corresponding xfer struct */415 rh->src_xfers[enqueue_ptr] = xfer;416 /* remember corresponding ring/entry for this xfer structure */417 xfer->ring_number = ring;418 xfer->ring_entry = enqueue_ptr;419 420 enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries;421 422 /*423 * We may update the hardware view of the enqueue pointer424 * only if we didn't reach its dequeue pointer.425 */426 op2_val = rh_reg_read(RING_OPERATION2);427 if (enqueue_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) {428 /* the ring is full */429 hci_dma_unmap_xfer(hci, xfer_list, i + 1);430 return -EBUSY;431 }432 }433 434 /* take care to update the hardware enqueue pointer atomically */435 spin_lock_irq(&rh->lock);436 op1_val = rh_reg_read(RING_OPERATION1);437 op1_val &= ~RING_OP1_CR_ENQ_PTR;438 op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr);439 rh_reg_write(RING_OPERATION1, op1_val);440 spin_unlock_irq(&rh->lock);441 442 return 0;443}444 445static bool hci_dma_dequeue_xfer(struct i3c_hci *hci,446 struct hci_xfer *xfer_list, int n)447{448 struct hci_rings_data *rings = hci->io_data;449 struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number];450 unsigned int i;451 bool did_unqueue = false;452 453 /* stop the ring */454 rh_reg_write(RING_CONTROL, RING_CTRL_ABORT);455 if (wait_for_completion_timeout(&rh->op_done, HZ) == 0) {456 /*457 * We're deep in it if ever this condition is ever met.458 * Hardware might still be writing to memory, etc.459 */460 dev_crit(&hci->master.dev, "unable to abort the ring\n");461 WARN_ON(1);462 }463 464 for (i = 0; i < n; i++) {465 struct hci_xfer *xfer = xfer_list + i;466 int idx = xfer->ring_entry;467 468 /*469 * At the time the abort happened, the xfer might have470 * completed already. If not then replace corresponding471 * descriptor entries with a no-op.472 */473 if (idx >= 0) {474 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx;475 476 /* store no-op cmd descriptor */477 *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7);478 *ring_data++ = 0;479 if (hci->cmd == &mipi_i3c_hci_cmd_v2) {480 *ring_data++ = 0;481 *ring_data++ = 0;482 }483 484 /* disassociate this xfer struct */485 rh->src_xfers[idx] = NULL;486 487 /* and unmap it */488 hci_dma_unmap_xfer(hci, xfer, 1);489 490 did_unqueue = true;491 }492 }493 494 /* restart the ring */495 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE);496 497 return did_unqueue;498}499 500static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh)501{502 u32 op1_val, op2_val, resp, *ring_resp;503 unsigned int tid, done_ptr = rh->done_ptr;504 struct hci_xfer *xfer;505 506 for (;;) {507 op2_val = rh_reg_read(RING_OPERATION2);508 if (done_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val))509 break;510 511 ring_resp = rh->resp + rh->resp_struct_sz * done_ptr;512 resp = *ring_resp;513 tid = RESP_TID(resp);514 DBG("resp = 0x%08x", resp);515 516 xfer = rh->src_xfers[done_ptr];517 if (!xfer) {518 DBG("orphaned ring entry");519 } else {520 hci_dma_unmap_xfer(hci, xfer, 1);521 xfer->ring_entry = -1;522 xfer->response = resp;523 if (tid != xfer->cmd_tid) {524 dev_err(&hci->master.dev,525 "response tid=%d when expecting %d\n",526 tid, xfer->cmd_tid);527 /* TODO: do something about it? */528 }529 if (xfer->completion)530 complete(xfer->completion);531 }532 533 done_ptr = (done_ptr + 1) % rh->xfer_entries;534 rh->done_ptr = done_ptr;535 }536 537 /* take care to update the software dequeue pointer atomically */538 spin_lock(&rh->lock);539 op1_val = rh_reg_read(RING_OPERATION1);540 op1_val &= ~RING_OP1_CR_SW_DEQ_PTR;541 op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr);542 rh_reg_write(RING_OPERATION1, op1_val);543 spin_unlock(&rh->lock);544}545 546static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev,547 const struct i3c_ibi_setup *req)548{549 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);550 struct i3c_generic_ibi_pool *pool;551 struct hci_dma_dev_ibi_data *dev_ibi;552 553 dev_ibi = kmalloc(sizeof(*dev_ibi), GFP_KERNEL);554 if (!dev_ibi)555 return -ENOMEM;556 pool = i3c_generic_ibi_alloc_pool(dev, req);557 if (IS_ERR(pool)) {558 kfree(dev_ibi);559 return PTR_ERR(pool);560 }561 dev_ibi->pool = pool;562 dev_ibi->max_len = req->max_payload_len;563 dev_data->ibi_data = dev_ibi;564 return 0;565}566 567static void hci_dma_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev)568{569 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);570 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data;571 572 dev_data->ibi_data = NULL;573 i3c_generic_ibi_free_pool(dev_ibi->pool);574 kfree(dev_ibi);575}576 577static void hci_dma_recycle_ibi_slot(struct i3c_hci *hci,578 struct i3c_dev_desc *dev,579 struct i3c_ibi_slot *slot)580{581 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);582 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data;583 584 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot);585}586 587static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh)588{589 struct i3c_dev_desc *dev;590 struct i3c_hci_dev_data *dev_data;591 struct hci_dma_dev_ibi_data *dev_ibi;592 struct i3c_ibi_slot *slot;593 u32 op1_val, op2_val, ibi_status_error;594 unsigned int ptr, enq_ptr, deq_ptr;595 unsigned int ibi_size, ibi_chunks, ibi_data_offset, first_part;596 int ibi_addr, last_ptr;597 void *ring_ibi_data;598 dma_addr_t ring_ibi_data_dma;599 600 op1_val = rh_reg_read(RING_OPERATION1);601 deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, op1_val);602 603 op2_val = rh_reg_read(RING_OPERATION2);604 enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val);605 606 ibi_status_error = 0;607 ibi_addr = -1;608 ibi_chunks = 0;609 ibi_size = 0;610 last_ptr = -1;611 612 /* let's find all we can about this IBI */613 for (ptr = deq_ptr; ptr != enq_ptr;614 ptr = (ptr + 1) % rh->ibi_status_entries) {615 u32 ibi_status, *ring_ibi_status;616 unsigned int chunks;617 618 ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr;619 ibi_status = *ring_ibi_status;620 DBG("status = %#x", ibi_status);621 622 if (ibi_status_error) {623 /* we no longer care */624 } else if (ibi_status & IBI_ERROR) {625 ibi_status_error = ibi_status;626 } else if (ibi_addr == -1) {627 ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status);628 } else if (ibi_addr != FIELD_GET(IBI_TARGET_ADDR, ibi_status)) {629 /* the address changed unexpectedly */630 ibi_status_error = ibi_status;631 }632 633 chunks = FIELD_GET(IBI_CHUNKS, ibi_status);634 ibi_chunks += chunks;635 if (!(ibi_status & IBI_LAST_STATUS)) {636 ibi_size += chunks * rh->ibi_chunk_sz;637 } else {638 ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status);639 last_ptr = ptr;640 break;641 }642 }643 644 /* validate what we've got */645 646 if (last_ptr == -1) {647 /* this IBI sequence is not yet complete */648 DBG("no LAST_STATUS available (e=%d d=%d)", enq_ptr, deq_ptr);649 return;650 }651 deq_ptr = last_ptr + 1;652 deq_ptr %= rh->ibi_status_entries;653 654 if (ibi_status_error) {655 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi_addr);656 goto done;657 }658 659 /* determine who this is for */660 dev = i3c_hci_addr_to_dev(hci, ibi_addr);661 if (!dev) {662 dev_err(&hci->master.dev,663 "IBI for unknown device %#x\n", ibi_addr);664 goto done;665 }666 667 dev_data = i3c_dev_get_master_data(dev);668 dev_ibi = dev_data->ibi_data;669 if (ibi_size > dev_ibi->max_len) {670 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n",671 ibi_size, dev_ibi->max_len);672 goto done;673 }674 675 /*676 * This ring model is not suitable for zero-copy processing of IBIs.677 * We have the data chunk ring wrap-around to deal with, meaning678 * that the payload might span multiple chunks beginning at the679 * end of the ring and wrap to the start of the ring. Furthermore680 * there is no guarantee that those chunks will be released in order681 * and in a timely manner by the upper driver. So let's just copy682 * them to a discrete buffer. In practice they're supposed to be683 * small anyway.684 */685 slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool);686 if (!slot) {687 dev_err(&hci->master.dev, "no free slot for IBI\n");688 goto done;689 }690 691 /* copy first part of the payload */692 ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr;693 ring_ibi_data = rh->ibi_data + ibi_data_offset;694 ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset;695 first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr)696 * rh->ibi_chunk_sz;697 if (first_part > ibi_size)698 first_part = ibi_size;699 dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma,700 first_part, DMA_FROM_DEVICE);701 memcpy(slot->data, ring_ibi_data, first_part);702 703 /* copy second part if any */704 if (ibi_size > first_part) {705 /* we wrap back to the start and copy remaining data */706 ring_ibi_data = rh->ibi_data;707 ring_ibi_data_dma = rh->ibi_data_dma;708 dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma,709 ibi_size - first_part, DMA_FROM_DEVICE);710 memcpy(slot->data + first_part, ring_ibi_data,711 ibi_size - first_part);712 }713 714 /* submit it */715 slot->dev = dev;716 slot->len = ibi_size;717 i3c_master_queue_ibi(dev, slot);718 719done:720 /* take care to update the ibi dequeue pointer atomically */721 spin_lock(&rh->lock);722 op1_val = rh_reg_read(RING_OPERATION1);723 op1_val &= ~RING_OP1_IBI_DEQ_PTR;724 op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr);725 rh_reg_write(RING_OPERATION1, op1_val);726 spin_unlock(&rh->lock);727 728 /* update the chunk pointer */729 rh->ibi_chunk_ptr += ibi_chunks;730 rh->ibi_chunk_ptr %= rh->ibi_chunks_total;731 732 /* and tell the hardware about freed chunks */733 rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks);734}735 736static bool hci_dma_irq_handler(struct i3c_hci *hci, unsigned int mask)737{738 struct hci_rings_data *rings = hci->io_data;739 unsigned int i;740 bool handled = false;741 742 for (i = 0; mask && i < rings->total; i++) {743 struct hci_rh_data *rh;744 u32 status;745 746 if (!(mask & BIT(i)))747 continue;748 mask &= ~BIT(i);749 750 rh = &rings->headers[i];751 status = rh_reg_read(INTR_STATUS);752 DBG("rh%d status: %#x", i, status);753 if (!status)754 continue;755 rh_reg_write(INTR_STATUS, status);756 757 if (status & INTR_IBI_READY)758 hci_dma_process_ibi(hci, rh);759 if (status & (INTR_TRANSFER_COMPLETION | INTR_TRANSFER_ERR))760 hci_dma_xfer_done(hci, rh);761 if (status & INTR_RING_OP)762 complete(&rh->op_done);763 764 if (status & INTR_TRANSFER_ABORT) {765 dev_notice_ratelimited(&hci->master.dev,766 "ring %d: Transfer Aborted\n", i);767 mipi_i3c_hci_resume(hci);768 }769 if (status & INTR_WARN_INS_STOP_MODE)770 dev_warn_ratelimited(&hci->master.dev,771 "ring %d: Inserted Stop on Mode Change\n", i);772 if (status & INTR_IBI_RING_FULL)773 dev_err_ratelimited(&hci->master.dev,774 "ring %d: IBI Ring Full Condition\n", i);775 776 handled = true;777 }778 779 return handled;780}781 782const struct hci_io_ops mipi_i3c_hci_dma = {783 .init = hci_dma_init,784 .cleanup = hci_dma_cleanup,785 .queue_xfer = hci_dma_queue_xfer,786 .dequeue_xfer = hci_dma_dequeue_xfer,787 .irq_handler = hci_dma_irq_handler,788 .request_ibi = hci_dma_request_ibi,789 .free_ibi = hci_dma_free_ibi,790 .recycle_ibi_slot = hci_dma_recycle_ibi_slot,791};792