1042 lines · c
1// SPDX-License-Identifier: BSD-3-Clause2/*3 * Copyright (c) 2020, MIPI Alliance, Inc.4 *5 * Author: Nicolas Pitre <npitre@baylibre.com>6 */7 8#include <linux/bitfield.h>9#include <linux/device.h>10#include <linux/errno.h>11#include <linux/i3c/master.h>12#include <linux/io.h>13 14#include "hci.h"15#include "cmd.h"16#include "ibi.h"17 18 19/*20 * PIO Access Area21 */22 23#define pio_reg_read(r) readl(hci->PIO_regs + (PIO_##r))24#define pio_reg_write(r, v) writel(v, hci->PIO_regs + (PIO_##r))25 26#define PIO_COMMAND_QUEUE_PORT 0x0027#define PIO_RESPONSE_QUEUE_PORT 0x0428#define PIO_XFER_DATA_PORT 0x0829#define PIO_IBI_PORT 0x0c30 31#define PIO_QUEUE_THLD_CTRL 0x1032#define QUEUE_IBI_STATUS_THLD GENMASK(31, 24)33#define QUEUE_IBI_DATA_THLD GENMASK(23, 16)34#define QUEUE_RESP_BUF_THLD GENMASK(15, 8)35#define QUEUE_CMD_EMPTY_BUF_THLD GENMASK(7, 0)36 37#define PIO_DATA_BUFFER_THLD_CTRL 0x1438#define DATA_RX_START_THLD GENMASK(26, 24)39#define DATA_TX_START_THLD GENMASK(18, 16)40#define DATA_RX_BUF_THLD GENMASK(10, 8)41#define DATA_TX_BUF_THLD GENMASK(2, 0)42 43#define PIO_QUEUE_SIZE 0x1844#define TX_DATA_BUFFER_SIZE GENMASK(31, 24)45#define RX_DATA_BUFFER_SIZE GENMASK(23, 16)46#define IBI_STATUS_SIZE GENMASK(15, 8)47#define CR_QUEUE_SIZE GENMASK(7, 0)48 49#define PIO_INTR_STATUS 0x2050#define PIO_INTR_STATUS_ENABLE 0x2451#define PIO_INTR_SIGNAL_ENABLE 0x2852#define PIO_INTR_FORCE 0x2c53#define STAT_TRANSFER_BLOCKED BIT(25)54#define STAT_PERR_RESP_UFLOW BIT(24)55#define STAT_PERR_CMD_OFLOW BIT(23)56#define STAT_PERR_IBI_UFLOW BIT(22)57#define STAT_PERR_RX_UFLOW BIT(21)58#define STAT_PERR_TX_OFLOW BIT(20)59#define STAT_ERR_RESP_QUEUE_FULL BIT(19)60#define STAT_WARN_RESP_QUEUE_FULL BIT(18)61#define STAT_ERR_IBI_QUEUE_FULL BIT(17)62#define STAT_WARN_IBI_QUEUE_FULL BIT(16)63#define STAT_ERR_RX_DATA_FULL BIT(15)64#define STAT_WARN_RX_DATA_FULL BIT(14)65#define STAT_ERR_TX_DATA_EMPTY BIT(13)66#define STAT_WARN_TX_DATA_EMPTY BIT(12)67#define STAT_TRANSFER_ERR BIT(9)68#define STAT_WARN_INS_STOP_MODE BIT(7)69#define STAT_TRANSFER_ABORT BIT(5)70#define STAT_RESP_READY BIT(4)71#define STAT_CMD_QUEUE_READY BIT(3)72#define STAT_IBI_STATUS_THLD BIT(2)73#define STAT_RX_THLD BIT(1)74#define STAT_TX_THLD BIT(0)75 76#define PIO_QUEUE_CUR_STATUS 0x3877#define CUR_IBI_Q_LEVEL GENMASK(28, 20)78#define CUR_RESP_Q_LEVEL GENMASK(18, 10)79#define CUR_CMD_Q_EMPTY_LEVEL GENMASK(8, 0)80 81#define PIO_DATA_BUFFER_CUR_STATUS 0x3c82#define CUR_RX_BUF_LVL GENMASK(26, 16)83#define CUR_TX_BUF_LVL GENMASK(10, 0)84 85/*86 * Handy status bit combinations87 */88 89#define STAT_LATENCY_WARNINGS (STAT_WARN_RESP_QUEUE_FULL | \90 STAT_WARN_IBI_QUEUE_FULL | \91 STAT_WARN_RX_DATA_FULL | \92 STAT_WARN_TX_DATA_EMPTY | \93 STAT_WARN_INS_STOP_MODE)94 95#define STAT_LATENCY_ERRORS (STAT_ERR_RESP_QUEUE_FULL | \96 STAT_ERR_IBI_QUEUE_FULL | \97 STAT_ERR_RX_DATA_FULL | \98 STAT_ERR_TX_DATA_EMPTY)99 100#define STAT_PROG_ERRORS (STAT_TRANSFER_BLOCKED | \101 STAT_PERR_RESP_UFLOW | \102 STAT_PERR_CMD_OFLOW | \103 STAT_PERR_IBI_UFLOW | \104 STAT_PERR_RX_UFLOW | \105 STAT_PERR_TX_OFLOW)106 107#define STAT_ALL_ERRORS (STAT_TRANSFER_ABORT | \108 STAT_TRANSFER_ERR | \109 STAT_LATENCY_ERRORS | \110 STAT_PROG_ERRORS)111 112struct hci_pio_dev_ibi_data {113 struct i3c_generic_ibi_pool *pool;114 unsigned int max_len;115};116 117struct hci_pio_ibi_data {118 struct i3c_ibi_slot *slot;119 void *data_ptr;120 unsigned int addr;121 unsigned int seg_len, seg_cnt;122 unsigned int max_len;123 bool last_seg;124};125 126struct hci_pio_data {127 spinlock_t lock;128 struct hci_xfer *curr_xfer, *xfer_queue;129 struct hci_xfer *curr_rx, *rx_queue;130 struct hci_xfer *curr_tx, *tx_queue;131 struct hci_xfer *curr_resp, *resp_queue;132 struct hci_pio_ibi_data ibi;133 unsigned int rx_thresh_size, tx_thresh_size;134 unsigned int max_ibi_thresh;135 u32 reg_queue_thresh;136 u32 enabled_irqs;137};138 139static int hci_pio_init(struct i3c_hci *hci)140{141 struct hci_pio_data *pio;142 u32 val, size_val, rx_thresh, tx_thresh, ibi_val;143 144 pio = kzalloc(sizeof(*pio), GFP_KERNEL);145 if (!pio)146 return -ENOMEM;147 148 hci->io_data = pio;149 spin_lock_init(&pio->lock);150 151 size_val = pio_reg_read(QUEUE_SIZE);152 dev_info(&hci->master.dev, "CMD/RESP FIFO = %ld entries\n",153 FIELD_GET(CR_QUEUE_SIZE, size_val));154 dev_info(&hci->master.dev, "IBI FIFO = %ld bytes\n",155 4 * FIELD_GET(IBI_STATUS_SIZE, size_val));156 dev_info(&hci->master.dev, "RX data FIFO = %d bytes\n",157 4 * (2 << FIELD_GET(RX_DATA_BUFFER_SIZE, size_val)));158 dev_info(&hci->master.dev, "TX data FIFO = %d bytes\n",159 4 * (2 << FIELD_GET(TX_DATA_BUFFER_SIZE, size_val)));160 161 /*162 * Let's initialize data thresholds to half of the actual FIFO size.163 * The start thresholds aren't used (set to 0) as the FIFO is always164 * serviced before the corresponding command is queued.165 */166 rx_thresh = FIELD_GET(RX_DATA_BUFFER_SIZE, size_val);167 tx_thresh = FIELD_GET(TX_DATA_BUFFER_SIZE, size_val);168 if (hci->version_major == 1) {169 /* those are expressed as 2^[n+1), so just sub 1 if not 0 */170 if (rx_thresh)171 rx_thresh -= 1;172 if (tx_thresh)173 tx_thresh -= 1;174 pio->rx_thresh_size = 2 << rx_thresh;175 pio->tx_thresh_size = 2 << tx_thresh;176 } else {177 /* size is 2^(n+1) and threshold is 2^n i.e. already halved */178 pio->rx_thresh_size = 1 << rx_thresh;179 pio->tx_thresh_size = 1 << tx_thresh;180 }181 val = FIELD_PREP(DATA_RX_BUF_THLD, rx_thresh) |182 FIELD_PREP(DATA_TX_BUF_THLD, tx_thresh);183 pio_reg_write(DATA_BUFFER_THLD_CTRL, val);184 185 /*186 * Let's raise an interrupt as soon as there is one free cmd slot187 * or one available response or IBI. For IBI data let's use half the188 * IBI queue size within allowed bounds.189 */190 ibi_val = FIELD_GET(IBI_STATUS_SIZE, size_val);191 pio->max_ibi_thresh = clamp_val(ibi_val/2, 1, 63);192 val = FIELD_PREP(QUEUE_IBI_STATUS_THLD, 1) |193 FIELD_PREP(QUEUE_IBI_DATA_THLD, pio->max_ibi_thresh) |194 FIELD_PREP(QUEUE_RESP_BUF_THLD, 1) |195 FIELD_PREP(QUEUE_CMD_EMPTY_BUF_THLD, 1);196 pio_reg_write(QUEUE_THLD_CTRL, val);197 pio->reg_queue_thresh = val;198 199 /* Disable all IRQs but allow all status bits */200 pio_reg_write(INTR_SIGNAL_ENABLE, 0x0);201 pio_reg_write(INTR_STATUS_ENABLE, 0xffffffff);202 203 /* Always accept error interrupts (will be activated on first xfer) */204 pio->enabled_irqs = STAT_ALL_ERRORS;205 206 return 0;207}208 209static void hci_pio_cleanup(struct i3c_hci *hci)210{211 struct hci_pio_data *pio = hci->io_data;212 213 pio_reg_write(INTR_SIGNAL_ENABLE, 0x0);214 215 if (pio) {216 DBG("status = %#x/%#x",217 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE));218 BUG_ON(pio->curr_xfer);219 BUG_ON(pio->curr_rx);220 BUG_ON(pio->curr_tx);221 BUG_ON(pio->curr_resp);222 kfree(pio);223 hci->io_data = NULL;224 }225}226 227static void hci_pio_write_cmd(struct i3c_hci *hci, struct hci_xfer *xfer)228{229 DBG("cmd_desc[%d] = 0x%08x", 0, xfer->cmd_desc[0]);230 DBG("cmd_desc[%d] = 0x%08x", 1, xfer->cmd_desc[1]);231 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[0]);232 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[1]);233 if (hci->cmd == &mipi_i3c_hci_cmd_v2) {234 DBG("cmd_desc[%d] = 0x%08x", 2, xfer->cmd_desc[2]);235 DBG("cmd_desc[%d] = 0x%08x", 3, xfer->cmd_desc[3]);236 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[2]);237 pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[3]);238 }239}240 241static bool hci_pio_do_rx(struct i3c_hci *hci, struct hci_pio_data *pio)242{243 struct hci_xfer *xfer = pio->curr_rx;244 unsigned int nr_words;245 u32 *p;246 247 p = xfer->data;248 p += (xfer->data_len - xfer->data_left) / 4;249 250 while (xfer->data_left >= 4) {251 /* bail out if FIFO hasn't reached the threshold value yet */252 if (!(pio_reg_read(INTR_STATUS) & STAT_RX_THLD))253 return false;254 nr_words = min(xfer->data_left / 4, pio->rx_thresh_size);255 /* extract data from FIFO */256 xfer->data_left -= nr_words * 4;257 DBG("now %d left %d", nr_words * 4, xfer->data_left);258 while (nr_words--)259 *p++ = pio_reg_read(XFER_DATA_PORT);260 }261 262 /* trailing data is retrieved upon response reception */263 return !xfer->data_left;264}265 266static void hci_pio_do_trailing_rx(struct i3c_hci *hci,267 struct hci_pio_data *pio, unsigned int count)268{269 struct hci_xfer *xfer = pio->curr_rx;270 u32 *p;271 272 DBG("%d remaining", count);273 274 p = xfer->data;275 p += (xfer->data_len - xfer->data_left) / 4;276 277 if (count >= 4) {278 unsigned int nr_words = count / 4;279 /* extract data from FIFO */280 xfer->data_left -= nr_words * 4;281 DBG("now %d left %d", nr_words * 4, xfer->data_left);282 while (nr_words--)283 *p++ = pio_reg_read(XFER_DATA_PORT);284 }285 286 count &= 3;287 if (count) {288 /*289 * There are trailing bytes in the last word.290 * Fetch it and extract bytes in an endian independent way.291 * Unlike the TX case, we must not write memory past the292 * end of the destination buffer.293 */294 u8 *p_byte = (u8 *)p;295 u32 data = pio_reg_read(XFER_DATA_PORT);296 297 xfer->data_word_before_partial = data;298 xfer->data_left -= count;299 data = (__force u32) cpu_to_le32(data);300 while (count--) {301 *p_byte++ = data;302 data >>= 8;303 }304 }305}306 307static bool hci_pio_do_tx(struct i3c_hci *hci, struct hci_pio_data *pio)308{309 struct hci_xfer *xfer = pio->curr_tx;310 unsigned int nr_words;311 u32 *p;312 313 p = xfer->data;314 p += (xfer->data_len - xfer->data_left) / 4;315 316 while (xfer->data_left >= 4) {317 /* bail out if FIFO free space is below set threshold */318 if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD))319 return false;320 /* we can fill up to that TX threshold */321 nr_words = min(xfer->data_left / 4, pio->tx_thresh_size);322 /* push data into the FIFO */323 xfer->data_left -= nr_words * 4;324 DBG("now %d left %d", nr_words * 4, xfer->data_left);325 while (nr_words--)326 pio_reg_write(XFER_DATA_PORT, *p++);327 }328 329 if (xfer->data_left) {330 /*331 * There are trailing bytes to send. We can simply load332 * them from memory as a word which will keep those bytes333 * in their proper place even on a BE system. This will334 * also get some bytes past the actual buffer but no one335 * should care as they won't be sent out.336 */337 if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD))338 return false;339 DBG("trailing %d", xfer->data_left);340 pio_reg_write(XFER_DATA_PORT, *p);341 xfer->data_left = 0;342 }343 344 return true;345}346 347static bool hci_pio_process_rx(struct i3c_hci *hci, struct hci_pio_data *pio)348{349 while (pio->curr_rx && hci_pio_do_rx(hci, pio))350 pio->curr_rx = pio->curr_rx->next_data;351 return !pio->curr_rx;352}353 354static bool hci_pio_process_tx(struct i3c_hci *hci, struct hci_pio_data *pio)355{356 while (pio->curr_tx && hci_pio_do_tx(hci, pio))357 pio->curr_tx = pio->curr_tx->next_data;358 return !pio->curr_tx;359}360 361static void hci_pio_queue_data(struct i3c_hci *hci, struct hci_pio_data *pio)362{363 struct hci_xfer *xfer = pio->curr_xfer;364 struct hci_xfer *prev_queue_tail;365 366 if (!xfer->data) {367 xfer->data_len = xfer->data_left = 0;368 return;369 }370 371 if (xfer->rnw) {372 prev_queue_tail = pio->rx_queue;373 pio->rx_queue = xfer;374 if (pio->curr_rx) {375 prev_queue_tail->next_data = xfer;376 } else {377 pio->curr_rx = xfer;378 if (!hci_pio_process_rx(hci, pio))379 pio->enabled_irqs |= STAT_RX_THLD;380 }381 } else {382 prev_queue_tail = pio->tx_queue;383 pio->tx_queue = xfer;384 if (pio->curr_tx) {385 prev_queue_tail->next_data = xfer;386 } else {387 pio->curr_tx = xfer;388 if (!hci_pio_process_tx(hci, pio))389 pio->enabled_irqs |= STAT_TX_THLD;390 }391 }392}393 394static void hci_pio_push_to_next_rx(struct i3c_hci *hci, struct hci_xfer *xfer,395 unsigned int words_to_keep)396{397 u32 *from = xfer->data;398 u32 from_last;399 unsigned int received, count;400 401 received = (xfer->data_len - xfer->data_left) / 4;402 if ((xfer->data_len - xfer->data_left) & 3) {403 from_last = xfer->data_word_before_partial;404 received += 1;405 } else {406 from_last = from[received];407 }408 from += words_to_keep;409 count = received - words_to_keep;410 411 while (count) {412 unsigned int room, left, chunk, bytes_to_move;413 u32 last_word;414 415 xfer = xfer->next_data;416 if (!xfer) {417 dev_err(&hci->master.dev, "pushing RX data to unexistent xfer\n");418 return;419 }420 421 room = DIV_ROUND_UP(xfer->data_len, 4);422 left = DIV_ROUND_UP(xfer->data_left, 4);423 chunk = min(count, room);424 if (chunk > left) {425 hci_pio_push_to_next_rx(hci, xfer, chunk - left);426 left = chunk;427 xfer->data_left = left * 4;428 }429 430 bytes_to_move = xfer->data_len - xfer->data_left;431 if (bytes_to_move & 3) {432 /* preserve word to become partial */433 u32 *p = xfer->data;434 435 xfer->data_word_before_partial = p[bytes_to_move / 4];436 }437 memmove(xfer->data + chunk, xfer->data, bytes_to_move);438 439 /* treat last word specially because of partial word issues */440 chunk -= 1;441 442 memcpy(xfer->data, from, chunk * 4);443 xfer->data_left -= chunk * 4;444 from += chunk;445 count -= chunk;446 447 last_word = (count == 1) ? from_last : *from++;448 if (xfer->data_left < 4) {449 /*450 * Like in hci_pio_do_trailing_rx(), preserve original451 * word to be stored partially then store bytes it452 * in an endian independent way.453 */454 u8 *p_byte = xfer->data;455 456 p_byte += chunk * 4;457 xfer->data_word_before_partial = last_word;458 last_word = (__force u32) cpu_to_le32(last_word);459 while (xfer->data_left--) {460 *p_byte++ = last_word;461 last_word >>= 8;462 }463 } else {464 u32 *p = xfer->data;465 466 p[chunk] = last_word;467 xfer->data_left -= 4;468 }469 count--;470 }471}472 473static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio,474 u32 status);475 476static bool hci_pio_process_resp(struct i3c_hci *hci, struct hci_pio_data *pio)477{478 while (pio->curr_resp &&479 (pio_reg_read(INTR_STATUS) & STAT_RESP_READY)) {480 struct hci_xfer *xfer = pio->curr_resp;481 u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT);482 unsigned int tid = RESP_TID(resp);483 484 DBG("resp = 0x%08x", resp);485 if (tid != xfer->cmd_tid) {486 dev_err(&hci->master.dev,487 "response tid=%d when expecting %d\n",488 tid, xfer->cmd_tid);489 /* let's pretend it is a prog error... any of them */490 hci_pio_err(hci, pio, STAT_PROG_ERRORS);491 return false;492 }493 xfer->response = resp;494 495 if (pio->curr_rx == xfer) {496 /*497 * Response availability implies RX completion.498 * Retrieve trailing RX data if any.499 * Note that short reads are possible.500 */501 unsigned int received, expected, to_keep;502 503 received = xfer->data_len - xfer->data_left;504 expected = RESP_DATA_LENGTH(xfer->response);505 if (expected > received) {506 hci_pio_do_trailing_rx(hci, pio,507 expected - received);508 } else if (received > expected) {509 /* we consumed data meant for next xfer */510 to_keep = DIV_ROUND_UP(expected, 4);511 hci_pio_push_to_next_rx(hci, xfer, to_keep);512 }513 514 /* then process the RX list pointer */515 if (hci_pio_process_rx(hci, pio))516 pio->enabled_irqs &= ~STAT_RX_THLD;517 }518 519 /*520 * We're about to give back ownership of the xfer structure521 * to the waiting instance. Make sure no reference to it522 * still exists.523 */524 if (pio->curr_rx == xfer) {525 DBG("short RX ?");526 pio->curr_rx = pio->curr_rx->next_data;527 } else if (pio->curr_tx == xfer) {528 DBG("short TX ?");529 pio->curr_tx = pio->curr_tx->next_data;530 } else if (xfer->data_left) {531 DBG("PIO xfer count = %d after response",532 xfer->data_left);533 }534 535 pio->curr_resp = xfer->next_resp;536 if (xfer->completion)537 complete(xfer->completion);538 }539 return !pio->curr_resp;540}541 542static void hci_pio_queue_resp(struct i3c_hci *hci, struct hci_pio_data *pio)543{544 struct hci_xfer *xfer = pio->curr_xfer;545 struct hci_xfer *prev_queue_tail;546 547 if (!(xfer->cmd_desc[0] & CMD_0_ROC))548 return;549 550 prev_queue_tail = pio->resp_queue;551 pio->resp_queue = xfer;552 if (pio->curr_resp) {553 prev_queue_tail->next_resp = xfer;554 } else {555 pio->curr_resp = xfer;556 if (!hci_pio_process_resp(hci, pio))557 pio->enabled_irqs |= STAT_RESP_READY;558 }559}560 561static bool hci_pio_process_cmd(struct i3c_hci *hci, struct hci_pio_data *pio)562{563 while (pio->curr_xfer &&564 (pio_reg_read(INTR_STATUS) & STAT_CMD_QUEUE_READY)) {565 /*566 * Always process the data FIFO before sending the command567 * so needed TX data or RX space is available upfront.568 */569 hci_pio_queue_data(hci, pio);570 /*571 * Then queue our response request. This will also process572 * the response FIFO in case it got suddenly filled up573 * with results from previous commands.574 */575 hci_pio_queue_resp(hci, pio);576 /*577 * Finally send the command.578 */579 hci_pio_write_cmd(hci, pio->curr_xfer);580 /*581 * And move on.582 */583 pio->curr_xfer = pio->curr_xfer->next_xfer;584 }585 return !pio->curr_xfer;586}587 588static int hci_pio_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n)589{590 struct hci_pio_data *pio = hci->io_data;591 struct hci_xfer *prev_queue_tail;592 int i;593 594 DBG("n = %d", n);595 596 /* link xfer instances together and initialize data count */597 for (i = 0; i < n; i++) {598 xfer[i].next_xfer = (i + 1 < n) ? &xfer[i + 1] : NULL;599 xfer[i].next_data = NULL;600 xfer[i].next_resp = NULL;601 xfer[i].data_left = xfer[i].data_len;602 }603 604 spin_lock_irq(&pio->lock);605 prev_queue_tail = pio->xfer_queue;606 pio->xfer_queue = &xfer[n - 1];607 if (pio->curr_xfer) {608 prev_queue_tail->next_xfer = xfer;609 } else {610 pio->curr_xfer = xfer;611 if (!hci_pio_process_cmd(hci, pio))612 pio->enabled_irqs |= STAT_CMD_QUEUE_READY;613 pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs);614 DBG("status = %#x/%#x",615 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE));616 }617 spin_unlock_irq(&pio->lock);618 return 0;619}620 621static bool hci_pio_dequeue_xfer_common(struct i3c_hci *hci,622 struct hci_pio_data *pio,623 struct hci_xfer *xfer, int n)624{625 struct hci_xfer *p, **p_prev_next;626 int i;627 628 /*629 * To safely dequeue a transfer request, it must be either entirely630 * processed, or not yet processed at all. If our request tail is631 * reachable from either the data or resp list that means the command632 * was submitted and not yet completed.633 */634 for (p = pio->curr_resp; p; p = p->next_resp)635 for (i = 0; i < n; i++)636 if (p == &xfer[i])637 goto pio_screwed;638 for (p = pio->curr_rx; p; p = p->next_data)639 for (i = 0; i < n; i++)640 if (p == &xfer[i])641 goto pio_screwed;642 for (p = pio->curr_tx; p; p = p->next_data)643 for (i = 0; i < n; i++)644 if (p == &xfer[i])645 goto pio_screwed;646 647 /*648 * The command was completed, or wasn't yet submitted.649 * Unlink it from the que if the later.650 */651 p_prev_next = &pio->curr_xfer;652 for (p = pio->curr_xfer; p; p = p->next_xfer) {653 if (p == &xfer[0]) {654 *p_prev_next = xfer[n - 1].next_xfer;655 break;656 }657 p_prev_next = &p->next_xfer;658 }659 660 /* return true if we actually unqueued something */661 return !!p;662 663pio_screwed:664 /*665 * Life is tough. We must invalidate the hardware state and666 * discard everything that is still queued.667 */668 for (p = pio->curr_resp; p; p = p->next_resp) {669 p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED);670 if (p->completion)671 complete(p->completion);672 }673 for (p = pio->curr_xfer; p; p = p->next_xfer) {674 p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED);675 if (p->completion)676 complete(p->completion);677 }678 pio->curr_xfer = pio->curr_rx = pio->curr_tx = pio->curr_resp = NULL;679 680 return true;681}682 683static bool hci_pio_dequeue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n)684{685 struct hci_pio_data *pio = hci->io_data;686 int ret;687 688 spin_lock_irq(&pio->lock);689 DBG("n=%d status=%#x/%#x", n,690 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE));691 DBG("main_status = %#x/%#x",692 readl(hci->base_regs + 0x20), readl(hci->base_regs + 0x28));693 694 ret = hci_pio_dequeue_xfer_common(hci, pio, xfer, n);695 spin_unlock_irq(&pio->lock);696 return ret;697}698 699static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio,700 u32 status)701{702 /* TODO: this ought to be more sophisticated eventually */703 704 if (pio_reg_read(INTR_STATUS) & STAT_RESP_READY) {705 /* this may happen when an error is signaled with ROC unset */706 u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT);707 708 dev_err(&hci->master.dev,709 "orphan response (%#x) on error\n", resp);710 }711 712 /* dump states on programming errors */713 if (status & STAT_PROG_ERRORS) {714 u32 queue = pio_reg_read(QUEUE_CUR_STATUS);715 u32 data = pio_reg_read(DATA_BUFFER_CUR_STATUS);716 717 dev_err(&hci->master.dev,718 "prog error %#lx (C/R/I = %ld/%ld/%ld, TX/RX = %ld/%ld)\n",719 status & STAT_PROG_ERRORS,720 FIELD_GET(CUR_CMD_Q_EMPTY_LEVEL, queue),721 FIELD_GET(CUR_RESP_Q_LEVEL, queue),722 FIELD_GET(CUR_IBI_Q_LEVEL, queue),723 FIELD_GET(CUR_TX_BUF_LVL, data),724 FIELD_GET(CUR_RX_BUF_LVL, data));725 }726 727 /* just bust out everything with pending responses for now */728 hci_pio_dequeue_xfer_common(hci, pio, pio->curr_resp, 1);729 /* ... and half-way TX transfers if any */730 if (pio->curr_tx && pio->curr_tx->data_left != pio->curr_tx->data_len)731 hci_pio_dequeue_xfer_common(hci, pio, pio->curr_tx, 1);732 /* then reset the hardware */733 mipi_i3c_hci_pio_reset(hci);734 mipi_i3c_hci_resume(hci);735 736 DBG("status=%#x/%#x",737 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE));738}739 740static void hci_pio_set_ibi_thresh(struct i3c_hci *hci,741 struct hci_pio_data *pio,742 unsigned int thresh_val)743{744 u32 regval = pio->reg_queue_thresh;745 746 regval &= ~QUEUE_IBI_STATUS_THLD;747 regval |= FIELD_PREP(QUEUE_IBI_STATUS_THLD, thresh_val);748 /* write the threshold reg only if it changes */749 if (regval != pio->reg_queue_thresh) {750 pio_reg_write(QUEUE_THLD_CTRL, regval);751 pio->reg_queue_thresh = regval;752 DBG("%d", thresh_val);753 }754}755 756static bool hci_pio_get_ibi_segment(struct i3c_hci *hci,757 struct hci_pio_data *pio)758{759 struct hci_pio_ibi_data *ibi = &pio->ibi;760 unsigned int nr_words, thresh_val;761 u32 *p;762 763 p = ibi->data_ptr;764 p += (ibi->seg_len - ibi->seg_cnt) / 4;765 766 while ((nr_words = ibi->seg_cnt/4)) {767 /* determine our IBI queue threshold value */768 thresh_val = min(nr_words, pio->max_ibi_thresh);769 hci_pio_set_ibi_thresh(hci, pio, thresh_val);770 /* bail out if we don't have that amount of data ready */771 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD))772 return false;773 /* extract the data from the IBI port */774 nr_words = thresh_val;775 ibi->seg_cnt -= nr_words * 4;776 DBG("now %d left %d", nr_words * 4, ibi->seg_cnt);777 while (nr_words--)778 *p++ = pio_reg_read(IBI_PORT);779 }780 781 if (ibi->seg_cnt) {782 /*783 * There are trailing bytes in the last word.784 * Fetch it and extract bytes in an endian independent way.785 * Unlike the TX case, we must not write past the end of786 * the destination buffer.787 */788 u32 data;789 u8 *p_byte = (u8 *)p;790 791 hci_pio_set_ibi_thresh(hci, pio, 1);792 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD))793 return false;794 DBG("trailing %d", ibi->seg_cnt);795 data = pio_reg_read(IBI_PORT);796 data = (__force u32) cpu_to_le32(data);797 while (ibi->seg_cnt--) {798 *p_byte++ = data;799 data >>= 8;800 }801 }802 803 return true;804}805 806static bool hci_pio_prep_new_ibi(struct i3c_hci *hci, struct hci_pio_data *pio)807{808 struct hci_pio_ibi_data *ibi = &pio->ibi;809 struct i3c_dev_desc *dev;810 struct i3c_hci_dev_data *dev_data;811 struct hci_pio_dev_ibi_data *dev_ibi;812 u32 ibi_status;813 814 /*815 * We have a new IBI. Try to set up its payload retrieval.816 * When returning true, the IBI data has to be consumed whether817 * or not we are set up to capture it. If we return true with818 * ibi->slot == NULL that means the data payload has to be819 * drained out of the IBI port and dropped.820 */821 822 ibi_status = pio_reg_read(IBI_PORT);823 DBG("status = %#x", ibi_status);824 ibi->addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status);825 if (ibi_status & IBI_ERROR) {826 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi->addr);827 return false;828 }829 830 ibi->last_seg = ibi_status & IBI_LAST_STATUS;831 ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status);832 ibi->seg_cnt = ibi->seg_len;833 834 dev = i3c_hci_addr_to_dev(hci, ibi->addr);835 if (!dev) {836 dev_err(&hci->master.dev,837 "IBI for unknown device %#x\n", ibi->addr);838 return true;839 }840 841 dev_data = i3c_dev_get_master_data(dev);842 dev_ibi = dev_data->ibi_data;843 ibi->max_len = dev_ibi->max_len;844 845 if (ibi->seg_len > ibi->max_len) {846 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n",847 ibi->seg_len, ibi->max_len);848 return true;849 }850 851 ibi->slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool);852 if (!ibi->slot) {853 dev_err(&hci->master.dev, "no free slot for IBI\n");854 } else {855 ibi->slot->len = 0;856 ibi->data_ptr = ibi->slot->data;857 }858 return true;859}860 861static void hci_pio_free_ibi_slot(struct i3c_hci *hci, struct hci_pio_data *pio)862{863 struct hci_pio_ibi_data *ibi = &pio->ibi;864 struct hci_pio_dev_ibi_data *dev_ibi;865 866 if (ibi->slot) {867 dev_ibi = ibi->slot->dev->common.master_priv;868 i3c_generic_ibi_recycle_slot(dev_ibi->pool, ibi->slot);869 ibi->slot = NULL;870 }871}872 873static bool hci_pio_process_ibi(struct i3c_hci *hci, struct hci_pio_data *pio)874{875 struct hci_pio_ibi_data *ibi = &pio->ibi;876 877 if (!ibi->slot && !ibi->seg_cnt && ibi->last_seg)878 if (!hci_pio_prep_new_ibi(hci, pio))879 return false;880 881 for (;;) {882 u32 ibi_status;883 unsigned int ibi_addr;884 885 if (ibi->slot) {886 if (!hci_pio_get_ibi_segment(hci, pio))887 return false;888 ibi->slot->len += ibi->seg_len;889 ibi->data_ptr += ibi->seg_len;890 if (ibi->last_seg) {891 /* was the last segment: submit it and leave */892 i3c_master_queue_ibi(ibi->slot->dev, ibi->slot);893 ibi->slot = NULL;894 hci_pio_set_ibi_thresh(hci, pio, 1);895 return true;896 }897 } else if (ibi->seg_cnt) {898 /*899 * No slot but a non-zero count. This is the result900 * of some error and the payload must be drained.901 * This normally does not happen therefore no need902 * to be extra optimized here.903 */904 hci_pio_set_ibi_thresh(hci, pio, 1);905 do {906 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD))907 return false;908 pio_reg_read(IBI_PORT);909 } while (--ibi->seg_cnt);910 if (ibi->last_seg)911 return true;912 }913 914 /* try to move to the next segment right away */915 hci_pio_set_ibi_thresh(hci, pio, 1);916 if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD))917 return false;918 ibi_status = pio_reg_read(IBI_PORT);919 ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status);920 if (ibi->addr != ibi_addr) {921 /* target address changed before last segment */922 dev_err(&hci->master.dev,923 "unexp IBI address changed from %d to %d\n",924 ibi->addr, ibi_addr);925 hci_pio_free_ibi_slot(hci, pio);926 }927 ibi->last_seg = ibi_status & IBI_LAST_STATUS;928 ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status);929 ibi->seg_cnt = ibi->seg_len;930 if (ibi->slot && ibi->slot->len + ibi->seg_len > ibi->max_len) {931 dev_err(&hci->master.dev,932 "IBI payload too big (%d > %d)\n",933 ibi->slot->len + ibi->seg_len, ibi->max_len);934 hci_pio_free_ibi_slot(hci, pio);935 }936 }937 938 return false;939}940 941static int hci_pio_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev,942 const struct i3c_ibi_setup *req)943{944 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);945 struct i3c_generic_ibi_pool *pool;946 struct hci_pio_dev_ibi_data *dev_ibi;947 948 dev_ibi = kmalloc(sizeof(*dev_ibi), GFP_KERNEL);949 if (!dev_ibi)950 return -ENOMEM;951 pool = i3c_generic_ibi_alloc_pool(dev, req);952 if (IS_ERR(pool)) {953 kfree(dev_ibi);954 return PTR_ERR(pool);955 }956 dev_ibi->pool = pool;957 dev_ibi->max_len = req->max_payload_len;958 dev_data->ibi_data = dev_ibi;959 return 0;960}961 962static void hci_pio_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev)963{964 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);965 struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data;966 967 dev_data->ibi_data = NULL;968 i3c_generic_ibi_free_pool(dev_ibi->pool);969 kfree(dev_ibi);970}971 972static void hci_pio_recycle_ibi_slot(struct i3c_hci *hci,973 struct i3c_dev_desc *dev,974 struct i3c_ibi_slot *slot)975{976 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);977 struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data;978 979 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot);980}981 982static bool hci_pio_irq_handler(struct i3c_hci *hci, unsigned int unused)983{984 struct hci_pio_data *pio = hci->io_data;985 u32 status;986 987 spin_lock(&pio->lock);988 status = pio_reg_read(INTR_STATUS);989 DBG("(in) status: %#x/%#x", status, pio->enabled_irqs);990 status &= pio->enabled_irqs | STAT_LATENCY_WARNINGS;991 if (!status) {992 spin_unlock(&pio->lock);993 return false;994 }995 996 if (status & STAT_IBI_STATUS_THLD)997 hci_pio_process_ibi(hci, pio);998 999 if (status & STAT_RX_THLD)1000 if (hci_pio_process_rx(hci, pio))1001 pio->enabled_irqs &= ~STAT_RX_THLD;1002 if (status & STAT_TX_THLD)1003 if (hci_pio_process_tx(hci, pio))1004 pio->enabled_irqs &= ~STAT_TX_THLD;1005 if (status & STAT_RESP_READY)1006 if (hci_pio_process_resp(hci, pio))1007 pio->enabled_irqs &= ~STAT_RESP_READY;1008 1009 if (unlikely(status & STAT_LATENCY_WARNINGS)) {1010 pio_reg_write(INTR_STATUS, status & STAT_LATENCY_WARNINGS);1011 dev_warn_ratelimited(&hci->master.dev,1012 "encountered warning condition %#lx\n",1013 status & STAT_LATENCY_WARNINGS);1014 }1015 1016 if (unlikely(status & STAT_ALL_ERRORS)) {1017 pio_reg_write(INTR_STATUS, status & STAT_ALL_ERRORS);1018 hci_pio_err(hci, pio, status & STAT_ALL_ERRORS);1019 }1020 1021 if (status & STAT_CMD_QUEUE_READY)1022 if (hci_pio_process_cmd(hci, pio))1023 pio->enabled_irqs &= ~STAT_CMD_QUEUE_READY;1024 1025 pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs);1026 DBG("(out) status: %#x/%#x",1027 pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE));1028 spin_unlock(&pio->lock);1029 return true;1030}1031 1032const struct hci_io_ops mipi_i3c_hci_pio = {1033 .init = hci_pio_init,1034 .cleanup = hci_pio_cleanup,1035 .queue_xfer = hci_pio_queue_xfer,1036 .dequeue_xfer = hci_pio_dequeue_xfer,1037 .irq_handler = hci_pio_irq_handler,1038 .request_ibi = hci_pio_request_ibi,1039 .free_ibi = hci_pio_free_ibi,1040 .recycle_ibi_slot = hci_pio_recycle_ibi_slot,1041};1042