722 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2014-2016, NVIDIA CORPORATION. All rights reserved.4 */5 6#include <soc/tegra/ivc.h>7 8#define TEGRA_IVC_ALIGN 649 10/*11 * IVC channel reset protocol.12 *13 * Each end uses its tx_channel.state to indicate its synchronization state.14 */15enum tegra_ivc_state {16 /*17 * This value is zero for backwards compatibility with services that18 * assume channels to be initially zeroed. Such channels are in an19 * initially valid state, but cannot be asynchronously reset, and must20 * maintain a valid state at all times.21 *22 * The transmitting end can enter the established state from the sync or23 * ack state when it observes the receiving endpoint in the ack or24 * established state, indicating that has cleared the counters in our25 * rx_channel.26 */27 TEGRA_IVC_STATE_ESTABLISHED = 0,28 29 /*30 * If an endpoint is observed in the sync state, the remote endpoint is31 * allowed to clear the counters it owns asynchronously with respect to32 * the current endpoint. Therefore, the current endpoint is no longer33 * allowed to communicate.34 */35 TEGRA_IVC_STATE_SYNC,36 37 /*38 * When the transmitting end observes the receiving end in the sync39 * state, it can clear the w_count and r_count and transition to the ack40 * state. If the remote endpoint observes us in the ack state, it can41 * return to the established state once it has cleared its counters.42 */43 TEGRA_IVC_STATE_ACK44};45 46/*47 * This structure is divided into two-cache aligned parts, the first is only48 * written through the tx.channel pointer, while the second is only written49 * through the rx.channel pointer. This delineates ownership of the cache50 * lines, which is critical to performance and necessary in non-cache coherent51 * implementations.52 */53struct tegra_ivc_header {54 union {55 struct {56 /* fields owned by the transmitting end */57 u32 count;58 u32 state;59 };60 61 u8 pad[TEGRA_IVC_ALIGN];62 } tx;63 64 union {65 /* fields owned by the receiving end */66 u32 count;67 u8 pad[TEGRA_IVC_ALIGN];68 } rx;69};70 71#define tegra_ivc_header_read_field(hdr, field) \72 iosys_map_rd_field(hdr, 0, struct tegra_ivc_header, field)73 74#define tegra_ivc_header_write_field(hdr, field, value) \75 iosys_map_wr_field(hdr, 0, struct tegra_ivc_header, field, value)76 77static inline void tegra_ivc_invalidate(struct tegra_ivc *ivc, dma_addr_t phys)78{79 if (!ivc->peer)80 return;81 82 dma_sync_single_for_cpu(ivc->peer, phys, TEGRA_IVC_ALIGN,83 DMA_FROM_DEVICE);84}85 86static inline void tegra_ivc_flush(struct tegra_ivc *ivc, dma_addr_t phys)87{88 if (!ivc->peer)89 return;90 91 dma_sync_single_for_device(ivc->peer, phys, TEGRA_IVC_ALIGN,92 DMA_TO_DEVICE);93}94 95static inline bool tegra_ivc_empty(struct tegra_ivc *ivc, struct iosys_map *map)96{97 /*98 * This function performs multiple checks on the same values with99 * security implications, so create snapshots with READ_ONCE() to100 * ensure that these checks use the same values.101 */102 u32 tx = tegra_ivc_header_read_field(map, tx.count);103 u32 rx = tegra_ivc_header_read_field(map, rx.count);104 105 /*106 * Perform an over-full check to prevent denial of service attacks107 * where a server could be easily fooled into believing that there's108 * an extremely large number of frames ready, since receivers are not109 * expected to check for full or over-full conditions.110 *111 * Although the channel isn't empty, this is an invalid case caused by112 * a potentially malicious peer, so returning empty is safer, because113 * it gives the impression that the channel has gone silent.114 */115 if (tx - rx > ivc->num_frames)116 return true;117 118 return tx == rx;119}120 121static inline bool tegra_ivc_full(struct tegra_ivc *ivc, struct iosys_map *map)122{123 u32 tx = tegra_ivc_header_read_field(map, tx.count);124 u32 rx = tegra_ivc_header_read_field(map, rx.count);125 126 /*127 * Invalid cases where the counters indicate that the queue is over128 * capacity also appear full.129 */130 return tx - rx >= ivc->num_frames;131}132 133static inline u32 tegra_ivc_available(struct tegra_ivc *ivc, struct iosys_map *map)134{135 u32 tx = tegra_ivc_header_read_field(map, tx.count);136 u32 rx = tegra_ivc_header_read_field(map, rx.count);137 138 /*139 * This function isn't expected to be used in scenarios where an140 * over-full situation can lead to denial of service attacks. See the141 * comment in tegra_ivc_empty() for an explanation about special142 * over-full considerations.143 */144 return tx - rx;145}146 147static inline void tegra_ivc_advance_tx(struct tegra_ivc *ivc)148{149 unsigned int count = tegra_ivc_header_read_field(&ivc->tx.map, tx.count);150 151 tegra_ivc_header_write_field(&ivc->tx.map, tx.count, count + 1);152 153 if (ivc->tx.position == ivc->num_frames - 1)154 ivc->tx.position = 0;155 else156 ivc->tx.position++;157}158 159static inline void tegra_ivc_advance_rx(struct tegra_ivc *ivc)160{161 unsigned int count = tegra_ivc_header_read_field(&ivc->rx.map, rx.count);162 163 tegra_ivc_header_write_field(&ivc->rx.map, rx.count, count + 1);164 165 if (ivc->rx.position == ivc->num_frames - 1)166 ivc->rx.position = 0;167 else168 ivc->rx.position++;169}170 171static inline int tegra_ivc_check_read(struct tegra_ivc *ivc)172{173 unsigned int offset = offsetof(struct tegra_ivc_header, tx.count);174 unsigned int state;175 176 /*177 * tx.channel->state is set locally, so it is not synchronized with178 * state from the remote peer. The remote peer cannot reset its179 * transmit counters until we've acknowledged its synchronization180 * request, so no additional synchronization is required because an181 * asynchronous transition of rx.channel->state to182 * TEGRA_IVC_STATE_ACK is not allowed.183 */184 state = tegra_ivc_header_read_field(&ivc->tx.map, tx.state);185 if (state != TEGRA_IVC_STATE_ESTABLISHED)186 return -ECONNRESET;187 188 /*189 * Avoid unnecessary invalidations when performing repeated accesses190 * to an IVC channel by checking the old queue pointers first.191 *192 * Synchronization is only necessary when these pointers indicate193 * empty or full.194 */195 if (!tegra_ivc_empty(ivc, &ivc->rx.map))196 return 0;197 198 tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);199 200 if (tegra_ivc_empty(ivc, &ivc->rx.map))201 return -ENOSPC;202 203 return 0;204}205 206static inline int tegra_ivc_check_write(struct tegra_ivc *ivc)207{208 unsigned int offset = offsetof(struct tegra_ivc_header, rx.count);209 unsigned int state;210 211 state = tegra_ivc_header_read_field(&ivc->tx.map, tx.state);212 if (state != TEGRA_IVC_STATE_ESTABLISHED)213 return -ECONNRESET;214 215 if (!tegra_ivc_full(ivc, &ivc->tx.map))216 return 0;217 218 tegra_ivc_invalidate(ivc, ivc->tx.phys + offset);219 220 if (tegra_ivc_full(ivc, &ivc->tx.map))221 return -ENOSPC;222 223 return 0;224}225 226static int tegra_ivc_frame_virt(struct tegra_ivc *ivc, const struct iosys_map *header,227 unsigned int frame, struct iosys_map *map)228{229 size_t offset = sizeof(struct tegra_ivc_header) + ivc->frame_size * frame;230 231 if (WARN_ON(frame >= ivc->num_frames))232 return -EINVAL;233 234 *map = IOSYS_MAP_INIT_OFFSET(header, offset);235 236 return 0;237}238 239static inline dma_addr_t tegra_ivc_frame_phys(struct tegra_ivc *ivc,240 dma_addr_t phys,241 unsigned int frame)242{243 unsigned long offset;244 245 offset = sizeof(struct tegra_ivc_header) + ivc->frame_size * frame;246 247 return phys + offset;248}249 250static inline void tegra_ivc_invalidate_frame(struct tegra_ivc *ivc,251 dma_addr_t phys,252 unsigned int frame,253 unsigned int offset,254 size_t size)255{256 if (!ivc->peer || WARN_ON(frame >= ivc->num_frames))257 return;258 259 phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset;260 261 dma_sync_single_for_cpu(ivc->peer, phys, size, DMA_FROM_DEVICE);262}263 264static inline void tegra_ivc_flush_frame(struct tegra_ivc *ivc,265 dma_addr_t phys,266 unsigned int frame,267 unsigned int offset,268 size_t size)269{270 if (!ivc->peer || WARN_ON(frame >= ivc->num_frames))271 return;272 273 phys = tegra_ivc_frame_phys(ivc, phys, frame) + offset;274 275 dma_sync_single_for_device(ivc->peer, phys, size, DMA_TO_DEVICE);276}277 278/* directly peek at the next frame rx'ed */279int tegra_ivc_read_get_next_frame(struct tegra_ivc *ivc, struct iosys_map *map)280{281 int err;282 283 if (WARN_ON(ivc == NULL))284 return -EINVAL;285 286 err = tegra_ivc_check_read(ivc);287 if (err < 0)288 return err;289 290 /*291 * Order observation of ivc->rx.position potentially indicating new292 * data before data read.293 */294 smp_rmb();295 296 tegra_ivc_invalidate_frame(ivc, ivc->rx.phys, ivc->rx.position, 0,297 ivc->frame_size);298 299 return tegra_ivc_frame_virt(ivc, &ivc->rx.map, ivc->rx.position, map);300}301EXPORT_SYMBOL(tegra_ivc_read_get_next_frame);302 303int tegra_ivc_read_advance(struct tegra_ivc *ivc)304{305 unsigned int rx = offsetof(struct tegra_ivc_header, rx.count);306 unsigned int tx = offsetof(struct tegra_ivc_header, tx.count);307 int err;308 309 /*310 * No read barriers or synchronization here: the caller is expected to311 * have already observed the channel non-empty. This check is just to312 * catch programming errors.313 */314 err = tegra_ivc_check_read(ivc);315 if (err < 0)316 return err;317 318 tegra_ivc_advance_rx(ivc);319 320 tegra_ivc_flush(ivc, ivc->rx.phys + rx);321 322 /*323 * Ensure our write to ivc->rx.position occurs before our read from324 * ivc->tx.position.325 */326 smp_mb();327 328 /*329 * Notify only upon transition from full to non-full. The available330 * count can only asynchronously increase, so the worst possible331 * side-effect will be a spurious notification.332 */333 tegra_ivc_invalidate(ivc, ivc->rx.phys + tx);334 335 if (tegra_ivc_available(ivc, &ivc->rx.map) == ivc->num_frames - 1)336 ivc->notify(ivc, ivc->notify_data);337 338 return 0;339}340EXPORT_SYMBOL(tegra_ivc_read_advance);341 342/* directly poke at the next frame to be tx'ed */343int tegra_ivc_write_get_next_frame(struct tegra_ivc *ivc, struct iosys_map *map)344{345 int err;346 347 err = tegra_ivc_check_write(ivc);348 if (err < 0)349 return err;350 351 return tegra_ivc_frame_virt(ivc, &ivc->tx.map, ivc->tx.position, map);352}353EXPORT_SYMBOL(tegra_ivc_write_get_next_frame);354 355/* advance the tx buffer */356int tegra_ivc_write_advance(struct tegra_ivc *ivc)357{358 unsigned int tx = offsetof(struct tegra_ivc_header, tx.count);359 unsigned int rx = offsetof(struct tegra_ivc_header, rx.count);360 int err;361 362 err = tegra_ivc_check_write(ivc);363 if (err < 0)364 return err;365 366 tegra_ivc_flush_frame(ivc, ivc->tx.phys, ivc->tx.position, 0,367 ivc->frame_size);368 369 /*370 * Order any possible stores to the frame before update of371 * ivc->tx.position.372 */373 smp_wmb();374 375 tegra_ivc_advance_tx(ivc);376 tegra_ivc_flush(ivc, ivc->tx.phys + tx);377 378 /*379 * Ensure our write to ivc->tx.position occurs before our read from380 * ivc->rx.position.381 */382 smp_mb();383 384 /*385 * Notify only upon transition from empty to non-empty. The available386 * count can only asynchronously decrease, so the worst possible387 * side-effect will be a spurious notification.388 */389 tegra_ivc_invalidate(ivc, ivc->tx.phys + rx);390 391 if (tegra_ivc_available(ivc, &ivc->tx.map) == 1)392 ivc->notify(ivc, ivc->notify_data);393 394 return 0;395}396EXPORT_SYMBOL(tegra_ivc_write_advance);397 398void tegra_ivc_reset(struct tegra_ivc *ivc)399{400 unsigned int offset = offsetof(struct tegra_ivc_header, tx.count);401 402 tegra_ivc_header_write_field(&ivc->tx.map, tx.state, TEGRA_IVC_STATE_SYNC);403 tegra_ivc_flush(ivc, ivc->tx.phys + offset);404 ivc->notify(ivc, ivc->notify_data);405}406EXPORT_SYMBOL(tegra_ivc_reset);407 408/*409 * =======================================================410 * IVC State Transition Table - see tegra_ivc_notified()411 * =======================================================412 *413 * local remote action414 * ----- ------ -----------------------------------415 * SYNC EST <none>416 * SYNC ACK reset counters; move to EST; notify417 * SYNC SYNC reset counters; move to ACK; notify418 * ACK EST move to EST; notify419 * ACK ACK move to EST; notify420 * ACK SYNC reset counters; move to ACK; notify421 * EST EST <none>422 * EST ACK <none>423 * EST SYNC reset counters; move to ACK; notify424 *425 * ===============================================================426 */427 428int tegra_ivc_notified(struct tegra_ivc *ivc)429{430 unsigned int offset = offsetof(struct tegra_ivc_header, tx.count);431 enum tegra_ivc_state rx_state, tx_state;432 433 /* Copy the receiver's state out of shared memory. */434 tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);435 rx_state = tegra_ivc_header_read_field(&ivc->rx.map, tx.state);436 tx_state = tegra_ivc_header_read_field(&ivc->tx.map, tx.state);437 438 if (rx_state == TEGRA_IVC_STATE_SYNC) {439 offset = offsetof(struct tegra_ivc_header, tx.count);440 441 /*442 * Order observation of TEGRA_IVC_STATE_SYNC before stores443 * clearing tx.channel.444 */445 smp_rmb();446 447 /*448 * Reset tx.channel counters. The remote end is in the SYNC449 * state and won't make progress until we change our state,450 * so the counters are not in use at this time.451 */452 tegra_ivc_header_write_field(&ivc->tx.map, tx.count, 0);453 tegra_ivc_header_write_field(&ivc->rx.map, rx.count, 0);454 455 ivc->tx.position = 0;456 ivc->rx.position = 0;457 458 /*459 * Ensure that counters appear cleared before new state can be460 * observed.461 */462 smp_wmb();463 464 /*465 * Move to ACK state. We have just cleared our counters, so it466 * is now safe for the remote end to start using these values.467 */468 tegra_ivc_header_write_field(&ivc->tx.map, tx.state, TEGRA_IVC_STATE_ACK);469 tegra_ivc_flush(ivc, ivc->tx.phys + offset);470 471 /*472 * Notify remote end to observe state transition.473 */474 ivc->notify(ivc, ivc->notify_data);475 476 } else if (tx_state == TEGRA_IVC_STATE_SYNC &&477 rx_state == TEGRA_IVC_STATE_ACK) {478 offset = offsetof(struct tegra_ivc_header, tx.count);479 480 /*481 * Order observation of ivc_state_sync before stores clearing482 * tx_channel.483 */484 smp_rmb();485 486 /*487 * Reset tx.channel counters. The remote end is in the ACK488 * state and won't make progress until we change our state,489 * so the counters are not in use at this time.490 */491 tegra_ivc_header_write_field(&ivc->tx.map, tx.count, 0);492 tegra_ivc_header_write_field(&ivc->rx.map, rx.count, 0);493 494 ivc->tx.position = 0;495 ivc->rx.position = 0;496 497 /*498 * Ensure that counters appear cleared before new state can be499 * observed.500 */501 smp_wmb();502 503 /*504 * Move to ESTABLISHED state. We know that the remote end has505 * already cleared its counters, so it is safe to start506 * writing/reading on this channel.507 */508 tegra_ivc_header_write_field(&ivc->tx.map, tx.state, TEGRA_IVC_STATE_ESTABLISHED);509 tegra_ivc_flush(ivc, ivc->tx.phys + offset);510 511 /*512 * Notify remote end to observe state transition.513 */514 ivc->notify(ivc, ivc->notify_data);515 516 } else if (tx_state == TEGRA_IVC_STATE_ACK) {517 offset = offsetof(struct tegra_ivc_header, tx.count);518 519 /*520 * At this point, we have observed the peer to be in either521 * the ACK or ESTABLISHED state. Next, order observation of522 * peer state before storing to tx.channel.523 */524 smp_rmb();525 526 /*527 * Move to ESTABLISHED state. We know that we have previously528 * cleared our counters, and we know that the remote end has529 * cleared its counters, so it is safe to start writing/reading530 * on this channel.531 */532 tegra_ivc_header_write_field(&ivc->tx.map, tx.state, TEGRA_IVC_STATE_ESTABLISHED);533 tegra_ivc_flush(ivc, ivc->tx.phys + offset);534 535 /*536 * Notify remote end to observe state transition.537 */538 ivc->notify(ivc, ivc->notify_data);539 540 } else {541 /*542 * There is no need to handle any further action. Either the543 * channel is already fully established, or we are waiting for544 * the remote end to catch up with our current state. Refer545 * to the diagram in "IVC State Transition Table" above.546 */547 }548 549 if (tx_state != TEGRA_IVC_STATE_ESTABLISHED)550 return -EAGAIN;551 552 return 0;553}554EXPORT_SYMBOL(tegra_ivc_notified);555 556size_t tegra_ivc_align(size_t size)557{558 return ALIGN(size, TEGRA_IVC_ALIGN);559}560EXPORT_SYMBOL(tegra_ivc_align);561 562unsigned tegra_ivc_total_queue_size(unsigned queue_size)563{564 if (!IS_ALIGNED(queue_size, TEGRA_IVC_ALIGN)) {565 pr_err("%s: queue_size (%u) must be %u-byte aligned\n",566 __func__, queue_size, TEGRA_IVC_ALIGN);567 return 0;568 }569 570 return queue_size + sizeof(struct tegra_ivc_header);571}572EXPORT_SYMBOL(tegra_ivc_total_queue_size);573 574static int tegra_ivc_check_params(unsigned long rx, unsigned long tx,575 unsigned int num_frames, size_t frame_size)576{577 BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct tegra_ivc_header, tx.count),578 TEGRA_IVC_ALIGN));579 BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct tegra_ivc_header, rx.count),580 TEGRA_IVC_ALIGN));581 BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct tegra_ivc_header),582 TEGRA_IVC_ALIGN));583 584 if ((uint64_t)num_frames * (uint64_t)frame_size >= 0x100000000UL) {585 pr_err("num_frames * frame_size overflows\n");586 return -EINVAL;587 }588 589 if (!IS_ALIGNED(frame_size, TEGRA_IVC_ALIGN)) {590 pr_err("frame size not adequately aligned: %zu\n", frame_size);591 return -EINVAL;592 }593 594 /*595 * The headers must at least be aligned enough for counters596 * to be accessed atomically.597 */598 if (!IS_ALIGNED(rx, TEGRA_IVC_ALIGN)) {599 pr_err("IVC channel start not aligned: %#lx\n", rx);600 return -EINVAL;601 }602 603 if (!IS_ALIGNED(tx, TEGRA_IVC_ALIGN)) {604 pr_err("IVC channel start not aligned: %#lx\n", tx);605 return -EINVAL;606 }607 608 if (rx < tx) {609 if (rx + frame_size * num_frames > tx) {610 pr_err("queue regions overlap: %#lx + %zx > %#lx\n",611 rx, frame_size * num_frames, tx);612 return -EINVAL;613 }614 } else {615 if (tx + frame_size * num_frames > rx) {616 pr_err("queue regions overlap: %#lx + %zx > %#lx\n",617 tx, frame_size * num_frames, rx);618 return -EINVAL;619 }620 }621 622 return 0;623}624 625static inline void iosys_map_copy(struct iosys_map *dst, const struct iosys_map *src)626{627 *dst = *src;628}629 630static inline unsigned long iosys_map_get_address(const struct iosys_map *map)631{632 if (map->is_iomem)633 return (unsigned long)map->vaddr_iomem;634 635 return (unsigned long)map->vaddr;636}637 638static inline void *iosys_map_get_vaddr(const struct iosys_map *map)639{640 if (WARN_ON(map->is_iomem))641 return NULL;642 643 return map->vaddr;644}645 646int tegra_ivc_init(struct tegra_ivc *ivc, struct device *peer, const struct iosys_map *rx,647 dma_addr_t rx_phys, const struct iosys_map *tx, dma_addr_t tx_phys,648 unsigned int num_frames, size_t frame_size,649 void (*notify)(struct tegra_ivc *ivc, void *data),650 void *data)651{652 size_t queue_size;653 int err;654 655 if (WARN_ON(!ivc || !notify))656 return -EINVAL;657 658 /*659 * All sizes that can be returned by communication functions should660 * fit in an int.661 */662 if (frame_size > INT_MAX)663 return -E2BIG;664 665 err = tegra_ivc_check_params(iosys_map_get_address(rx), iosys_map_get_address(tx),666 num_frames, frame_size);667 if (err < 0)668 return err;669 670 queue_size = tegra_ivc_total_queue_size(num_frames * frame_size);671 672 if (peer) {673 ivc->rx.phys = dma_map_single(peer, iosys_map_get_vaddr(rx), queue_size,674 DMA_BIDIRECTIONAL);675 if (dma_mapping_error(peer, ivc->rx.phys))676 return -ENOMEM;677 678 ivc->tx.phys = dma_map_single(peer, iosys_map_get_vaddr(tx), queue_size,679 DMA_BIDIRECTIONAL);680 if (dma_mapping_error(peer, ivc->tx.phys)) {681 dma_unmap_single(peer, ivc->rx.phys, queue_size,682 DMA_BIDIRECTIONAL);683 return -ENOMEM;684 }685 } else {686 ivc->rx.phys = rx_phys;687 ivc->tx.phys = tx_phys;688 }689 690 iosys_map_copy(&ivc->rx.map, rx);691 iosys_map_copy(&ivc->tx.map, tx);692 ivc->peer = peer;693 ivc->notify = notify;694 ivc->notify_data = data;695 ivc->frame_size = frame_size;696 ivc->num_frames = num_frames;697 698 /*699 * These values aren't necessarily correct until the channel has been700 * reset.701 */702 ivc->tx.position = 0;703 ivc->rx.position = 0;704 705 return 0;706}707EXPORT_SYMBOL(tegra_ivc_init);708 709void tegra_ivc_cleanup(struct tegra_ivc *ivc)710{711 if (ivc->peer) {712 size_t size = tegra_ivc_total_queue_size(ivc->num_frames *713 ivc->frame_size);714 715 dma_unmap_single(ivc->peer, ivc->rx.phys, size,716 DMA_BIDIRECTIONAL);717 dma_unmap_single(ivc->peer, ivc->tx.phys, size,718 DMA_BIDIRECTIONAL);719 }720}721EXPORT_SYMBOL(tegra_ivc_cleanup);722