2087 lines · c
1// SPDX-License-Identifier: GPL-2.0+2/*3 * SSH packet transport layer.4 *5 * Copyright (C) 2019-2022 Maximilian Luz <luzmaximilian@gmail.com>6 */7 8#include <linux/unaligned.h>9#include <linux/atomic.h>10#include <linux/error-injection.h>11#include <linux/jiffies.h>12#include <linux/kfifo.h>13#include <linux/kref.h>14#include <linux/kthread.h>15#include <linux/ktime.h>16#include <linux/limits.h>17#include <linux/list.h>18#include <linux/lockdep.h>19#include <linux/serdev.h>20#include <linux/slab.h>21#include <linux/spinlock.h>22#include <linux/workqueue.h>23 24#include <linux/surface_aggregator/serial_hub.h>25 26#include "ssh_msgb.h"27#include "ssh_packet_layer.h"28#include "ssh_parser.h"29 30#include "trace.h"31 32/*33 * To simplify reasoning about the code below, we define a few concepts. The34 * system below is similar to a state-machine for packets, however, there are35 * too many states to explicitly write them down. To (somewhat) manage the36 * states and packets we rely on flags, reference counting, and some simple37 * concepts. State transitions are triggered by actions.38 *39 * >> Actions <<40 *41 * - submit42 * - transmission start (process next item in queue)43 * - transmission finished (guaranteed to never be parallel to transmission44 * start)45 * - ACK received46 * - NAK received (this is equivalent to issuing re-submit for all pending47 * packets)48 * - timeout (this is equivalent to re-issuing a submit or canceling)49 * - cancel (non-pending and pending)50 *51 * >> Data Structures, Packet Ownership, General Overview <<52 *53 * The code below employs two main data structures: The packet queue,54 * containing all packets scheduled for transmission, and the set of pending55 * packets, containing all packets awaiting an ACK.56 *57 * Shared ownership of a packet is controlled via reference counting. Inside58 * the transport system are a total of five packet owners:59 *60 * - the packet queue,61 * - the pending set,62 * - the transmitter thread,63 * - the receiver thread (via ACKing), and64 * - the timeout work item.65 *66 * Normal operation is as follows: The initial reference of the packet is67 * obtained by submitting the packet and queuing it. The receiver thread takes68 * packets from the queue. By doing this, it does not increment the refcount69 * but takes over the reference (removing it from the queue). If the packet is70 * sequenced (i.e. needs to be ACKed by the client), the transmitter thread71 * sets-up the timeout and adds the packet to the pending set before starting72 * to transmit it. As the timeout is handled by a reaper task, no additional73 * reference for it is needed. After the transmit is done, the reference held74 * by the transmitter thread is dropped. If the packet is unsequenced (i.e.75 * does not need an ACK), the packet is completed by the transmitter thread76 * before dropping that reference.77 *78 * On receival of an ACK, the receiver thread removes and obtains the79 * reference to the packet from the pending set. The receiver thread will then80 * complete the packet and drop its reference.81 *82 * On receival of a NAK, the receiver thread re-submits all currently pending83 * packets.84 *85 * Packet timeouts are detected by the timeout reaper. This is a task,86 * scheduled depending on the earliest packet timeout expiration date,87 * checking all currently pending packets if their timeout has expired. If the88 * timeout of a packet has expired, it is re-submitted and the number of tries89 * of this packet is incremented. If this number reaches its limit, the packet90 * will be completed with a failure.91 *92 * On transmission failure (such as repeated packet timeouts), the completion93 * callback is immediately run by on thread on which the error was detected.94 *95 * To ensure that a packet eventually leaves the system it is marked as96 * "locked" directly before it is going to be completed or when it is97 * canceled. Marking a packet as "locked" has the effect that passing and98 * creating new references of the packet is disallowed. This means that the99 * packet cannot be added to the queue, the pending set, and the timeout, or100 * be picked up by the transmitter thread or receiver thread. To remove a101 * packet from the system it has to be marked as locked and subsequently all102 * references from the data structures (queue, pending) have to be removed.103 * References held by threads will eventually be dropped automatically as104 * their execution progresses.105 *106 * Note that the packet completion callback is, in case of success and for a107 * sequenced packet, guaranteed to run on the receiver thread, thus providing108 * a way to reliably identify responses to the packet. The packet completion109 * callback is only run once and it does not indicate that the packet has110 * fully left the system (for this, one should rely on the release method,111 * triggered when the reference count of the packet reaches zero). In case of112 * re-submission (and with somewhat unlikely timing), it may be possible that113 * the packet is being re-transmitted while the completion callback runs.114 * Completion will occur both on success and internal error, as well as when115 * the packet is canceled.116 *117 * >> Flags <<118 *119 * Flags are used to indicate the state and progression of a packet. Some flags120 * have stricter guarantees than other:121 *122 * - locked123 * Indicates if the packet is locked. If the packet is locked, passing and/or124 * creating additional references to the packet is forbidden. The packet thus125 * may not be queued, dequeued, or removed or added to the pending set. Note126 * that the packet state flags may still change (e.g. it may be marked as127 * ACKed, transmitted, ...).128 *129 * - completed130 * Indicates if the packet completion callback has been executed or is about131 * to be executed. This flag is used to ensure that the packet completion132 * callback is only run once.133 *134 * - queued135 * Indicates if a packet is present in the submission queue or not. This flag136 * must only be modified with the queue lock held, and must be coherent to the137 * presence of the packet in the queue.138 *139 * - pending140 * Indicates if a packet is present in the set of pending packets or not.141 * This flag must only be modified with the pending lock held, and must be142 * coherent to the presence of the packet in the pending set.143 *144 * - transmitting145 * Indicates if the packet is currently transmitting. In case of146 * re-transmissions, it is only safe to wait on the "transmitted" completion147 * after this flag has been set. The completion will be set both in success148 * and error case.149 *150 * - transmitted151 * Indicates if the packet has been transmitted. This flag is not cleared by152 * the system, thus it indicates the first transmission only.153 *154 * - acked155 * Indicates if the packet has been acknowledged by the client. There are no156 * other guarantees given. For example, the packet may still be canceled157 * and/or the completion may be triggered an error even though this bit is158 * set. Rely on the status provided to the completion callback instead.159 *160 * - canceled161 * Indicates if the packet has been canceled from the outside. There are no162 * other guarantees given. Specifically, the packet may be completed by163 * another part of the system before the cancellation attempts to complete it.164 *165 * >> General Notes <<166 *167 * - To avoid deadlocks, if both queue and pending locks are required, the168 * pending lock must be acquired before the queue lock.169 *170 * - The packet priority must be accessed only while holding the queue lock.171 *172 * - The packet timestamp must be accessed only while holding the pending173 * lock.174 */175 176/*177 * SSH_PTL_MAX_PACKET_TRIES - Maximum transmission attempts for packet.178 *179 * Maximum number of transmission attempts per sequenced packet in case of180 * time-outs. Must be smaller than 16. If the packet times out after this181 * amount of tries, the packet will be completed with %-ETIMEDOUT as status182 * code.183 */184#define SSH_PTL_MAX_PACKET_TRIES 3185 186/*187 * SSH_PTL_TX_TIMEOUT - Packet transmission timeout.188 *189 * Timeout in jiffies for packet transmission via the underlying serial190 * device. If transmitting the packet takes longer than this timeout, the191 * packet will be completed with -ETIMEDOUT. It will not be re-submitted.192 */193#define SSH_PTL_TX_TIMEOUT HZ194 195/*196 * SSH_PTL_PACKET_TIMEOUT - Packet response timeout.197 *198 * Timeout as ktime_t delta for ACKs. If we have not received an ACK in this199 * time-frame after starting transmission, the packet will be re-submitted.200 */201#define SSH_PTL_PACKET_TIMEOUT ms_to_ktime(1000)202 203/*204 * SSH_PTL_PACKET_TIMEOUT_RESOLUTION - Packet timeout granularity.205 *206 * Time-resolution for timeouts. Should be larger than one jiffy to avoid207 * direct re-scheduling of reaper work_struct.208 */209#define SSH_PTL_PACKET_TIMEOUT_RESOLUTION ms_to_ktime(max(2000 / HZ, 50))210 211/*212 * SSH_PTL_MAX_PENDING - Maximum number of pending packets.213 *214 * Maximum number of sequenced packets concurrently waiting for an ACK.215 * Packets marked as blocking will not be transmitted while this limit is216 * reached.217 */218#define SSH_PTL_MAX_PENDING 1219 220/*221 * SSH_PTL_RX_BUF_LEN - Evaluation-buffer size in bytes.222 */223#define SSH_PTL_RX_BUF_LEN 4096224 225/*226 * SSH_PTL_RX_FIFO_LEN - Fifo input-buffer size in bytes.227 */228#define SSH_PTL_RX_FIFO_LEN 4096229 230#ifdef CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION231 232/**233 * ssh_ptl_should_drop_ack_packet() - Error injection hook to drop ACK packets.234 *235 * Useful to test detection and handling of automated re-transmits by the EC.236 * Specifically of packets that the EC considers not-ACKed but the driver237 * already considers ACKed (due to dropped ACK). In this case, the EC238 * re-transmits the packet-to-be-ACKed and the driver should detect it as239 * duplicate/already handled. Note that the driver should still send an ACK240 * for the re-transmitted packet.241 */242static noinline bool ssh_ptl_should_drop_ack_packet(void)243{244 return false;245}246ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_ack_packet, TRUE);247 248/**249 * ssh_ptl_should_drop_nak_packet() - Error injection hook to drop NAK packets.250 *251 * Useful to test/force automated (timeout-based) re-transmit by the EC.252 * Specifically, packets that have not reached the driver completely/with valid253 * checksums. Only useful in combination with receival of (injected) bad data.254 */255static noinline bool ssh_ptl_should_drop_nak_packet(void)256{257 return false;258}259ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_nak_packet, TRUE);260 261/**262 * ssh_ptl_should_drop_dsq_packet() - Error injection hook to drop sequenced263 * data packet.264 *265 * Useful to test re-transmit timeout of the driver. If the data packet has not266 * been ACKed after a certain time, the driver should re-transmit the packet up267 * to limited number of times defined in SSH_PTL_MAX_PACKET_TRIES.268 */269static noinline bool ssh_ptl_should_drop_dsq_packet(void)270{271 return false;272}273ALLOW_ERROR_INJECTION(ssh_ptl_should_drop_dsq_packet, TRUE);274 275/**276 * ssh_ptl_should_fail_write() - Error injection hook to make277 * serdev_device_write() fail.278 *279 * Hook to simulate errors in serdev_device_write when transmitting packets.280 */281static noinline int ssh_ptl_should_fail_write(void)282{283 return 0;284}285ALLOW_ERROR_INJECTION(ssh_ptl_should_fail_write, ERRNO);286 287/**288 * ssh_ptl_should_corrupt_tx_data() - Error injection hook to simulate invalid289 * data being sent to the EC.290 *291 * Hook to simulate corrupt/invalid data being sent from host (driver) to EC.292 * Causes the packet data to be actively corrupted by overwriting it with293 * pre-defined values, such that it becomes invalid, causing the EC to respond294 * with a NAK packet. Useful to test handling of NAK packets received by the295 * driver.296 */297static noinline bool ssh_ptl_should_corrupt_tx_data(void)298{299 return false;300}301ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_tx_data, TRUE);302 303/**304 * ssh_ptl_should_corrupt_rx_syn() - Error injection hook to simulate invalid305 * data being sent by the EC.306 *307 * Hook to simulate invalid SYN bytes, i.e. an invalid start of messages and308 * test handling thereof in the driver.309 */310static noinline bool ssh_ptl_should_corrupt_rx_syn(void)311{312 return false;313}314ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_rx_syn, TRUE);315 316/**317 * ssh_ptl_should_corrupt_rx_data() - Error injection hook to simulate invalid318 * data being sent by the EC.319 *320 * Hook to simulate invalid data/checksum of the message frame and test handling321 * thereof in the driver.322 */323static noinline bool ssh_ptl_should_corrupt_rx_data(void)324{325 return false;326}327ALLOW_ERROR_INJECTION(ssh_ptl_should_corrupt_rx_data, TRUE);328 329static bool __ssh_ptl_should_drop_ack_packet(struct ssh_packet *packet)330{331 if (likely(!ssh_ptl_should_drop_ack_packet()))332 return false;333 334 trace_ssam_ei_tx_drop_ack_packet(packet);335 ptl_info(packet->ptl, "packet error injection: dropping ACK packet %p\n",336 packet);337 338 return true;339}340 341static bool __ssh_ptl_should_drop_nak_packet(struct ssh_packet *packet)342{343 if (likely(!ssh_ptl_should_drop_nak_packet()))344 return false;345 346 trace_ssam_ei_tx_drop_nak_packet(packet);347 ptl_info(packet->ptl, "packet error injection: dropping NAK packet %p\n",348 packet);349 350 return true;351}352 353static bool __ssh_ptl_should_drop_dsq_packet(struct ssh_packet *packet)354{355 if (likely(!ssh_ptl_should_drop_dsq_packet()))356 return false;357 358 trace_ssam_ei_tx_drop_dsq_packet(packet);359 ptl_info(packet->ptl,360 "packet error injection: dropping sequenced data packet %p\n",361 packet);362 363 return true;364}365 366static bool ssh_ptl_should_drop_packet(struct ssh_packet *packet)367{368 /* Ignore packets that don't carry any data (i.e. flush). */369 if (!packet->data.ptr || !packet->data.len)370 return false;371 372 switch (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)]) {373 case SSH_FRAME_TYPE_ACK:374 return __ssh_ptl_should_drop_ack_packet(packet);375 376 case SSH_FRAME_TYPE_NAK:377 return __ssh_ptl_should_drop_nak_packet(packet);378 379 case SSH_FRAME_TYPE_DATA_SEQ:380 return __ssh_ptl_should_drop_dsq_packet(packet);381 382 default:383 return false;384 }385}386 387static int ssh_ptl_write_buf(struct ssh_ptl *ptl, struct ssh_packet *packet,388 const unsigned char *buf, size_t count)389{390 int status;391 392 status = ssh_ptl_should_fail_write();393 if (unlikely(status)) {394 trace_ssam_ei_tx_fail_write(packet, status);395 ptl_info(packet->ptl,396 "packet error injection: simulating transmit error %d, packet %p\n",397 status, packet);398 399 return status;400 }401 402 return serdev_device_write_buf(ptl->serdev, buf, count);403}404 405static void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet)406{407 /* Ignore packets that don't carry any data (i.e. flush). */408 if (!packet->data.ptr || !packet->data.len)409 return;410 411 /* Only allow sequenced data packets to be modified. */412 if (packet->data.ptr[SSH_MSGOFFSET_FRAME(type)] != SSH_FRAME_TYPE_DATA_SEQ)413 return;414 415 if (likely(!ssh_ptl_should_corrupt_tx_data()))416 return;417 418 trace_ssam_ei_tx_corrupt_data(packet);419 ptl_info(packet->ptl,420 "packet error injection: simulating invalid transmit data on packet %p\n",421 packet);422 423 /*424 * NB: The value 0xb3 has been chosen more or less randomly so that it425 * doesn't have any (major) overlap with the SYN bytes (aa 55) and is426 * non-trivial (i.e. non-zero, non-0xff).427 */428 memset(packet->data.ptr, 0xb3, packet->data.len);429}430 431static void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl,432 struct ssam_span *data)433{434 struct ssam_span frame;435 436 /* Check if there actually is something to corrupt. */437 if (!sshp_find_syn(data, &frame))438 return;439 440 if (likely(!ssh_ptl_should_corrupt_rx_syn()))441 return;442 443 trace_ssam_ei_rx_corrupt_syn(data->len);444 445 data->ptr[1] = 0xb3; /* Set second byte of SYN to "random" value. */446}447 448static void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl,449 struct ssam_span *frame)450{451 size_t payload_len, message_len;452 struct ssh_frame *sshf;453 454 /* Ignore incomplete messages, will get handled once it's complete. */455 if (frame->len < SSH_MESSAGE_LENGTH(0))456 return;457 458 /* Ignore incomplete messages, part 2. */459 payload_len = get_unaligned_le16(&frame->ptr[SSH_MSGOFFSET_FRAME(len)]);460 message_len = SSH_MESSAGE_LENGTH(payload_len);461 if (frame->len < message_len)462 return;463 464 if (likely(!ssh_ptl_should_corrupt_rx_data()))465 return;466 467 sshf = (struct ssh_frame *)&frame->ptr[SSH_MSGOFFSET_FRAME(type)];468 trace_ssam_ei_rx_corrupt_data(sshf);469 470 /*471 * Flip bits in first byte of payload checksum. This is basically472 * equivalent to a payload/frame data error without us having to worry473 * about (the, arguably pretty small, probability of) accidental474 * checksum collisions.475 */476 frame->ptr[frame->len - 2] = ~frame->ptr[frame->len - 2];477}478 479#else /* CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION */480 481static inline bool ssh_ptl_should_drop_packet(struct ssh_packet *packet)482{483 return false;484}485 486static inline int ssh_ptl_write_buf(struct ssh_ptl *ptl,487 struct ssh_packet *packet,488 const unsigned char *buf,489 size_t count)490{491 return serdev_device_write_buf(ptl->serdev, buf, count);492}493 494static inline void ssh_ptl_tx_inject_invalid_data(struct ssh_packet *packet)495{496}497 498static inline void ssh_ptl_rx_inject_invalid_syn(struct ssh_ptl *ptl,499 struct ssam_span *data)500{501}502 503static inline void ssh_ptl_rx_inject_invalid_data(struct ssh_ptl *ptl,504 struct ssam_span *frame)505{506}507 508#endif /* CONFIG_SURFACE_AGGREGATOR_ERROR_INJECTION */509 510static void __ssh_ptl_packet_release(struct kref *kref)511{512 struct ssh_packet *p = container_of(kref, struct ssh_packet, refcnt);513 514 trace_ssam_packet_release(p);515 516 ptl_dbg_cond(p->ptl, "ptl: releasing packet %p\n", p);517 p->ops->release(p);518}519 520/**521 * ssh_packet_get() - Increment reference count of packet.522 * @packet: The packet to increment the reference count of.523 *524 * Increments the reference count of the given packet. See ssh_packet_put()525 * for the counter-part of this function.526 *527 * Return: Returns the packet provided as input.528 */529struct ssh_packet *ssh_packet_get(struct ssh_packet *packet)530{531 if (packet)532 kref_get(&packet->refcnt);533 return packet;534}535EXPORT_SYMBOL_GPL(ssh_packet_get);536 537/**538 * ssh_packet_put() - Decrement reference count of packet.539 * @packet: The packet to decrement the reference count of.540 *541 * If the reference count reaches zero, the ``release`` callback specified in542 * the packet's &struct ssh_packet_ops, i.e. ``packet->ops->release``, will be543 * called.544 *545 * See ssh_packet_get() for the counter-part of this function.546 */547void ssh_packet_put(struct ssh_packet *packet)548{549 if (packet)550 kref_put(&packet->refcnt, __ssh_ptl_packet_release);551}552EXPORT_SYMBOL_GPL(ssh_packet_put);553 554static u8 ssh_packet_get_seq(struct ssh_packet *packet)555{556 return packet->data.ptr[SSH_MSGOFFSET_FRAME(seq)];557}558 559/**560 * ssh_packet_init() - Initialize SSH packet.561 * @packet: The packet to initialize.562 * @type: Type-flags of the packet.563 * @priority: Priority of the packet. See SSH_PACKET_PRIORITY() for details.564 * @ops: Packet operations.565 *566 * Initializes the given SSH packet. Sets the transmission buffer pointer to567 * %NULL and the transmission buffer length to zero. For data-type packets,568 * this buffer has to be set separately via ssh_packet_set_data() before569 * submission, and must contain a valid SSH message, i.e. frame with optional570 * payload of any type.571 */572void ssh_packet_init(struct ssh_packet *packet, unsigned long type,573 u8 priority, const struct ssh_packet_ops *ops)574{575 kref_init(&packet->refcnt);576 577 packet->ptl = NULL;578 INIT_LIST_HEAD(&packet->queue_node);579 INIT_LIST_HEAD(&packet->pending_node);580 581 packet->state = type & SSH_PACKET_FLAGS_TY_MASK;582 packet->priority = priority;583 packet->timestamp = KTIME_MAX;584 585 packet->data.ptr = NULL;586 packet->data.len = 0;587 588 packet->ops = ops;589}590 591static struct kmem_cache *ssh_ctrl_packet_cache;592 593/**594 * ssh_ctrl_packet_cache_init() - Initialize the control packet cache.595 */596int ssh_ctrl_packet_cache_init(void)597{598 const unsigned int size = sizeof(struct ssh_packet) + SSH_MSG_LEN_CTRL;599 const unsigned int align = __alignof__(struct ssh_packet);600 struct kmem_cache *cache;601 602 cache = kmem_cache_create("ssam_ctrl_packet", size, align, 0, NULL);603 if (!cache)604 return -ENOMEM;605 606 ssh_ctrl_packet_cache = cache;607 return 0;608}609 610/**611 * ssh_ctrl_packet_cache_destroy() - Deinitialize the control packet cache.612 */613void ssh_ctrl_packet_cache_destroy(void)614{615 kmem_cache_destroy(ssh_ctrl_packet_cache);616 ssh_ctrl_packet_cache = NULL;617}618 619/**620 * ssh_ctrl_packet_alloc() - Allocate packet from control packet cache.621 * @packet: Where the pointer to the newly allocated packet should be stored.622 * @buffer: The buffer corresponding to this packet.623 * @flags: Flags used for allocation.624 *625 * Allocates a packet and corresponding transport buffer from the control626 * packet cache. Sets the packet's buffer reference to the allocated buffer.627 * The packet must be freed via ssh_ctrl_packet_free(), which will also free628 * the corresponding buffer. The corresponding buffer must not be freed629 * separately. Intended to be used with %ssh_ptl_ctrl_packet_ops as packet630 * operations.631 *632 * Return: Returns zero on success, %-ENOMEM if the allocation failed.633 */634static int ssh_ctrl_packet_alloc(struct ssh_packet **packet,635 struct ssam_span *buffer, gfp_t flags)636{637 *packet = kmem_cache_alloc(ssh_ctrl_packet_cache, flags);638 if (!*packet)639 return -ENOMEM;640 641 buffer->ptr = (u8 *)(*packet + 1);642 buffer->len = SSH_MSG_LEN_CTRL;643 644 trace_ssam_ctrl_packet_alloc(*packet, buffer->len);645 return 0;646}647 648/**649 * ssh_ctrl_packet_free() - Free packet allocated from control packet cache.650 * @p: The packet to free.651 */652static void ssh_ctrl_packet_free(struct ssh_packet *p)653{654 trace_ssam_ctrl_packet_free(p);655 kmem_cache_free(ssh_ctrl_packet_cache, p);656}657 658static const struct ssh_packet_ops ssh_ptl_ctrl_packet_ops = {659 .complete = NULL,660 .release = ssh_ctrl_packet_free,661};662 663static void ssh_ptl_timeout_reaper_mod(struct ssh_ptl *ptl, ktime_t now,664 ktime_t expires)665{666 unsigned long delta = msecs_to_jiffies(ktime_ms_delta(expires, now));667 ktime_t aexp = ktime_add(expires, SSH_PTL_PACKET_TIMEOUT_RESOLUTION);668 669 spin_lock(&ptl->rtx_timeout.lock);670 671 /* Re-adjust / schedule reaper only if it is above resolution delta. */672 if (ktime_before(aexp, ptl->rtx_timeout.expires)) {673 ptl->rtx_timeout.expires = expires;674 mod_delayed_work(system_wq, &ptl->rtx_timeout.reaper, delta);675 }676 677 spin_unlock(&ptl->rtx_timeout.lock);678}679 680/* Must be called with queue lock held. */681static void ssh_packet_next_try(struct ssh_packet *p)682{683 u8 base = ssh_packet_priority_get_base(p->priority);684 u8 try = ssh_packet_priority_get_try(p->priority);685 686 lockdep_assert_held(&p->ptl->queue.lock);687 688 /*689 * Ensure that we write the priority in one go via WRITE_ONCE() so we690 * can access it via READ_ONCE() for tracing. Note that other access691 * is guarded by the queue lock, so no need to use READ_ONCE() there.692 */693 WRITE_ONCE(p->priority, __SSH_PACKET_PRIORITY(base, try + 1));694}695 696/* Must be called with queue lock held. */697static struct list_head *__ssh_ptl_queue_find_entrypoint(struct ssh_packet *p)698{699 struct list_head *head;700 struct ssh_packet *q;701 702 lockdep_assert_held(&p->ptl->queue.lock);703 704 /*705 * We generally assume that there are less control (ACK/NAK) packets706 * and re-submitted data packets as there are normal data packets (at707 * least in situations in which many packets are queued; if there708 * aren't many packets queued the decision on how to iterate should be709 * basically irrelevant; the number of control/data packets is more or710 * less limited via the maximum number of pending packets). Thus, when711 * inserting a control or re-submitted data packet, (determined by712 * their priority), we search from front to back. Normal data packets713 * are, usually queued directly at the tail of the queue, so for those714 * search from back to front.715 */716 717 if (p->priority > SSH_PACKET_PRIORITY(DATA, 0)) {718 list_for_each(head, &p->ptl->queue.head) {719 q = list_entry(head, struct ssh_packet, queue_node);720 721 if (q->priority < p->priority)722 break;723 }724 } else {725 list_for_each_prev(head, &p->ptl->queue.head) {726 q = list_entry(head, struct ssh_packet, queue_node);727 728 if (q->priority >= p->priority) {729 head = head->next;730 break;731 }732 }733 }734 735 return head;736}737 738/* Must be called with queue lock held. */739static int __ssh_ptl_queue_push(struct ssh_packet *packet)740{741 struct ssh_ptl *ptl = packet->ptl;742 struct list_head *head;743 744 lockdep_assert_held(&ptl->queue.lock);745 746 if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))747 return -ESHUTDOWN;748 749 /* Avoid further transitions when canceling/completing. */750 if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state))751 return -EINVAL;752 753 /* If this packet has already been queued, do not add it. */754 if (test_and_set_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state))755 return -EALREADY;756 757 head = __ssh_ptl_queue_find_entrypoint(packet);758 759 list_add_tail(&ssh_packet_get(packet)->queue_node, head);760 return 0;761}762 763static int ssh_ptl_queue_push(struct ssh_packet *packet)764{765 int status;766 767 spin_lock(&packet->ptl->queue.lock);768 status = __ssh_ptl_queue_push(packet);769 spin_unlock(&packet->ptl->queue.lock);770 771 return status;772}773 774static void ssh_ptl_queue_remove(struct ssh_packet *packet)775{776 struct ssh_ptl *ptl = packet->ptl;777 778 spin_lock(&ptl->queue.lock);779 780 if (!test_and_clear_bit(SSH_PACKET_SF_QUEUED_BIT, &packet->state)) {781 spin_unlock(&ptl->queue.lock);782 return;783 }784 785 list_del(&packet->queue_node);786 787 spin_unlock(&ptl->queue.lock);788 ssh_packet_put(packet);789}790 791static void ssh_ptl_pending_push(struct ssh_packet *p)792{793 struct ssh_ptl *ptl = p->ptl;794 const ktime_t timestamp = ktime_get_coarse_boottime();795 const ktime_t timeout = ptl->rtx_timeout.timeout;796 797 /*798 * Note: We can get the time for the timestamp before acquiring the799 * lock as this is the only place we're setting it and this function800 * is called only from the transmitter thread. Thus it is not possible801 * to overwrite the timestamp with an outdated value below.802 */803 804 spin_lock(&ptl->pending.lock);805 806 /* If we are canceling/completing this packet, do not add it. */807 if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state)) {808 spin_unlock(&ptl->pending.lock);809 return;810 }811 812 /*813 * On re-submission, the packet has already been added the pending814 * set. We still need to update the timestamp as the packet timeout is815 * reset for each (re-)submission.816 */817 p->timestamp = timestamp;818 819 /* In case it is already pending (e.g. re-submission), do not add it. */820 if (!test_and_set_bit(SSH_PACKET_SF_PENDING_BIT, &p->state)) {821 atomic_inc(&ptl->pending.count);822 list_add_tail(&ssh_packet_get(p)->pending_node, &ptl->pending.head);823 }824 825 spin_unlock(&ptl->pending.lock);826 827 /* Arm/update timeout reaper. */828 ssh_ptl_timeout_reaper_mod(ptl, timestamp, timestamp + timeout);829}830 831static void ssh_ptl_pending_remove(struct ssh_packet *packet)832{833 struct ssh_ptl *ptl = packet->ptl;834 835 spin_lock(&ptl->pending.lock);836 837 if (!test_and_clear_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state)) {838 spin_unlock(&ptl->pending.lock);839 return;840 }841 842 list_del(&packet->pending_node);843 atomic_dec(&ptl->pending.count);844 845 spin_unlock(&ptl->pending.lock);846 847 ssh_packet_put(packet);848}849 850/* Warning: Does not check/set "completed" bit. */851static void __ssh_ptl_complete(struct ssh_packet *p, int status)852{853 struct ssh_ptl *ptl = READ_ONCE(p->ptl);854 855 trace_ssam_packet_complete(p, status);856 ptl_dbg_cond(ptl, "ptl: completing packet %p (status: %d)\n", p, status);857 858 if (p->ops->complete)859 p->ops->complete(p, status);860}861 862static void ssh_ptl_remove_and_complete(struct ssh_packet *p, int status)863{864 /*865 * A call to this function should in general be preceded by866 * set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->flags) to avoid re-adding the867 * packet to the structures it's going to be removed from.868 *869 * The set_bit call does not need explicit memory barriers as the870 * implicit barrier of the test_and_set_bit() call below ensure that the871 * flag is visible before we actually attempt to remove the packet.872 */873 874 if (test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state))875 return;876 877 ssh_ptl_queue_remove(p);878 ssh_ptl_pending_remove(p);879 880 __ssh_ptl_complete(p, status);881}882 883static bool ssh_ptl_tx_can_process(struct ssh_packet *packet)884{885 struct ssh_ptl *ptl = packet->ptl;886 887 if (test_bit(SSH_PACKET_TY_FLUSH_BIT, &packet->state))888 return !atomic_read(&ptl->pending.count);889 890 /* We can always process non-blocking packets. */891 if (!test_bit(SSH_PACKET_TY_BLOCKING_BIT, &packet->state))892 return true;893 894 /* If we are already waiting for this packet, send it again. */895 if (test_bit(SSH_PACKET_SF_PENDING_BIT, &packet->state))896 return true;897 898 /* Otherwise: Check if we have the capacity to send. */899 return atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING;900}901 902static struct ssh_packet *ssh_ptl_tx_pop(struct ssh_ptl *ptl)903{904 struct ssh_packet *packet = ERR_PTR(-ENOENT);905 struct ssh_packet *p, *n;906 907 spin_lock(&ptl->queue.lock);908 list_for_each_entry_safe(p, n, &ptl->queue.head, queue_node) {909 /*910 * If we are canceling or completing this packet, ignore it.911 * It's going to be removed from this queue shortly.912 */913 if (test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))914 continue;915 916 /*917 * Packets should be ordered non-blocking/to-be-resent first.918 * If we cannot process this packet, assume that we can't919 * process any following packet either and abort.920 */921 if (!ssh_ptl_tx_can_process(p)) {922 packet = ERR_PTR(-EBUSY);923 break;924 }925 926 /*927 * We are allowed to change the state now. Remove it from the928 * queue and mark it as being transmitted.929 */930 931 list_del(&p->queue_node);932 933 set_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &p->state);934 /* Ensure that state never gets zero. */935 smp_mb__before_atomic();936 clear_bit(SSH_PACKET_SF_QUEUED_BIT, &p->state);937 938 /*939 * Update number of tries. This directly influences the940 * priority in case the packet is re-submitted (e.g. via941 * timeout/NAK). Note that all reads and writes to the942 * priority after the first submission are guarded by the943 * queue lock.944 */945 ssh_packet_next_try(p);946 947 packet = p;948 break;949 }950 spin_unlock(&ptl->queue.lock);951 952 return packet;953}954 955static struct ssh_packet *ssh_ptl_tx_next(struct ssh_ptl *ptl)956{957 struct ssh_packet *p;958 959 p = ssh_ptl_tx_pop(ptl);960 if (IS_ERR(p))961 return p;962 963 if (test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &p->state)) {964 ptl_dbg(ptl, "ptl: transmitting sequenced packet %p\n", p);965 ssh_ptl_pending_push(p);966 } else {967 ptl_dbg(ptl, "ptl: transmitting non-sequenced packet %p\n", p);968 }969 970 return p;971}972 973static void ssh_ptl_tx_compl_success(struct ssh_packet *packet)974{975 struct ssh_ptl *ptl = packet->ptl;976 977 ptl_dbg(ptl, "ptl: successfully transmitted packet %p\n", packet);978 979 /* Transition state to "transmitted". */980 set_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &packet->state);981 /* Ensure that state never gets zero. */982 smp_mb__before_atomic();983 clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state);984 985 /* If the packet is unsequenced, we're done: Lock and complete. */986 if (!test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &packet->state)) {987 set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state);988 ssh_ptl_remove_and_complete(packet, 0);989 }990 991 /*992 * Notify that a packet transmission has finished. In general we're only993 * waiting for one packet (if any), so wake_up_all should be fine.994 */995 wake_up_all(&ptl->tx.packet_wq);996}997 998static void ssh_ptl_tx_compl_error(struct ssh_packet *packet, int status)999{1000 /* Transmission failure: Lock the packet and try to complete it. */1001 set_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state);1002 /* Ensure that state never gets zero. */1003 smp_mb__before_atomic();1004 clear_bit(SSH_PACKET_SF_TRANSMITTING_BIT, &packet->state);1005 1006 ptl_err(packet->ptl, "ptl: transmission error: %d\n", status);1007 ptl_dbg(packet->ptl, "ptl: failed to transmit packet: %p\n", packet);1008 1009 ssh_ptl_remove_and_complete(packet, status);1010 1011 /*1012 * Notify that a packet transmission has finished. In general we're only1013 * waiting for one packet (if any), so wake_up_all should be fine.1014 */1015 wake_up_all(&packet->ptl->tx.packet_wq);1016}1017 1018static long ssh_ptl_tx_wait_packet(struct ssh_ptl *ptl)1019{1020 int status;1021 1022 status = wait_for_completion_interruptible(&ptl->tx.thread_cplt_pkt);1023 reinit_completion(&ptl->tx.thread_cplt_pkt);1024 1025 /*1026 * Ensure completion is cleared before continuing to avoid lost update1027 * problems.1028 */1029 smp_mb__after_atomic();1030 1031 return status;1032}1033 1034static long ssh_ptl_tx_wait_transfer(struct ssh_ptl *ptl, long timeout)1035{1036 long status;1037 1038 status = wait_for_completion_interruptible_timeout(&ptl->tx.thread_cplt_tx,1039 timeout);1040 reinit_completion(&ptl->tx.thread_cplt_tx);1041 1042 /*1043 * Ensure completion is cleared before continuing to avoid lost update1044 * problems.1045 */1046 smp_mb__after_atomic();1047 1048 return status;1049}1050 1051static int ssh_ptl_tx_packet(struct ssh_ptl *ptl, struct ssh_packet *packet)1052{1053 long timeout = SSH_PTL_TX_TIMEOUT;1054 size_t offset = 0;1055 1056 /* Note: Flush-packets don't have any data. */1057 if (unlikely(!packet->data.ptr))1058 return 0;1059 1060 /* Error injection: drop packet to simulate transmission problem. */1061 if (ssh_ptl_should_drop_packet(packet))1062 return 0;1063 1064 /* Error injection: simulate invalid packet data. */1065 ssh_ptl_tx_inject_invalid_data(packet);1066 1067 ptl_dbg(ptl, "tx: sending data (length: %zu)\n", packet->data.len);1068 print_hex_dump_debug("tx: ", DUMP_PREFIX_OFFSET, 16, 1,1069 packet->data.ptr, packet->data.len, false);1070 1071 do {1072 ssize_t status, len;1073 u8 *buf;1074 1075 buf = packet->data.ptr + offset;1076 len = packet->data.len - offset;1077 1078 status = ssh_ptl_write_buf(ptl, packet, buf, len);1079 if (status < 0)1080 return status;1081 1082 if (status == len)1083 return 0;1084 1085 offset += status;1086 1087 timeout = ssh_ptl_tx_wait_transfer(ptl, timeout);1088 if (kthread_should_stop() || !atomic_read(&ptl->tx.running))1089 return -ESHUTDOWN;1090 1091 if (timeout < 0)1092 return -EINTR;1093 1094 if (timeout == 0)1095 return -ETIMEDOUT;1096 } while (true);1097}1098 1099static int ssh_ptl_tx_threadfn(void *data)1100{1101 struct ssh_ptl *ptl = data;1102 1103 while (!kthread_should_stop() && atomic_read(&ptl->tx.running)) {1104 struct ssh_packet *packet;1105 int status;1106 1107 /* Try to get the next packet. */1108 packet = ssh_ptl_tx_next(ptl);1109 1110 /* If no packet can be processed, we are done. */1111 if (IS_ERR(packet)) {1112 ssh_ptl_tx_wait_packet(ptl);1113 continue;1114 }1115 1116 /* Transfer and complete packet. */1117 status = ssh_ptl_tx_packet(ptl, packet);1118 if (status)1119 ssh_ptl_tx_compl_error(packet, status);1120 else1121 ssh_ptl_tx_compl_success(packet);1122 1123 ssh_packet_put(packet);1124 }1125 1126 return 0;1127}1128 1129/**1130 * ssh_ptl_tx_wakeup_packet() - Wake up packet transmitter thread for new1131 * packet.1132 * @ptl: The packet transport layer.1133 *1134 * Wakes up the packet transmitter thread, notifying it that a new packet has1135 * arrived and is ready for transfer. If the packet transport layer has been1136 * shut down, calls to this function will be ignored.1137 */1138static void ssh_ptl_tx_wakeup_packet(struct ssh_ptl *ptl)1139{1140 if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))1141 return;1142 1143 complete(&ptl->tx.thread_cplt_pkt);1144}1145 1146/**1147 * ssh_ptl_tx_start() - Start packet transmitter thread.1148 * @ptl: The packet transport layer.1149 *1150 * Return: Returns zero on success, a negative error code on failure.1151 */1152int ssh_ptl_tx_start(struct ssh_ptl *ptl)1153{1154 atomic_set_release(&ptl->tx.running, 1);1155 1156 ptl->tx.thread = kthread_run(ssh_ptl_tx_threadfn, ptl, "ssam_serial_hub-tx");1157 if (IS_ERR(ptl->tx.thread))1158 return PTR_ERR(ptl->tx.thread);1159 1160 return 0;1161}1162 1163/**1164 * ssh_ptl_tx_stop() - Stop packet transmitter thread.1165 * @ptl: The packet transport layer.1166 *1167 * Return: Returns zero on success, a negative error code on failure.1168 */1169int ssh_ptl_tx_stop(struct ssh_ptl *ptl)1170{1171 int status = 0;1172 1173 if (!IS_ERR_OR_NULL(ptl->tx.thread)) {1174 /* Tell thread to stop. */1175 atomic_set_release(&ptl->tx.running, 0);1176 1177 /*1178 * Wake up thread in case it is paused. Do not use wakeup1179 * helpers as this may be called when the shutdown bit has1180 * already been set.1181 */1182 complete(&ptl->tx.thread_cplt_pkt);1183 complete(&ptl->tx.thread_cplt_tx);1184 1185 /* Finally, wait for thread to stop. */1186 status = kthread_stop(ptl->tx.thread);1187 ptl->tx.thread = NULL;1188 }1189 1190 return status;1191}1192 1193static struct ssh_packet *ssh_ptl_ack_pop(struct ssh_ptl *ptl, u8 seq_id)1194{1195 struct ssh_packet *packet = ERR_PTR(-ENOENT);1196 struct ssh_packet *p, *n;1197 1198 spin_lock(&ptl->pending.lock);1199 list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {1200 /*1201 * We generally expect packets to be in order, so first packet1202 * to be added to pending is first to be sent, is first to be1203 * ACKed.1204 */1205 if (unlikely(ssh_packet_get_seq(p) != seq_id))1206 continue;1207 1208 /*1209 * In case we receive an ACK while handling a transmission1210 * error completion. The packet will be removed shortly.1211 */1212 if (unlikely(test_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))) {1213 packet = ERR_PTR(-EPERM);1214 break;1215 }1216 1217 /*1218 * Mark the packet as ACKed and remove it from pending by1219 * removing its node and decrementing the pending counter.1220 */1221 set_bit(SSH_PACKET_SF_ACKED_BIT, &p->state);1222 /* Ensure that state never gets zero. */1223 smp_mb__before_atomic();1224 clear_bit(SSH_PACKET_SF_PENDING_BIT, &p->state);1225 1226 atomic_dec(&ptl->pending.count);1227 list_del(&p->pending_node);1228 packet = p;1229 1230 break;1231 }1232 spin_unlock(&ptl->pending.lock);1233 1234 return packet;1235}1236 1237static void ssh_ptl_wait_until_transmitted(struct ssh_packet *packet)1238{1239 wait_event(packet->ptl->tx.packet_wq,1240 test_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &packet->state) ||1241 test_bit(SSH_PACKET_SF_LOCKED_BIT, &packet->state));1242}1243 1244static void ssh_ptl_acknowledge(struct ssh_ptl *ptl, u8 seq)1245{1246 struct ssh_packet *p;1247 1248 p = ssh_ptl_ack_pop(ptl, seq);1249 if (IS_ERR(p)) {1250 if (PTR_ERR(p) == -ENOENT) {1251 /*1252 * The packet has not been found in the set of pending1253 * packets.1254 */1255 ptl_warn(ptl, "ptl: received ACK for non-pending packet\n");1256 } else {1257 /*1258 * The packet is pending, but we are not allowed to take1259 * it because it has been locked.1260 */1261 WARN_ON(PTR_ERR(p) != -EPERM);1262 }1263 return;1264 }1265 1266 ptl_dbg(ptl, "ptl: received ACK for packet %p\n", p);1267 1268 /*1269 * It is possible that the packet has been transmitted, but the state1270 * has not been updated from "transmitting" to "transmitted" yet.1271 * In that case, we need to wait for this transition to occur in order1272 * to determine between success or failure.1273 *1274 * On transmission failure, the packet will be locked after this call.1275 * On success, the transmitted bit will be set.1276 */1277 ssh_ptl_wait_until_transmitted(p);1278 1279 /*1280 * The packet will already be locked in case of a transmission error or1281 * cancellation. Let the transmitter or cancellation issuer complete the1282 * packet.1283 */1284 if (unlikely(test_and_set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))) {1285 if (unlikely(!test_bit(SSH_PACKET_SF_TRANSMITTED_BIT, &p->state)))1286 ptl_err(ptl, "ptl: received ACK before packet had been fully transmitted\n");1287 1288 ssh_packet_put(p);1289 return;1290 }1291 1292 ssh_ptl_remove_and_complete(p, 0);1293 ssh_packet_put(p);1294 1295 if (atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING)1296 ssh_ptl_tx_wakeup_packet(ptl);1297}1298 1299/**1300 * ssh_ptl_submit() - Submit a packet to the transport layer.1301 * @ptl: The packet transport layer to submit the packet to.1302 * @p: The packet to submit.1303 *1304 * Submits a new packet to the transport layer, queuing it to be sent. This1305 * function should not be used for re-submission.1306 *1307 * Return: Returns zero on success, %-EINVAL if a packet field is invalid or1308 * the packet has been canceled prior to submission, %-EALREADY if the packet1309 * has already been submitted, or %-ESHUTDOWN if the packet transport layer1310 * has been shut down.1311 */1312int ssh_ptl_submit(struct ssh_ptl *ptl, struct ssh_packet *p)1313{1314 struct ssh_ptl *ptl_old;1315 int status;1316 1317 trace_ssam_packet_submit(p);1318 1319 /* Validate packet fields. */1320 if (test_bit(SSH_PACKET_TY_FLUSH_BIT, &p->state)) {1321 if (p->data.ptr || test_bit(SSH_PACKET_TY_SEQUENCED_BIT, &p->state))1322 return -EINVAL;1323 } else if (!p->data.ptr) {1324 return -EINVAL;1325 }1326 1327 /*1328 * The ptl reference only gets set on or before the first submission.1329 * After the first submission, it has to be read-only.1330 *1331 * Note that ptl may already be set from upper-layer request1332 * submission, thus we cannot expect it to be NULL.1333 */1334 ptl_old = READ_ONCE(p->ptl);1335 if (!ptl_old)1336 WRITE_ONCE(p->ptl, ptl);1337 else if (WARN_ON(ptl_old != ptl))1338 return -EALREADY; /* Submitted on different PTL. */1339 1340 status = ssh_ptl_queue_push(p);1341 if (status)1342 return status;1343 1344 if (!test_bit(SSH_PACKET_TY_BLOCKING_BIT, &p->state) ||1345 (atomic_read(&ptl->pending.count) < SSH_PTL_MAX_PENDING))1346 ssh_ptl_tx_wakeup_packet(ptl);1347 1348 return 0;1349}1350 1351/*1352 * __ssh_ptl_resubmit() - Re-submit a packet to the transport layer.1353 * @packet: The packet to re-submit.1354 *1355 * Re-submits the given packet: Checks if it can be re-submitted and queues it1356 * if it can, resetting the packet timestamp in the process. Must be called1357 * with the pending lock held.1358 *1359 * Return: Returns %-ECANCELED if the packet has exceeded its number of tries,1360 * %-EINVAL if the packet has been locked, %-EALREADY if the packet is already1361 * on the queue, and %-ESHUTDOWN if the transmission layer has been shut down.1362 */1363static int __ssh_ptl_resubmit(struct ssh_packet *packet)1364{1365 int status;1366 u8 try;1367 1368 lockdep_assert_held(&packet->ptl->pending.lock);1369 1370 trace_ssam_packet_resubmit(packet);1371 1372 spin_lock(&packet->ptl->queue.lock);1373 1374 /* Check if the packet is out of tries. */1375 try = ssh_packet_priority_get_try(packet->priority);1376 if (try >= SSH_PTL_MAX_PACKET_TRIES) {1377 spin_unlock(&packet->ptl->queue.lock);1378 return -ECANCELED;1379 }1380 1381 status = __ssh_ptl_queue_push(packet);1382 if (status) {1383 /*1384 * An error here indicates that the packet has either already1385 * been queued, been locked, or the transport layer is being1386 * shut down. In all cases: Ignore the error.1387 */1388 spin_unlock(&packet->ptl->queue.lock);1389 return status;1390 }1391 1392 packet->timestamp = KTIME_MAX;1393 1394 spin_unlock(&packet->ptl->queue.lock);1395 return 0;1396}1397 1398static void ssh_ptl_resubmit_pending(struct ssh_ptl *ptl)1399{1400 struct ssh_packet *p;1401 bool resub = false;1402 1403 /*1404 * Note: We deliberately do not remove/attempt to cancel and complete1405 * packets that are out of tires in this function. The packet will be1406 * eventually canceled and completed by the timeout. Removing the packet1407 * here could lead to overly eager cancellation if the packet has not1408 * been re-transmitted yet but the tries-counter already updated (i.e1409 * ssh_ptl_tx_next() removed the packet from the queue and updated the1410 * counter, but re-transmission for the last try has not actually1411 * started yet).1412 */1413 1414 spin_lock(&ptl->pending.lock);1415 1416 /* Re-queue all pending packets. */1417 list_for_each_entry(p, &ptl->pending.head, pending_node) {1418 /*1419 * Re-submission fails if the packet is out of tries, has been1420 * locked, is already queued, or the layer is being shut down.1421 * No need to re-schedule tx-thread in those cases.1422 */1423 if (!__ssh_ptl_resubmit(p))1424 resub = true;1425 }1426 1427 spin_unlock(&ptl->pending.lock);1428 1429 if (resub)1430 ssh_ptl_tx_wakeup_packet(ptl);1431}1432 1433/**1434 * ssh_ptl_cancel() - Cancel a packet.1435 * @p: The packet to cancel.1436 *1437 * Cancels a packet. There are no guarantees on when completion and release1438 * callbacks will be called. This may occur during execution of this function1439 * or may occur at any point later.1440 *1441 * Note that it is not guaranteed that the packet will actually be canceled if1442 * the packet is concurrently completed by another process. The only guarantee1443 * of this function is that the packet will be completed (with success,1444 * failure, or cancellation) and released from the transport layer in a1445 * reasonable time-frame.1446 *1447 * May be called before the packet has been submitted, in which case any later1448 * packet submission fails.1449 */1450void ssh_ptl_cancel(struct ssh_packet *p)1451{1452 if (test_and_set_bit(SSH_PACKET_SF_CANCELED_BIT, &p->state))1453 return;1454 1455 trace_ssam_packet_cancel(p);1456 1457 /*1458 * Lock packet and commit with memory barrier. If this packet has1459 * already been locked, it's going to be removed and completed by1460 * another party, which should have precedence.1461 */1462 if (test_and_set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))1463 return;1464 1465 /*1466 * By marking the packet as locked and employing the implicit memory1467 * barrier of test_and_set_bit, we have guaranteed that, at this point,1468 * the packet cannot be added to the queue any more.1469 *1470 * In case the packet has never been submitted, packet->ptl is NULL. If1471 * the packet is currently being submitted, packet->ptl may be NULL or1472 * non-NULL. Due marking the packet as locked above and committing with1473 * the memory barrier, we have guaranteed that, if packet->ptl is NULL,1474 * the packet will never be added to the queue. If packet->ptl is1475 * non-NULL, we don't have any guarantees.1476 */1477 1478 if (READ_ONCE(p->ptl)) {1479 ssh_ptl_remove_and_complete(p, -ECANCELED);1480 1481 if (atomic_read(&p->ptl->pending.count) < SSH_PTL_MAX_PENDING)1482 ssh_ptl_tx_wakeup_packet(p->ptl);1483 1484 } else if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state)) {1485 __ssh_ptl_complete(p, -ECANCELED);1486 }1487}1488 1489/* Must be called with pending lock held */1490static ktime_t ssh_packet_get_expiration(struct ssh_packet *p, ktime_t timeout)1491{1492 lockdep_assert_held(&p->ptl->pending.lock);1493 1494 if (p->timestamp != KTIME_MAX)1495 return ktime_add(p->timestamp, timeout);1496 else1497 return KTIME_MAX;1498}1499 1500static void ssh_ptl_timeout_reap(struct work_struct *work)1501{1502 struct ssh_ptl *ptl = to_ssh_ptl(work, rtx_timeout.reaper.work);1503 struct ssh_packet *p, *n;1504 LIST_HEAD(claimed);1505 ktime_t now = ktime_get_coarse_boottime();1506 ktime_t timeout = ptl->rtx_timeout.timeout;1507 ktime_t next = KTIME_MAX;1508 bool resub = false;1509 int status;1510 1511 trace_ssam_ptl_timeout_reap(atomic_read(&ptl->pending.count));1512 1513 /*1514 * Mark reaper as "not pending". This is done before checking any1515 * packets to avoid lost-update type problems.1516 */1517 spin_lock(&ptl->rtx_timeout.lock);1518 ptl->rtx_timeout.expires = KTIME_MAX;1519 spin_unlock(&ptl->rtx_timeout.lock);1520 1521 spin_lock(&ptl->pending.lock);1522 1523 list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {1524 ktime_t expires = ssh_packet_get_expiration(p, timeout);1525 1526 /*1527 * Check if the timeout hasn't expired yet. Find out next1528 * expiration date to be handled after this run.1529 */1530 if (ktime_after(expires, now)) {1531 next = ktime_before(expires, next) ? expires : next;1532 continue;1533 }1534 1535 trace_ssam_packet_timeout(p);1536 1537 status = __ssh_ptl_resubmit(p);1538 1539 /*1540 * Re-submission fails if the packet is out of tries, has been1541 * locked, is already queued, or the layer is being shut down.1542 * No need to re-schedule tx-thread in those cases.1543 */1544 if (!status)1545 resub = true;1546 1547 /* Go to next packet if this packet is not out of tries. */1548 if (status != -ECANCELED)1549 continue;1550 1551 /* No more tries left: Cancel the packet. */1552 1553 /*1554 * If someone else has locked the packet already, don't use it1555 * and let the other party complete it.1556 */1557 if (test_and_set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state))1558 continue;1559 1560 /*1561 * We have now marked the packet as locked. Thus it cannot be1562 * added to the pending list again after we've removed it here.1563 * We can therefore re-use the pending_node of this packet1564 * temporarily.1565 */1566 1567 clear_bit(SSH_PACKET_SF_PENDING_BIT, &p->state);1568 1569 atomic_dec(&ptl->pending.count);1570 list_move_tail(&p->pending_node, &claimed);1571 }1572 1573 spin_unlock(&ptl->pending.lock);1574 1575 /* Cancel and complete the packet. */1576 list_for_each_entry_safe(p, n, &claimed, pending_node) {1577 if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state)) {1578 ssh_ptl_queue_remove(p);1579 __ssh_ptl_complete(p, -ETIMEDOUT);1580 }1581 1582 /*1583 * Drop the reference we've obtained by removing it from1584 * the pending set.1585 */1586 list_del(&p->pending_node);1587 ssh_packet_put(p);1588 }1589 1590 /* Ensure that reaper doesn't run again immediately. */1591 next = max(next, ktime_add(now, SSH_PTL_PACKET_TIMEOUT_RESOLUTION));1592 if (next != KTIME_MAX)1593 ssh_ptl_timeout_reaper_mod(ptl, now, next);1594 1595 if (resub)1596 ssh_ptl_tx_wakeup_packet(ptl);1597}1598 1599static bool ssh_ptl_rx_retransmit_check(struct ssh_ptl *ptl, const struct ssh_frame *frame)1600{1601 int i;1602 1603 /*1604 * Ignore unsequenced packets. On some devices (notably Surface Pro 9),1605 * unsequenced events will always be sent with SEQ=0x00. Attempting to1606 * detect retransmission would thus just block all events.1607 *1608 * While sequence numbers would also allow detection of retransmitted1609 * packets in unsequenced communication, they have only ever been used1610 * to cover edge-cases in sequenced transmission. In particular, the1611 * only instance of packets being retransmitted (that we are aware of)1612 * is due to an ACK timeout. As this does not happen in unsequenced1613 * communication, skip the retransmission check for those packets1614 * entirely.1615 */1616 if (frame->type == SSH_FRAME_TYPE_DATA_NSQ)1617 return false;1618 1619 /*1620 * Check if SEQ has been seen recently (i.e. packet was1621 * re-transmitted and we should ignore it).1622 */1623 for (i = 0; i < ARRAY_SIZE(ptl->rx.blocked.seqs); i++) {1624 if (likely(ptl->rx.blocked.seqs[i] != frame->seq))1625 continue;1626 1627 ptl_dbg(ptl, "ptl: ignoring repeated data packet\n");1628 return true;1629 }1630 1631 /* Update list of blocked sequence IDs. */1632 ptl->rx.blocked.seqs[ptl->rx.blocked.offset] = frame->seq;1633 ptl->rx.blocked.offset = (ptl->rx.blocked.offset + 1)1634 % ARRAY_SIZE(ptl->rx.blocked.seqs);1635 1636 return false;1637}1638 1639static void ssh_ptl_rx_dataframe(struct ssh_ptl *ptl,1640 const struct ssh_frame *frame,1641 const struct ssam_span *payload)1642{1643 if (ssh_ptl_rx_retransmit_check(ptl, frame))1644 return;1645 1646 ptl->ops.data_received(ptl, payload);1647}1648 1649static void ssh_ptl_send_ack(struct ssh_ptl *ptl, u8 seq)1650{1651 struct ssh_packet *packet;1652 struct ssam_span buf;1653 struct msgbuf msgb;1654 int status;1655 1656 status = ssh_ctrl_packet_alloc(&packet, &buf, GFP_KERNEL);1657 if (status) {1658 ptl_err(ptl, "ptl: failed to allocate ACK packet\n");1659 return;1660 }1661 1662 ssh_packet_init(packet, 0, SSH_PACKET_PRIORITY(ACK, 0),1663 &ssh_ptl_ctrl_packet_ops);1664 1665 msgb_init(&msgb, buf.ptr, buf.len);1666 msgb_push_ack(&msgb, seq);1667 ssh_packet_set_data(packet, msgb.begin, msgb_bytes_used(&msgb));1668 1669 ssh_ptl_submit(ptl, packet);1670 ssh_packet_put(packet);1671}1672 1673static void ssh_ptl_send_nak(struct ssh_ptl *ptl)1674{1675 struct ssh_packet *packet;1676 struct ssam_span buf;1677 struct msgbuf msgb;1678 int status;1679 1680 status = ssh_ctrl_packet_alloc(&packet, &buf, GFP_KERNEL);1681 if (status) {1682 ptl_err(ptl, "ptl: failed to allocate NAK packet\n");1683 return;1684 }1685 1686 ssh_packet_init(packet, 0, SSH_PACKET_PRIORITY(NAK, 0),1687 &ssh_ptl_ctrl_packet_ops);1688 1689 msgb_init(&msgb, buf.ptr, buf.len);1690 msgb_push_nak(&msgb);1691 ssh_packet_set_data(packet, msgb.begin, msgb_bytes_used(&msgb));1692 1693 ssh_ptl_submit(ptl, packet);1694 ssh_packet_put(packet);1695}1696 1697static size_t ssh_ptl_rx_eval(struct ssh_ptl *ptl, struct ssam_span *source)1698{1699 struct ssh_frame *frame;1700 struct ssam_span payload;1701 struct ssam_span aligned;1702 bool syn_found;1703 int status;1704 1705 /* Error injection: Modify data to simulate corrupt SYN bytes. */1706 ssh_ptl_rx_inject_invalid_syn(ptl, source);1707 1708 /* Find SYN. */1709 syn_found = sshp_find_syn(source, &aligned);1710 1711 if (unlikely(aligned.ptr != source->ptr)) {1712 /*1713 * We expect aligned.ptr == source->ptr. If this is not the1714 * case, then aligned.ptr > source->ptr and we've encountered1715 * some unexpected data where we'd expect the start of a new1716 * message (i.e. the SYN sequence).1717 *1718 * This can happen when a CRC check for the previous message1719 * failed and we start actively searching for the next one1720 * (via the call to sshp_find_syn() above), or the first bytes1721 * of a message got dropped or corrupted.1722 *1723 * In any case, we issue a warning, send a NAK to the EC to1724 * request re-transmission of any data we haven't acknowledged1725 * yet, and finally, skip everything up to the next SYN1726 * sequence.1727 */1728 1729 ptl_warn(ptl, "rx: parser: invalid start of frame, skipping\n");1730 1731 /*1732 * Notes:1733 * - This might send multiple NAKs in case the communication1734 * starts with an invalid SYN and is broken down into multiple1735 * pieces. This should generally be handled fine, we just1736 * might receive duplicate data in this case, which is1737 * detected when handling data frames.1738 * - This path will also be executed on invalid CRCs: When an1739 * invalid CRC is encountered, the code below will skip data1740 * until directly after the SYN. This causes the search for1741 * the next SYN, which is generally not placed directly after1742 * the last one.1743 *1744 * Open question: Should we send this in case of invalid1745 * payload CRCs if the frame-type is non-sequential (current1746 * implementation) or should we drop that frame without1747 * telling the EC?1748 */1749 ssh_ptl_send_nak(ptl);1750 }1751 1752 if (unlikely(!syn_found))1753 return aligned.ptr - source->ptr;1754 1755 /* Error injection: Modify data to simulate corruption. */1756 ssh_ptl_rx_inject_invalid_data(ptl, &aligned);1757 1758 /* Parse and validate frame. */1759 status = sshp_parse_frame(&ptl->serdev->dev, &aligned, &frame, &payload,1760 SSH_PTL_RX_BUF_LEN);1761 if (status) /* Invalid frame: skip to next SYN. */1762 return aligned.ptr - source->ptr + sizeof(u16);1763 if (!frame) /* Not enough data. */1764 return aligned.ptr - source->ptr;1765 1766 trace_ssam_rx_frame_received(frame);1767 1768 switch (frame->type) {1769 case SSH_FRAME_TYPE_ACK:1770 ssh_ptl_acknowledge(ptl, frame->seq);1771 break;1772 1773 case SSH_FRAME_TYPE_NAK:1774 ssh_ptl_resubmit_pending(ptl);1775 break;1776 1777 case SSH_FRAME_TYPE_DATA_SEQ:1778 ssh_ptl_send_ack(ptl, frame->seq);1779 fallthrough;1780 1781 case SSH_FRAME_TYPE_DATA_NSQ:1782 ssh_ptl_rx_dataframe(ptl, frame, &payload);1783 break;1784 1785 default:1786 ptl_warn(ptl, "ptl: received frame with unknown type %#04x\n",1787 frame->type);1788 break;1789 }1790 1791 return aligned.ptr - source->ptr + SSH_MESSAGE_LENGTH(payload.len);1792}1793 1794static int ssh_ptl_rx_threadfn(void *data)1795{1796 struct ssh_ptl *ptl = data;1797 1798 while (true) {1799 struct ssam_span span;1800 size_t offs = 0;1801 size_t n;1802 1803 wait_event_interruptible(ptl->rx.wq,1804 !kfifo_is_empty(&ptl->rx.fifo) ||1805 kthread_should_stop());1806 if (kthread_should_stop())1807 break;1808 1809 /* Copy from fifo to evaluation buffer. */1810 n = sshp_buf_read_from_fifo(&ptl->rx.buf, &ptl->rx.fifo);1811 1812 ptl_dbg(ptl, "rx: received data (size: %zu)\n", n);1813 print_hex_dump_debug("rx: ", DUMP_PREFIX_OFFSET, 16, 1,1814 ptl->rx.buf.ptr + ptl->rx.buf.len - n,1815 n, false);1816 1817 /* Parse until we need more bytes or buffer is empty. */1818 while (offs < ptl->rx.buf.len) {1819 sshp_buf_span_from(&ptl->rx.buf, offs, &span);1820 n = ssh_ptl_rx_eval(ptl, &span);1821 if (n == 0)1822 break; /* Need more bytes. */1823 1824 offs += n;1825 }1826 1827 /* Throw away the evaluated parts. */1828 sshp_buf_drop(&ptl->rx.buf, offs);1829 }1830 1831 return 0;1832}1833 1834static void ssh_ptl_rx_wakeup(struct ssh_ptl *ptl)1835{1836 wake_up(&ptl->rx.wq);1837}1838 1839/**1840 * ssh_ptl_rx_start() - Start packet transport layer receiver thread.1841 * @ptl: The packet transport layer.1842 *1843 * Return: Returns zero on success, a negative error code on failure.1844 */1845int ssh_ptl_rx_start(struct ssh_ptl *ptl)1846{1847 if (ptl->rx.thread)1848 return 0;1849 1850 ptl->rx.thread = kthread_run(ssh_ptl_rx_threadfn, ptl,1851 "ssam_serial_hub-rx");1852 if (IS_ERR(ptl->rx.thread))1853 return PTR_ERR(ptl->rx.thread);1854 1855 return 0;1856}1857 1858/**1859 * ssh_ptl_rx_stop() - Stop packet transport layer receiver thread.1860 * @ptl: The packet transport layer.1861 *1862 * Return: Returns zero on success, a negative error code on failure.1863 */1864int ssh_ptl_rx_stop(struct ssh_ptl *ptl)1865{1866 int status = 0;1867 1868 if (ptl->rx.thread) {1869 status = kthread_stop(ptl->rx.thread);1870 ptl->rx.thread = NULL;1871 }1872 1873 return status;1874}1875 1876/**1877 * ssh_ptl_rx_rcvbuf() - Push data from lower-layer transport to the packet1878 * layer.1879 * @ptl: The packet transport layer.1880 * @buf: Pointer to the data to push to the layer.1881 * @n: Size of the data to push to the layer, in bytes.1882 *1883 * Pushes data from a lower-layer transport to the receiver fifo buffer of the1884 * packet layer and notifies the receiver thread. Calls to this function are1885 * ignored once the packet layer has been shut down.1886 *1887 * Return: Returns the number of bytes transferred (positive or zero) on1888 * success. Returns %-ESHUTDOWN if the packet layer has been shut down.1889 */1890ssize_t ssh_ptl_rx_rcvbuf(struct ssh_ptl *ptl, const u8 *buf, size_t n)1891{1892 size_t used;1893 1894 if (test_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state))1895 return -ESHUTDOWN;1896 1897 used = kfifo_in(&ptl->rx.fifo, buf, n);1898 if (used)1899 ssh_ptl_rx_wakeup(ptl);1900 1901 return used;1902}1903 1904/**1905 * ssh_ptl_shutdown() - Shut down the packet transport layer.1906 * @ptl: The packet transport layer.1907 *1908 * Shuts down the packet transport layer, removing and canceling all queued1909 * and pending packets. Packets canceled by this operation will be completed1910 * with %-ESHUTDOWN as status. Receiver and transmitter threads will be1911 * stopped.1912 *1913 * As a result of this function, the transport layer will be marked as shut1914 * down. Submission of packets after the transport layer has been shut down1915 * will fail with %-ESHUTDOWN.1916 */1917void ssh_ptl_shutdown(struct ssh_ptl *ptl)1918{1919 LIST_HEAD(complete_q);1920 LIST_HEAD(complete_p);1921 struct ssh_packet *p, *n;1922 int status;1923 1924 /* Ensure that no new packets (including ACK/NAK) can be submitted. */1925 set_bit(SSH_PTL_SF_SHUTDOWN_BIT, &ptl->state);1926 /*1927 * Ensure that the layer gets marked as shut-down before actually1928 * stopping it. In combination with the check in ssh_ptl_queue_push(),1929 * this guarantees that no new packets can be added and all already1930 * queued packets are properly canceled. In combination with the check1931 * in ssh_ptl_rx_rcvbuf(), this guarantees that received data is1932 * properly cut off.1933 */1934 smp_mb__after_atomic();1935 1936 status = ssh_ptl_rx_stop(ptl);1937 if (status)1938 ptl_err(ptl, "ptl: failed to stop receiver thread\n");1939 1940 status = ssh_ptl_tx_stop(ptl);1941 if (status)1942 ptl_err(ptl, "ptl: failed to stop transmitter thread\n");1943 1944 cancel_delayed_work_sync(&ptl->rtx_timeout.reaper);1945 1946 /*1947 * At this point, all threads have been stopped. This means that the1948 * only references to packets from inside the system are in the queue1949 * and pending set.1950 *1951 * Note: We still need locks here because someone could still be1952 * canceling packets.1953 *1954 * Note 2: We can re-use queue_node (or pending_node) if we mark the1955 * packet as locked an then remove it from the queue (or pending set1956 * respectively). Marking the packet as locked avoids re-queuing1957 * (which should already be prevented by having stopped the treads...)1958 * and not setting QUEUED_BIT (or PENDING_BIT) prevents removal from a1959 * new list via other threads (e.g. cancellation).1960 *1961 * Note 3: There may be overlap between complete_p and complete_q.1962 * This is handled via test_and_set_bit() on the "completed" flag1963 * (also handles cancellation).1964 */1965 1966 /* Mark queued packets as locked and move them to complete_q. */1967 spin_lock(&ptl->queue.lock);1968 list_for_each_entry_safe(p, n, &ptl->queue.head, queue_node) {1969 set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state);1970 /* Ensure that state does not get zero. */1971 smp_mb__before_atomic();1972 clear_bit(SSH_PACKET_SF_QUEUED_BIT, &p->state);1973 1974 list_move_tail(&p->queue_node, &complete_q);1975 }1976 spin_unlock(&ptl->queue.lock);1977 1978 /* Mark pending packets as locked and move them to complete_p. */1979 spin_lock(&ptl->pending.lock);1980 list_for_each_entry_safe(p, n, &ptl->pending.head, pending_node) {1981 set_bit(SSH_PACKET_SF_LOCKED_BIT, &p->state);1982 /* Ensure that state does not get zero. */1983 smp_mb__before_atomic();1984 clear_bit(SSH_PACKET_SF_PENDING_BIT, &p->state);1985 1986 list_move_tail(&p->pending_node, &complete_q);1987 }1988 atomic_set(&ptl->pending.count, 0);1989 spin_unlock(&ptl->pending.lock);1990 1991 /* Complete and drop packets on complete_q. */1992 list_for_each_entry(p, &complete_q, queue_node) {1993 if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state))1994 __ssh_ptl_complete(p, -ESHUTDOWN);1995 1996 ssh_packet_put(p);1997 }1998 1999 /* Complete and drop packets on complete_p. */2000 list_for_each_entry(p, &complete_p, pending_node) {2001 if (!test_and_set_bit(SSH_PACKET_SF_COMPLETED_BIT, &p->state))2002 __ssh_ptl_complete(p, -ESHUTDOWN);2003 2004 ssh_packet_put(p);2005 }2006 2007 /*2008 * At this point we have guaranteed that the system doesn't reference2009 * any packets any more.2010 */2011}2012 2013/**2014 * ssh_ptl_init() - Initialize packet transport layer.2015 * @ptl: The packet transport layer to initialize.2016 * @serdev: The underlying serial device, i.e. the lower-level transport.2017 * @ops: Packet layer operations.2018 *2019 * Initializes the given packet transport layer. Transmitter and receiver2020 * threads must be started separately via ssh_ptl_tx_start() and2021 * ssh_ptl_rx_start(), after the packet-layer has been initialized and the2022 * lower-level transport layer has been set up.2023 *2024 * Return: Returns zero on success and a nonzero error code on failure.2025 */2026int ssh_ptl_init(struct ssh_ptl *ptl, struct serdev_device *serdev,2027 struct ssh_ptl_ops *ops)2028{2029 int i, status;2030 2031 ptl->serdev = serdev;2032 ptl->state = 0;2033 2034 spin_lock_init(&ptl->queue.lock);2035 INIT_LIST_HEAD(&ptl->queue.head);2036 2037 spin_lock_init(&ptl->pending.lock);2038 INIT_LIST_HEAD(&ptl->pending.head);2039 atomic_set_release(&ptl->pending.count, 0);2040 2041 ptl->tx.thread = NULL;2042 atomic_set(&ptl->tx.running, 0);2043 init_completion(&ptl->tx.thread_cplt_pkt);2044 init_completion(&ptl->tx.thread_cplt_tx);2045 init_waitqueue_head(&ptl->tx.packet_wq);2046 2047 ptl->rx.thread = NULL;2048 init_waitqueue_head(&ptl->rx.wq);2049 2050 spin_lock_init(&ptl->rtx_timeout.lock);2051 ptl->rtx_timeout.timeout = SSH_PTL_PACKET_TIMEOUT;2052 ptl->rtx_timeout.expires = KTIME_MAX;2053 INIT_DELAYED_WORK(&ptl->rtx_timeout.reaper, ssh_ptl_timeout_reap);2054 2055 ptl->ops = *ops;2056 2057 /* Initialize list of recent/blocked SEQs with invalid sequence IDs. */2058 for (i = 0; i < ARRAY_SIZE(ptl->rx.blocked.seqs); i++)2059 ptl->rx.blocked.seqs[i] = U16_MAX;2060 ptl->rx.blocked.offset = 0;2061 2062 status = kfifo_alloc(&ptl->rx.fifo, SSH_PTL_RX_FIFO_LEN, GFP_KERNEL);2063 if (status)2064 return status;2065 2066 status = sshp_buf_alloc(&ptl->rx.buf, SSH_PTL_RX_BUF_LEN, GFP_KERNEL);2067 if (status)2068 kfifo_free(&ptl->rx.fifo);2069 2070 return status;2071}2072 2073/**2074 * ssh_ptl_destroy() - Deinitialize packet transport layer.2075 * @ptl: The packet transport layer to deinitialize.2076 *2077 * Deinitializes the given packet transport layer and frees resources2078 * associated with it. If receiver and/or transmitter threads have been2079 * started, the layer must first be shut down via ssh_ptl_shutdown() before2080 * this function can be called.2081 */2082void ssh_ptl_destroy(struct ssh_ptl *ptl)2083{2084 kfifo_free(&ptl->rx.fifo);2085 sshp_buf_free(&ptl->rx.buf);2086}2087