2272 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */3 4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt5 6#include <linux/skbuff.h>7#include <linux/module.h>8#include <linux/highmem.h>9#include <linux/ip.h>10#include <net/ipv6.h>11#include <linux/netdevice.h>12#include <crypto/aes.h>13#include <linux/skbuff_ref.h>14#include "chcr_ktls.h"15 16static LIST_HEAD(uld_ctx_list);17static DEFINE_MUTEX(dev_mutex);18 19/* chcr_get_nfrags_to_send: get the remaining nfrags after start offset20 * @skb: skb21 * @start: start offset.22 * @len: how much data to send after @start23 */24static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len)25{26 struct skb_shared_info *si = skb_shinfo(skb);27 u32 frag_size, skb_linear_data_len = skb_headlen(skb);28 u8 nfrags = 0, frag_idx = 0;29 skb_frag_t *frag;30 31 /* if its a linear skb then return 1 */32 if (!skb_is_nonlinear(skb))33 return 1;34 35 if (unlikely(start < skb_linear_data_len)) {36 frag_size = min(len, skb_linear_data_len - start);37 } else {38 start -= skb_linear_data_len;39 40 frag = &si->frags[frag_idx];41 frag_size = skb_frag_size(frag);42 while (start >= frag_size) {43 start -= frag_size;44 frag_idx++;45 frag = &si->frags[frag_idx];46 frag_size = skb_frag_size(frag);47 }48 frag_size = min(len, skb_frag_size(frag) - start);49 }50 len -= frag_size;51 nfrags++;52 53 while (len) {54 frag_size = min(len, skb_frag_size(&si->frags[frag_idx]));55 len -= frag_size;56 nfrags++;57 frag_idx++;58 }59 return nfrags;60}61 62static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);63static void clear_conn_resources(struct chcr_ktls_info *tx_info);64/*65 * chcr_ktls_save_keys: calculate and save crypto keys.66 * @tx_info - driver specific tls info.67 * @crypto_info - tls crypto information.68 * @direction - TX/RX direction.69 * return - SUCCESS/FAILURE.70 */71static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,72 struct tls_crypto_info *crypto_info,73 enum tls_offload_ctx_dir direction)74{75 int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;76 unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];77 struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;78 struct ktls_key_ctx *kctx = &tx_info->key_ctx;79 struct crypto_aes_ctx aes_ctx;80 unsigned char *key, *salt;81 82 switch (crypto_info->cipher_type) {83 case TLS_CIPHER_AES_GCM_128:84 info_128_gcm =85 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;86 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;87 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;88 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;89 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;90 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;91 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);92 93 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;94 key = info_128_gcm->key;95 salt = info_128_gcm->salt;96 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;97 98 /* The SCMD fields used when encrypting a full TLS99 * record. Its a one time calculation till the100 * connection exists.101 */102 tx_info->scmd0_seqno_numivs =103 SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |104 SCMD_CIPH_AUTH_SEQ_CTRL_F |105 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |106 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |107 SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |108 SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |109 SCMD_NUM_IVS_V(1);110 111 /* keys will be sent inline. */112 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;113 114 /* The SCMD fields used when encrypting a partial TLS115 * record (no trailer and possibly a truncated payload).116 */117 tx_info->scmd0_short_seqno_numivs =118 SCMD_CIPH_AUTH_SEQ_CTRL_F |119 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |120 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |121 SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);122 123 tx_info->scmd0_short_ivgen_hdrlen =124 tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;125 126 break;127 128 default:129 pr_err("GCM: cipher type 0x%x not supported\n",130 crypto_info->cipher_type);131 ret = -EINVAL;132 goto out;133 }134 135 key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +136 roundup(keylen, 16) + ghash_size;137 /* Calculate the H = CIPH(K, 0 repeated 16 times).138 * It will go in key context139 */140 141 ret = aes_expandkey(&aes_ctx, key, keylen);142 if (ret)143 goto out;144 145 memset(ghash_h, 0, ghash_size);146 aes_encrypt(&aes_ctx, ghash_h, ghash_h);147 memzero_explicit(&aes_ctx, sizeof(aes_ctx));148 149 /* fill the Key context */150 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {151 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,152 mac_key_size,153 key_ctx_size >> 4);154 } else {155 ret = -EINVAL;156 goto out;157 }158 159 memcpy(kctx->salt, salt, tx_info->salt_size);160 memcpy(kctx->key, key, keylen);161 memcpy(kctx->key + keylen, ghash_h, ghash_size);162 tx_info->key_ctx_len = key_ctx_size;163 164out:165 return ret;166}167 168/*169 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.170 * @sk - tcp socket.171 * @tx_info - driver specific tls info.172 * @atid - connection active tid.173 * return - send success/failure.174 */175static int chcr_ktls_act_open_req(struct sock *sk,176 struct chcr_ktls_info *tx_info,177 int atid)178{179 struct inet_sock *inet = inet_sk(sk);180 struct cpl_t6_act_open_req *cpl6;181 struct cpl_act_open_req *cpl;182 struct sk_buff *skb;183 unsigned int len;184 int qid_atid;185 u64 options;186 187 len = sizeof(*cpl6);188 skb = alloc_skb(len, GFP_KERNEL);189 if (unlikely(!skb))190 return -ENOMEM;191 /* mark it a control pkt */192 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);193 194 cpl6 = __skb_put_zero(skb, len);195 cpl = (struct cpl_act_open_req *)cpl6;196 INIT_TP_WR(cpl6, 0);197 qid_atid = TID_QID_V(tx_info->rx_qid) |198 TID_TID_V(atid);199 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));200 cpl->local_port = inet->inet_sport;201 cpl->peer_port = inet->inet_dport;202 cpl->local_ip = inet->inet_rcv_saddr;203 cpl->peer_ip = inet->inet_daddr;204 205 /* fill first 64 bit option field. */206 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |207 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);208 cpl->opt0 = cpu_to_be64(options);209 210 /* next 64 bit option field. */211 options =212 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);213 cpl->opt2 = htonl(options);214 215 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);216}217 218#if IS_ENABLED(CONFIG_IPV6)219/*220 * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.221 * @sk - tcp socket.222 * @tx_info - driver specific tls info.223 * @atid - connection active tid.224 * return - send success/failure.225 */226static int chcr_ktls_act_open_req6(struct sock *sk,227 struct chcr_ktls_info *tx_info,228 int atid)229{230 struct inet_sock *inet = inet_sk(sk);231 struct cpl_t6_act_open_req6 *cpl6;232 struct cpl_act_open_req6 *cpl;233 struct sk_buff *skb;234 unsigned int len;235 int qid_atid;236 u64 options;237 238 len = sizeof(*cpl6);239 skb = alloc_skb(len, GFP_KERNEL);240 if (unlikely(!skb))241 return -ENOMEM;242 /* mark it a control pkt */243 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);244 245 cpl6 = __skb_put_zero(skb, len);246 cpl = (struct cpl_act_open_req6 *)cpl6;247 INIT_TP_WR(cpl6, 0);248 qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);249 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));250 cpl->local_port = inet->inet_sport;251 cpl->peer_port = inet->inet_dport;252 cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];253 cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];254 cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];255 cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];256 257 /* first 64 bit option field. */258 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |259 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);260 cpl->opt0 = cpu_to_be64(options);261 /* next 64 bit option field. */262 options =263 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);264 cpl->opt2 = htonl(options);265 266 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);267}268#endif /* #if IS_ENABLED(CONFIG_IPV6) */269 270/*271 * chcr_setup_connection: create a TCB entry so that TP will form tcp packets.272 * @sk - tcp socket.273 * @tx_info - driver specific tls info.274 * return: NET_TX_OK/NET_XMIT_DROP275 */276static int chcr_setup_connection(struct sock *sk,277 struct chcr_ktls_info *tx_info)278{279 struct tid_info *t = &tx_info->adap->tids;280 int atid, ret = 0;281 282 atid = cxgb4_alloc_atid(t, tx_info);283 if (atid == -1)284 return -EINVAL;285 286 tx_info->atid = atid;287 288 if (tx_info->ip_family == AF_INET) {289 ret = chcr_ktls_act_open_req(sk, tx_info, atid);290#if IS_ENABLED(CONFIG_IPV6)291 } else {292 ret = cxgb4_clip_get(tx_info->netdev, (const u32 *)293 &sk->sk_v6_rcv_saddr,294 1);295 if (ret)296 return ret;297 ret = chcr_ktls_act_open_req6(sk, tx_info, atid);298#endif299 }300 301 /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret302 * success, if any other return type clear atid and return that failure.303 */304 if (ret) {305 if (ret == NET_XMIT_CN) {306 ret = 0;307 } else {308#if IS_ENABLED(CONFIG_IPV6)309 /* clear clip entry */310 if (tx_info->ip_family == AF_INET6)311 cxgb4_clip_release(tx_info->netdev,312 (const u32 *)313 &sk->sk_v6_rcv_saddr,314 1);315#endif316 cxgb4_free_atid(t, atid);317 }318 }319 320 return ret;321}322 323/*324 * chcr_set_tcb_field: update tcb fields.325 * @tx_info - driver specific tls info.326 * @word - TCB word.327 * @mask - TCB word related mask.328 * @val - TCB word related value.329 * @no_reply - set 1 if not looking for TP response.330 */331static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,332 u64 mask, u64 val, int no_reply)333{334 struct cpl_set_tcb_field *req;335 struct sk_buff *skb;336 337 skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);338 if (!skb)339 return -ENOMEM;340 341 req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));342 INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);343 req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |344 NO_REPLY_V(no_reply));345 req->word_cookie = htons(TCB_WORD_V(word));346 req->mask = cpu_to_be64(mask);347 req->val = cpu_to_be64(val);348 349 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);350 return cxgb4_ofld_send(tx_info->netdev, skb);351}352 353/*354 * chcr_ktls_dev_del: call back for tls_dev_del.355 * Remove the tid and l2t entry and close the connection.356 * it per connection basis.357 * @netdev - net device.358 * @tls_cts - tls context.359 * @direction - TX/RX crypto direction360 */361static void chcr_ktls_dev_del(struct net_device *netdev,362 struct tls_context *tls_ctx,363 enum tls_offload_ctx_dir direction)364{365 struct chcr_ktls_info *tx_info = chcr_get_ktls_tx_info(tls_ctx);366 struct ch_ktls_port_stats_debug *port_stats;367 struct chcr_ktls_uld_ctx *u_ctx;368 369 if (!tx_info)370 return;371 372 u_ctx = tx_info->adap->uld[CXGB4_ULD_KTLS].handle;373 if (u_ctx && u_ctx->detach)374 return;375 /* clear l2t entry */376 if (tx_info->l2te)377 cxgb4_l2t_release(tx_info->l2te);378 379#if IS_ENABLED(CONFIG_IPV6)380 /* clear clip entry */381 if (tx_info->ip_family == AF_INET6)382 cxgb4_clip_release(netdev, (const u32 *)383 &tx_info->sk->sk_v6_rcv_saddr,384 1);385#endif386 387 /* clear tid */388 if (tx_info->tid != -1) {389 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,390 tx_info->tid, tx_info->ip_family);391 392 xa_erase(&u_ctx->tid_list, tx_info->tid);393 }394 395 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];396 atomic64_inc(&port_stats->ktls_tx_connection_close);397 kvfree(tx_info);398 chcr_set_ktls_tx_info(tls_ctx, NULL);399 /* release module refcount */400 module_put(THIS_MODULE);401}402 403/*404 * chcr_ktls_dev_add: call back for tls_dev_add.405 * Create a tcb entry for TP. Also add l2t entry for the connection. And406 * generate keys & save those keys locally.407 * @netdev - net device.408 * @tls_cts - tls context.409 * @direction - TX/RX crypto direction410 * return: SUCCESS/FAILURE.411 */412static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,413 enum tls_offload_ctx_dir direction,414 struct tls_crypto_info *crypto_info,415 u32 start_offload_tcp_sn)416{417 struct tls_context *tls_ctx = tls_get_ctx(sk);418 struct ch_ktls_port_stats_debug *port_stats;419 struct chcr_ktls_uld_ctx *u_ctx;420 struct chcr_ktls_info *tx_info;421 struct dst_entry *dst;422 struct adapter *adap;423 struct port_info *pi;424 struct neighbour *n;425 u8 daaddr[16];426 int ret = -1;427 428 pi = netdev_priv(netdev);429 adap = pi->adapter;430 port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id];431 atomic64_inc(&port_stats->ktls_tx_connection_open);432 u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;433 434 if (direction == TLS_OFFLOAD_CTX_DIR_RX) {435 pr_err("not expecting for RX direction\n");436 goto out;437 }438 439 if (chcr_get_ktls_tx_info(tls_ctx))440 goto out;441 442 if (u_ctx && u_ctx->detach)443 goto out;444 445 tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);446 if (!tx_info)447 goto out;448 449 tx_info->sk = sk;450 spin_lock_init(&tx_info->lock);451 /* initialize tid and atid to -1, 0 is a also a valid id. */452 tx_info->tid = -1;453 tx_info->atid = -1;454 455 tx_info->adap = adap;456 tx_info->netdev = netdev;457 tx_info->first_qset = pi->first_qset;458 tx_info->tx_chan = pi->tx_chan;459 tx_info->smt_idx = pi->smt_idx;460 tx_info->port_id = pi->port_id;461 tx_info->prev_ack = 0;462 tx_info->prev_win = 0;463 464 tx_info->rx_qid = chcr_get_first_rx_qid(adap);465 if (unlikely(tx_info->rx_qid < 0))466 goto free_tx_info;467 468 tx_info->prev_seq = start_offload_tcp_sn;469 tx_info->tcp_start_seq_number = start_offload_tcp_sn;470 471 /* save crypto keys */472 ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);473 if (ret < 0)474 goto free_tx_info;475 476 /* get peer ip */477 if (sk->sk_family == AF_INET) {478 memcpy(daaddr, &sk->sk_daddr, 4);479 tx_info->ip_family = AF_INET;480#if IS_ENABLED(CONFIG_IPV6)481 } else {482 if (!ipv6_only_sock(sk) &&483 ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {484 memcpy(daaddr, &sk->sk_daddr, 4);485 tx_info->ip_family = AF_INET;486 } else {487 memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);488 tx_info->ip_family = AF_INET6;489 }490#endif491 }492 493 /* get the l2t index */494 dst = sk_dst_get(sk);495 if (!dst) {496 pr_err("DST entry not found\n");497 goto free_tx_info;498 }499 n = dst_neigh_lookup(dst, daaddr);500 if (!n || !n->dev) {501 pr_err("neighbour not found\n");502 dst_release(dst);503 goto free_tx_info;504 }505 tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);506 507 neigh_release(n);508 dst_release(dst);509 510 if (!tx_info->l2te) {511 pr_err("l2t entry not found\n");512 goto free_tx_info;513 }514 515 /* Driver shouldn't be removed until any single connection exists */516 if (!try_module_get(THIS_MODULE))517 goto free_l2t;518 519 init_completion(&tx_info->completion);520 /* create a filter and call cxgb4_l2t_send to send the packet out, which521 * will take care of updating l2t entry in hw if not already done.522 */523 tx_info->open_state = CH_KTLS_OPEN_PENDING;524 525 if (chcr_setup_connection(sk, tx_info))526 goto put_module;527 528 /* Wait for reply */529 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);530 spin_lock_bh(&tx_info->lock);531 if (tx_info->open_state) {532 /* need to wait for hw response, can't free tx_info yet. */533 if (tx_info->open_state == CH_KTLS_OPEN_PENDING)534 tx_info->pending_close = true;535 else536 spin_unlock_bh(&tx_info->lock);537 /* if in pending close, free the lock after the cleanup */538 goto put_module;539 }540 spin_unlock_bh(&tx_info->lock);541 542 /* initialize tcb */543 reinit_completion(&tx_info->completion);544 /* mark it pending for hw response */545 tx_info->open_state = CH_KTLS_OPEN_PENDING;546 547 if (chcr_init_tcb_fields(tx_info))548 goto free_tid;549 550 /* Wait for reply */551 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);552 spin_lock_bh(&tx_info->lock);553 if (tx_info->open_state) {554 /* need to wait for hw response, can't free tx_info yet. */555 tx_info->pending_close = true;556 /* free the lock after cleanup */557 goto free_tid;558 }559 spin_unlock_bh(&tx_info->lock);560 561 if (!cxgb4_check_l2t_valid(tx_info->l2te))562 goto free_tid;563 564 atomic64_inc(&port_stats->ktls_tx_ctx);565 chcr_set_ktls_tx_info(tls_ctx, tx_info);566 567 return 0;568 569free_tid:570#if IS_ENABLED(CONFIG_IPV6)571 /* clear clip entry */572 if (tx_info->ip_family == AF_INET6)573 cxgb4_clip_release(netdev, (const u32 *)574 &sk->sk_v6_rcv_saddr,575 1);576#endif577 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,578 tx_info->tid, tx_info->ip_family);579 580 xa_erase(&u_ctx->tid_list, tx_info->tid);581 582put_module:583 /* release module refcount */584 module_put(THIS_MODULE);585free_l2t:586 cxgb4_l2t_release(tx_info->l2te);587free_tx_info:588 if (tx_info->pending_close)589 spin_unlock_bh(&tx_info->lock);590 else591 kvfree(tx_info);592out:593 atomic64_inc(&port_stats->ktls_tx_connection_fail);594 return -1;595}596 597/*598 * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number599 * handling.600 * @tx_info - driver specific tls info.601 * return: NET_TX_OK/NET_XMIT_DROP602 */603static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)604{605 int ret = 0;606 607 /* set tcb in offload and bypass */608 ret =609 chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,610 TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),611 TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);612 if (ret)613 return ret;614 /* reset snd_una and snd_next fields in tcb */615 ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,616 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |617 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),618 0, 1);619 if (ret)620 return ret;621 622 /* reset send max */623 ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,624 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),625 0, 1);626 if (ret)627 return ret;628 629 /* update l2t index and request for tp reply to confirm tcb is630 * initialised to handle tx traffic.631 */632 ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,633 TCB_L2T_IX_V(TCB_L2T_IX_M),634 TCB_L2T_IX_V(tx_info->l2te->idx), 0);635 return ret;636}637 638/*639 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.640 */641static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap,642 unsigned char *input)643{644 const struct cpl_act_open_rpl *p = (void *)input;645 struct chcr_ktls_info *tx_info = NULL;646 struct tls_offload_context_tx *tx_ctx;647 struct chcr_ktls_uld_ctx *u_ctx;648 unsigned int atid, tid, status;649 struct tls_context *tls_ctx;650 struct tid_info *t;651 int ret = 0;652 653 tid = GET_TID(p);654 status = AOPEN_STATUS_G(ntohl(p->atid_status));655 atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));656 657 t = &adap->tids;658 tx_info = lookup_atid(t, atid);659 660 if (!tx_info || tx_info->atid != atid) {661 pr_err("%s: incorrect tx_info or atid\n", __func__);662 return -1;663 }664 665 cxgb4_free_atid(t, atid);666 tx_info->atid = -1;667 668 spin_lock(&tx_info->lock);669 /* HW response is very close, finish pending cleanup */670 if (tx_info->pending_close) {671 spin_unlock(&tx_info->lock);672 if (!status) {673 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,674 tid, tx_info->ip_family);675 }676 kvfree(tx_info);677 return 0;678 }679 680 if (!status) {681 tx_info->tid = tid;682 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);683 /* Adding tid */684 tls_ctx = tls_get_ctx(tx_info->sk);685 tx_ctx = tls_offload_ctx_tx(tls_ctx);686 u_ctx = adap->uld[CXGB4_ULD_KTLS].handle;687 if (u_ctx) {688 ret = xa_insert_bh(&u_ctx->tid_list, tid, tx_ctx,689 GFP_NOWAIT);690 if (ret < 0) {691 pr_err("%s: Failed to allocate tid XA entry = %d\n",692 __func__, tx_info->tid);693 tx_info->open_state = CH_KTLS_OPEN_FAILURE;694 goto out;695 }696 }697 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;698 } else {699 tx_info->open_state = CH_KTLS_OPEN_FAILURE;700 }701out:702 spin_unlock(&tx_info->lock);703 704 complete(&tx_info->completion);705 return ret;706}707 708/*709 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.710 */711static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)712{713 const struct cpl_set_tcb_rpl *p = (void *)input;714 struct chcr_ktls_info *tx_info = NULL;715 struct tid_info *t;716 u32 tid;717 718 tid = GET_TID(p);719 720 t = &adap->tids;721 tx_info = lookup_tid(t, tid);722 723 if (!tx_info || tx_info->tid != tid) {724 pr_err("%s: incorrect tx_info or tid\n", __func__);725 return -1;726 }727 728 spin_lock(&tx_info->lock);729 if (tx_info->pending_close) {730 spin_unlock(&tx_info->lock);731 kvfree(tx_info);732 return 0;733 }734 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;735 spin_unlock(&tx_info->lock);736 737 complete(&tx_info->completion);738 return 0;739}740 741static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,742 u32 tid, void *pos, u16 word,743 struct sge_eth_txq *q, u64 mask,744 u64 val, u32 reply)745{746 struct cpl_set_tcb_field_core *cpl;747 struct ulptx_idata *idata;748 struct ulp_txpkt *txpkt;749 750 /* ULP_TXPKT */751 txpkt = pos;752 txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |753 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |754 ULP_TXPKT_FID_V(q->q.cntxt_id) |755 ULP_TXPKT_RO_F);756 txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));757 758 /* ULPTX_IDATA sub-command */759 idata = (struct ulptx_idata *)(txpkt + 1);760 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));761 idata->len = htonl(sizeof(*cpl));762 pos = idata + 1;763 764 cpl = pos;765 /* CPL_SET_TCB_FIELD */766 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));767 cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |768 NO_REPLY_V(!reply));769 cpl->word_cookie = htons(TCB_WORD_V(word));770 cpl->mask = cpu_to_be64(mask);771 cpl->val = cpu_to_be64(val);772 773 /* ULPTX_NOOP */774 idata = (struct ulptx_idata *)(cpl + 1);775 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));776 idata->len = htonl(0);777 pos = idata + 1;778 779 return pos;780}781 782 783/*784 * chcr_write_cpl_set_tcb_ulp: update tcb values.785 * TCB is responsible to create tcp headers, so all the related values786 * should be correctly updated.787 * @tx_info - driver specific tls info.788 * @q - tx queue on which packet is going out.789 * @tid - TCB identifier.790 * @pos - current index where should we start writing.791 * @word - TCB word.792 * @mask - TCB word related mask.793 * @val - TCB word related value.794 * @reply - set 1 if looking for TP response.795 * return - next position to write.796 */797static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,798 struct sge_eth_txq *q, u32 tid,799 void *pos, u16 word, u64 mask,800 u64 val, u32 reply)801{802 int left = (void *)q->q.stat - pos;803 804 if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {805 if (!left) {806 pos = q->q.desc;807 } else {808 u8 buf[48] = {0};809 810 __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q,811 mask, val, reply);812 813 return chcr_copy_to_txd(buf, &q->q, pos,814 CHCR_SET_TCB_FIELD_LEN);815 }816 }817 818 pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q,819 mask, val, reply);820 821 /* check again if we are at the end of the queue */822 if (left == CHCR_SET_TCB_FIELD_LEN)823 pos = q->q.desc;824 825 return pos;826}827 828/*829 * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header830 * with updated values like tcp seq, ack, window etc.831 * @tx_info - driver specific tls info.832 * @q - TX queue.833 * @tcp_seq834 * @tcp_ack835 * @tcp_win836 * return: NETDEV_TX_BUSY/NET_TX_OK.837 */838static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,839 struct sge_eth_txq *q, u64 tcp_seq,840 u64 tcp_ack, u64 tcp_win, bool offset)841{842 bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));843 struct ch_ktls_port_stats_debug *port_stats;844 u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0;845 struct fw_ulptx_wr *wr;846 int credits;847 void *pos;848 849 wr_len = sizeof(*wr);850 /* there can be max 4 cpls, check if we have enough credits */851 len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);852 ndesc = DIV_ROUND_UP(len, 64);853 854 credits = chcr_txq_avail(&q->q) - ndesc;855 if (unlikely(credits < 0)) {856 chcr_eth_txq_stop(q);857 return NETDEV_TX_BUSY;858 }859 860 if (unlikely(credits < ETHTXQ_STOP_THRES)) {861 chcr_eth_txq_stop(q);862 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;863 }864 865 pos = &q->q.desc[q->q.pidx];866 /* make space for WR, we'll fill it later when we know all the cpls867 * being sent out and have complete length.868 */869 wr = pos;870 pos += wr_len;871 /* update tx_max if its a re-transmit or the first wr */872 if (first_wr || tcp_seq != tx_info->prev_seq) {873 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,874 TCB_TX_MAX_W,875 TCB_TX_MAX_V(TCB_TX_MAX_M),876 TCB_TX_MAX_V(tcp_seq), 0);877 cpl++;878 }879 /* reset snd una if it's a re-transmit pkt */880 if (tcp_seq != tx_info->prev_seq || offset) {881 /* reset snd_una */882 port_stats =883 &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];884 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,885 TCB_SND_UNA_RAW_W,886 TCB_SND_UNA_RAW_V887 (TCB_SND_UNA_RAW_M),888 TCB_SND_UNA_RAW_V(0), 0);889 if (tcp_seq != tx_info->prev_seq)890 atomic64_inc(&port_stats->ktls_tx_ooo);891 cpl++;892 }893 /* update ack */894 if (first_wr || tx_info->prev_ack != tcp_ack) {895 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,896 TCB_RCV_NXT_W,897 TCB_RCV_NXT_V(TCB_RCV_NXT_M),898 TCB_RCV_NXT_V(tcp_ack), 0);899 tx_info->prev_ack = tcp_ack;900 cpl++;901 }902 /* update receive window */903 if (first_wr || tx_info->prev_win != tcp_win) {904 chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,905 TCB_RCV_WND_W,906 TCB_RCV_WND_V(TCB_RCV_WND_M),907 TCB_RCV_WND_V(tcp_win), 0);908 tx_info->prev_win = tcp_win;909 cpl++;910 }911 912 if (cpl) {913 /* get the actual length */914 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);915 /* ULPTX wr */916 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));917 wr->cookie = 0;918 /* fill len in wr field */919 wr->flowid_len16 = htonl(wr_mid |920 FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));921 922 ndesc = DIV_ROUND_UP(len, 64);923 chcr_txq_advance(&q->q, ndesc);924 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);925 }926 return 0;927}928 929/*930 * chcr_ktls_get_tx_flits931 * returns number of flits to be sent out, it includes key context length, WR932 * size and skb fragments.933 */934static unsigned int935chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len)936{937 return chcr_sgl_len(nr_frags) +938 DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);939}940 941/*942 * chcr_ktls_check_tcp_options: To check if there is any TCP option available943 * other than timestamp.944 * @skb - skb contains partial record..945 * return: 1 / 0946 */947static int948chcr_ktls_check_tcp_options(struct tcphdr *tcp)949{950 int cnt, opt, optlen;951 u_char *cp;952 953 cp = (u_char *)(tcp + 1);954 cnt = (tcp->doff << 2) - sizeof(struct tcphdr);955 for (; cnt > 0; cnt -= optlen, cp += optlen) {956 opt = cp[0];957 if (opt == TCPOPT_EOL)958 break;959 if (opt == TCPOPT_NOP) {960 optlen = 1;961 } else {962 if (cnt < 2)963 break;964 optlen = cp[1];965 if (optlen < 2 || optlen > cnt)966 break;967 }968 switch (opt) {969 case TCPOPT_NOP:970 break;971 default:972 return 1;973 }974 }975 return 0;976}977 978/*979 * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to980 * send out separately.981 * @tx_info - driver specific tls info.982 * @skb - skb contains partial record..983 * @q - TX queue.984 * @tx_chan - channel number.985 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.986 */987static int988chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,989 struct sge_eth_txq *q, uint32_t tx_chan)990{991 struct fw_eth_tx_pkt_wr *wr;992 struct cpl_tx_pkt_core *cpl;993 u32 ctrl, iplen, maclen;994 struct ipv6hdr *ip6;995 unsigned int ndesc;996 struct tcphdr *tcp;997 int len16, pktlen;998 struct iphdr *ip;999 u32 wr_mid = 0;1000 int credits;1001 u8 buf[150];1002 u64 cntrl1;1003 void *pos;1004 1005 iplen = skb_network_header_len(skb);1006 maclen = skb_mac_header_len(skb);1007 1008 /* packet length = eth hdr len + ip hdr len + tcp hdr len1009 * (including options).1010 */1011 pktlen = skb_tcp_all_headers(skb);1012 1013 ctrl = sizeof(*cpl) + pktlen;1014 len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);1015 /* check how many descriptors needed */1016 ndesc = DIV_ROUND_UP(len16, 4);1017 1018 credits = chcr_txq_avail(&q->q) - ndesc;1019 if (unlikely(credits < 0)) {1020 chcr_eth_txq_stop(q);1021 return NETDEV_TX_BUSY;1022 }1023 1024 if (unlikely(credits < ETHTXQ_STOP_THRES)) {1025 chcr_eth_txq_stop(q);1026 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;1027 }1028 1029 pos = &q->q.desc[q->q.pidx];1030 wr = pos;1031 1032 /* Firmware work request header */1033 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |1034 FW_WR_IMMDLEN_V(ctrl));1035 1036 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));1037 wr->r3 = 0;1038 1039 cpl = (void *)(wr + 1);1040 1041 /* CPL header */1042 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |1043 TXPKT_PF_V(tx_info->adap->pf));1044 cpl->pack = 0;1045 cpl->len = htons(pktlen);1046 1047 memcpy(buf, skb->data, pktlen);1048 if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) {1049 /* we need to correct ip header len */1050 ip = (struct iphdr *)(buf + maclen);1051 ip->tot_len = htons(pktlen - maclen);1052 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP);1053 } else {1054 ip6 = (struct ipv6hdr *)(buf + maclen);1055 ip6->payload_len = htons(pktlen - maclen - iplen);1056 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6);1057 }1058 1059 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |1060 TXPKT_IPHDR_LEN_V(iplen);1061 /* checksum offload */1062 cpl->ctrl1 = cpu_to_be64(cntrl1);1063 1064 pos = cpl + 1;1065 1066 /* now take care of the tcp header, if fin is not set then clear push1067 * bit as well, and if fin is set, it will be sent at the last so we1068 * need to update the tcp sequence number as per the last packet.1069 */1070 tcp = (struct tcphdr *)(buf + maclen + iplen);1071 1072 if (!tcp->fin)1073 tcp->psh = 0;1074 else1075 tcp->seq = htonl(tx_info->prev_seq);1076 1077 chcr_copy_to_txd(buf, &q->q, pos, pktlen);1078 1079 chcr_txq_advance(&q->q, ndesc);1080 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);1081 return 0;1082}1083 1084/*1085 * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb1086 * received has partial end part of the record, send out the complete record, so1087 * that crypto block will be able to generate TAG/HASH.1088 * @skb - segment which has complete or partial end part.1089 * @tx_info - driver specific tls info.1090 * @q - TX queue.1091 * @tcp_seq1092 * @tcp_push - tcp push bit.1093 * @mss - segment size.1094 * return: NETDEV_TX_BUSY/NET_TX_OK.1095 */1096static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,1097 struct chcr_ktls_info *tx_info,1098 struct sge_eth_txq *q, u32 tcp_seq,1099 bool is_last_wr, u32 data_len,1100 u32 skb_offset, u32 nfrags,1101 bool tcp_push, u32 mss)1102{1103 u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;1104 struct adapter *adap = tx_info->adap;1105 int credits, left, last_desc;1106 struct tx_sw_desc *sgl_sdesc;1107 struct cpl_tx_data *tx_data;1108 struct cpl_tx_sec_pdu *cpl;1109 struct ulptx_idata *idata;1110 struct ulp_txpkt *ulptx;1111 struct fw_ulptx_wr *wr;1112 void *pos;1113 u64 *end;1114 1115 /* get the number of flits required */1116 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len);1117 /* number of descriptors */1118 ndesc = chcr_flits_to_desc(flits);1119 /* check if enough credits available */1120 credits = chcr_txq_avail(&q->q) - ndesc;1121 if (unlikely(credits < 0)) {1122 chcr_eth_txq_stop(q);1123 return NETDEV_TX_BUSY;1124 }1125 1126 if (unlikely(credits < ETHTXQ_STOP_THRES)) {1127 /* Credits are below the threshold values, stop the queue after1128 * injecting the Work Request for this packet.1129 */1130 chcr_eth_txq_stop(q);1131 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;1132 }1133 1134 last_desc = q->q.pidx + ndesc - 1;1135 if (last_desc >= q->q.size)1136 last_desc -= q->q.size;1137 sgl_sdesc = &q->q.sdesc[last_desc];1138 1139 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {1140 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));1141 q->mapping_err++;1142 return NETDEV_TX_BUSY;1143 }1144 1145 if (!is_last_wr)1146 skb_get(skb);1147 1148 pos = &q->q.desc[q->q.pidx];1149 end = (u64 *)pos + flits;1150 /* FW_ULPTX_WR */1151 wr = pos;1152 /* WR will need len16 */1153 len16 = DIV_ROUND_UP(flits, 2);1154 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));1155 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));1156 wr->cookie = 0;1157 pos += sizeof(*wr);1158 /* ULP_TXPKT */1159 ulptx = pos;1160 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |1161 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |1162 ULP_TXPKT_FID_V(q->q.cntxt_id) |1163 ULP_TXPKT_RO_F);1164 ulptx->len = htonl(len16 - 1);1165 /* ULPTX_IDATA sub-command */1166 idata = (struct ulptx_idata *)(ulptx + 1);1167 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);1168 /* idata length will include cpl_tx_sec_pdu + key context size +1169 * cpl_tx_data header.1170 */1171 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +1172 sizeof(*tx_data));1173 /* SEC CPL */1174 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);1175 cpl->op_ivinsrtofst =1176 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |1177 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |1178 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |1179 CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));1180 cpl->pldlen = htonl(data_len);1181 1182 /* encryption should start after tls header size + iv size */1183 cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;1184 1185 cpl->aadstart_cipherstop_hi =1186 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |1187 CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |1188 CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));1189 1190 /* authentication will also start after tls header + iv size */1191 cpl->cipherstop_lo_authinsert =1192 htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |1193 CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |1194 CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));1195 1196 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */1197 cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);1198 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);1199 cpl->scmd1 = cpu_to_be64(tx_info->record_no);1200 1201 pos = cpl + 1;1202 /* check if space left to fill the keys */1203 left = (void *)q->q.stat - pos;1204 if (!left) {1205 left = (void *)end - (void *)q->q.stat;1206 pos = q->q.desc;1207 end = pos + left;1208 }1209 1210 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,1211 tx_info->key_ctx_len);1212 left = (void *)q->q.stat - pos;1213 1214 if (!left) {1215 left = (void *)end - (void *)q->q.stat;1216 pos = q->q.desc;1217 end = pos + left;1218 }1219 /* CPL_TX_DATA */1220 tx_data = (void *)pos;1221 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));1222 tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len));1223 1224 tx_data->rsvd = htonl(tcp_seq);1225 1226 tx_data->flags = htonl(TX_BYPASS_F);1227 if (tcp_push)1228 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);1229 1230 /* check left again, it might go beyond queue limit */1231 pos = tx_data + 1;1232 left = (void *)q->q.stat - pos;1233 1234 /* check the position again */1235 if (!left) {1236 left = (void *)end - (void *)q->q.stat;1237 pos = q->q.desc;1238 end = pos + left;1239 }1240 1241 /* send the complete packet except the header */1242 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,1243 skb_offset, data_len);1244 sgl_sdesc->skb = skb;1245 1246 chcr_txq_advance(&q->q, ndesc);1247 cxgb4_ring_tx_db(adap, &q->q, ndesc);1248 atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records);1249 1250 return 0;1251}1252 1253/*1254 * chcr_ktls_xmit_wr_short: This is to send out partial records. If its1255 * a middle part of a record, fetch the prior data to make it 16 byte aligned1256 * and then only send it out.1257 *1258 * @skb - skb contains partial record..1259 * @tx_info - driver specific tls info.1260 * @q - TX queue.1261 * @tcp_seq1262 * @tcp_push - tcp push bit.1263 * @mss - segment size.1264 * @tls_rec_offset - offset from start of the tls record.1265 * @perior_data - data before the current segment, required to make this record1266 * 16 byte aligned.1267 * @prior_data_len - prior_data length (less than 16)1268 * return: NETDEV_TX_BUSY/NET_TX_OK.1269 */1270static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,1271 struct chcr_ktls_info *tx_info,1272 struct sge_eth_txq *q,1273 u32 tcp_seq, bool tcp_push, u32 mss,1274 u32 tls_rec_offset, u8 *prior_data,1275 u32 prior_data_len, u32 data_len,1276 u32 skb_offset)1277{1278 u32 len16, wr_mid = 0, cipher_start, nfrags;1279 struct adapter *adap = tx_info->adap;1280 unsigned int flits = 0, ndesc;1281 int credits, left, last_desc;1282 struct tx_sw_desc *sgl_sdesc;1283 struct cpl_tx_data *tx_data;1284 struct cpl_tx_sec_pdu *cpl;1285 struct ulptx_idata *idata;1286 struct ulp_txpkt *ulptx;1287 struct fw_ulptx_wr *wr;1288 __be64 iv_record;1289 void *pos;1290 u64 *end;1291 1292 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);1293 /* get the number of flits required, it's a partial record so 2 flits1294 * (AES_BLOCK_SIZE) will be added.1295 */1296 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2;1297 /* get the correct 8 byte IV of this record */1298 iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);1299 /* If it's a middle record and not 16 byte aligned to run AES CTR, need1300 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate1301 * data will be added.1302 */1303 if (prior_data_len)1304 flits += 2;1305 /* number of descriptors */1306 ndesc = chcr_flits_to_desc(flits);1307 /* check if enough credits available */1308 credits = chcr_txq_avail(&q->q) - ndesc;1309 if (unlikely(credits < 0)) {1310 chcr_eth_txq_stop(q);1311 return NETDEV_TX_BUSY;1312 }1313 1314 if (unlikely(credits < ETHTXQ_STOP_THRES)) {1315 chcr_eth_txq_stop(q);1316 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;1317 }1318 1319 last_desc = q->q.pidx + ndesc - 1;1320 if (last_desc >= q->q.size)1321 last_desc -= q->q.size;1322 sgl_sdesc = &q->q.sdesc[last_desc];1323 1324 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {1325 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));1326 q->mapping_err++;1327 return NETDEV_TX_BUSY;1328 }1329 1330 pos = &q->q.desc[q->q.pidx];1331 end = (u64 *)pos + flits;1332 /* FW_ULPTX_WR */1333 wr = pos;1334 /* WR will need len16 */1335 len16 = DIV_ROUND_UP(flits, 2);1336 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));1337 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));1338 wr->cookie = 0;1339 pos += sizeof(*wr);1340 /* ULP_TXPKT */1341 ulptx = pos;1342 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |1343 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |1344 ULP_TXPKT_FID_V(q->q.cntxt_id) |1345 ULP_TXPKT_RO_F);1346 ulptx->len = htonl(len16 - 1);1347 /* ULPTX_IDATA sub-command */1348 idata = (struct ulptx_idata *)(ulptx + 1);1349 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);1350 /* idata length will include cpl_tx_sec_pdu + key context size +1351 * cpl_tx_data header.1352 */1353 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +1354 sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);1355 /* SEC CPL */1356 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);1357 /* cipher start will have tls header + iv size extra if its a header1358 * part of tls record. else only 16 byte IV will be added.1359 */1360 cipher_start =1361 AES_BLOCK_LEN + 1 +1362 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);1363 1364 cpl->op_ivinsrtofst =1365 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |1366 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |1367 CPL_TX_SEC_PDU_IVINSRTOFST_V(1));1368 cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len);1369 cpl->aadstart_cipherstop_hi =1370 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));1371 cpl->cipherstop_lo_authinsert = 0;1372 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */1373 cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);1374 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);1375 cpl->scmd1 = 0;1376 1377 pos = cpl + 1;1378 /* check if space left to fill the keys */1379 left = (void *)q->q.stat - pos;1380 if (!left) {1381 left = (void *)end - (void *)q->q.stat;1382 pos = q->q.desc;1383 end = pos + left;1384 }1385 1386 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,1387 tx_info->key_ctx_len);1388 left = (void *)q->q.stat - pos;1389 1390 if (!left) {1391 left = (void *)end - (void *)q->q.stat;1392 pos = q->q.desc;1393 end = pos + left;1394 }1395 /* CPL_TX_DATA */1396 tx_data = (void *)pos;1397 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));1398 tx_data->len = htonl(TX_DATA_MSS_V(mss) |1399 TX_LENGTH_V(data_len + prior_data_len));1400 tx_data->rsvd = htonl(tcp_seq);1401 tx_data->flags = htonl(TX_BYPASS_F);1402 if (tcp_push)1403 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);1404 1405 /* check left again, it might go beyond queue limit */1406 pos = tx_data + 1;1407 left = (void *)q->q.stat - pos;1408 1409 /* check the position again */1410 if (!left) {1411 left = (void *)end - (void *)q->q.stat;1412 pos = q->q.desc;1413 end = pos + left;1414 }1415 /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 81416 * bytes of actual IV and 4 bytes of 16 byte-sequence.1417 */1418 memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);1419 memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);1420 *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =1421 htonl(2 + (tls_rec_offset ? ((tls_rec_offset -1422 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));1423 1424 pos += 16;1425 /* Prior_data_len will always be less than 16 bytes, fill the1426 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length1427 * to 0.1428 */1429 if (prior_data_len)1430 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);1431 /* send the complete packet except the header */1432 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,1433 skb_offset, data_len);1434 sgl_sdesc->skb = skb;1435 1436 chcr_txq_advance(&q->q, ndesc);1437 cxgb4_ring_tx_db(adap, &q->q, ndesc);1438 1439 return 0;1440}1441 1442/*1443 * chcr_ktls_tx_plaintxt: This handler will take care of the records which has1444 * only plain text (only tls header and iv)1445 * @tx_info - driver specific tls info.1446 * @skb - skb contains partial record..1447 * @tcp_seq1448 * @mss - segment size.1449 * @tcp_push - tcp push bit.1450 * @q - TX queue.1451 * @port_id : port number1452 * @perior_data - data before the current segment, required to make this record1453 * 16 byte aligned.1454 * @prior_data_len - prior_data length (less than 16)1455 * return: NETDEV_TX_BUSY/NET_TX_OK.1456 */1457static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,1458 struct sk_buff *skb, u32 tcp_seq, u32 mss,1459 bool tcp_push, struct sge_eth_txq *q,1460 u32 port_id, u8 *prior_data,1461 u32 data_len, u32 skb_offset,1462 u32 prior_data_len)1463{1464 int credits, left, len16, last_desc;1465 unsigned int flits = 0, ndesc;1466 struct tx_sw_desc *sgl_sdesc;1467 struct cpl_tx_data *tx_data;1468 struct ulptx_idata *idata;1469 struct ulp_txpkt *ulptx;1470 struct fw_ulptx_wr *wr;1471 u32 wr_mid = 0, nfrags;1472 void *pos;1473 u64 *end;1474 1475 flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);1476 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);1477 flits += chcr_sgl_len(nfrags);1478 if (prior_data_len)1479 flits += 2;1480 1481 /* WR will need len16 */1482 len16 = DIV_ROUND_UP(flits, 2);1483 /* check how many descriptors needed */1484 ndesc = DIV_ROUND_UP(flits, 8);1485 1486 credits = chcr_txq_avail(&q->q) - ndesc;1487 if (unlikely(credits < 0)) {1488 chcr_eth_txq_stop(q);1489 return NETDEV_TX_BUSY;1490 }1491 1492 if (unlikely(credits < ETHTXQ_STOP_THRES)) {1493 chcr_eth_txq_stop(q);1494 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;1495 }1496 1497 last_desc = q->q.pidx + ndesc - 1;1498 if (last_desc >= q->q.size)1499 last_desc -= q->q.size;1500 sgl_sdesc = &q->q.sdesc[last_desc];1501 1502 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,1503 sgl_sdesc->addr) < 0)) {1504 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));1505 q->mapping_err++;1506 return NETDEV_TX_BUSY;1507 }1508 1509 pos = &q->q.desc[q->q.pidx];1510 end = (u64 *)pos + flits;1511 /* FW_ULPTX_WR */1512 wr = pos;1513 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));1514 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));1515 wr->cookie = 0;1516 /* ULP_TXPKT */1517 ulptx = (struct ulp_txpkt *)(wr + 1);1518 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |1519 ULP_TXPKT_DATAMODIFY_V(0) |1520 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |1521 ULP_TXPKT_DEST_V(0) |1522 ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));1523 ulptx->len = htonl(len16 - 1);1524 /* ULPTX_IDATA sub-command */1525 idata = (struct ulptx_idata *)(ulptx + 1);1526 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);1527 idata->len = htonl(sizeof(*tx_data) + prior_data_len);1528 /* CPL_TX_DATA */1529 tx_data = (struct cpl_tx_data *)(idata + 1);1530 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));1531 tx_data->len = htonl(TX_DATA_MSS_V(mss) |1532 TX_LENGTH_V(data_len + prior_data_len));1533 /* set tcp seq number */1534 tx_data->rsvd = htonl(tcp_seq);1535 tx_data->flags = htonl(TX_BYPASS_F);1536 if (tcp_push)1537 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);1538 1539 pos = tx_data + 1;1540 /* apart from prior_data_len, we should set remaining part of 16 bytes1541 * to be zero.1542 */1543 if (prior_data_len)1544 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);1545 1546 /* check left again, it might go beyond queue limit */1547 left = (void *)q->q.stat - pos;1548 1549 /* check the position again */1550 if (!left) {1551 left = (void *)end - (void *)q->q.stat;1552 pos = q->q.desc;1553 end = pos + left;1554 }1555 /* send the complete packet including the header */1556 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,1557 skb_offset, data_len);1558 sgl_sdesc->skb = skb;1559 1560 chcr_txq_advance(&q->q, ndesc);1561 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);1562 return 0;1563}1564 1565static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info,1566 struct sk_buff *skb,1567 struct sge_eth_txq *q)1568{1569 u32 ctrl, iplen, maclen, wr_mid = 0, len16;1570 struct tx_sw_desc *sgl_sdesc;1571 struct fw_eth_tx_pkt_wr *wr;1572 struct cpl_tx_pkt_core *cpl;1573 unsigned int flits, ndesc;1574 int credits, last_desc;1575 u64 cntrl1, *end;1576 void *pos;1577 1578 ctrl = sizeof(*cpl);1579 flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8);1580 1581 flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1);1582 len16 = DIV_ROUND_UP(flits, 2);1583 /* check how many descriptors needed */1584 ndesc = DIV_ROUND_UP(flits, 8);1585 1586 credits = chcr_txq_avail(&q->q) - ndesc;1587 if (unlikely(credits < 0)) {1588 chcr_eth_txq_stop(q);1589 return -ENOMEM;1590 }1591 1592 if (unlikely(credits < ETHTXQ_STOP_THRES)) {1593 chcr_eth_txq_stop(q);1594 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;1595 }1596 1597 last_desc = q->q.pidx + ndesc - 1;1598 if (last_desc >= q->q.size)1599 last_desc -= q->q.size;1600 sgl_sdesc = &q->q.sdesc[last_desc];1601 1602 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,1603 sgl_sdesc->addr) < 0)) {1604 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));1605 q->mapping_err++;1606 return -ENOMEM;1607 }1608 1609 iplen = skb_network_header_len(skb);1610 maclen = skb_mac_header_len(skb);1611 1612 pos = &q->q.desc[q->q.pidx];1613 end = (u64 *)pos + flits;1614 wr = pos;1615 1616 /* Firmware work request header */1617 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |1618 FW_WR_IMMDLEN_V(ctrl));1619 1620 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));1621 wr->r3 = 0;1622 1623 cpl = (void *)(wr + 1);1624 1625 /* CPL header */1626 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) |1627 TXPKT_INTF_V(tx_info->tx_chan) |1628 TXPKT_PF_V(tx_info->adap->pf));1629 cpl->pack = 0;1630 cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ?1631 TX_CSUM_TCPIP : TX_CSUM_TCPIP6);1632 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |1633 TXPKT_IPHDR_LEN_V(iplen);1634 /* checksum offload */1635 cpl->ctrl1 = cpu_to_be64(cntrl1);1636 cpl->len = htons(skb->len);1637 1638 pos = cpl + 1;1639 1640 cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr);1641 sgl_sdesc->skb = skb;1642 chcr_txq_advance(&q->q, ndesc);1643 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);1644 return 0;1645}1646 1647/*1648 * chcr_ktls_copy_record_in_skb1649 * @nskb - new skb where the frags to be added.1650 * @skb - old skb, to copy socket and destructor details.1651 * @record - specific record which has complete 16k record in frags.1652 */1653static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,1654 struct sk_buff *skb,1655 struct tls_record_info *record)1656{1657 int i = 0;1658 1659 for (i = 0; i < record->num_frags; i++) {1660 skb_shinfo(nskb)->frags[i] = record->frags[i];1661 /* increase the frag ref count */1662 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);1663 }1664 1665 skb_shinfo(nskb)->nr_frags = record->num_frags;1666 nskb->data_len = record->len;1667 nskb->len += record->len;1668 nskb->truesize += record->len;1669 nskb->sk = skb->sk;1670 nskb->destructor = skb->destructor;1671 refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc);1672}1673 1674/*1675 * chcr_end_part_handler: This handler will handle the record which1676 * is complete or if record's end part is received. T6 adapter has a issue that1677 * it can't send out TAG with partial record so if its an end part then we have1678 * to send TAG as well and for which we need to fetch the complete record and1679 * send it to crypto module.1680 * @tx_info - driver specific tls info.1681 * @skb - skb contains partial record.1682 * @record - complete record of 16K size.1683 * @tcp_seq1684 * @mss - segment size in which TP needs to chop a packet.1685 * @tcp_push_no_fin - tcp push if fin is not set.1686 * @q - TX queue.1687 * @tls_end_offset - offset from end of the record.1688 * @last wr : check if this is the last part of the skb going out.1689 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.1690 */1691static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,1692 struct sk_buff *skb,1693 struct tls_record_info *record,1694 u32 tcp_seq, int mss, bool tcp_push_no_fin,1695 struct sge_eth_txq *q, u32 skb_offset,1696 u32 tls_end_offset, bool last_wr)1697{1698 bool free_skb_if_tx_fails = false;1699 struct sk_buff *nskb = NULL;1700 1701 /* check if it is a complete record */1702 if (tls_end_offset == record->len) {1703 nskb = skb;1704 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts);1705 } else {1706 nskb = alloc_skb(0, GFP_ATOMIC);1707 if (!nskb) {1708 dev_kfree_skb_any(skb);1709 return NETDEV_TX_BUSY;1710 }1711 1712 /* copy complete record in skb */1713 chcr_ktls_copy_record_in_skb(nskb, skb, record);1714 /* packet is being sent from the beginning, update the tcp_seq1715 * accordingly.1716 */1717 tcp_seq = tls_record_start_seq(record);1718 /* reset skb offset */1719 skb_offset = 0;1720 1721 if (last_wr)1722 dev_kfree_skb_any(skb);1723 else1724 free_skb_if_tx_fails = true;1725 1726 last_wr = true;1727 1728 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts);1729 }1730 1731 if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,1732 last_wr, record->len, skb_offset,1733 record->num_frags,1734 (last_wr && tcp_push_no_fin),1735 mss)) {1736 if (free_skb_if_tx_fails)1737 dev_kfree_skb_any(skb);1738 goto out;1739 }1740 tx_info->prev_seq = record->end_seq;1741 return 0;1742out:1743 dev_kfree_skb_any(nskb);1744 return NETDEV_TX_BUSY;1745}1746 1747/*1748 * chcr_short_record_handler: This handler will take care of the records which1749 * doesn't have end part (1st part or the middle part(/s) of a record). In such1750 * cases, AES CTR will be used in place of AES GCM to send out partial packet.1751 * This partial record might be the first part of the record, or the middle1752 * part. In case of middle record we should fetch the prior data to make it 161753 * byte aligned. If it has a partial tls header or iv then get to the start of1754 * tls header. And if it has partial TAG, then remove the complete TAG and send1755 * only the payload.1756 * There is one more possibility that it gets a partial header, send that1757 * portion as a plaintext.1758 * @tx_info - driver specific tls info.1759 * @skb - skb contains partial record..1760 * @record - complete record of 16K size.1761 * @tcp_seq1762 * @mss - segment size in which TP needs to chop a packet.1763 * @tcp_push_no_fin - tcp push if fin is not set.1764 * @q - TX queue.1765 * @tls_end_offset - offset from end of the record.1766 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.1767 */1768static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,1769 struct sk_buff *skb,1770 struct tls_record_info *record,1771 u32 tcp_seq, int mss, bool tcp_push_no_fin,1772 u32 data_len, u32 skb_offset,1773 struct sge_eth_txq *q, u32 tls_end_offset)1774{1775 u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);1776 u8 prior_data[16] = {0};1777 u32 prior_data_len = 0;1778 1779 /* check if the skb is ending in middle of tag/HASH, its a big1780 * trouble, send the packet before the HASH.1781 */1782 int remaining_record = tls_end_offset - data_len;1783 1784 if (remaining_record > 0 &&1785 remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {1786 int trimmed_len = 0;1787 1788 if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE)1789 trimmed_len = data_len -1790 (TLS_CIPHER_AES_GCM_128_TAG_SIZE -1791 remaining_record);1792 if (!trimmed_len)1793 return FALLBACK;1794 1795 WARN_ON(trimmed_len > data_len);1796 1797 data_len = trimmed_len;1798 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts);1799 }1800 1801 /* check if it is only the header part. */1802 if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) {1803 if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,1804 tcp_push_no_fin, q,1805 tx_info->port_id, prior_data,1806 data_len, skb_offset, prior_data_len))1807 goto out;1808 1809 tx_info->prev_seq = tcp_seq + data_len;1810 return 0;1811 }1812 1813 /* check if the middle record's start point is 16 byte aligned. CTR1814 * needs 16 byte aligned start point to start encryption.1815 */1816 if (tls_rec_offset) {1817 /* there is an offset from start, means its a middle record */1818 int remaining = 0;1819 1820 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {1821 prior_data_len = tls_rec_offset;1822 tls_rec_offset = 0;1823 remaining = 0;1824 } else {1825 prior_data_len =1826 (tls_rec_offset -1827 (TLS_HEADER_SIZE + tx_info->iv_size))1828 % AES_BLOCK_LEN;1829 remaining = tls_rec_offset - prior_data_len;1830 }1831 1832 /* if prior_data_len is not zero, means we need to fetch prior1833 * data to make this record 16 byte aligned, or we need to reach1834 * to start offset.1835 */1836 if (prior_data_len) {1837 int i = 0;1838 skb_frag_t *f;1839 int frag_size = 0, frag_delta = 0;1840 1841 while (remaining > 0) {1842 frag_size = skb_frag_size(&record->frags[i]);1843 if (remaining < frag_size)1844 break;1845 1846 remaining -= frag_size;1847 i++;1848 }1849 f = &record->frags[i];1850 frag_delta = skb_frag_size(f) - remaining;1851 1852 if (frag_delta >= prior_data_len) {1853 memcpy_from_page(prior_data, skb_frag_page(f),1854 skb_frag_off(f) + remaining,1855 prior_data_len);1856 } else {1857 memcpy_from_page(prior_data, skb_frag_page(f),1858 skb_frag_off(f) + remaining,1859 frag_delta);1860 1861 /* get the next page */1862 f = &record->frags[i + 1];1863 1864 memcpy_from_page(prior_data + frag_delta,1865 skb_frag_page(f),1866 skb_frag_off(f),1867 prior_data_len - frag_delta);1868 }1869 /* reset tcp_seq as per the prior_data_required len */1870 tcp_seq -= prior_data_len;1871 }1872 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts);1873 } else {1874 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts);1875 }1876 1877 if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,1878 mss, tls_rec_offset, prior_data,1879 prior_data_len, data_len, skb_offset)) {1880 goto out;1881 }1882 1883 tx_info->prev_seq = tcp_seq + data_len + prior_data_len;1884 return 0;1885out:1886 dev_kfree_skb_any(skb);1887 return NETDEV_TX_BUSY;1888}1889 1890static int chcr_ktls_sw_fallback(struct sk_buff *skb,1891 struct chcr_ktls_info *tx_info,1892 struct sge_eth_txq *q)1893{1894 u32 data_len, skb_offset;1895 struct sk_buff *nskb;1896 struct tcphdr *th;1897 1898 nskb = tls_encrypt_skb(skb);1899 1900 if (!nskb)1901 return 0;1902 1903 th = tcp_hdr(nskb);1904 skb_offset = skb_tcp_all_headers(nskb);1905 data_len = nskb->len - skb_offset;1906 skb_tx_timestamp(nskb);1907 1908 if (chcr_ktls_tunnel_pkt(tx_info, nskb, q))1909 goto out;1910 1911 tx_info->prev_seq = ntohl(th->seq) + data_len;1912 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback);1913 return 0;1914out:1915 dev_kfree_skb_any(nskb);1916 return 0;1917}1918/* nic tls TX handler */1919static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)1920{1921 u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset;1922 struct ch_ktls_port_stats_debug *port_stats;1923 struct tls_offload_context_tx *tx_ctx;1924 struct ch_ktls_stats_debug *stats;1925 struct tcphdr *th = tcp_hdr(skb);1926 int data_len, qidx, ret = 0, mss;1927 struct tls_record_info *record;1928 struct chcr_ktls_info *tx_info;1929 struct net_device *tls_netdev;1930 struct tls_context *tls_ctx;1931 struct sge_eth_txq *q;1932 struct adapter *adap;1933 unsigned long flags;1934 1935 tcp_seq = ntohl(th->seq);1936 skb_offset = skb_tcp_all_headers(skb);1937 skb_data_len = skb->len - skb_offset;1938 data_len = skb_data_len;1939 1940 mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len;1941 1942 tls_ctx = tls_get_ctx(skb->sk);1943 tx_ctx = tls_offload_ctx_tx(tls_ctx);1944 tls_netdev = rcu_dereference_bh(tls_ctx->netdev);1945 /* Don't quit on NULL: if tls_device_down is running in parallel,1946 * netdev might become NULL, even if tls_is_skb_tx_device_offloaded was1947 * true. Rather continue processing this packet.1948 */1949 if (unlikely(tls_netdev && tls_netdev != dev))1950 goto out;1951 1952 tx_info = chcr_get_ktls_tx_info(tls_ctx);1953 1954 if (unlikely(!tx_info))1955 goto out;1956 1957 adap = tx_info->adap;1958 stats = &adap->ch_ktls_stats;1959 port_stats = &stats->ktls_port[tx_info->port_id];1960 1961 qidx = skb->queue_mapping;1962 q = &adap->sge.ethtxq[qidx + tx_info->first_qset];1963 cxgb4_reclaim_completed_tx(adap, &q->q, true);1964 /* if tcp options are set but finish is not send the options first */1965 if (!th->fin && chcr_ktls_check_tcp_options(th)) {1966 ret = chcr_ktls_write_tcp_options(tx_info, skb, q,1967 tx_info->tx_chan);1968 if (ret)1969 return NETDEV_TX_BUSY;1970 }1971 1972 /* TCP segments can be in received either complete or partial.1973 * chcr_end_part_handler will handle cases if complete record or end1974 * part of the record is received. In case of partial end part of record,1975 * we will send the complete record again.1976 */1977 1978 spin_lock_irqsave(&tx_ctx->lock, flags);1979 1980 do {1981 1982 cxgb4_reclaim_completed_tx(adap, &q->q, true);1983 /* fetch the tls record */1984 record = tls_get_record(tx_ctx, tcp_seq,1985 &tx_info->record_no);1986 /* By the time packet reached to us, ACK is received, and record1987 * won't be found in that case, handle it gracefully.1988 */1989 if (unlikely(!record)) {1990 spin_unlock_irqrestore(&tx_ctx->lock, flags);1991 atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data);1992 goto out;1993 }1994 1995 tls_end_offset = record->end_seq - tcp_seq;1996 1997 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",1998 tcp_seq, record->end_seq, tx_info->prev_seq, data_len);1999 /* update tcb for the skb */2000 if (skb_data_len == data_len) {2001 u32 tx_max = tcp_seq;2002 2003 if (!tls_record_is_start_marker(record) &&2004 tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE)2005 tx_max = record->end_seq -2006 TLS_CIPHER_AES_GCM_128_TAG_SIZE;2007 2008 ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max,2009 ntohl(th->ack_seq),2010 ntohs(th->window),2011 tls_end_offset !=2012 record->len);2013 if (ret) {2014 spin_unlock_irqrestore(&tx_ctx->lock,2015 flags);2016 goto out;2017 }2018 2019 if (th->fin)2020 skb_get(skb);2021 }2022 2023 if (unlikely(tls_record_is_start_marker(record))) {2024 atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data);2025 /* If tls_end_offset < data_len, means there is some2026 * data after start marker, which needs encryption, send2027 * plaintext first and take skb refcount. else send out2028 * complete pkt as plaintext.2029 */2030 if (tls_end_offset < data_len)2031 skb_get(skb);2032 else2033 tls_end_offset = data_len;2034 2035 ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,2036 (!th->fin && th->psh), q,2037 tx_info->port_id, NULL,2038 tls_end_offset, skb_offset,2039 0);2040 2041 if (ret) {2042 /* free the refcount taken earlier */2043 if (tls_end_offset < data_len)2044 dev_kfree_skb_any(skb);2045 spin_unlock_irqrestore(&tx_ctx->lock, flags);2046 goto out;2047 }2048 2049 data_len -= tls_end_offset;2050 tcp_seq = record->end_seq;2051 skb_offset += tls_end_offset;2052 continue;2053 }2054 2055 /* if a tls record is finishing in this SKB */2056 if (tls_end_offset <= data_len) {2057 ret = chcr_end_part_handler(tx_info, skb, record,2058 tcp_seq, mss,2059 (!th->fin && th->psh), q,2060 skb_offset,2061 tls_end_offset,2062 skb_offset +2063 tls_end_offset == skb->len);2064 2065 data_len -= tls_end_offset;2066 /* tcp_seq increment is required to handle next record.2067 */2068 tcp_seq += tls_end_offset;2069 skb_offset += tls_end_offset;2070 } else {2071 ret = chcr_short_record_handler(tx_info, skb,2072 record, tcp_seq, mss,2073 (!th->fin && th->psh),2074 data_len, skb_offset,2075 q, tls_end_offset);2076 data_len = 0;2077 }2078 2079 /* if any failure, come out from the loop. */2080 if (ret) {2081 spin_unlock_irqrestore(&tx_ctx->lock, flags);2082 if (th->fin)2083 dev_kfree_skb_any(skb);2084 2085 if (ret == FALLBACK)2086 return chcr_ktls_sw_fallback(skb, tx_info, q);2087 2088 return NETDEV_TX_OK;2089 }2090 2091 /* length should never be less than 0 */2092 WARN_ON(data_len < 0);2093 2094 } while (data_len > 0);2095 2096 spin_unlock_irqrestore(&tx_ctx->lock, flags);2097 atomic64_inc(&port_stats->ktls_tx_encrypted_packets);2098 atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes);2099 2100 /* tcp finish is set, send a separate tcp msg including all the options2101 * as well.2102 */2103 if (th->fin) {2104 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);2105 dev_kfree_skb_any(skb);2106 }2107 2108 return NETDEV_TX_OK;2109out:2110 dev_kfree_skb_any(skb);2111 return NETDEV_TX_OK;2112}2113 2114static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi)2115{2116 struct chcr_ktls_uld_ctx *u_ctx;2117 2118 pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC,2119 CHCR_KTLS_DRV_VERSION);2120 u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);2121 if (!u_ctx) {2122 u_ctx = ERR_PTR(-ENOMEM);2123 goto out;2124 }2125 u_ctx->lldi = *lldi;2126 u_ctx->detach = false;2127 xa_init_flags(&u_ctx->tid_list, XA_FLAGS_LOCK_BH);2128out:2129 return u_ctx;2130}2131 2132static const struct tlsdev_ops chcr_ktls_ops = {2133 .tls_dev_add = chcr_ktls_dev_add,2134 .tls_dev_del = chcr_ktls_dev_del,2135};2136 2137static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {2138 [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,2139 [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,2140};2141 2142static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp,2143 const struct pkt_gl *pgl)2144{2145 const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp;2146 struct chcr_ktls_uld_ctx *u_ctx = handle;2147 u8 opcode = rpl->ot.opcode;2148 struct adapter *adap;2149 2150 adap = pci_get_drvdata(u_ctx->lldi.pdev);2151 2152 if (!work_handlers[opcode]) {2153 pr_err("Unsupported opcode %d received\n", opcode);2154 return 0;2155 }2156 2157 work_handlers[opcode](adap, (unsigned char *)&rsp[1]);2158 return 0;2159}2160 2161static void clear_conn_resources(struct chcr_ktls_info *tx_info)2162{2163 /* clear l2t entry */2164 if (tx_info->l2te)2165 cxgb4_l2t_release(tx_info->l2te);2166 2167#if IS_ENABLED(CONFIG_IPV6)2168 /* clear clip entry */2169 if (tx_info->ip_family == AF_INET6)2170 cxgb4_clip_release(tx_info->netdev, (const u32 *)2171 &tx_info->sk->sk_v6_rcv_saddr,2172 1);2173#endif2174 2175 /* clear tid */2176 if (tx_info->tid != -1)2177 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,2178 tx_info->tid, tx_info->ip_family);2179}2180 2181static void ch_ktls_reset_all_conn(struct chcr_ktls_uld_ctx *u_ctx)2182{2183 struct ch_ktls_port_stats_debug *port_stats;2184 struct tls_offload_context_tx *tx_ctx;2185 struct chcr_ktls_info *tx_info;2186 unsigned long index;2187 2188 xa_for_each(&u_ctx->tid_list, index, tx_ctx) {2189 tx_info = __chcr_get_ktls_tx_info(tx_ctx);2190 clear_conn_resources(tx_info);2191 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];2192 atomic64_inc(&port_stats->ktls_tx_connection_close);2193 kvfree(tx_info);2194 memset(tx_ctx->driver_state, 0, TLS_DRIVER_STATE_SIZE_TX);2195 /* release module refcount */2196 module_put(THIS_MODULE);2197 }2198}2199 2200static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state)2201{2202 struct chcr_ktls_uld_ctx *u_ctx = handle;2203 2204 switch (new_state) {2205 case CXGB4_STATE_UP:2206 pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));2207 mutex_lock(&dev_mutex);2208 list_add_tail(&u_ctx->entry, &uld_ctx_list);2209 mutex_unlock(&dev_mutex);2210 break;2211 case CXGB4_STATE_START_RECOVERY:2212 case CXGB4_STATE_DOWN:2213 case CXGB4_STATE_DETACH:2214 pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));2215 mutex_lock(&dev_mutex);2216 u_ctx->detach = true;2217 list_del(&u_ctx->entry);2218 ch_ktls_reset_all_conn(u_ctx);2219 xa_destroy(&u_ctx->tid_list);2220 mutex_unlock(&dev_mutex);2221 break;2222 default:2223 break;2224 }2225 2226 return 0;2227}2228 2229static struct cxgb4_uld_info chcr_ktls_uld_info = {2230 .name = CHCR_KTLS_DRV_MODULE_NAME,2231 .nrxq = 1,2232 .rxq_size = 1024,2233 .add = chcr_ktls_uld_add,2234 .tx_handler = chcr_ktls_xmit,2235 .rx_handler = chcr_ktls_uld_rx_handler,2236 .state_change = chcr_ktls_uld_state_change,2237 .tlsdev_ops = &chcr_ktls_ops,2238};2239 2240static int __init chcr_ktls_init(void)2241{2242 cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info);2243 return 0;2244}2245 2246static void __exit chcr_ktls_exit(void)2247{2248 struct chcr_ktls_uld_ctx *u_ctx, *tmp;2249 struct adapter *adap;2250 2251 pr_info("driver unloaded\n");2252 2253 mutex_lock(&dev_mutex);2254 list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {2255 adap = pci_get_drvdata(u_ctx->lldi.pdev);2256 memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats));2257 list_del(&u_ctx->entry);2258 xa_destroy(&u_ctx->tid_list);2259 kfree(u_ctx);2260 }2261 mutex_unlock(&dev_mutex);2262 cxgb4_unregister_uld(CXGB4_ULD_KTLS);2263}2264 2265module_init(chcr_ktls_init);2266module_exit(chcr_ktls_exit);2267 2268MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver");2269MODULE_LICENSE("GPL");2270MODULE_AUTHOR("Chelsio Communications");2271MODULE_VERSION(CHCR_KTLS_DRV_VERSION);2272