2335 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Copyright (c) 2018 Chelsio Communications, Inc.4 *5 * Written by: Atul Gupta (atul.gupta@chelsio.com)6 */7 8#include <linux/module.h>9#include <linux/list.h>10#include <linux/workqueue.h>11#include <linux/skbuff.h>12#include <linux/timer.h>13#include <linux/notifier.h>14#include <linux/inetdevice.h>15#include <linux/ip.h>16#include <linux/tcp.h>17#include <linux/sched/signal.h>18#include <linux/kallsyms.h>19#include <linux/kprobes.h>20#include <linux/if_vlan.h>21#include <linux/ipv6.h>22#include <net/ipv6.h>23#include <net/transp_v6.h>24#include <net/ip6_route.h>25#include <net/inet_common.h>26#include <net/tcp.h>27#include <net/dst.h>28#include <net/tls.h>29#include <net/addrconf.h>30#include <net/secure_seq.h>31 32#include "chtls.h"33#include "chtls_cm.h"34#include "clip_tbl.h"35#include "t4_tcb.h"36 37/*38 * State transitions and actions for close. Note that if we are in SYN_SENT39 * we remain in that state as we cannot control a connection while it's in40 * SYN_SENT; such connections are allowed to establish and are then aborted.41 */42static unsigned char new_state[16] = {43 /* current state: new state: action: */44 /* (Invalid) */ TCP_CLOSE,45 /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,46 /* TCP_SYN_SENT */ TCP_SYN_SENT,47 /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,48 /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,49 /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,50 /* TCP_TIME_WAIT */ TCP_CLOSE,51 /* TCP_CLOSE */ TCP_CLOSE,52 /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,53 /* TCP_LAST_ACK */ TCP_LAST_ACK,54 /* TCP_LISTEN */ TCP_CLOSE,55 /* TCP_CLOSING */ TCP_CLOSING,56};57 58static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)59{60 struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);61 62 if (!csk)63 return NULL;64 65 csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);66 if (!csk->txdata_skb_cache) {67 kfree(csk);68 return NULL;69 }70 71 kref_init(&csk->kref);72 csk->cdev = cdev;73 skb_queue_head_init(&csk->txq);74 csk->wr_skb_head = NULL;75 csk->wr_skb_tail = NULL;76 csk->mss = MAX_MSS;77 csk->tlshws.ofld = 1;78 csk->tlshws.txkey = -1;79 csk->tlshws.rxkey = -1;80 csk->tlshws.mfs = TLS_MFS;81 skb_queue_head_init(&csk->tlshws.sk_recv_queue);82 return csk;83}84 85static void chtls_sock_release(struct kref *ref)86{87 struct chtls_sock *csk =88 container_of(ref, struct chtls_sock, kref);89 90 kfree(csk);91}92 93static struct net_device *chtls_find_netdev(struct chtls_dev *cdev,94 struct sock *sk)95{96 struct adapter *adap = pci_get_drvdata(cdev->pdev);97 struct net_device *ndev = cdev->ports[0];98#if IS_ENABLED(CONFIG_IPV6)99 struct net_device *temp;100 int addr_type;101#endif102 int i;103 104 switch (sk->sk_family) {105 case PF_INET:106 if (likely(!inet_sk(sk)->inet_rcv_saddr))107 return ndev;108 ndev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false);109 break;110#if IS_ENABLED(CONFIG_IPV6)111 case PF_INET6:112 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);113 if (likely(addr_type == IPV6_ADDR_ANY))114 return ndev;115 116 for_each_netdev_rcu(&init_net, temp) {117 if (ipv6_chk_addr(&init_net, (struct in6_addr *)118 &sk->sk_v6_rcv_saddr, temp, 1)) {119 ndev = temp;120 break;121 }122 }123 break;124#endif125 default:126 return NULL;127 }128 129 if (!ndev)130 return NULL;131 132 if (is_vlan_dev(ndev))133 ndev = vlan_dev_real_dev(ndev);134 135 for_each_port(adap, i)136 if (cdev->ports[i] == ndev)137 return ndev;138 return NULL;139}140 141static void assign_rxopt(struct sock *sk, unsigned int opt)142{143 const struct chtls_dev *cdev;144 struct chtls_sock *csk;145 struct tcp_sock *tp;146 147 csk = rcu_dereference_sk_user_data(sk);148 tp = tcp_sk(sk);149 150 cdev = csk->cdev;151 tp->tcp_header_len = sizeof(struct tcphdr);152 tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;153 tp->mss_cache = tp->rx_opt.mss_clamp;154 tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);155 tp->rx_opt.snd_wscale = TCPOPT_SACK_G(opt);156 tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);157 SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);158 if (!tp->rx_opt.wscale_ok)159 tp->rx_opt.rcv_wscale = 0;160 if (tp->rx_opt.tstamp_ok) {161 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;162 tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;163 } else if (csk->opt2 & TSTAMPS_EN_F) {164 csk->opt2 &= ~TSTAMPS_EN_F;165 csk->mtu_idx = TCPOPT_MSS_G(opt);166 }167}168 169static void chtls_purge_receive_queue(struct sock *sk)170{171 struct sk_buff *skb;172 173 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {174 skb_dst_set(skb, (void *)NULL);175 kfree_skb(skb);176 }177}178 179static void chtls_purge_write_queue(struct sock *sk)180{181 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);182 struct sk_buff *skb;183 184 while ((skb = __skb_dequeue(&csk->txq))) {185 sk->sk_wmem_queued -= skb->truesize;186 __kfree_skb(skb);187 }188}189 190static void chtls_purge_recv_queue(struct sock *sk)191{192 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);193 struct chtls_hws *tlsk = &csk->tlshws;194 struct sk_buff *skb;195 196 while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {197 skb_dst_set(skb, NULL);198 kfree_skb(skb);199 }200}201 202static void abort_arp_failure(void *handle, struct sk_buff *skb)203{204 struct cpl_abort_req *req = cplhdr(skb);205 struct chtls_dev *cdev;206 207 cdev = (struct chtls_dev *)handle;208 req->cmd = CPL_ABORT_NO_RST;209 cxgb4_ofld_send(cdev->lldi->ports[0], skb);210}211 212static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)213{214 if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {215 __skb_trim(skb, 0);216 refcount_inc(&skb->users);217 } else {218 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);219 }220 return skb;221}222 223static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)224{225 struct cpl_abort_req *req;226 struct chtls_sock *csk;227 struct tcp_sock *tp;228 229 csk = rcu_dereference_sk_user_data(sk);230 tp = tcp_sk(sk);231 232 if (!skb)233 skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));234 235 req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));236 INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);237 skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);238 req->rsvd0 = htonl(tp->snd_nxt);239 req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);240 req->cmd = mode;241 t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);242 send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);243}244 245static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)246{247 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);248 249 if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||250 !csk->cdev)) {251 if (sk->sk_state == TCP_SYN_RECV)252 csk_set_flag(csk, CSK_RST_ABORTED);253 goto out;254 }255 256 if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {257 struct tcp_sock *tp = tcp_sk(sk);258 259 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)260 WARN_ONCE(1, "send tx flowc error");261 csk_set_flag(csk, CSK_TX_DATA_SENT);262 }263 264 csk_set_flag(csk, CSK_ABORT_RPL_PENDING);265 chtls_purge_write_queue(sk);266 267 csk_set_flag(csk, CSK_ABORT_SHUTDOWN);268 if (sk->sk_state != TCP_SYN_RECV)269 chtls_send_abort(sk, mode, skb);270 else271 chtls_set_tcb_field_rpl_skb(sk, TCB_T_FLAGS_W,272 TCB_T_FLAGS_V(TCB_T_FLAGS_M), 0,273 TCB_FIELD_COOKIE_TFLAG, 1);274 275 return;276out:277 kfree_skb(skb);278}279 280static void release_tcp_port(struct sock *sk)281{282 if (inet_csk(sk)->icsk_bind_hash)283 inet_put_port(sk);284}285 286static void tcp_uncork(struct sock *sk)287{288 struct tcp_sock *tp = tcp_sk(sk);289 290 if (tp->nonagle & TCP_NAGLE_CORK) {291 tp->nonagle &= ~TCP_NAGLE_CORK;292 chtls_tcp_push(sk, 0);293 }294}295 296static void chtls_close_conn(struct sock *sk)297{298 struct cpl_close_con_req *req;299 struct chtls_sock *csk;300 struct sk_buff *skb;301 unsigned int tid;302 unsigned int len;303 304 len = roundup(sizeof(struct cpl_close_con_req), 16);305 csk = rcu_dereference_sk_user_data(sk);306 tid = csk->tid;307 308 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);309 req = (struct cpl_close_con_req *)__skb_put(skb, len);310 memset(req, 0, len);311 req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |312 FW_WR_IMMDLEN_V(sizeof(*req) -313 sizeof(req->wr)));314 req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |315 FW_WR_FLOWID_V(tid));316 317 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));318 319 tcp_uncork(sk);320 skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);321 if (sk->sk_state != TCP_SYN_SENT)322 chtls_push_frames(csk, 1);323}324 325/*326 * Perform a state transition during close and return the actions indicated327 * for the transition. Do not make this function inline, the main reason328 * it exists at all is to avoid multiple inlining of tcp_set_state.329 */330static int make_close_transition(struct sock *sk)331{332 int next = (int)new_state[sk->sk_state];333 334 tcp_set_state(sk, next & TCP_STATE_MASK);335 return next & TCP_ACTION_FIN;336}337 338void chtls_close(struct sock *sk, long timeout)339{340 int data_lost, prev_state;341 struct chtls_sock *csk;342 343 csk = rcu_dereference_sk_user_data(sk);344 345 lock_sock(sk);346 sk->sk_shutdown |= SHUTDOWN_MASK;347 348 data_lost = skb_queue_len(&sk->sk_receive_queue);349 data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);350 chtls_purge_recv_queue(sk);351 chtls_purge_receive_queue(sk);352 353 if (sk->sk_state == TCP_CLOSE) {354 goto wait;355 } else if (data_lost || sk->sk_state == TCP_SYN_SENT) {356 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);357 release_tcp_port(sk);358 goto unlock;359 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {360 sk->sk_prot->disconnect(sk, 0);361 } else if (make_close_transition(sk)) {362 chtls_close_conn(sk);363 }364wait:365 if (timeout)366 sk_stream_wait_close(sk, timeout);367 368unlock:369 prev_state = sk->sk_state;370 sock_hold(sk);371 sock_orphan(sk);372 373 release_sock(sk);374 375 local_bh_disable();376 bh_lock_sock(sk);377 378 if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)379 goto out;380 381 if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&382 !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {383 struct sk_buff *skb;384 385 skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);386 if (skb)387 chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);388 }389 390 if (sk->sk_state == TCP_CLOSE)391 inet_csk_destroy_sock(sk);392 393out:394 bh_unlock_sock(sk);395 local_bh_enable();396 sock_put(sk);397}398 399/*400 * Wait until a socket enters on of the given states.401 */402static int wait_for_states(struct sock *sk, unsigned int states)403{404 DECLARE_WAITQUEUE(wait, current);405 struct socket_wq _sk_wq;406 long current_timeo;407 int err = 0;408 409 current_timeo = 200;410 411 /*412 * We want this to work even when there's no associated struct socket.413 * In that case we provide a temporary wait_queue_head_t.414 */415 if (!sk->sk_wq) {416 init_waitqueue_head(&_sk_wq.wait);417 _sk_wq.fasync_list = NULL;418 init_rcu_head_on_stack(&_sk_wq.rcu);419 RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);420 }421 422 add_wait_queue(sk_sleep(sk), &wait);423 while (!sk_in_state(sk, states)) {424 if (!current_timeo) {425 err = -EBUSY;426 break;427 }428 if (signal_pending(current)) {429 err = sock_intr_errno(current_timeo);430 break;431 }432 set_current_state(TASK_UNINTERRUPTIBLE);433 release_sock(sk);434 if (!sk_in_state(sk, states))435 current_timeo = schedule_timeout(current_timeo);436 __set_current_state(TASK_RUNNING);437 lock_sock(sk);438 }439 remove_wait_queue(sk_sleep(sk), &wait);440 441 if (rcu_dereference(sk->sk_wq) == &_sk_wq)442 sk->sk_wq = NULL;443 return err;444}445 446int chtls_disconnect(struct sock *sk, int flags)447{448 struct tcp_sock *tp;449 int err;450 451 tp = tcp_sk(sk);452 chtls_purge_recv_queue(sk);453 chtls_purge_receive_queue(sk);454 chtls_purge_write_queue(sk);455 456 if (sk->sk_state != TCP_CLOSE) {457 sk->sk_err = ECONNRESET;458 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);459 err = wait_for_states(sk, TCPF_CLOSE);460 if (err)461 return err;462 }463 chtls_purge_recv_queue(sk);464 chtls_purge_receive_queue(sk);465 tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);466 return tcp_disconnect(sk, flags);467}468 469#define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \470 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)471void chtls_shutdown(struct sock *sk, int how)472{473 if ((how & SEND_SHUTDOWN) &&474 sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&475 make_close_transition(sk))476 chtls_close_conn(sk);477}478 479void chtls_destroy_sock(struct sock *sk)480{481 struct chtls_sock *csk;482 483 csk = rcu_dereference_sk_user_data(sk);484 chtls_purge_recv_queue(sk);485 csk->ulp_mode = ULP_MODE_NONE;486 chtls_purge_write_queue(sk);487 free_tls_keyid(sk);488 kref_put(&csk->kref, chtls_sock_release);489 if (sk->sk_family == AF_INET)490 sk->sk_prot = &tcp_prot;491#if IS_ENABLED(CONFIG_IPV6)492 else493 sk->sk_prot = &tcpv6_prot;494#endif495 sk->sk_prot->destroy(sk);496}497 498static void reset_listen_child(struct sock *child)499{500 struct chtls_sock *csk = rcu_dereference_sk_user_data(child);501 struct sk_buff *skb;502 503 skb = alloc_ctrl_skb(csk->txdata_skb_cache,504 sizeof(struct cpl_abort_req));505 506 chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);507 sock_orphan(child);508 INC_ORPHAN_COUNT(child);509 if (child->sk_state == TCP_CLOSE)510 inet_csk_destroy_sock(child);511}512 513static void chtls_disconnect_acceptq(struct sock *listen_sk)514{515 struct request_sock **pprev;516 517 pprev = ACCEPT_QUEUE(listen_sk);518 while (*pprev) {519 struct request_sock *req = *pprev;520 521 if (req->rsk_ops == &chtls_rsk_ops ||522 req->rsk_ops == &chtls_rsk_opsv6) {523 struct sock *child = req->sk;524 525 *pprev = req->dl_next;526 sk_acceptq_removed(listen_sk);527 reqsk_put(req);528 sock_hold(child);529 local_bh_disable();530 bh_lock_sock(child);531 release_tcp_port(child);532 reset_listen_child(child);533 bh_unlock_sock(child);534 local_bh_enable();535 sock_put(child);536 } else {537 pprev = &req->dl_next;538 }539 }540}541 542static int listen_hashfn(const struct sock *sk)543{544 return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);545}546 547static struct listen_info *listen_hash_add(struct chtls_dev *cdev,548 struct sock *sk,549 unsigned int stid)550{551 struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL);552 553 if (p) {554 int key = listen_hashfn(sk);555 556 p->sk = sk;557 p->stid = stid;558 spin_lock(&cdev->listen_lock);559 p->next = cdev->listen_hash_tab[key];560 cdev->listen_hash_tab[key] = p;561 spin_unlock(&cdev->listen_lock);562 }563 return p;564}565 566static int listen_hash_find(struct chtls_dev *cdev,567 struct sock *sk)568{569 struct listen_info *p;570 int stid = -1;571 int key;572 573 key = listen_hashfn(sk);574 575 spin_lock(&cdev->listen_lock);576 for (p = cdev->listen_hash_tab[key]; p; p = p->next)577 if (p->sk == sk) {578 stid = p->stid;579 break;580 }581 spin_unlock(&cdev->listen_lock);582 return stid;583}584 585static int listen_hash_del(struct chtls_dev *cdev,586 struct sock *sk)587{588 struct listen_info *p, **prev;589 int stid = -1;590 int key;591 592 key = listen_hashfn(sk);593 prev = &cdev->listen_hash_tab[key];594 595 spin_lock(&cdev->listen_lock);596 for (p = *prev; p; prev = &p->next, p = p->next)597 if (p->sk == sk) {598 stid = p->stid;599 *prev = p->next;600 kfree(p);601 break;602 }603 spin_unlock(&cdev->listen_lock);604 return stid;605}606 607static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)608{609 struct request_sock *req;610 struct chtls_sock *csk;611 612 csk = rcu_dereference_sk_user_data(child);613 req = csk->passive_reap_next;614 615 reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);616 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);617 chtls_reqsk_free(req);618 csk->passive_reap_next = NULL;619}620 621static void chtls_reset_synq(struct listen_ctx *listen_ctx)622{623 struct sock *listen_sk = listen_ctx->lsk;624 625 while (!skb_queue_empty(&listen_ctx->synq)) {626 struct chtls_sock *csk =627 container_of((struct synq *)skb_peek628 (&listen_ctx->synq), struct chtls_sock, synq);629 struct sock *child = csk->sk;630 631 cleanup_syn_rcv_conn(child, listen_sk);632 sock_hold(child);633 local_bh_disable();634 bh_lock_sock(child);635 release_tcp_port(child);636 reset_listen_child(child);637 bh_unlock_sock(child);638 local_bh_enable();639 sock_put(child);640 }641}642 643int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)644{645 struct net_device *ndev;646#if IS_ENABLED(CONFIG_IPV6)647 bool clip_valid = false;648#endif649 struct listen_ctx *ctx;650 struct adapter *adap;651 struct port_info *pi;652 int ret = 0;653 int stid;654 655 rcu_read_lock();656 ndev = chtls_find_netdev(cdev, sk);657 rcu_read_unlock();658 if (!ndev)659 return -EBADF;660 661 pi = netdev_priv(ndev);662 adap = pi->adapter;663 if (!(adap->flags & CXGB4_FULL_INIT_DONE))664 return -EBADF;665 666 if (listen_hash_find(cdev, sk) >= 0) /* already have it */667 return -EADDRINUSE;668 669 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);670 if (!ctx)671 return -ENOMEM;672 673 __module_get(THIS_MODULE);674 ctx->lsk = sk;675 ctx->cdev = cdev;676 ctx->state = T4_LISTEN_START_PENDING;677 skb_queue_head_init(&ctx->synq);678 679 stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);680 if (stid < 0)681 goto free_ctx;682 683 sock_hold(sk);684 if (!listen_hash_add(cdev, sk, stid))685 goto free_stid;686 687 if (sk->sk_family == PF_INET) {688 ret = cxgb4_create_server(ndev, stid,689 inet_sk(sk)->inet_rcv_saddr,690 inet_sk(sk)->inet_sport, 0,691 cdev->lldi->rxq_ids[0]);692#if IS_ENABLED(CONFIG_IPV6)693 } else {694 int addr_type;695 696 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);697 if (addr_type != IPV6_ADDR_ANY) {698 ret = cxgb4_clip_get(ndev, (const u32 *)699 &sk->sk_v6_rcv_saddr, 1);700 if (ret)701 goto del_hash;702 clip_valid = true;703 }704 ret = cxgb4_create_server6(ndev, stid,705 &sk->sk_v6_rcv_saddr,706 inet_sk(sk)->inet_sport,707 cdev->lldi->rxq_ids[0]);708#endif709 }710 if (ret > 0)711 ret = net_xmit_errno(ret);712 if (ret)713 goto del_hash;714 return 0;715del_hash:716#if IS_ENABLED(CONFIG_IPV6)717 if (clip_valid)718 cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1);719#endif720 listen_hash_del(cdev, sk);721free_stid:722 cxgb4_free_stid(cdev->tids, stid, sk->sk_family);723 sock_put(sk);724free_ctx:725 kfree(ctx);726 module_put(THIS_MODULE);727 return -EBADF;728}729 730void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)731{732 struct listen_ctx *listen_ctx;733 int stid;734 735 stid = listen_hash_del(cdev, sk);736 if (stid < 0)737 return;738 739 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);740 chtls_reset_synq(listen_ctx);741 742 cxgb4_remove_server(cdev->lldi->ports[0], stid,743 cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6);744 745#if IS_ENABLED(CONFIG_IPV6)746 if (sk->sk_family == PF_INET6) {747 struct net_device *ndev = chtls_find_netdev(cdev, sk);748 int addr_type = 0;749 750 addr_type = ipv6_addr_type((const struct in6_addr *)751 &sk->sk_v6_rcv_saddr);752 if (addr_type != IPV6_ADDR_ANY)753 cxgb4_clip_release(ndev, (const u32 *)754 &sk->sk_v6_rcv_saddr, 1);755 }756#endif757 chtls_disconnect_acceptq(sk);758}759 760static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)761{762 struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;763 unsigned int stid = GET_TID(rpl);764 struct listen_ctx *listen_ctx;765 766 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);767 if (!listen_ctx)768 return CPL_RET_BUF_DONE;769 770 if (listen_ctx->state == T4_LISTEN_START_PENDING) {771 listen_ctx->state = T4_LISTEN_STARTED;772 return CPL_RET_BUF_DONE;773 }774 775 if (rpl->status != CPL_ERR_NONE) {776 pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",777 rpl->status, stid);778 } else {779 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);780 sock_put(listen_ctx->lsk);781 kfree(listen_ctx);782 module_put(THIS_MODULE);783 }784 return CPL_RET_BUF_DONE;785}786 787static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)788{789 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;790 struct listen_ctx *listen_ctx;791 unsigned int stid;792 void *data;793 794 stid = GET_TID(rpl);795 data = lookup_stid(cdev->tids, stid);796 listen_ctx = (struct listen_ctx *)data;797 798 if (rpl->status != CPL_ERR_NONE) {799 pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",800 rpl->status, stid);801 } else {802 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);803 sock_put(listen_ctx->lsk);804 kfree(listen_ctx);805 module_put(THIS_MODULE);806 }807 return CPL_RET_BUF_DONE;808}809 810static void chtls_purge_wr_queue(struct sock *sk)811{812 struct sk_buff *skb;813 814 while ((skb = dequeue_wr(sk)) != NULL)815 kfree_skb(skb);816}817 818static void chtls_release_resources(struct sock *sk)819{820 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);821 struct chtls_dev *cdev = csk->cdev;822 unsigned int tid = csk->tid;823 struct tid_info *tids;824 825 if (!cdev)826 return;827 828 tids = cdev->tids;829 kfree_skb(csk->txdata_skb_cache);830 csk->txdata_skb_cache = NULL;831 832 if (csk->wr_credits != csk->wr_max_credits) {833 chtls_purge_wr_queue(sk);834 chtls_reset_wr_list(csk);835 }836 837 if (csk->l2t_entry) {838 cxgb4_l2t_release(csk->l2t_entry);839 csk->l2t_entry = NULL;840 }841 842 if (sk->sk_state != TCP_SYN_SENT) {843 cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);844 sock_put(sk);845 }846}847 848static void chtls_conn_done(struct sock *sk)849{850 if (sock_flag(sk, SOCK_DEAD))851 chtls_purge_receive_queue(sk);852 sk_wakeup_sleepers(sk, 0);853 tcp_done(sk);854}855 856static void do_abort_syn_rcv(struct sock *child, struct sock *parent)857{858 /*859 * If the server is still open we clean up the child connection,860 * otherwise the server already did the clean up as it was purging861 * its SYN queue and the skb was just sitting in its backlog.862 */863 if (likely(parent->sk_state == TCP_LISTEN)) {864 cleanup_syn_rcv_conn(child, parent);865 /* Without the below call to sock_orphan,866 * we leak the socket resource with syn_flood test867 * as inet_csk_destroy_sock will not be called868 * in tcp_done since SOCK_DEAD flag is not set.869 * Kernel handles this differently where new socket is870 * created only after 3 way handshake is done.871 */872 sock_orphan(child);873 INC_ORPHAN_COUNT(child);874 chtls_release_resources(child);875 chtls_conn_done(child);876 } else {877 if (csk_flag(child, CSK_RST_ABORTED)) {878 chtls_release_resources(child);879 chtls_conn_done(child);880 }881 }882}883 884static void pass_open_abort(struct sock *child, struct sock *parent,885 struct sk_buff *skb)886{887 do_abort_syn_rcv(child, parent);888 kfree_skb(skb);889}890 891static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)892{893 pass_open_abort(skb->sk, lsk, skb);894}895 896static void chtls_pass_open_arp_failure(struct sock *sk,897 struct sk_buff *skb)898{899 const struct request_sock *oreq;900 struct chtls_sock *csk;901 struct chtls_dev *cdev;902 struct sock *parent;903 void *data;904 905 csk = rcu_dereference_sk_user_data(sk);906 cdev = csk->cdev;907 908 /*909 * If the connection is being aborted due to the parent listening910 * socket going away there's nothing to do, the ABORT_REQ will close911 * the connection.912 */913 if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {914 kfree_skb(skb);915 return;916 }917 918 oreq = csk->passive_reap_next;919 data = lookup_stid(cdev->tids, oreq->ts_recent);920 parent = ((struct listen_ctx *)data)->lsk;921 922 bh_lock_sock(parent);923 if (!sock_owned_by_user(parent)) {924 pass_open_abort(sk, parent, skb);925 } else {926 BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;927 __sk_add_backlog(parent, skb);928 }929 bh_unlock_sock(parent);930}931 932static void chtls_accept_rpl_arp_failure(void *handle,933 struct sk_buff *skb)934{935 struct sock *sk = (struct sock *)handle;936 937 sock_hold(sk);938 process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);939 sock_put(sk);940}941 942static unsigned int chtls_select_mss(const struct chtls_sock *csk,943 unsigned int pmtu,944 struct cpl_pass_accept_req *req)945{946 struct chtls_dev *cdev;947 struct dst_entry *dst;948 unsigned int tcpoptsz;949 unsigned int iphdrsz;950 unsigned int mtu_idx;951 struct tcp_sock *tp;952 unsigned int mss;953 struct sock *sk;954 955 mss = ntohs(req->tcpopt.mss);956 sk = csk->sk;957 dst = __sk_dst_get(sk);958 cdev = csk->cdev;959 tp = tcp_sk(sk);960 tcpoptsz = 0;961 962#if IS_ENABLED(CONFIG_IPV6)963 if (sk->sk_family == AF_INET6)964 iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr);965 else966#endif967 iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);968 if (req->tcpopt.tstamp)969 tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);970 971 tp->advmss = dst_metric_advmss(dst);972 if (USER_MSS(tp) && tp->advmss > USER_MSS(tp))973 tp->advmss = USER_MSS(tp);974 if (tp->advmss > pmtu - iphdrsz)975 tp->advmss = pmtu - iphdrsz;976 if (mss && tp->advmss > mss)977 tp->advmss = mss;978 979 tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,980 iphdrsz + tcpoptsz,981 tp->advmss - tcpoptsz,982 8, &mtu_idx);983 tp->advmss -= iphdrsz;984 985 inet_csk(sk)->icsk_pmtu_cookie = pmtu;986 return mtu_idx;987}988 989static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)990{991 int wscale = 0;992 993 if (space > MAX_RCV_WND)994 space = MAX_RCV_WND;995 if (win_clamp && win_clamp < space)996 space = win_clamp;997 998 if (wscale_ok) {999 while (wscale < 14 && (65535 << wscale) < space)1000 wscale++;1001 }1002 return wscale;1003}1004 1005static void chtls_pass_accept_rpl(struct sk_buff *skb,1006 struct cpl_pass_accept_req *req,1007 unsigned int tid)1008 1009{1010 struct cpl_t5_pass_accept_rpl *rpl5;1011 struct cxgb4_lld_info *lldi;1012 const struct tcphdr *tcph;1013 const struct tcp_sock *tp;1014 struct chtls_sock *csk;1015 unsigned int len;1016 struct sock *sk;1017 u32 opt2, hlen;1018 u64 opt0;1019 1020 sk = skb->sk;1021 tp = tcp_sk(sk);1022 csk = sk->sk_user_data;1023 csk->tid = tid;1024 lldi = csk->cdev->lldi;1025 len = roundup(sizeof(*rpl5), 16);1026 1027 rpl5 = __skb_put_zero(skb, len);1028 INIT_TP_WR(rpl5, tid);1029 1030 OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,1031 csk->tid));1032 csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),1033 req);1034 opt0 = TCAM_BYPASS_F |1035 WND_SCALE_V(RCV_WSCALE(tp)) |1036 MSS_IDX_V(csk->mtu_idx) |1037 L2T_IDX_V(csk->l2t_entry->idx) |1038 NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |1039 TX_CHAN_V(csk->tx_chan) |1040 SMAC_SEL_V(csk->smac_idx) |1041 DSCP_V(csk->tos >> 2) |1042 ULP_MODE_V(ULP_MODE_TLS) |1043 RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));1044 1045 opt2 = RX_CHANNEL_V(0) |1046 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);1047 1048 if (!is_t5(lldi->adapter_type))1049 opt2 |= RX_FC_DISABLE_F;1050 if (req->tcpopt.tstamp)1051 opt2 |= TSTAMPS_EN_F;1052 if (req->tcpopt.sack)1053 opt2 |= SACK_EN_F;1054 hlen = ntohl(req->hdr_len);1055 1056 tcph = (struct tcphdr *)((u8 *)(req + 1) +1057 T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));1058 if (tcph->ece && tcph->cwr)1059 opt2 |= CCTRL_ECN_V(1);1060 opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);1061 opt2 |= T5_ISS_F;1062 opt2 |= T5_OPT_2_VALID_F;1063 opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp));1064 rpl5->opt0 = cpu_to_be64(opt0);1065 rpl5->opt2 = cpu_to_be32(opt2);1066 rpl5->iss = cpu_to_be32((get_random_u32() & ~7UL) - 1);1067 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);1068 t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);1069 cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);1070}1071 1072static void inet_inherit_port(struct sock *lsk, struct sock *newsk)1073{1074 local_bh_disable();1075 __inet_inherit_port(lsk, newsk);1076 local_bh_enable();1077}1078 1079static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)1080{1081 if (skb->protocol) {1082 kfree_skb(skb);1083 return 0;1084 }1085 BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);1086 return 0;1087}1088 1089static void chtls_set_tcp_window(struct chtls_sock *csk)1090{1091 struct net_device *ndev = csk->egress_dev;1092 struct port_info *pi = netdev_priv(ndev);1093 unsigned int linkspeed;1094 u8 scale;1095 1096 linkspeed = pi->link_cfg.speed;1097 scale = linkspeed / SPEED_10000;1098#define CHTLS_10G_RCVWIN (256 * 1024)1099 csk->rcv_win = CHTLS_10G_RCVWIN;1100 if (scale)1101 csk->rcv_win *= scale;1102#define CHTLS_10G_SNDWIN (256 * 1024)1103 csk->snd_win = CHTLS_10G_SNDWIN;1104 if (scale)1105 csk->snd_win *= scale;1106}1107 1108static struct sock *chtls_recv_sock(struct sock *lsk,1109 struct request_sock *oreq,1110 void *network_hdr,1111 const struct cpl_pass_accept_req *req,1112 struct chtls_dev *cdev)1113{1114 struct adapter *adap = pci_get_drvdata(cdev->pdev);1115 struct neighbour *n = NULL;1116 struct inet_sock *newinet;1117 const struct iphdr *iph;1118 struct tls_context *ctx;1119 struct net_device *ndev;1120 struct chtls_sock *csk;1121 struct dst_entry *dst;1122 struct tcp_sock *tp;1123 struct sock *newsk;1124 bool found = false;1125 u16 port_id;1126 int rxq_idx;1127 int step, i;1128 1129 iph = (const struct iphdr *)network_hdr;1130 newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);1131 if (!newsk)1132 goto free_oreq;1133 1134 if (lsk->sk_family == AF_INET) {1135 dst = inet_csk_route_child_sock(lsk, newsk, oreq);1136 if (!dst)1137 goto free_sk;1138 1139 n = dst_neigh_lookup(dst, &iph->saddr);1140#if IS_ENABLED(CONFIG_IPV6)1141 } else {1142 const struct ipv6hdr *ip6h;1143 struct flowi6 fl6;1144 1145 ip6h = (const struct ipv6hdr *)network_hdr;1146 memset(&fl6, 0, sizeof(fl6));1147 fl6.flowi6_proto = IPPROTO_TCP;1148 fl6.saddr = ip6h->daddr;1149 fl6.daddr = ip6h->saddr;1150 fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port;1151 fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num);1152 security_req_classify_flow(oreq, flowi6_to_flowi_common(&fl6));1153 dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL);1154 if (IS_ERR(dst))1155 goto free_sk;1156 n = dst_neigh_lookup(dst, &ip6h->saddr);1157#endif1158 }1159 if (!n || !n->dev)1160 goto free_dst;1161 1162 ndev = n->dev;1163 if (is_vlan_dev(ndev))1164 ndev = vlan_dev_real_dev(ndev);1165 1166 for_each_port(adap, i)1167 if (cdev->ports[i] == ndev)1168 found = true;1169 1170 if (!found)1171 goto free_dst;1172 1173 port_id = cxgb4_port_idx(ndev);1174 1175 csk = chtls_sock_create(cdev);1176 if (!csk)1177 goto free_dst;1178 1179 csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);1180 if (!csk->l2t_entry)1181 goto free_csk;1182 1183 newsk->sk_user_data = csk;1184 newsk->sk_backlog_rcv = chtls_backlog_rcv;1185 1186 tp = tcp_sk(newsk);1187 newinet = inet_sk(newsk);1188 1189 if (iph->version == 0x4) {1190 newinet->inet_daddr = iph->saddr;1191 newinet->inet_rcv_saddr = iph->daddr;1192 newinet->inet_saddr = iph->daddr;1193#if IS_ENABLED(CONFIG_IPV6)1194 } else {1195 struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk;1196 struct inet_request_sock *treq = inet_rsk(oreq);1197 struct ipv6_pinfo *newnp = inet6_sk(newsk);1198 struct ipv6_pinfo *np = inet6_sk(lsk);1199 1200 inet_sk(newsk)->pinet6 = &newtcp6sk->inet6;1201 memcpy(newnp, np, sizeof(struct ipv6_pinfo));1202 newsk->sk_v6_daddr = treq->ir_v6_rmt_addr;1203 newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr;1204 inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr;1205 newnp->ipv6_fl_list = NULL;1206 newnp->pktoptions = NULL;1207 newsk->sk_bound_dev_if = treq->ir_iif;1208 newinet->inet_opt = NULL;1209 newinet->inet_daddr = LOOPBACK4_IPV6;1210 newinet->inet_saddr = LOOPBACK4_IPV6;1211#endif1212 }1213 1214 oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));1215 sk_setup_caps(newsk, dst);1216 ctx = tls_get_ctx(lsk);1217 newsk->sk_destruct = ctx->sk_destruct;1218 newsk->sk_prot_creator = lsk->sk_prot_creator;1219 csk->sk = newsk;1220 csk->passive_reap_next = oreq;1221 csk->tx_chan = cxgb4_port_chan(ndev);1222 csk->port_id = port_id;1223 csk->egress_dev = ndev;1224 csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));1225 chtls_set_tcp_window(csk);1226 tp->rcv_wnd = csk->rcv_win;1227 csk->sndbuf = csk->snd_win;1228 csk->ulp_mode = ULP_MODE_TLS;1229 step = cdev->lldi->nrxq / cdev->lldi->nchan;1230 rxq_idx = port_id * step;1231 rxq_idx += cdev->round_robin_cnt++ % step;1232 csk->rss_qid = cdev->lldi->rxq_ids[rxq_idx];1233 csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :1234 port_id * step;1235 csk->sndbuf = newsk->sk_sndbuf;1236 csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;1237 RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),1238 READ_ONCE(sock_net(newsk)->1239 ipv4.sysctl_tcp_window_scaling),1240 tp->window_clamp);1241 neigh_release(n);1242 inet_inherit_port(lsk, newsk);1243 csk_set_flag(csk, CSK_CONN_INLINE);1244 bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */1245 1246 return newsk;1247free_csk:1248 chtls_sock_release(&csk->kref);1249free_dst:1250 if (n)1251 neigh_release(n);1252 dst_release(dst);1253free_sk:1254 inet_csk_prepare_forced_close(newsk);1255 tcp_done(newsk);1256free_oreq:1257 chtls_reqsk_free(oreq);1258 return NULL;1259}1260 1261/*1262 * Populate a TID_RELEASE WR. The skb must be already propely sized.1263 */1264static void mk_tid_release(struct sk_buff *skb,1265 unsigned int chan, unsigned int tid)1266{1267 struct cpl_tid_release *req;1268 unsigned int len;1269 1270 len = roundup(sizeof(struct cpl_tid_release), 16);1271 req = (struct cpl_tid_release *)__skb_put(skb, len);1272 memset(req, 0, len);1273 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);1274 INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);1275}1276 1277static int chtls_get_module(struct sock *sk)1278{1279 struct inet_connection_sock *icsk = inet_csk(sk);1280 1281 if (!try_module_get(icsk->icsk_ulp_ops->owner))1282 return -1;1283 1284 return 0;1285}1286 1287static void chtls_pass_accept_request(struct sock *sk,1288 struct sk_buff *skb)1289{1290 struct cpl_t5_pass_accept_rpl *rpl;1291 struct cpl_pass_accept_req *req;1292 struct listen_ctx *listen_ctx;1293 struct vlan_ethhdr *vlan_eh;1294 struct request_sock *oreq;1295 struct sk_buff *reply_skb;1296 struct chtls_sock *csk;1297 struct chtls_dev *cdev;1298 struct ipv6hdr *ip6h;1299 struct tcphdr *tcph;1300 struct sock *newsk;1301 struct ethhdr *eh;1302 struct iphdr *iph;1303 void *network_hdr;1304 unsigned int stid;1305 unsigned int len;1306 unsigned int tid;1307 bool th_ecn, ect;1308 __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */1309 u16 eth_hdr_len;1310 bool ecn_ok;1311 1312 req = cplhdr(skb) + RSS_HDR;1313 tid = GET_TID(req);1314 cdev = BLOG_SKB_CB(skb)->cdev;1315 newsk = lookup_tid(cdev->tids, tid);1316 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));1317 if (newsk) {1318 pr_info("tid (%d) already in use\n", tid);1319 return;1320 }1321 1322 len = roundup(sizeof(*rpl), 16);1323 reply_skb = alloc_skb(len, GFP_ATOMIC);1324 if (!reply_skb) {1325 cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);1326 kfree_skb(skb);1327 return;1328 }1329 1330 if (sk->sk_state != TCP_LISTEN)1331 goto reject;1332 1333 if (inet_csk_reqsk_queue_is_full(sk))1334 goto reject;1335 1336 if (sk_acceptq_is_full(sk))1337 goto reject;1338 1339 1340 eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len));1341 if (eth_hdr_len == ETH_HLEN) {1342 eh = (struct ethhdr *)(req + 1);1343 iph = (struct iphdr *)(eh + 1);1344 ip6h = (struct ipv6hdr *)(eh + 1);1345 network_hdr = (void *)(eh + 1);1346 } else {1347 vlan_eh = (struct vlan_ethhdr *)(req + 1);1348 iph = (struct iphdr *)(vlan_eh + 1);1349 ip6h = (struct ipv6hdr *)(vlan_eh + 1);1350 network_hdr = (void *)(vlan_eh + 1);1351 }1352 1353 if (iph->version == 0x4) {1354 tcph = (struct tcphdr *)(iph + 1);1355 skb_set_network_header(skb, (void *)iph - (void *)req);1356 oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);1357 } else {1358 tcph = (struct tcphdr *)(ip6h + 1);1359 skb_set_network_header(skb, (void *)ip6h - (void *)req);1360 oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false);1361 }1362 1363 if (!oreq)1364 goto reject;1365 1366 oreq->rsk_rcv_wnd = 0;1367 oreq->rsk_window_clamp = 0;1368 oreq->syncookie = 0;1369 oreq->mss = 0;1370 oreq->ts_recent = 0;1371 1372 tcp_rsk(oreq)->tfo_listener = false;1373 tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);1374 chtls_set_req_port(oreq, tcph->source, tcph->dest);1375 if (iph->version == 0x4) {1376 chtls_set_req_addr(oreq, iph->daddr, iph->saddr);1377 ip_dsfield = ipv4_get_dsfield(iph);1378#if IS_ENABLED(CONFIG_IPV6)1379 } else {1380 inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;1381 inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;1382 ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb));1383#endif1384 }1385 if (req->tcpopt.wsf <= 14 &&1386 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling)) {1387 inet_rsk(oreq)->wscale_ok = 1;1388 inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;1389 }1390 inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;1391 th_ecn = tcph->ece && tcph->cwr;1392 if (th_ecn) {1393 ect = !INET_ECN_is_not_ect(ip_dsfield);1394 ecn_ok = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn);1395 if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk))1396 inet_rsk(oreq)->ecn_ok = 1;1397 }1398 1399 newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);1400 if (!newsk)1401 goto reject;1402 1403 if (chtls_get_module(newsk))1404 goto reject;1405 inet_csk_reqsk_queue_added(sk);1406 reply_skb->sk = newsk;1407 chtls_install_cpl_ops(newsk);1408 cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);1409 csk = rcu_dereference_sk_user_data(newsk);1410 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);1411 csk->listen_ctx = listen_ctx;1412 __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);1413 chtls_pass_accept_rpl(reply_skb, req, tid);1414 kfree_skb(skb);1415 return;1416 1417reject:1418 mk_tid_release(reply_skb, 0, tid);1419 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);1420 kfree_skb(skb);1421}1422 1423/*1424 * Handle a CPL_PASS_ACCEPT_REQ message.1425 */1426static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)1427{1428 struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;1429 struct listen_ctx *ctx;1430 unsigned int stid;1431 unsigned int tid;1432 struct sock *lsk;1433 void *data;1434 1435 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));1436 tid = GET_TID(req);1437 1438 data = lookup_stid(cdev->tids, stid);1439 if (!data)1440 return 1;1441 1442 ctx = (struct listen_ctx *)data;1443 lsk = ctx->lsk;1444 1445 if (unlikely(tid_out_of_range(cdev->tids, tid))) {1446 pr_info("passive open TID %u too large\n", tid);1447 return 1;1448 }1449 1450 BLOG_SKB_CB(skb)->cdev = cdev;1451 process_cpl_msg(chtls_pass_accept_request, lsk, skb);1452 return 0;1453}1454 1455/*1456 * Completes some final bits of initialization for just established connections1457 * and changes their state to TCP_ESTABLISHED.1458 *1459 * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.1460 */1461static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)1462{1463 struct tcp_sock *tp = tcp_sk(sk);1464 1465 tp->pushed_seq = snd_isn;1466 tp->write_seq = snd_isn;1467 tp->snd_nxt = snd_isn;1468 tp->snd_una = snd_isn;1469 atomic_set(&inet_sk(sk)->inet_id, get_random_u16());1470 assign_rxopt(sk, opt);1471 1472 if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))1473 tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);1474 1475 smp_mb();1476 tcp_set_state(sk, TCP_ESTABLISHED);1477}1478 1479static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)1480{1481 struct sk_buff *abort_skb;1482 1483 abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);1484 if (abort_skb)1485 chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);1486}1487 1488static struct sock *reap_list;1489static DEFINE_SPINLOCK(reap_list_lock);1490 1491/*1492 * Process the reap list.1493 */1494DECLARE_TASK_FUNC(process_reap_list, task_param)1495{1496 spin_lock_bh(&reap_list_lock);1497 while (reap_list) {1498 struct sock *sk = reap_list;1499 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);1500 1501 reap_list = csk->passive_reap_next;1502 csk->passive_reap_next = NULL;1503 spin_unlock(&reap_list_lock);1504 sock_hold(sk);1505 1506 bh_lock_sock(sk);1507 chtls_abort_conn(sk, NULL);1508 sock_orphan(sk);1509 if (sk->sk_state == TCP_CLOSE)1510 inet_csk_destroy_sock(sk);1511 bh_unlock_sock(sk);1512 sock_put(sk);1513 spin_lock(&reap_list_lock);1514 }1515 spin_unlock_bh(&reap_list_lock);1516}1517 1518static DECLARE_WORK(reap_task, process_reap_list);1519 1520static void add_to_reap_list(struct sock *sk)1521{1522 struct chtls_sock *csk = sk->sk_user_data;1523 1524 local_bh_disable();1525 release_tcp_port(sk); /* release the port immediately */1526 1527 spin_lock(&reap_list_lock);1528 csk->passive_reap_next = reap_list;1529 reap_list = sk;1530 if (!csk->passive_reap_next)1531 schedule_work(&reap_task);1532 spin_unlock(&reap_list_lock);1533 local_bh_enable();1534}1535 1536static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,1537 struct chtls_dev *cdev)1538{1539 struct request_sock *oreq;1540 struct chtls_sock *csk;1541 1542 if (lsk->sk_state != TCP_LISTEN)1543 return;1544 1545 csk = child->sk_user_data;1546 oreq = csk->passive_reap_next;1547 csk->passive_reap_next = NULL;1548 1549 reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);1550 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);1551 1552 if (sk_acceptq_is_full(lsk)) {1553 chtls_reqsk_free(oreq);1554 add_to_reap_list(child);1555 } else {1556 refcount_set(&oreq->rsk_refcnt, 1);1557 inet_csk_reqsk_queue_add(lsk, oreq, child);1558 lsk->sk_data_ready(lsk);1559 }1560}1561 1562static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)1563{1564 struct sock *child = skb->sk;1565 1566 skb->sk = NULL;1567 add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);1568 kfree_skb(skb);1569}1570 1571static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)1572{1573 struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;1574 struct chtls_sock *csk;1575 struct sock *lsk, *sk;1576 unsigned int hwtid;1577 1578 hwtid = GET_TID(req);1579 sk = lookup_tid(cdev->tids, hwtid);1580 if (!sk)1581 return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);1582 1583 bh_lock_sock(sk);1584 if (unlikely(sock_owned_by_user(sk))) {1585 kfree_skb(skb);1586 } else {1587 unsigned int stid;1588 void *data;1589 1590 csk = sk->sk_user_data;1591 csk->wr_max_credits = 64;1592 csk->wr_credits = 64;1593 csk->wr_unacked = 0;1594 make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));1595 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));1596 sk->sk_state_change(sk);1597 if (unlikely(sk->sk_socket))1598 sk_wake_async(sk, 0, POLL_OUT);1599 1600 data = lookup_stid(cdev->tids, stid);1601 if (!data) {1602 /* listening server close */1603 kfree_skb(skb);1604 goto unlock;1605 }1606 lsk = ((struct listen_ctx *)data)->lsk;1607 1608 bh_lock_sock(lsk);1609 if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {1610 /* removed from synq */1611 bh_unlock_sock(lsk);1612 kfree_skb(skb);1613 goto unlock;1614 }1615 1616 if (likely(!sock_owned_by_user(lsk))) {1617 kfree_skb(skb);1618 add_pass_open_to_parent(sk, lsk, cdev);1619 } else {1620 skb->sk = sk;1621 BLOG_SKB_CB(skb)->cdev = cdev;1622 BLOG_SKB_CB(skb)->backlog_rcv =1623 bl_add_pass_open_to_parent;1624 __sk_add_backlog(lsk, skb);1625 }1626 bh_unlock_sock(lsk);1627 }1628unlock:1629 bh_unlock_sock(sk);1630 return 0;1631}1632 1633/*1634 * Handle receipt of an urgent pointer.1635 */1636static void handle_urg_ptr(struct sock *sk, u32 urg_seq)1637{1638 struct tcp_sock *tp = tcp_sk(sk);1639 1640 urg_seq--;1641 if (tp->urg_data && !after(urg_seq, tp->urg_seq))1642 return; /* duplicate pointer */1643 1644 sk_send_sigurg(sk);1645 if (tp->urg_seq == tp->copied_seq && tp->urg_data &&1646 !sock_flag(sk, SOCK_URGINLINE) &&1647 tp->copied_seq != tp->rcv_nxt) {1648 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);1649 1650 tp->copied_seq++;1651 if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)1652 chtls_free_skb(sk, skb);1653 }1654 1655 tp->urg_data = TCP_URG_NOTYET;1656 tp->urg_seq = urg_seq;1657}1658 1659static void check_sk_callbacks(struct chtls_sock *csk)1660{1661 struct sock *sk = csk->sk;1662 1663 if (unlikely(sk->sk_user_data &&1664 !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))1665 csk_set_flag(csk, CSK_CALLBACKS_CHKD);1666}1667 1668/*1669 * Handles Rx data that arrives in a state where the socket isn't accepting1670 * new data.1671 */1672static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)1673{1674 if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))1675 chtls_abort_conn(sk, skb);1676 1677 kfree_skb(skb);1678}1679 1680static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)1681{1682 struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;1683 struct chtls_sock *csk;1684 struct tcp_sock *tp;1685 1686 csk = rcu_dereference_sk_user_data(sk);1687 tp = tcp_sk(sk);1688 1689 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {1690 handle_excess_rx(sk, skb);1691 return;1692 }1693 1694 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);1695 ULP_SKB_CB(skb)->psh = hdr->psh;1696 skb_ulp_mode(skb) = ULP_MODE_NONE;1697 1698 skb_reset_transport_header(skb);1699 __skb_pull(skb, sizeof(*hdr) + RSS_HDR);1700 if (!skb->data_len)1701 __skb_trim(skb, ntohs(hdr->len));1702 1703 if (unlikely(hdr->urg))1704 handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));1705 if (unlikely(tp->urg_data == TCP_URG_NOTYET &&1706 tp->urg_seq - tp->rcv_nxt < skb->len))1707 tp->urg_data = TCP_URG_VALID |1708 skb->data[tp->urg_seq - tp->rcv_nxt];1709 1710 if (unlikely(hdr->dack_mode != csk->delack_mode)) {1711 csk->delack_mode = hdr->dack_mode;1712 csk->delack_seq = tp->rcv_nxt;1713 }1714 1715 tcp_hdr(skb)->fin = 0;1716 tp->rcv_nxt += skb->len;1717 1718 __skb_queue_tail(&sk->sk_receive_queue, skb);1719 1720 if (!sock_flag(sk, SOCK_DEAD)) {1721 check_sk_callbacks(csk);1722 sk->sk_data_ready(sk);1723 }1724}1725 1726static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)1727{1728 struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;1729 unsigned int hwtid = GET_TID(req);1730 struct sock *sk;1731 1732 sk = lookup_tid(cdev->tids, hwtid);1733 if (unlikely(!sk)) {1734 pr_err("can't find conn. for hwtid %u.\n", hwtid);1735 return -EINVAL;1736 }1737 skb_dst_set(skb, NULL);1738 process_cpl_msg(chtls_recv_data, sk, skb);1739 return 0;1740}1741 1742static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)1743{1744 struct cpl_tls_data *hdr = cplhdr(skb);1745 struct chtls_sock *csk;1746 struct chtls_hws *tlsk;1747 struct tcp_sock *tp;1748 1749 csk = rcu_dereference_sk_user_data(sk);1750 tlsk = &csk->tlshws;1751 tp = tcp_sk(sk);1752 1753 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {1754 handle_excess_rx(sk, skb);1755 return;1756 }1757 1758 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);1759 ULP_SKB_CB(skb)->flags = 0;1760 skb_ulp_mode(skb) = ULP_MODE_TLS;1761 1762 skb_reset_transport_header(skb);1763 __skb_pull(skb, sizeof(*hdr));1764 if (!skb->data_len)1765 __skb_trim(skb,1766 CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));1767 1768 if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -1769 tp->rcv_nxt < skb->len))1770 tp->urg_data = TCP_URG_VALID |1771 skb->data[tp->urg_seq - tp->rcv_nxt];1772 1773 tcp_hdr(skb)->fin = 0;1774 tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));1775 __skb_queue_tail(&tlsk->sk_recv_queue, skb);1776}1777 1778static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)1779{1780 struct cpl_tls_data *req = cplhdr(skb);1781 unsigned int hwtid = GET_TID(req);1782 struct sock *sk;1783 1784 sk = lookup_tid(cdev->tids, hwtid);1785 if (unlikely(!sk)) {1786 pr_err("can't find conn. for hwtid %u.\n", hwtid);1787 return -EINVAL;1788 }1789 skb_dst_set(skb, NULL);1790 process_cpl_msg(chtls_recv_pdu, sk, skb);1791 return 0;1792}1793 1794static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)1795{1796 struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);1797 1798 skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);1799 tls_cmp_hdr->length = ntohs((__force __be16)nlen);1800}1801 1802static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)1803{1804 struct tlsrx_cmp_hdr *tls_hdr_pkt;1805 struct cpl_rx_tls_cmp *cmp_cpl;1806 struct sk_buff *skb_rec;1807 struct chtls_sock *csk;1808 struct chtls_hws *tlsk;1809 struct tcp_sock *tp;1810 1811 cmp_cpl = cplhdr(skb);1812 csk = rcu_dereference_sk_user_data(sk);1813 tlsk = &csk->tlshws;1814 tp = tcp_sk(sk);1815 1816 ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);1817 ULP_SKB_CB(skb)->flags = 0;1818 1819 skb_reset_transport_header(skb);1820 __skb_pull(skb, sizeof(*cmp_cpl));1821 tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;1822 if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)1823 tls_hdr_pkt->type = CONTENT_TYPE_ERROR;1824 if (!skb->data_len)1825 __skb_trim(skb, TLS_HEADER_LENGTH);1826 1827 tp->rcv_nxt +=1828 CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));1829 1830 ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;1831 skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);1832 if (!skb_rec) {1833 __skb_queue_tail(&sk->sk_receive_queue, skb);1834 } else {1835 chtls_set_hdrlen(skb, tlsk->pldlen);1836 tlsk->pldlen = 0;1837 __skb_queue_tail(&sk->sk_receive_queue, skb);1838 __skb_queue_tail(&sk->sk_receive_queue, skb_rec);1839 }1840 1841 if (!sock_flag(sk, SOCK_DEAD)) {1842 check_sk_callbacks(csk);1843 sk->sk_data_ready(sk);1844 }1845}1846 1847static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)1848{1849 struct cpl_rx_tls_cmp *req = cplhdr(skb);1850 unsigned int hwtid = GET_TID(req);1851 struct sock *sk;1852 1853 sk = lookup_tid(cdev->tids, hwtid);1854 if (unlikely(!sk)) {1855 pr_err("can't find conn. for hwtid %u.\n", hwtid);1856 return -EINVAL;1857 }1858 skb_dst_set(skb, NULL);1859 process_cpl_msg(chtls_rx_hdr, sk, skb);1860 1861 return 0;1862}1863 1864static void chtls_timewait(struct sock *sk)1865{1866 struct tcp_sock *tp = tcp_sk(sk);1867 1868 tp->rcv_nxt++;1869 tp->rx_opt.ts_recent_stamp = ktime_get_seconds();1870 tp->srtt_us = 0;1871 tcp_time_wait(sk, TCP_TIME_WAIT, 0);1872}1873 1874static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)1875{1876 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);1877 1878 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))1879 goto out;1880 1881 sk->sk_shutdown |= RCV_SHUTDOWN;1882 sock_set_flag(sk, SOCK_DONE);1883 1884 switch (sk->sk_state) {1885 case TCP_SYN_RECV:1886 case TCP_ESTABLISHED:1887 tcp_set_state(sk, TCP_CLOSE_WAIT);1888 break;1889 case TCP_FIN_WAIT1:1890 tcp_set_state(sk, TCP_CLOSING);1891 break;1892 case TCP_FIN_WAIT2:1893 chtls_release_resources(sk);1894 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))1895 chtls_conn_done(sk);1896 else1897 chtls_timewait(sk);1898 break;1899 default:1900 pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);1901 }1902 1903 if (!sock_flag(sk, SOCK_DEAD)) {1904 sk->sk_state_change(sk);1905 /* Do not send POLL_HUP for half duplex close. */1906 1907 if ((sk->sk_shutdown & SEND_SHUTDOWN) ||1908 sk->sk_state == TCP_CLOSE)1909 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);1910 else1911 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);1912 }1913out:1914 kfree_skb(skb);1915}1916 1917static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)1918{1919 struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;1920 struct chtls_sock *csk;1921 struct tcp_sock *tp;1922 1923 csk = rcu_dereference_sk_user_data(sk);1924 1925 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))1926 goto out;1927 1928 tp = tcp_sk(sk);1929 1930 tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */1931 1932 switch (sk->sk_state) {1933 case TCP_CLOSING:1934 chtls_release_resources(sk);1935 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))1936 chtls_conn_done(sk);1937 else1938 chtls_timewait(sk);1939 break;1940 case TCP_LAST_ACK:1941 chtls_release_resources(sk);1942 chtls_conn_done(sk);1943 break;1944 case TCP_FIN_WAIT1:1945 tcp_set_state(sk, TCP_FIN_WAIT2);1946 sk->sk_shutdown |= SEND_SHUTDOWN;1947 1948 if (!sock_flag(sk, SOCK_DEAD))1949 sk->sk_state_change(sk);1950 else if (tcp_sk(sk)->linger2 < 0 &&1951 !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))1952 chtls_abort_conn(sk, skb);1953 else if (csk_flag_nochk(csk, CSK_TX_DATA_SENT))1954 chtls_set_quiesce_ctrl(sk, 0);1955 break;1956 default:1957 pr_info("close_con_rpl in bad state %d\n", sk->sk_state);1958 }1959out:1960 kfree_skb(skb);1961}1962 1963static struct sk_buff *get_cpl_skb(struct sk_buff *skb,1964 size_t len, gfp_t gfp)1965{1966 if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {1967 WARN_ONCE(skb->len < len, "skb alloc error");1968 __skb_trim(skb, len);1969 skb_get(skb);1970 } else {1971 skb = alloc_skb(len, gfp);1972 if (skb)1973 __skb_put(skb, len);1974 }1975 return skb;1976}1977 1978static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,1979 int cmd)1980{1981 struct cpl_abort_rpl *rpl = cplhdr(skb);1982 1983 INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);1984 rpl->cmd = cmd;1985}1986 1987static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)1988{1989 struct cpl_abort_req_rss *req = cplhdr(skb);1990 struct sk_buff *reply_skb;1991 1992 reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),1993 GFP_KERNEL | __GFP_NOFAIL);1994 __skb_put(reply_skb, sizeof(struct cpl_abort_rpl));1995 set_abort_rpl_wr(reply_skb, GET_TID(req),1996 (req->status & CPL_ABORT_NO_RST));1997 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);1998 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);1999 kfree_skb(skb);2000}2001 2002/*2003 * Add an skb to the deferred skb queue for processing from process context.2004 */2005static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,2006 defer_handler_t handler)2007{2008 DEFERRED_SKB_CB(skb)->handler = handler;2009 spin_lock_bh(&cdev->deferq.lock);2010 __skb_queue_tail(&cdev->deferq, skb);2011 if (skb_queue_len(&cdev->deferq) == 1)2012 schedule_work(&cdev->deferq_task);2013 spin_unlock_bh(&cdev->deferq.lock);2014}2015 2016static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,2017 struct chtls_dev *cdev,2018 int status, int queue)2019{2020 struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;2021 struct sk_buff *reply_skb;2022 struct chtls_sock *csk;2023 unsigned int tid;2024 2025 csk = rcu_dereference_sk_user_data(sk);2026 tid = GET_TID(req);2027 2028 reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());2029 if (!reply_skb) {2030 req->status = (queue << 1) | status;2031 t4_defer_reply(skb, cdev, send_defer_abort_rpl);2032 return;2033 }2034 2035 set_abort_rpl_wr(reply_skb, tid, status);2036 kfree_skb(skb);2037 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);2038 if (csk_conn_inline(csk)) {2039 struct l2t_entry *e = csk->l2t_entry;2040 2041 if (e && sk->sk_state != TCP_SYN_RECV) {2042 cxgb4_l2t_send(csk->egress_dev, reply_skb, e);2043 return;2044 }2045 }2046 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);2047}2048 2049/*2050 * This is run from a listener's backlog to abort a child connection in2051 * SYN_RCV state (i.e., one on the listener's SYN queue).2052 */2053static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)2054{2055 struct chtls_sock *csk;2056 struct sock *child;2057 int queue;2058 2059 child = skb->sk;2060 csk = rcu_dereference_sk_user_data(child);2061 queue = csk->txq_idx;2062 2063 skb->sk = NULL;2064 chtls_send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,2065 CPL_ABORT_NO_RST, queue);2066 do_abort_syn_rcv(child, lsk);2067}2068 2069static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)2070{2071 const struct request_sock *oreq;2072 struct listen_ctx *listen_ctx;2073 struct chtls_sock *csk;2074 struct chtls_dev *cdev;2075 struct sock *psk;2076 void *ctx;2077 2078 csk = sk->sk_user_data;2079 oreq = csk->passive_reap_next;2080 cdev = csk->cdev;2081 2082 if (!oreq)2083 return -1;2084 2085 ctx = lookup_stid(cdev->tids, oreq->ts_recent);2086 if (!ctx)2087 return -1;2088 2089 listen_ctx = (struct listen_ctx *)ctx;2090 psk = listen_ctx->lsk;2091 2092 bh_lock_sock(psk);2093 if (!sock_owned_by_user(psk)) {2094 int queue = csk->txq_idx;2095 2096 chtls_send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);2097 do_abort_syn_rcv(sk, psk);2098 } else {2099 skb->sk = sk;2100 BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;2101 __sk_add_backlog(psk, skb);2102 }2103 bh_unlock_sock(psk);2104 return 0;2105}2106 2107static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)2108{2109 const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;2110 struct chtls_sock *csk = sk->sk_user_data;2111 int rst_status = CPL_ABORT_NO_RST;2112 int queue = csk->txq_idx;2113 2114 if (is_neg_adv(req->status)) {2115 kfree_skb(skb);2116 return;2117 }2118 2119 csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);2120 2121 if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&2122 !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {2123 struct tcp_sock *tp = tcp_sk(sk);2124 2125 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)2126 WARN_ONCE(1, "send_tx_flowc error");2127 csk_set_flag(csk, CSK_TX_DATA_SENT);2128 }2129 2130 csk_set_flag(csk, CSK_ABORT_SHUTDOWN);2131 2132 if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {2133 sk->sk_err = ETIMEDOUT;2134 2135 if (!sock_flag(sk, SOCK_DEAD))2136 sk_error_report(sk);2137 2138 if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))2139 return;2140 2141 }2142 2143 chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev,2144 rst_status, queue);2145 chtls_release_resources(sk);2146 chtls_conn_done(sk);2147}2148 2149static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)2150{2151 struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;2152 struct chtls_sock *csk;2153 struct chtls_dev *cdev;2154 2155 csk = rcu_dereference_sk_user_data(sk);2156 cdev = csk->cdev;2157 2158 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {2159 csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);2160 if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {2161 if (sk->sk_state == TCP_SYN_SENT) {2162 cxgb4_remove_tid(cdev->tids,2163 csk->port_id,2164 GET_TID(rpl),2165 sk->sk_family);2166 sock_put(sk);2167 }2168 chtls_release_resources(sk);2169 chtls_conn_done(sk);2170 }2171 }2172 kfree_skb(skb);2173}2174 2175static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)2176{2177 struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;2178 void (*fn)(struct sock *sk, struct sk_buff *skb);2179 unsigned int hwtid = GET_TID(req);2180 struct chtls_sock *csk;2181 struct sock *sk;2182 u8 opcode;2183 2184 opcode = ((const struct rss_header *)cplhdr(skb))->opcode;2185 2186 sk = lookup_tid(cdev->tids, hwtid);2187 if (!sk)2188 goto rel_skb;2189 2190 csk = sk->sk_user_data;2191 2192 switch (opcode) {2193 case CPL_PEER_CLOSE:2194 fn = chtls_peer_close;2195 break;2196 case CPL_CLOSE_CON_RPL:2197 fn = chtls_close_con_rpl;2198 break;2199 case CPL_ABORT_REQ_RSS:2200 /*2201 * Save the offload device in the skb, we may process this2202 * message after the socket has closed.2203 */2204 BLOG_SKB_CB(skb)->cdev = csk->cdev;2205 fn = chtls_abort_req_rss;2206 break;2207 case CPL_ABORT_RPL_RSS:2208 fn = chtls_abort_rpl_rss;2209 break;2210 default:2211 goto rel_skb;2212 }2213 2214 process_cpl_msg(fn, sk, skb);2215 return 0;2216 2217rel_skb:2218 kfree_skb(skb);2219 return 0;2220}2221 2222static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)2223{2224 struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;2225 struct chtls_sock *csk = sk->sk_user_data;2226 struct tcp_sock *tp = tcp_sk(sk);2227 u32 credits = hdr->credits;2228 u32 snd_una;2229 2230 snd_una = ntohl(hdr->snd_una);2231 csk->wr_credits += credits;2232 2233 if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)2234 csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;2235 2236 while (credits) {2237 struct sk_buff *pskb = csk->wr_skb_head;2238 u32 csum;2239 2240 if (unlikely(!pskb)) {2241 if (csk->wr_nondata)2242 csk->wr_nondata -= credits;2243 break;2244 }2245 csum = (__force u32)pskb->csum;2246 if (unlikely(credits < csum)) {2247 pskb->csum = (__force __wsum)(csum - credits);2248 break;2249 }2250 dequeue_wr(sk);2251 credits -= csum;2252 kfree_skb(pskb);2253 }2254 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {2255 if (unlikely(before(snd_una, tp->snd_una))) {2256 kfree_skb(skb);2257 return;2258 }2259 2260 if (tp->snd_una != snd_una) {2261 tp->snd_una = snd_una;2262 tp->rcv_tstamp = tcp_jiffies32;2263 if (tp->snd_una == tp->snd_nxt &&2264 !csk_flag_nochk(csk, CSK_TX_FAILOVER))2265 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);2266 }2267 }2268 2269 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {2270 unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);2271 2272 csk->wr_credits -= fclen16;2273 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);2274 csk_reset_flag(csk, CSK_TX_FAILOVER);2275 }2276 if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))2277 sk->sk_write_space(sk);2278 2279 kfree_skb(skb);2280}2281 2282static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)2283{2284 struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;2285 unsigned int hwtid = GET_TID(rpl);2286 struct sock *sk;2287 2288 sk = lookup_tid(cdev->tids, hwtid);2289 if (unlikely(!sk)) {2290 pr_err("can't find conn. for hwtid %u.\n", hwtid);2291 return -EINVAL;2292 }2293 process_cpl_msg(chtls_rx_ack, sk, skb);2294 2295 return 0;2296}2297 2298static int chtls_set_tcb_rpl(struct chtls_dev *cdev, struct sk_buff *skb)2299{2300 struct cpl_set_tcb_rpl *rpl = cplhdr(skb) + RSS_HDR;2301 unsigned int hwtid = GET_TID(rpl);2302 struct sock *sk;2303 2304 sk = lookup_tid(cdev->tids, hwtid);2305 2306 /* return EINVAL if socket doesn't exist */2307 if (!sk)2308 return -EINVAL;2309 2310 /* Reusing the skb as size of cpl_set_tcb_field structure2311 * is greater than cpl_abort_req2312 */2313 if (TCB_COOKIE_G(rpl->cookie) == TCB_FIELD_COOKIE_TFLAG)2314 chtls_send_abort(sk, CPL_ABORT_SEND_RST, NULL);2315 2316 kfree_skb(skb);2317 return 0;2318}2319 2320chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {2321 [CPL_PASS_OPEN_RPL] = chtls_pass_open_rpl,2322 [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,2323 [CPL_PASS_ACCEPT_REQ] = chtls_pass_accept_req,2324 [CPL_PASS_ESTABLISH] = chtls_pass_establish,2325 [CPL_RX_DATA] = chtls_rx_data,2326 [CPL_TLS_DATA] = chtls_rx_pdu,2327 [CPL_RX_TLS_CMP] = chtls_rx_cmp,2328 [CPL_PEER_CLOSE] = chtls_conn_cpl,2329 [CPL_CLOSE_CON_RPL] = chtls_conn_cpl,2330 [CPL_ABORT_REQ_RSS] = chtls_conn_cpl,2331 [CPL_ABORT_RPL_RSS] = chtls_conn_cpl,2332 [CPL_FW4_ACK] = chtls_wr_ack,2333 [CPL_SET_TCB_RPL] = chtls_set_tcb_rpl,2334};2335