828 lines · c
1/*2 * This file is part of the Chelsio T6 Crypto driver for Linux.3 *4 * Copyright (c) 2003-2017 Chelsio Communications, Inc. All rights reserved.5 *6 * This software is available to you under a choice of one of two7 * licenses. You may choose to be licensed under the terms of the GNU8 * General Public License (GPL) Version 2, available from the file9 * COPYING in the main directory of this source tree, or the10 * OpenIB.org BSD license below:11 *12 * Redistribution and use in source and binary forms, with or13 * without modification, are permitted provided that the following14 * conditions are met:15 *16 * - Redistributions of source code must retain the above17 * copyright notice, this list of conditions and the following18 * disclaimer.19 *20 * - Redistributions in binary form must reproduce the above21 * copyright notice, this list of conditions and the following22 * disclaimer in the documentation and/or other materials23 * provided with the distribution.24 *25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE32 * SOFTWARE.33 *34 * Written and Maintained by:35 * Atul Gupta (atul.gupta@chelsio.com)36 */37 38#define pr_fmt(fmt) "ch_ipsec: " fmt39 40#include <linux/kernel.h>41#include <linux/module.h>42#include <linux/skbuff.h>43#include <linux/rtnetlink.h>44#include <linux/highmem.h>45#include <linux/if_vlan.h>46#include <linux/ip.h>47#include <linux/netdevice.h>48#include <net/esp.h>49#include <net/xfrm.h>50#include <crypto/aes.h>51#include <crypto/hash.h>52#include <crypto/sha1.h>53#include <crypto/sha2.h>54#include <crypto/authenc.h>55#include <crypto/internal/aead.h>56#include <crypto/null.h>57#include <crypto/internal/skcipher.h>58#include <crypto/aead.h>59#include <crypto/scatterwalk.h>60#include <crypto/internal/hash.h>61 62#include "chcr_ipsec.h"63 64/*65 * Max Tx descriptor space we allow for an Ethernet packet to be inlined66 * into a WR.67 */68#define MAX_IMM_TX_PKT_LEN 25669#define GCM_ESP_IV_SIZE 870 71static LIST_HEAD(uld_ctx_list);72static DEFINE_MUTEX(dev_mutex);73 74static bool ch_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x);75static int ch_ipsec_uld_state_change(void *handle, enum cxgb4_state new_state);76static int ch_ipsec_xmit(struct sk_buff *skb, struct net_device *dev);77static void *ch_ipsec_uld_add(const struct cxgb4_lld_info *infop);78static void ch_ipsec_advance_esn_state(struct xfrm_state *x);79static void ch_ipsec_xfrm_free_state(struct xfrm_state *x);80static void ch_ipsec_xfrm_del_state(struct xfrm_state *x);81static int ch_ipsec_xfrm_add_state(struct xfrm_state *x,82 struct netlink_ext_ack *extack);83 84static const struct xfrmdev_ops ch_ipsec_xfrmdev_ops = {85 .xdo_dev_state_add = ch_ipsec_xfrm_add_state,86 .xdo_dev_state_delete = ch_ipsec_xfrm_del_state,87 .xdo_dev_state_free = ch_ipsec_xfrm_free_state,88 .xdo_dev_offload_ok = ch_ipsec_offload_ok,89 .xdo_dev_state_advance_esn = ch_ipsec_advance_esn_state,90};91 92static struct cxgb4_uld_info ch_ipsec_uld_info = {93 .name = CHIPSEC_DRV_MODULE_NAME,94 .add = ch_ipsec_uld_add,95 .state_change = ch_ipsec_uld_state_change,96 .tx_handler = ch_ipsec_xmit,97 .xfrmdev_ops = &ch_ipsec_xfrmdev_ops,98};99 100static void *ch_ipsec_uld_add(const struct cxgb4_lld_info *infop)101{102 struct ipsec_uld_ctx *u_ctx;103 104 pr_info_once("%s - version %s\n", CHIPSEC_DRV_DESC,105 CHIPSEC_DRV_VERSION);106 u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);107 if (!u_ctx) {108 u_ctx = ERR_PTR(-ENOMEM);109 goto out;110 }111 u_ctx->lldi = *infop;112out:113 return u_ctx;114}115 116static int ch_ipsec_uld_state_change(void *handle, enum cxgb4_state new_state)117{118 struct ipsec_uld_ctx *u_ctx = handle;119 120 pr_debug("new_state %u\n", new_state);121 switch (new_state) {122 case CXGB4_STATE_UP:123 pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));124 mutex_lock(&dev_mutex);125 list_add_tail(&u_ctx->entry, &uld_ctx_list);126 mutex_unlock(&dev_mutex);127 break;128 case CXGB4_STATE_START_RECOVERY:129 case CXGB4_STATE_DOWN:130 case CXGB4_STATE_DETACH:131 pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));132 list_del(&u_ctx->entry);133 break;134 default:135 break;136 }137 138 return 0;139}140 141static int ch_ipsec_setauthsize(struct xfrm_state *x,142 struct ipsec_sa_entry *sa_entry)143{144 int hmac_ctrl;145 int authsize = x->aead->alg_icv_len / 8;146 147 sa_entry->authsize = authsize;148 149 switch (authsize) {150 case ICV_8:151 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;152 break;153 case ICV_12:154 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;155 break;156 case ICV_16:157 hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;158 break;159 default:160 return -EINVAL;161 }162 return hmac_ctrl;163}164 165static int ch_ipsec_setkey(struct xfrm_state *x,166 struct ipsec_sa_entry *sa_entry)167{168 int keylen = (x->aead->alg_key_len + 7) / 8;169 unsigned char *key = x->aead->alg_key;170 int ck_size, key_ctx_size = 0;171 unsigned char ghash_h[AEAD_H_SIZE];172 struct crypto_aes_ctx aes;173 int ret = 0;174 175 if (keylen > 3) {176 keylen -= 4; /* nonce/salt is present in the last 4 bytes */177 memcpy(sa_entry->salt, key + keylen, 4);178 }179 180 if (keylen == AES_KEYSIZE_128) {181 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;182 } else if (keylen == AES_KEYSIZE_192) {183 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;184 } else if (keylen == AES_KEYSIZE_256) {185 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;186 } else {187 pr_err("GCM: Invalid key length %d\n", keylen);188 ret = -EINVAL;189 goto out;190 }191 192 memcpy(sa_entry->key, key, keylen);193 sa_entry->enckey_len = keylen;194 key_ctx_size = sizeof(struct _key_ctx) +195 ((DIV_ROUND_UP(keylen, 16)) << 4) +196 AEAD_H_SIZE;197 198 sa_entry->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,199 CHCR_KEYCTX_MAC_KEY_SIZE_128,200 0, 0,201 key_ctx_size >> 4);202 203 /* Calculate the H = CIPH(K, 0 repeated 16 times).204 * It will go in key context205 */206 ret = aes_expandkey(&aes, key, keylen);207 if (ret) {208 sa_entry->enckey_len = 0;209 goto out;210 }211 memset(ghash_h, 0, AEAD_H_SIZE);212 aes_encrypt(&aes, ghash_h, ghash_h);213 memzero_explicit(&aes, sizeof(aes));214 215 memcpy(sa_entry->key + (DIV_ROUND_UP(sa_entry->enckey_len, 16) *216 16), ghash_h, AEAD_H_SIZE);217 sa_entry->kctx_len = ((DIV_ROUND_UP(sa_entry->enckey_len, 16)) << 4) +218 AEAD_H_SIZE;219out:220 return ret;221}222 223/*224 * ch_ipsec_xfrm_add_state225 * returns 0 on success, negative error if failed to send message to FPGA226 * positive error if FPGA returned a bad response227 */228static int ch_ipsec_xfrm_add_state(struct xfrm_state *x,229 struct netlink_ext_ack *extack)230{231 struct ipsec_sa_entry *sa_entry;232 int res = 0;233 234 if (x->props.aalgo != SADB_AALG_NONE) {235 NL_SET_ERR_MSG_MOD(extack, "Cannot offload authenticated xfrm states");236 return -EINVAL;237 }238 if (x->props.calgo != SADB_X_CALG_NONE) {239 NL_SET_ERR_MSG_MOD(extack, "Cannot offload compressed xfrm states");240 return -EINVAL;241 }242 if (x->props.family != AF_INET &&243 x->props.family != AF_INET6) {244 NL_SET_ERR_MSG_MOD(extack, "Only IPv4/6 xfrm state offloaded");245 return -EINVAL;246 }247 if (x->props.mode != XFRM_MODE_TRANSPORT &&248 x->props.mode != XFRM_MODE_TUNNEL) {249 NL_SET_ERR_MSG_MOD(extack, "Only transport and tunnel xfrm offload");250 return -EINVAL;251 }252 if (x->id.proto != IPPROTO_ESP) {253 NL_SET_ERR_MSG_MOD(extack, "Only ESP xfrm state offloaded");254 return -EINVAL;255 }256 if (x->encap) {257 NL_SET_ERR_MSG_MOD(extack, "Encapsulated xfrm state not offloaded");258 return -EINVAL;259 }260 if (!x->aead) {261 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states without aead");262 return -EINVAL;263 }264 if (x->aead->alg_icv_len != 128 &&265 x->aead->alg_icv_len != 96) {266 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with AEAD ICV length other than 96b & 128b");267 return -EINVAL;268 }269 if ((x->aead->alg_key_len != 128 + 32) &&270 (x->aead->alg_key_len != 256 + 32)) {271 NL_SET_ERR_MSG_MOD(extack, "cannot offload xfrm states with AEAD key length other than 128/256 bit");272 return -EINVAL;273 }274 if (x->tfcpad) {275 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with tfc padding");276 return -EINVAL;277 }278 if (!x->geniv) {279 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states without geniv");280 return -EINVAL;281 }282 if (strcmp(x->geniv, "seqiv")) {283 NL_SET_ERR_MSG_MOD(extack, "Cannot offload xfrm states with geniv other than seqiv");284 return -EINVAL;285 }286 if (x->xso.type != XFRM_DEV_OFFLOAD_CRYPTO) {287 NL_SET_ERR_MSG_MOD(extack, "Unsupported xfrm offload");288 return -EINVAL;289 }290 291 sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);292 if (!sa_entry) {293 res = -ENOMEM;294 goto out;295 }296 297 sa_entry->hmac_ctrl = ch_ipsec_setauthsize(x, sa_entry);298 if (x->props.flags & XFRM_STATE_ESN)299 sa_entry->esn = 1;300 ch_ipsec_setkey(x, sa_entry);301 x->xso.offload_handle = (unsigned long)sa_entry;302 try_module_get(THIS_MODULE);303out:304 return res;305}306 307static void ch_ipsec_xfrm_del_state(struct xfrm_state *x)308{309 /* do nothing */310 if (!x->xso.offload_handle)311 return;312}313 314static void ch_ipsec_xfrm_free_state(struct xfrm_state *x)315{316 struct ipsec_sa_entry *sa_entry;317 318 if (!x->xso.offload_handle)319 return;320 321 sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;322 kfree(sa_entry);323 module_put(THIS_MODULE);324}325 326static bool ch_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)327{328 if (x->props.family == AF_INET) {329 /* Offload with IP options is not supported yet */330 if (ip_hdr(skb)->ihl > 5)331 return false;332 } else {333 /* Offload with IPv6 extension headers is not support yet */334 if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))335 return false;336 }337 return true;338}339 340static void ch_ipsec_advance_esn_state(struct xfrm_state *x)341{342 /* do nothing */343 if (!x->xso.offload_handle)344 return;345}346 347static int is_eth_imm(const struct sk_buff *skb,348 struct ipsec_sa_entry *sa_entry)349{350 unsigned int kctx_len;351 int hdrlen;352 353 kctx_len = sa_entry->kctx_len;354 hdrlen = sizeof(struct fw_ulptx_wr) +355 sizeof(struct chcr_ipsec_req) + kctx_len;356 357 hdrlen += sizeof(struct cpl_tx_pkt);358 if (sa_entry->esn)359 hdrlen += (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16)360 << 4);361 if (skb->len <= MAX_IMM_TX_PKT_LEN - hdrlen)362 return hdrlen;363 return 0;364}365 366static unsigned int calc_tx_sec_flits(const struct sk_buff *skb,367 struct ipsec_sa_entry *sa_entry,368 bool *immediate)369{370 unsigned int kctx_len;371 unsigned int flits;372 int aadivlen;373 int hdrlen;374 375 kctx_len = sa_entry->kctx_len;376 hdrlen = is_eth_imm(skb, sa_entry);377 aadivlen = sa_entry->esn ? DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),378 16) : 0;379 aadivlen <<= 4;380 381 /* If the skb is small enough, we can pump it out as a work request382 * with only immediate data. In that case we just have to have the383 * TX Packet header plus the skb data in the Work Request.384 */385 386 if (hdrlen) {387 *immediate = true;388 return DIV_ROUND_UP(skb->len + hdrlen, sizeof(__be64));389 }390 391 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1);392 393 /* Otherwise, we're going to have to construct a Scatter gather list394 * of the skb body and fragments. We also include the flits necessary395 * for the TX Packet Work Request and CPL. We always have a firmware396 * Write Header (incorporated as part of the cpl_tx_pkt_lso and397 * cpl_tx_pkt structures), followed by either a TX Packet Write CPL398 * message or, if we're doing a Large Send Offload, an LSO CPL message399 * with an embedded TX Packet Write CPL message.400 */401 flits += (sizeof(struct fw_ulptx_wr) +402 sizeof(struct chcr_ipsec_req) +403 kctx_len +404 sizeof(struct cpl_tx_pkt_core) +405 aadivlen) / sizeof(__be64);406 return flits;407}408 409static void *copy_esn_pktxt(struct sk_buff *skb,410 struct net_device *dev,411 void *pos,412 struct ipsec_sa_entry *sa_entry)413{414 struct chcr_ipsec_aadiv *aadiv;415 struct ulptx_idata *sc_imm;416 struct ip_esp_hdr *esphdr;417 struct xfrm_offload *xo;418 struct sge_eth_txq *q;419 struct adapter *adap;420 struct port_info *pi;421 __be64 seqno;422 u32 qidx;423 u32 seqlo;424 u8 *iv;425 int eoq;426 int len;427 428 pi = netdev_priv(dev);429 adap = pi->adapter;430 qidx = skb->queue_mapping;431 q = &adap->sge.ethtxq[qidx + pi->first_qset];432 433 /* end of queue, reset pos to start of queue */434 eoq = (void *)q->q.stat - pos;435 if (!eoq)436 pos = q->q.desc;437 438 len = DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv), 16) << 4;439 memset(pos, 0, len);440 aadiv = (struct chcr_ipsec_aadiv *)pos;441 esphdr = (struct ip_esp_hdr *)skb_transport_header(skb);442 iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);443 xo = xfrm_offload(skb);444 445 aadiv->spi = (esphdr->spi);446 seqlo = ntohl(esphdr->seq_no);447 seqno = cpu_to_be64(seqlo + ((u64)xo->seq.hi << 32));448 memcpy(aadiv->seq_no, &seqno, 8);449 iv = skb_transport_header(skb) + sizeof(struct ip_esp_hdr);450 memcpy(aadiv->iv, iv, 8);451 452 if (is_eth_imm(skb, sa_entry) && !skb_is_nonlinear(skb)) {453 sc_imm = (struct ulptx_idata *)(pos +454 (DIV_ROUND_UP(sizeof(struct chcr_ipsec_aadiv),455 sizeof(__be64)) << 3));456 sc_imm->cmd_more = FILL_CMD_MORE(0);457 sc_imm->len = cpu_to_be32(skb->len);458 }459 pos += len;460 return pos;461}462 463static void *copy_cpltx_pktxt(struct sk_buff *skb,464 struct net_device *dev,465 void *pos,466 struct ipsec_sa_entry *sa_entry)467{468 struct cpl_tx_pkt_core *cpl;469 struct sge_eth_txq *q;470 struct adapter *adap;471 struct port_info *pi;472 u32 ctrl0, qidx;473 u64 cntrl = 0;474 int left;475 476 pi = netdev_priv(dev);477 adap = pi->adapter;478 qidx = skb->queue_mapping;479 q = &adap->sge.ethtxq[qidx + pi->first_qset];480 481 left = (void *)q->q.stat - pos;482 if (!left)483 pos = q->q.desc;484 485 cpl = (struct cpl_tx_pkt_core *)pos;486 487 cntrl = TXPKT_L4CSUM_DIS_F | TXPKT_IPCSUM_DIS_F;488 ctrl0 = TXPKT_OPCODE_V(CPL_TX_PKT_XT) | TXPKT_INTF_V(pi->tx_chan) |489 TXPKT_PF_V(adap->pf);490 if (skb_vlan_tag_present(skb)) {491 q->vlan_ins++;492 cntrl |= TXPKT_VLAN_VLD_F | TXPKT_VLAN_V(skb_vlan_tag_get(skb));493 }494 495 cpl->ctrl0 = htonl(ctrl0);496 cpl->pack = htons(0);497 cpl->len = htons(skb->len);498 cpl->ctrl1 = cpu_to_be64(cntrl);499 500 pos += sizeof(struct cpl_tx_pkt_core);501 /* Copy ESN info for HW */502 if (sa_entry->esn)503 pos = copy_esn_pktxt(skb, dev, pos, sa_entry);504 return pos;505}506 507static void *copy_key_cpltx_pktxt(struct sk_buff *skb,508 struct net_device *dev,509 void *pos,510 struct ipsec_sa_entry *sa_entry)511{512 struct _key_ctx *key_ctx;513 int left, eoq, key_len;514 struct sge_eth_txq *q;515 struct adapter *adap;516 struct port_info *pi;517 unsigned int qidx;518 519 pi = netdev_priv(dev);520 adap = pi->adapter;521 qidx = skb->queue_mapping;522 q = &adap->sge.ethtxq[qidx + pi->first_qset];523 key_len = sa_entry->kctx_len;524 525 /* end of queue, reset pos to start of queue */526 eoq = (void *)q->q.stat - pos;527 left = eoq;528 if (!eoq) {529 pos = q->q.desc;530 left = 64 * q->q.size;531 }532 533 /* Copy the Key context header */534 key_ctx = (struct _key_ctx *)pos;535 key_ctx->ctx_hdr = sa_entry->key_ctx_hdr;536 memcpy(key_ctx->salt, sa_entry->salt, MAX_SALT);537 pos += sizeof(struct _key_ctx);538 left -= sizeof(struct _key_ctx);539 540 if (likely(key_len <= left)) {541 memcpy(key_ctx->key, sa_entry->key, key_len);542 pos += key_len;543 } else {544 memcpy(pos, sa_entry->key, left);545 memcpy(q->q.desc, sa_entry->key + left,546 key_len - left);547 pos = (u8 *)q->q.desc + (key_len - left);548 }549 /* Copy CPL TX PKT XT */550 pos = copy_cpltx_pktxt(skb, dev, pos, sa_entry);551 552 return pos;553}554 555static void *ch_ipsec_crypto_wreq(struct sk_buff *skb,556 struct net_device *dev,557 void *pos,558 int credits,559 struct ipsec_sa_entry *sa_entry)560{561 struct port_info *pi = netdev_priv(dev);562 struct adapter *adap = pi->adapter;563 unsigned int ivsize = GCM_ESP_IV_SIZE;564 struct chcr_ipsec_wr *wr;565 bool immediate = false;566 u16 immdatalen = 0;567 unsigned int flits;568 u32 ivinoffset;569 u32 aadstart;570 u32 aadstop;571 u32 ciphstart;572 u16 sc_more = 0;573 u32 ivdrop = 0;574 u32 esnlen = 0;575 u32 wr_mid;576 u16 ndesc;577 int qidx = skb_get_queue_mapping(skb);578 struct sge_eth_txq *q = &adap->sge.ethtxq[qidx + pi->first_qset];579 unsigned int kctx_len = sa_entry->kctx_len;580 int qid = q->q.cntxt_id;581 582 atomic_inc(&adap->ch_ipsec_stats.ipsec_cnt);583 584 flits = calc_tx_sec_flits(skb, sa_entry, &immediate);585 ndesc = DIV_ROUND_UP(flits, 2);586 if (sa_entry->esn)587 ivdrop = 1;588 589 if (immediate)590 immdatalen = skb->len;591 592 if (sa_entry->esn) {593 esnlen = sizeof(struct chcr_ipsec_aadiv);594 if (!skb_is_nonlinear(skb))595 sc_more = 1;596 }597 598 /* WR Header */599 wr = (struct chcr_ipsec_wr *)pos;600 wr->wreq.op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));601 wr_mid = FW_CRYPTO_LOOKASIDE_WR_LEN16_V(ndesc);602 603 if (unlikely(credits < ETHTXQ_STOP_THRES)) {604 netif_tx_stop_queue(q->txq);605 q->q.stops++;606 if (!q->dbqt)607 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;608 }609 wr_mid |= FW_ULPTX_WR_DATA_F;610 wr->wreq.flowid_len16 = htonl(wr_mid);611 612 /* ULPTX */613 wr->req.ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(pi->port_id, qid);614 wr->req.ulptx.len = htonl(ndesc - 1);615 616 /* Sub-command */617 wr->req.sc_imm.cmd_more = FILL_CMD_MORE(!immdatalen || sc_more);618 wr->req.sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +619 sizeof(wr->req.key_ctx) +620 kctx_len +621 sizeof(struct cpl_tx_pkt_core) +622 esnlen +623 (esnlen ? 0 : immdatalen));624 625 /* CPL_SEC_PDU */626 ivinoffset = sa_entry->esn ? (ESN_IV_INSERT_OFFSET + 1) :627 (skb_transport_offset(skb) +628 sizeof(struct ip_esp_hdr) + 1);629 wr->req.sec_cpl.op_ivinsrtofst = htonl(630 CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |631 CPL_TX_SEC_PDU_CPLLEN_V(2) |632 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |633 CPL_TX_SEC_PDU_IVINSRTOFST_V(634 ivinoffset));635 636 wr->req.sec_cpl.pldlen = htonl(skb->len + esnlen);637 aadstart = sa_entry->esn ? 1 : (skb_transport_offset(skb) + 1);638 aadstop = sa_entry->esn ? ESN_IV_INSERT_OFFSET :639 (skb_transport_offset(skb) +640 sizeof(struct ip_esp_hdr));641 ciphstart = skb_transport_offset(skb) + sizeof(struct ip_esp_hdr) +642 GCM_ESP_IV_SIZE + 1;643 ciphstart += sa_entry->esn ? esnlen : 0;644 645 wr->req.sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(646 aadstart,647 aadstop,648 ciphstart, 0);649 650 wr->req.sec_cpl.cipherstop_lo_authinsert =651 FILL_SEC_CPL_AUTHINSERT(0, ciphstart,652 sa_entry->authsize,653 sa_entry->authsize);654 wr->req.sec_cpl.seqno_numivs =655 FILL_SEC_CPL_SCMD0_SEQNO(CHCR_ENCRYPT_OP, 1,656 CHCR_SCMD_CIPHER_MODE_AES_GCM,657 CHCR_SCMD_AUTH_MODE_GHASH,658 sa_entry->hmac_ctrl,659 ivsize >> 1);660 wr->req.sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,661 0, ivdrop, 0);662 663 pos += sizeof(struct fw_ulptx_wr) +664 sizeof(struct ulp_txpkt) +665 sizeof(struct ulptx_idata) +666 sizeof(struct cpl_tx_sec_pdu);667 668 pos = copy_key_cpltx_pktxt(skb, dev, pos, sa_entry);669 670 return pos;671}672 673/**674 * flits_to_desc - returns the num of Tx descriptors for the given flits675 * @n: the number of flits676 *677 * Returns the number of Tx descriptors needed for the supplied number678 * of flits.679 */680static unsigned int flits_to_desc(unsigned int n)681{682 WARN_ON(n > SGE_MAX_WR_LEN / 8);683 return DIV_ROUND_UP(n, 8);684}685 686static unsigned int txq_avail(const struct sge_txq *q)687{688 return q->size - 1 - q->in_use;689}690 691static void eth_txq_stop(struct sge_eth_txq *q)692{693 netif_tx_stop_queue(q->txq);694 q->q.stops++;695}696 697static void txq_advance(struct sge_txq *q, unsigned int n)698{699 q->in_use += n;700 q->pidx += n;701 if (q->pidx >= q->size)702 q->pidx -= q->size;703}704 705/*706 * ch_ipsec_xmit called from ULD Tx handler707 */708int ch_ipsec_xmit(struct sk_buff *skb, struct net_device *dev)709{710 struct xfrm_state *x = xfrm_input_state(skb);711 unsigned int last_desc, ndesc, flits = 0;712 struct ipsec_sa_entry *sa_entry;713 u64 *pos, *end, *before, *sgl;714 struct tx_sw_desc *sgl_sdesc;715 int qidx, left, credits;716 bool immediate = false;717 struct sge_eth_txq *q;718 struct adapter *adap;719 struct port_info *pi;720 struct sec_path *sp;721 722 if (!x->xso.offload_handle)723 return NETDEV_TX_BUSY;724 725 sa_entry = (struct ipsec_sa_entry *)x->xso.offload_handle;726 727 sp = skb_sec_path(skb);728 if (sp->len != 1) {729out_free: dev_kfree_skb_any(skb);730 return NETDEV_TX_OK;731 }732 733 pi = netdev_priv(dev);734 adap = pi->adapter;735 qidx = skb->queue_mapping;736 q = &adap->sge.ethtxq[qidx + pi->first_qset];737 738 cxgb4_reclaim_completed_tx(adap, &q->q, true);739 740 flits = calc_tx_sec_flits(skb, sa_entry, &immediate);741 ndesc = flits_to_desc(flits);742 credits = txq_avail(&q->q) - ndesc;743 744 if (unlikely(credits < 0)) {745 eth_txq_stop(q);746 dev_err(adap->pdev_dev,747 "%s: Tx ring %u full while queue awake! cred:%d %d %d flits:%d\n",748 dev->name, qidx, credits, ndesc, txq_avail(&q->q),749 flits);750 return NETDEV_TX_BUSY;751 }752 753 last_desc = q->q.pidx + ndesc - 1;754 if (last_desc >= q->q.size)755 last_desc -= q->q.size;756 sgl_sdesc = &q->q.sdesc[last_desc];757 758 if (!immediate &&759 unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {760 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));761 q->mapping_err++;762 goto out_free;763 }764 765 pos = (u64 *)&q->q.desc[q->q.pidx];766 before = (u64 *)pos;767 end = (u64 *)pos + flits;768 /* Setup IPSec CPL */769 pos = (void *)ch_ipsec_crypto_wreq(skb, dev, (void *)pos,770 credits, sa_entry);771 if (before > (u64 *)pos) {772 left = (u8 *)end - (u8 *)q->q.stat;773 end = (void *)q->q.desc + left;774 }775 if (pos == (u64 *)q->q.stat) {776 left = (u8 *)end - (u8 *)q->q.stat;777 end = (void *)q->q.desc + left;778 pos = (void *)q->q.desc;779 }780 781 sgl = (void *)pos;782 if (immediate) {783 cxgb4_inline_tx_skb(skb, &q->q, sgl);784 dev_consume_skb_any(skb);785 } else {786 cxgb4_write_sgl(skb, &q->q, (void *)sgl, end,787 0, sgl_sdesc->addr);788 skb_orphan(skb);789 sgl_sdesc->skb = skb;790 }791 txq_advance(&q->q, ndesc);792 793 cxgb4_ring_tx_db(adap, &q->q, ndesc);794 return NETDEV_TX_OK;795}796 797static int __init ch_ipsec_init(void)798{799 cxgb4_register_uld(CXGB4_ULD_IPSEC, &ch_ipsec_uld_info);800 801 return 0;802}803 804static void __exit ch_ipsec_exit(void)805{806 struct ipsec_uld_ctx *u_ctx, *tmp;807 struct adapter *adap;808 809 mutex_lock(&dev_mutex);810 list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {811 adap = pci_get_drvdata(u_ctx->lldi.pdev);812 atomic_set(&adap->ch_ipsec_stats.ipsec_cnt, 0);813 list_del(&u_ctx->entry);814 kfree(u_ctx);815 }816 mutex_unlock(&dev_mutex);817 cxgb4_unregister_uld(CXGB4_ULD_IPSEC);818}819 820module_init(ch_ipsec_init);821module_exit(ch_ipsec_exit);822 823MODULE_DESCRIPTION("Crypto IPSEC for Chelsio Terminator cards.");824MODULE_LICENSE("GPL");825MODULE_AUTHOR("Chelsio Communications");826MODULE_VERSION(CHIPSEC_DRV_VERSION);827 828