2541 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB2/* Copyright (c) 2015 - 2021 Intel Corporation */3#include "main.h"4 5/**6 * irdma_arp_table -manage arp table7 * @rf: RDMA PCI function8 * @ip_addr: ip address for device9 * @ipv4: IPv4 flag10 * @mac_addr: mac address ptr11 * @action: modify, delete or add12 */13int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,14 const u8 *mac_addr, u32 action)15{16 unsigned long flags;17 int arp_index;18 u32 ip[4] = {};19 20 if (ipv4)21 ip[0] = *ip_addr;22 else23 memcpy(ip, ip_addr, sizeof(ip));24 25 spin_lock_irqsave(&rf->arp_lock, flags);26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))28 break;29 }30 31 switch (action) {32 case IRDMA_ARP_ADD:33 if (arp_index != rf->arp_table_size) {34 arp_index = -1;35 break;36 }37 38 arp_index = 0;39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,40 (u32 *)&arp_index, &rf->next_arp_index)) {41 arp_index = -1;42 break;43 }44 45 memcpy(rf->arp_table[arp_index].ip_addr, ip,46 sizeof(rf->arp_table[arp_index].ip_addr));47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);48 break;49 case IRDMA_ARP_RESOLVE:50 if (arp_index == rf->arp_table_size)51 arp_index = -1;52 break;53 case IRDMA_ARP_DELETE:54 if (arp_index == rf->arp_table_size) {55 arp_index = -1;56 break;57 }58 59 memset(rf->arp_table[arp_index].ip_addr, 0,60 sizeof(rf->arp_table[arp_index].ip_addr));61 eth_zero_addr(rf->arp_table[arp_index].mac_addr);62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index);63 break;64 default:65 arp_index = -1;66 break;67 }68 69 spin_unlock_irqrestore(&rf->arp_lock, flags);70 return arp_index;71}72 73/**74 * irdma_add_arp - add a new arp entry if needed75 * @rf: RDMA function76 * @ip: IP address77 * @ipv4: IPv4 flag78 * @mac: MAC address79 */80int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac)81{82 int arpidx;83 84 arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);85 if (arpidx >= 0) {86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))87 return arpidx;88 89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,90 ipv4, IRDMA_ARP_DELETE);91 }92 93 irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);94 95 return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);96}97 98/**99 * wr32 - write 32 bits to hw register100 * @hw: hardware information including registers101 * @reg: register offset102 * @val: value to write to register103 */104inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)105{106 writel(val, hw->hw_addr + reg);107}108 109/**110 * rd32 - read a 32 bit hw register111 * @hw: hardware information including registers112 * @reg: register offset113 *114 * Return value of register content115 */116inline u32 rd32(struct irdma_hw *hw, u32 reg)117{118 return readl(hw->hw_addr + reg);119}120 121/**122 * rd64 - read a 64 bit hw register123 * @hw: hardware information including registers124 * @reg: register offset125 *126 * Return value of register content127 */128inline u64 rd64(struct irdma_hw *hw, u32 reg)129{130 return readq(hw->hw_addr + reg);131}132 133static void irdma_gid_change_event(struct ib_device *ibdev)134{135 struct ib_event ib_event;136 137 ib_event.event = IB_EVENT_GID_CHANGE;138 ib_event.device = ibdev;139 ib_event.element.port_num = 1;140 ib_dispatch_event(&ib_event);141}142 143/**144 * irdma_inetaddr_event - system notifier for ipv4 addr events145 * @notifier: not used146 * @event: event for notifier147 * @ptr: if address148 */149int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,150 void *ptr)151{152 struct in_ifaddr *ifa = ptr;153 struct net_device *real_dev, *netdev = ifa->ifa_dev->dev;154 struct irdma_device *iwdev;155 struct ib_device *ibdev;156 u32 local_ipaddr;157 158 real_dev = rdma_vlan_dev_real_dev(netdev);159 if (!real_dev)160 real_dev = netdev;161 162 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);163 if (!ibdev)164 return NOTIFY_DONE;165 166 iwdev = to_iwdev(ibdev);167 local_ipaddr = ntohl(ifa->ifa_address);168 ibdev_dbg(&iwdev->ibdev,169 "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,170 event, &local_ipaddr, real_dev->dev_addr);171 switch (event) {172 case NETDEV_DOWN:173 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,174 &local_ipaddr, true, IRDMA_ARP_DELETE);175 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false);176 irdma_gid_change_event(&iwdev->ibdev);177 break;178 case NETDEV_UP:179 case NETDEV_CHANGEADDR:180 irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr);181 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true);182 irdma_gid_change_event(&iwdev->ibdev);183 break;184 default:185 break;186 }187 188 ib_device_put(ibdev);189 190 return NOTIFY_DONE;191}192 193/**194 * irdma_inet6addr_event - system notifier for ipv6 addr events195 * @notifier: not used196 * @event: event for notifier197 * @ptr: if address198 */199int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,200 void *ptr)201{202 struct inet6_ifaddr *ifa = ptr;203 struct net_device *real_dev, *netdev = ifa->idev->dev;204 struct irdma_device *iwdev;205 struct ib_device *ibdev;206 u32 local_ipaddr6[4];207 208 real_dev = rdma_vlan_dev_real_dev(netdev);209 if (!real_dev)210 real_dev = netdev;211 212 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);213 if (!ibdev)214 return NOTIFY_DONE;215 216 iwdev = to_iwdev(ibdev);217 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);218 ibdev_dbg(&iwdev->ibdev,219 "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,220 event, local_ipaddr6, real_dev->dev_addr);221 switch (event) {222 case NETDEV_DOWN:223 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,224 local_ipaddr6, false, IRDMA_ARP_DELETE);225 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false);226 irdma_gid_change_event(&iwdev->ibdev);227 break;228 case NETDEV_UP:229 case NETDEV_CHANGEADDR:230 irdma_add_arp(iwdev->rf, local_ipaddr6, false,231 real_dev->dev_addr);232 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true);233 irdma_gid_change_event(&iwdev->ibdev);234 break;235 default:236 break;237 }238 239 ib_device_put(ibdev);240 241 return NOTIFY_DONE;242}243 244/**245 * irdma_net_event - system notifier for net events246 * @notifier: not used247 * @event: event for notifier248 * @ptr: neighbor249 */250int irdma_net_event(struct notifier_block *notifier, unsigned long event,251 void *ptr)252{253 struct neighbour *neigh = ptr;254 struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev;255 struct irdma_device *iwdev;256 struct ib_device *ibdev;257 __be32 *p;258 u32 local_ipaddr[4] = {};259 bool ipv4 = true;260 261 switch (event) {262 case NETEVENT_NEIGH_UPDATE:263 real_dev = rdma_vlan_dev_real_dev(netdev);264 if (!real_dev)265 real_dev = netdev;266 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);267 if (!ibdev)268 return NOTIFY_DONE;269 270 iwdev = to_iwdev(ibdev);271 p = (__be32 *)neigh->primary_key;272 if (neigh->tbl->family == AF_INET6) {273 ipv4 = false;274 irdma_copy_ip_ntohl(local_ipaddr, p);275 } else {276 local_ipaddr[0] = ntohl(*p);277 }278 279 ibdev_dbg(&iwdev->ibdev,280 "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",281 iwdev->netdev, neigh->nud_state, local_ipaddr,282 neigh->ha);283 284 if (neigh->nud_state & NUD_VALID)285 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha);286 287 else288 irdma_manage_arp_cache(iwdev->rf, neigh->ha,289 local_ipaddr, ipv4,290 IRDMA_ARP_DELETE);291 ib_device_put(ibdev);292 break;293 default:294 break;295 }296 297 return NOTIFY_DONE;298}299 300/**301 * irdma_netdevice_event - system notifier for netdev events302 * @notifier: not used303 * @event: event for notifier304 * @ptr: netdev305 */306int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,307 void *ptr)308{309 struct irdma_device *iwdev;310 struct ib_device *ibdev;311 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);312 313 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);314 if (!ibdev)315 return NOTIFY_DONE;316 317 iwdev = to_iwdev(ibdev);318 iwdev->iw_status = 1;319 switch (event) {320 case NETDEV_DOWN:321 iwdev->iw_status = 0;322 fallthrough;323 case NETDEV_UP:324 irdma_port_ibevent(iwdev);325 break;326 default:327 break;328 }329 ib_device_put(ibdev);330 331 return NOTIFY_DONE;332}333 334/**335 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table336 * @iwdev: irdma device337 */338static void irdma_add_ipv6_addr(struct irdma_device *iwdev)339{340 struct net_device *ip_dev;341 struct inet6_dev *idev;342 struct inet6_ifaddr *ifp, *tmp;343 u32 local_ipaddr6[4];344 345 rcu_read_lock();346 for_each_netdev_rcu (&init_net, ip_dev) {347 if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF &&348 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) ||349 ip_dev == iwdev->netdev) &&350 (READ_ONCE(ip_dev->flags) & IFF_UP)) {351 idev = __in6_dev_get(ip_dev);352 if (!idev) {353 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n");354 break;355 }356 list_for_each_entry_safe (ifp, tmp, &idev->addr_list,357 if_list) {358 ibdev_dbg(&iwdev->ibdev,359 "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",360 &ifp->addr,361 rdma_vlan_dev_vlan_id(ip_dev),362 ip_dev->dev_addr);363 364 irdma_copy_ip_ntohl(local_ipaddr6,365 ifp->addr.in6_u.u6_addr32);366 irdma_manage_arp_cache(iwdev->rf,367 ip_dev->dev_addr,368 local_ipaddr6, false,369 IRDMA_ARP_ADD);370 }371 }372 }373 rcu_read_unlock();374}375 376/**377 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table378 * @iwdev: irdma device379 */380static void irdma_add_ipv4_addr(struct irdma_device *iwdev)381{382 struct net_device *dev;383 struct in_device *idev;384 u32 ip_addr;385 386 rcu_read_lock();387 for_each_netdev_rcu (&init_net, dev) {388 if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF &&389 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) ||390 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) {391 const struct in_ifaddr *ifa;392 393 idev = __in_dev_get_rcu(dev);394 if (!idev)395 continue;396 397 in_dev_for_each_ifa_rcu(ifa, idev) {398 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",399 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev),400 dev->dev_addr);401 402 ip_addr = ntohl(ifa->ifa_address);403 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr,404 &ip_addr, true,405 IRDMA_ARP_ADD);406 }407 }408 }409 rcu_read_unlock();410}411 412/**413 * irdma_add_ip - add ip addresses414 * @iwdev: irdma device415 *416 * Add ipv4/ipv6 addresses to the arp cache417 */418void irdma_add_ip(struct irdma_device *iwdev)419{420 irdma_add_ipv4_addr(iwdev);421 irdma_add_ipv6_addr(iwdev);422}423 424/**425 * irdma_alloc_and_get_cqp_request - get cqp struct426 * @cqp: device cqp ptr427 * @wait: cqp to be used in wait mode428 */429struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,430 bool wait)431{432 struct irdma_cqp_request *cqp_request = NULL;433 unsigned long flags;434 435 spin_lock_irqsave(&cqp->req_lock, flags);436 if (!list_empty(&cqp->cqp_avail_reqs)) {437 cqp_request = list_first_entry(&cqp->cqp_avail_reqs,438 struct irdma_cqp_request, list);439 list_del_init(&cqp_request->list);440 }441 spin_unlock_irqrestore(&cqp->req_lock, flags);442 if (!cqp_request) {443 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);444 if (cqp_request) {445 cqp_request->dynamic = true;446 if (wait)447 init_waitqueue_head(&cqp_request->waitq);448 }449 }450 if (!cqp_request) {451 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory");452 return NULL;453 }454 455 cqp_request->waiting = wait;456 refcount_set(&cqp_request->refcnt, 1);457 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));458 459 return cqp_request;460}461 462/**463 * irdma_get_cqp_request - increase refcount for cqp_request464 * @cqp_request: pointer to cqp_request instance465 */466static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)467{468 refcount_inc(&cqp_request->refcnt);469}470 471/**472 * irdma_free_cqp_request - free cqp request473 * @cqp: cqp ptr474 * @cqp_request: to be put back in cqp list475 */476void irdma_free_cqp_request(struct irdma_cqp *cqp,477 struct irdma_cqp_request *cqp_request)478{479 unsigned long flags;480 481 if (cqp_request->dynamic) {482 kfree(cqp_request);483 } else {484 WRITE_ONCE(cqp_request->request_done, false);485 cqp_request->callback_fcn = NULL;486 cqp_request->waiting = false;487 488 spin_lock_irqsave(&cqp->req_lock, flags);489 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);490 spin_unlock_irqrestore(&cqp->req_lock, flags);491 }492 wake_up(&cqp->remove_wq);493}494 495/**496 * irdma_put_cqp_request - dec ref count and free if 0497 * @cqp: cqp ptr498 * @cqp_request: to be put back in cqp list499 */500void irdma_put_cqp_request(struct irdma_cqp *cqp,501 struct irdma_cqp_request *cqp_request)502{503 if (refcount_dec_and_test(&cqp_request->refcnt))504 irdma_free_cqp_request(cqp, cqp_request);505}506 507/**508 * irdma_free_pending_cqp_request -free pending cqp request objs509 * @cqp: cqp ptr510 * @cqp_request: to be put back in cqp list511 */512static void513irdma_free_pending_cqp_request(struct irdma_cqp *cqp,514 struct irdma_cqp_request *cqp_request)515{516 if (cqp_request->waiting) {517 cqp_request->compl_info.error = true;518 WRITE_ONCE(cqp_request->request_done, true);519 wake_up(&cqp_request->waitq);520 }521 wait_event_timeout(cqp->remove_wq,522 refcount_read(&cqp_request->refcnt) == 1, 1000);523 irdma_put_cqp_request(cqp, cqp_request);524}525 526/**527 * irdma_cleanup_pending_cqp_op - clean-up cqp with no528 * completions529 * @rf: RDMA PCI function530 */531void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)532{533 struct irdma_sc_dev *dev = &rf->sc_dev;534 struct irdma_cqp *cqp = &rf->cqp;535 struct irdma_cqp_request *cqp_request = NULL;536 struct cqp_cmds_info *pcmdinfo = NULL;537 u32 i, pending_work, wqe_idx;538 539 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);540 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);541 for (i = 0; i < pending_work; i++) {542 cqp_request = (struct irdma_cqp_request *)(unsigned long)543 cqp->scratch_array[wqe_idx];544 if (cqp_request)545 irdma_free_pending_cqp_request(cqp, cqp_request);546 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);547 }548 549 while (!list_empty(&dev->cqp_cmd_head)) {550 pcmdinfo = irdma_remove_cqp_head(dev);551 cqp_request =552 container_of(pcmdinfo, struct irdma_cqp_request, info);553 if (cqp_request)554 irdma_free_pending_cqp_request(cqp, cqp_request);555 }556}557 558/**559 * irdma_wait_event - wait for completion560 * @rf: RDMA PCI function561 * @cqp_request: cqp request to wait562 */563static int irdma_wait_event(struct irdma_pci_f *rf,564 struct irdma_cqp_request *cqp_request)565{566 struct irdma_cqp_timeout cqp_timeout = {};567 bool cqp_error = false;568 int err_code = 0;569 570 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);571 do {572 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);573 if (wait_event_timeout(cqp_request->waitq,574 READ_ONCE(cqp_request->request_done),575 msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))576 break;577 578 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);579 580 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)581 continue;582 583 if (!rf->reset) {584 rf->reset = true;585 rf->gen_ops.request_reset(rf);586 }587 return -ETIMEDOUT;588 } while (1);589 590 cqp_error = cqp_request->compl_info.error;591 if (cqp_error) {592 err_code = -EIO;593 if (cqp_request->compl_info.maj_err_code == 0xFFFF) {594 if (cqp_request->compl_info.min_err_code == 0x8002)595 err_code = -EBUSY;596 else if (cqp_request->compl_info.min_err_code == 0x8029) {597 if (!rf->reset) {598 rf->reset = true;599 rf->gen_ops.request_reset(rf);600 }601 }602 }603 }604 605 return err_code;606}607 608static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {609 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",610 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",611 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",612 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",613 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",614 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",615 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",616 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",617 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",618 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",619 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",620 [IRDMA_OP_QP_CREATE] = "Create QP Cmd",621 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",622 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",623 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",624 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",625 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",626 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",627 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",628 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",629 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",630 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",631 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd",632 [IRDMA_OP_RESUME] = "Resume QP Cmd",633 [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",634 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",635 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",636 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",637 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",638 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",639 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",640 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",641 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",642 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",643 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",644 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",645 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",646 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",647 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",648 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",649 [IRDMA_OP_GEN_AE] = "Generate AE Cmd",650 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",651 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",652 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",653 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",654 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",655};656 657static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {658 {0xffff, 0x8002, "Invalid State"},659 {0xffff, 0x8006, "Flush No Wqe Pending"},660 {0xffff, 0x8007, "Modify QP Bad Close"},661 {0xffff, 0x8009, "LLP Closed"},662 {0xffff, 0x800a, "Reset Not Sent"}663};664 665/**666 * irdma_cqp_crit_err - check if CQP error is critical667 * @dev: pointer to dev structure668 * @cqp_cmd: code for last CQP operation669 * @maj_err_code: major error code670 * @min_err_code: minot error code671 */672bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,673 u16 maj_err_code, u16 min_err_code)674{675 int i;676 677 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {678 if (maj_err_code == irdma_noncrit_err_list[i].maj &&679 min_err_code == irdma_noncrit_err_list[i].min) {680 ibdev_dbg(to_ibdev(dev),681 "CQP: [%s Error][%s] maj=0x%x min=0x%x\n",682 irdma_noncrit_err_list[i].desc,683 irdma_cqp_cmd_names[cqp_cmd], maj_err_code,684 min_err_code);685 return false;686 }687 }688 return true;689}690 691/**692 * irdma_handle_cqp_op - process cqp command693 * @rf: RDMA PCI function694 * @cqp_request: cqp request to process695 */696int irdma_handle_cqp_op(struct irdma_pci_f *rf,697 struct irdma_cqp_request *cqp_request)698{699 struct irdma_sc_dev *dev = &rf->sc_dev;700 struct cqp_cmds_info *info = &cqp_request->info;701 int status;702 bool put_cqp_request = true;703 704 if (rf->reset)705 return -EBUSY;706 707 irdma_get_cqp_request(cqp_request);708 status = irdma_process_cqp_cmd(dev, info);709 if (status)710 goto err;711 712 if (cqp_request->waiting) {713 put_cqp_request = false;714 status = irdma_wait_event(rf, cqp_request);715 if (status)716 goto err;717 }718 719 return 0;720 721err:722 if (irdma_cqp_crit_err(dev, info->cqp_cmd,723 cqp_request->compl_info.maj_err_code,724 cqp_request->compl_info.min_err_code))725 ibdev_err(&rf->iwdev->ibdev,726 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",727 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,728 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,729 cqp_request->compl_info.min_err_code);730 731 if (put_cqp_request)732 irdma_put_cqp_request(&rf->cqp, cqp_request);733 734 return status;735}736 737void irdma_qp_add_ref(struct ib_qp *ibqp)738{739 struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;740 741 refcount_inc(&iwqp->refcnt);742}743 744void irdma_qp_rem_ref(struct ib_qp *ibqp)745{746 struct irdma_qp *iwqp = to_iwqp(ibqp);747 struct irdma_device *iwdev = iwqp->iwdev;748 u32 qp_num;749 unsigned long flags;750 751 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);752 if (!refcount_dec_and_test(&iwqp->refcnt)) {753 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);754 return;755 }756 757 qp_num = iwqp->ibqp.qp_num;758 iwdev->rf->qp_table[qp_num] = NULL;759 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);760 complete(&iwqp->free_qp);761}762 763void irdma_cq_add_ref(struct ib_cq *ibcq)764{765 struct irdma_cq *iwcq = to_iwcq(ibcq);766 767 refcount_inc(&iwcq->refcnt);768}769 770void irdma_cq_rem_ref(struct ib_cq *ibcq)771{772 struct ib_device *ibdev = ibcq->device;773 struct irdma_device *iwdev = to_iwdev(ibdev);774 struct irdma_cq *iwcq = to_iwcq(ibcq);775 unsigned long flags;776 777 spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags);778 if (!refcount_dec_and_test(&iwcq->refcnt)) {779 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);780 return;781 }782 783 iwdev->rf->cq_table[iwcq->cq_num] = NULL;784 spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);785 complete(&iwcq->free_cq);786}787 788struct ib_device *to_ibdev(struct irdma_sc_dev *dev)789{790 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;791}792 793/**794 * irdma_get_qp - get qp address795 * @device: iwarp device796 * @qpn: qp number797 */798struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)799{800 struct irdma_device *iwdev = to_iwdev(device);801 802 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)803 return NULL;804 805 return &iwdev->rf->qp_table[qpn]->ibqp;806}807 808/**809 * irdma_remove_cqp_head - return head entry and remove810 * @dev: device811 */812void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)813{814 struct list_head *entry;815 struct list_head *list = &dev->cqp_cmd_head;816 817 if (list_empty(list))818 return NULL;819 820 entry = list->next;821 list_del(entry);822 823 return entry;824}825 826/**827 * irdma_cqp_sds_cmd - create cqp command for sd828 * @dev: hardware control device structure829 * @sdinfo: information for sd cqp830 *831 */832int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,833 struct irdma_update_sds_info *sdinfo)834{835 struct irdma_cqp_request *cqp_request;836 struct cqp_cmds_info *cqp_info;837 struct irdma_pci_f *rf = dev_to_rf(dev);838 int status;839 840 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);841 if (!cqp_request)842 return -ENOMEM;843 844 cqp_info = &cqp_request->info;845 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,846 sizeof(cqp_info->in.u.update_pe_sds.info));847 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;848 cqp_info->post_sq = 1;849 cqp_info->in.u.update_pe_sds.dev = dev;850 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;851 852 status = irdma_handle_cqp_op(rf, cqp_request);853 irdma_put_cqp_request(&rf->cqp, cqp_request);854 855 return status;856}857 858/**859 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume860 * @qp: hardware control qp861 * @op: suspend or resume862 */863int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)864{865 struct irdma_sc_dev *dev = qp->dev;866 struct irdma_cqp_request *cqp_request;867 struct irdma_sc_cqp *cqp = dev->cqp;868 struct cqp_cmds_info *cqp_info;869 struct irdma_pci_f *rf = dev_to_rf(dev);870 int status;871 872 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);873 if (!cqp_request)874 return -ENOMEM;875 876 cqp_info = &cqp_request->info;877 cqp_info->cqp_cmd = op;878 cqp_info->in.u.suspend_resume.cqp = cqp;879 cqp_info->in.u.suspend_resume.qp = qp;880 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;881 882 status = irdma_handle_cqp_op(rf, cqp_request);883 irdma_put_cqp_request(&rf->cqp, cqp_request);884 885 return status;886}887 888/**889 * irdma_term_modify_qp - modify qp for term message890 * @qp: hardware control qp891 * @next_state: qp's next state892 * @term: terminate code893 * @term_len: length894 */895void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,896 u8 term_len)897{898 struct irdma_qp *iwqp;899 900 iwqp = qp->qp_uk.back_qp;901 irdma_next_iw_state(iwqp, next_state, 0, term, term_len);902};903 904/**905 * irdma_terminate_done - after terminate is completed906 * @qp: hardware control qp907 * @timeout_occurred: indicates if terminate timer expired908 */909void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)910{911 struct irdma_qp *iwqp;912 u8 hte = 0;913 bool first_time;914 unsigned long flags;915 916 iwqp = qp->qp_uk.back_qp;917 spin_lock_irqsave(&iwqp->lock, flags);918 if (iwqp->hte_added) {919 iwqp->hte_added = 0;920 hte = 1;921 }922 first_time = !(qp->term_flags & IRDMA_TERM_DONE);923 qp->term_flags |= IRDMA_TERM_DONE;924 spin_unlock_irqrestore(&iwqp->lock, flags);925 if (first_time) {926 if (!timeout_occurred)927 irdma_terminate_del_timer(qp);928 929 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);930 irdma_cm_disconn(iwqp);931 }932}933 934static void irdma_terminate_timeout(struct timer_list *t)935{936 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);937 struct irdma_sc_qp *qp = &iwqp->sc_qp;938 939 irdma_terminate_done(qp, 1);940 irdma_qp_rem_ref(&iwqp->ibqp);941}942 943/**944 * irdma_terminate_start_timer - start terminate timeout945 * @qp: hardware control qp946 */947void irdma_terminate_start_timer(struct irdma_sc_qp *qp)948{949 struct irdma_qp *iwqp;950 951 iwqp = qp->qp_uk.back_qp;952 irdma_qp_add_ref(&iwqp->ibqp);953 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);954 iwqp->terminate_timer.expires = jiffies + HZ;955 956 add_timer(&iwqp->terminate_timer);957}958 959/**960 * irdma_terminate_del_timer - delete terminate timeout961 * @qp: hardware control qp962 */963void irdma_terminate_del_timer(struct irdma_sc_qp *qp)964{965 struct irdma_qp *iwqp;966 int ret;967 968 iwqp = qp->qp_uk.back_qp;969 ret = del_timer(&iwqp->terminate_timer);970 if (ret)971 irdma_qp_rem_ref(&iwqp->ibqp);972}973 974/**975 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm976 * @dev: function device struct977 * @val_mem: buffer for fpm978 * @hmc_fn_id: function id for fpm979 */980int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,981 struct irdma_dma_mem *val_mem, u8 hmc_fn_id)982{983 struct irdma_cqp_request *cqp_request;984 struct cqp_cmds_info *cqp_info;985 struct irdma_pci_f *rf = dev_to_rf(dev);986 int status;987 988 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);989 if (!cqp_request)990 return -ENOMEM;991 992 cqp_info = &cqp_request->info;993 cqp_request->param = NULL;994 cqp_info->in.u.query_fpm_val.cqp = dev->cqp;995 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa;996 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va;997 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id;998 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL;999 cqp_info->post_sq = 1;1000 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;1001 1002 status = irdma_handle_cqp_op(rf, cqp_request);1003 irdma_put_cqp_request(&rf->cqp, cqp_request);1004 1005 return status;1006}1007 1008/**1009 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw1010 * @dev: hardware control device structure1011 * @val_mem: buffer with fpm values1012 * @hmc_fn_id: function id for fpm1013 */1014int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,1015 struct irdma_dma_mem *val_mem, u8 hmc_fn_id)1016{1017 struct irdma_cqp_request *cqp_request;1018 struct cqp_cmds_info *cqp_info;1019 struct irdma_pci_f *rf = dev_to_rf(dev);1020 int status;1021 1022 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);1023 if (!cqp_request)1024 return -ENOMEM;1025 1026 cqp_info = &cqp_request->info;1027 cqp_request->param = NULL;1028 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp;1029 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa;1030 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va;1031 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id;1032 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL;1033 cqp_info->post_sq = 1;1034 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;1035 1036 status = irdma_handle_cqp_op(rf, cqp_request);1037 irdma_put_cqp_request(&rf->cqp, cqp_request);1038 1039 return status;1040}1041 1042/**1043 * irdma_cqp_cq_create_cmd - create a cq for the cqp1044 * @dev: device pointer1045 * @cq: pointer to created cq1046 */1047int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)1048{1049 struct irdma_pci_f *rf = dev_to_rf(dev);1050 struct irdma_cqp *iwcqp = &rf->cqp;1051 struct irdma_cqp_request *cqp_request;1052 struct cqp_cmds_info *cqp_info;1053 int status;1054 1055 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);1056 if (!cqp_request)1057 return -ENOMEM;1058 1059 cqp_info = &cqp_request->info;1060 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;1061 cqp_info->post_sq = 1;1062 cqp_info->in.u.cq_create.cq = cq;1063 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;1064 1065 status = irdma_handle_cqp_op(rf, cqp_request);1066 irdma_put_cqp_request(iwcqp, cqp_request);1067 1068 return status;1069}1070 1071/**1072 * irdma_cqp_qp_create_cmd - create a qp for the cqp1073 * @dev: device pointer1074 * @qp: pointer to created qp1075 */1076int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)1077{1078 struct irdma_pci_f *rf = dev_to_rf(dev);1079 struct irdma_cqp *iwcqp = &rf->cqp;1080 struct irdma_cqp_request *cqp_request;1081 struct cqp_cmds_info *cqp_info;1082 struct irdma_create_qp_info *qp_info;1083 int status;1084 1085 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);1086 if (!cqp_request)1087 return -ENOMEM;1088 1089 cqp_info = &cqp_request->info;1090 qp_info = &cqp_request->info.in.u.qp_create.info;1091 memset(qp_info, 0, sizeof(*qp_info));1092 qp_info->cq_num_valid = true;1093 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;1094 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;1095 cqp_info->post_sq = 1;1096 cqp_info->in.u.qp_create.qp = qp;1097 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;1098 1099 status = irdma_handle_cqp_op(rf, cqp_request);1100 irdma_put_cqp_request(iwcqp, cqp_request);1101 1102 return status;1103}1104 1105/**1106 * irdma_dealloc_push_page - free a push page for qp1107 * @rf: RDMA PCI function1108 * @qp: hardware control qp1109 */1110static void irdma_dealloc_push_page(struct irdma_pci_f *rf,1111 struct irdma_sc_qp *qp)1112{1113 struct irdma_cqp_request *cqp_request;1114 struct cqp_cmds_info *cqp_info;1115 int status;1116 1117 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)1118 return;1119 1120 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);1121 if (!cqp_request)1122 return;1123 1124 cqp_info = &cqp_request->info;1125 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;1126 cqp_info->post_sq = 1;1127 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;1128 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;1129 cqp_info->in.u.manage_push_page.info.free_page = 1;1130 cqp_info->in.u.manage_push_page.info.push_page_type = 0;1131 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;1132 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;1133 status = irdma_handle_cqp_op(rf, cqp_request);1134 if (!status)1135 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;1136 irdma_put_cqp_request(&rf->cqp, cqp_request);1137}1138 1139/**1140 * irdma_free_qp_rsrc - free up memory resources for qp1141 * @iwqp: qp ptr (user or kernel)1142 */1143void irdma_free_qp_rsrc(struct irdma_qp *iwqp)1144{1145 struct irdma_device *iwdev = iwqp->iwdev;1146 struct irdma_pci_f *rf = iwdev->rf;1147 u32 qp_num = iwqp->ibqp.qp_num;1148 1149 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);1150 irdma_dealloc_push_page(rf, &iwqp->sc_qp);1151 if (iwqp->sc_qp.vsi) {1152 irdma_qp_rem_qos(&iwqp->sc_qp);1153 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,1154 iwqp->sc_qp.user_pri);1155 }1156 1157 if (qp_num > 2)1158 irdma_free_rsrc(rf, rf->allocated_qps, qp_num);1159 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size,1160 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa);1161 iwqp->q2_ctx_mem.va = NULL;1162 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size,1163 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa);1164 iwqp->kqp.dma_mem.va = NULL;1165 kfree(iwqp->kqp.sq_wrid_mem);1166 kfree(iwqp->kqp.rq_wrid_mem);1167}1168 1169/**1170 * irdma_cq_wq_destroy - send cq destroy cqp1171 * @rf: RDMA PCI function1172 * @cq: hardware control cq1173 */1174void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)1175{1176 struct irdma_cqp_request *cqp_request;1177 struct cqp_cmds_info *cqp_info;1178 1179 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);1180 if (!cqp_request)1181 return;1182 1183 cqp_info = &cqp_request->info;1184 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;1185 cqp_info->post_sq = 1;1186 cqp_info->in.u.cq_destroy.cq = cq;1187 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;1188 1189 irdma_handle_cqp_op(rf, cqp_request);1190 irdma_put_cqp_request(&rf->cqp, cqp_request);1191}1192 1193/**1194 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait1195 * @cqp_request: modify QP completion1196 */1197static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)1198{1199 struct cqp_cmds_info *cqp_info;1200 struct irdma_qp *iwqp;1201 1202 cqp_info = &cqp_request->info;1203 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;1204 atomic_dec(&iwqp->hw_mod_qp_pend);1205 wake_up(&iwqp->mod_qp_waitq);1206}1207 1208/**1209 * irdma_hw_modify_qp - setup cqp for modify qp1210 * @iwdev: RDMA device1211 * @iwqp: qp ptr (user or kernel)1212 * @info: info for modify qp1213 * @wait: flag to wait or not for modify qp completion1214 */1215int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,1216 struct irdma_modify_qp_info *info, bool wait)1217{1218 int status;1219 struct irdma_pci_f *rf = iwdev->rf;1220 struct irdma_cqp_request *cqp_request;1221 struct cqp_cmds_info *cqp_info;1222 struct irdma_modify_qp_info *m_info;1223 1224 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);1225 if (!cqp_request)1226 return -ENOMEM;1227 1228 if (!wait) {1229 cqp_request->callback_fcn = irdma_hw_modify_qp_callback;1230 atomic_inc(&iwqp->hw_mod_qp_pend);1231 }1232 cqp_info = &cqp_request->info;1233 m_info = &cqp_info->in.u.qp_modify.info;1234 memcpy(m_info, info, sizeof(*m_info));1235 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;1236 cqp_info->post_sq = 1;1237 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;1238 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;1239 status = irdma_handle_cqp_op(rf, cqp_request);1240 irdma_put_cqp_request(&rf->cqp, cqp_request);1241 if (status) {1242 if (rdma_protocol_roce(&iwdev->ibdev, 1))1243 return status;1244 1245 switch (m_info->next_iwarp_state) {1246 struct irdma_gen_ae_info ae_info;1247 1248 case IRDMA_QP_STATE_RTS:1249 case IRDMA_QP_STATE_IDLE:1250 case IRDMA_QP_STATE_TERMINATE:1251 case IRDMA_QP_STATE_CLOSING:1252 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)1253 irdma_send_reset(iwqp->cm_node);1254 else1255 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;1256 if (!wait) {1257 ae_info.ae_code = IRDMA_AE_BAD_CLOSE;1258 ae_info.ae_src = 0;1259 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);1260 } else {1261 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,1262 wait);1263 if (!cqp_request)1264 return -ENOMEM;1265 1266 cqp_info = &cqp_request->info;1267 m_info = &cqp_info->in.u.qp_modify.info;1268 memcpy(m_info, info, sizeof(*m_info));1269 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;1270 cqp_info->post_sq = 1;1271 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;1272 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;1273 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;1274 m_info->reset_tcp_conn = true;1275 irdma_handle_cqp_op(rf, cqp_request);1276 irdma_put_cqp_request(&rf->cqp, cqp_request);1277 }1278 break;1279 case IRDMA_QP_STATE_ERROR:1280 default:1281 break;1282 }1283 }1284 1285 return status;1286}1287 1288/**1289 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq1290 * @dev: device pointer1291 * @cq: pointer to cq1292 */1293void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)1294{1295 struct irdma_pci_f *rf = dev_to_rf(dev);1296 1297 irdma_cq_wq_destroy(rf, cq);1298}1299 1300/**1301 * irdma_cqp_qp_destroy_cmd - destroy the cqp1302 * @dev: device pointer1303 * @qp: pointer to qp1304 */1305int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)1306{1307 struct irdma_pci_f *rf = dev_to_rf(dev);1308 struct irdma_cqp *iwcqp = &rf->cqp;1309 struct irdma_cqp_request *cqp_request;1310 struct cqp_cmds_info *cqp_info;1311 int status;1312 1313 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);1314 if (!cqp_request)1315 return -ENOMEM;1316 1317 cqp_info = &cqp_request->info;1318 memset(cqp_info, 0, sizeof(*cqp_info));1319 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;1320 cqp_info->post_sq = 1;1321 cqp_info->in.u.qp_destroy.qp = qp;1322 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;1323 cqp_info->in.u.qp_destroy.remove_hash_idx = true;1324 1325 status = irdma_handle_cqp_op(rf, cqp_request);1326 irdma_put_cqp_request(&rf->cqp, cqp_request);1327 1328 return status;1329}1330 1331/**1332 * irdma_ieq_mpa_crc_ae - generate AE for crc error1333 * @dev: hardware control device structure1334 * @qp: hardware control qp1335 */1336void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)1337{1338 struct irdma_gen_ae_info info = {};1339 struct irdma_pci_f *rf = dev_to_rf(dev);1340 1341 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n");1342 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;1343 info.ae_src = IRDMA_AE_SOURCE_RQ;1344 irdma_gen_ae(rf, qp, &info, false);1345}1346 1347/**1348 * irdma_init_hash_desc - initialize hash for crc calculation1349 * @desc: cryption type1350 */1351int irdma_init_hash_desc(struct shash_desc **desc)1352{1353 struct crypto_shash *tfm;1354 struct shash_desc *tdesc;1355 1356 tfm = crypto_alloc_shash("crc32c", 0, 0);1357 if (IS_ERR(tfm))1358 return -EINVAL;1359 1360 tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),1361 GFP_KERNEL);1362 if (!tdesc) {1363 crypto_free_shash(tfm);1364 return -EINVAL;1365 }1366 1367 tdesc->tfm = tfm;1368 *desc = tdesc;1369 1370 return 0;1371}1372 1373/**1374 * irdma_free_hash_desc - free hash desc1375 * @desc: to be freed1376 */1377void irdma_free_hash_desc(struct shash_desc *desc)1378{1379 if (desc) {1380 crypto_free_shash(desc->tfm);1381 kfree(desc);1382 }1383}1384 1385/**1386 * irdma_ieq_check_mpacrc - check if mpa crc is OK1387 * @desc: desc for hash1388 * @addr: address of buffer for crc1389 * @len: length of buffer1390 * @val: value to be compared1391 */1392int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len,1393 u32 val)1394{1395 u32 crc = 0;1396 1397 crypto_shash_digest(desc, addr, len, (u8 *)&crc);1398 if (crc != val)1399 return -EINVAL;1400 1401 return 0;1402}1403 1404/**1405 * irdma_ieq_get_qp - get qp based on quad in puda buffer1406 * @dev: hardware control device structure1407 * @buf: receive puda buffer on exception q1408 */1409struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,1410 struct irdma_puda_buf *buf)1411{1412 struct irdma_qp *iwqp;1413 struct irdma_cm_node *cm_node;1414 struct irdma_device *iwdev = buf->vsi->back_vsi;1415 u32 loc_addr[4] = {};1416 u32 rem_addr[4] = {};1417 u16 loc_port, rem_port;1418 struct ipv6hdr *ip6h;1419 struct iphdr *iph = (struct iphdr *)buf->iph;1420 struct tcphdr *tcph = (struct tcphdr *)buf->tcph;1421 1422 if (iph->version == 4) {1423 loc_addr[0] = ntohl(iph->daddr);1424 rem_addr[0] = ntohl(iph->saddr);1425 } else {1426 ip6h = (struct ipv6hdr *)buf->iph;1427 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);1428 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);1429 }1430 loc_port = ntohs(tcph->dest);1431 rem_port = ntohs(tcph->source);1432 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,1433 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);1434 if (!cm_node)1435 return NULL;1436 1437 iwqp = cm_node->iwqp;1438 irdma_rem_ref_cm_node(cm_node);1439 1440 return &iwqp->sc_qp;1441}1442 1443/**1444 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs1445 * @qp: qp ptr1446 */1447void irdma_send_ieq_ack(struct irdma_sc_qp *qp)1448{1449 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;1450 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;1451 struct tcphdr *tcph = (struct tcphdr *)buf->tcph;1452 1453 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;1454 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);1455 1456 irdma_send_ack(cm_node);1457}1458 1459/**1460 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer1461 * @qp: qp pointer1462 * @ah_info: AH info pointer1463 */1464void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,1465 struct irdma_ah_info *ah_info)1466{1467 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;1468 struct iphdr *iph;1469 struct ipv6hdr *ip6h;1470 1471 memset(ah_info, 0, sizeof(*ah_info));1472 ah_info->do_lpbk = true;1473 ah_info->vlan_tag = buf->vlan_id;1474 ah_info->insert_vlan_tag = buf->vlan_valid;1475 ah_info->ipv4_valid = buf->ipv4;1476 ah_info->vsi = qp->vsi;1477 1478 if (buf->smac_valid)1479 ether_addr_copy(ah_info->mac_addr, buf->smac);1480 1481 if (buf->ipv4) {1482 ah_info->ipv4_valid = true;1483 iph = (struct iphdr *)buf->iph;1484 ah_info->hop_ttl = iph->ttl;1485 ah_info->tc_tos = iph->tos;1486 ah_info->dest_ip_addr[0] = ntohl(iph->daddr);1487 ah_info->src_ip_addr[0] = ntohl(iph->saddr);1488 } else {1489 ip6h = (struct ipv6hdr *)buf->iph;1490 ah_info->hop_ttl = ip6h->hop_limit;1491 ah_info->tc_tos = ip6h->priority;1492 irdma_copy_ip_ntohl(ah_info->dest_ip_addr,1493 ip6h->daddr.in6_u.u6_addr32);1494 irdma_copy_ip_ntohl(ah_info->src_ip_addr,1495 ip6h->saddr.in6_u.u6_addr32);1496 }1497 1498 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),1499 ah_info->dest_ip_addr,1500 ah_info->ipv4_valid,1501 NULL, IRDMA_ARP_RESOLVE);1502}1503 1504/**1505 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer1506 * @buf: puda to update1507 * @len: length of buffer1508 * @seqnum: seq number for tcp1509 */1510static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,1511 u16 len, u32 seqnum)1512{1513 struct tcphdr *tcph;1514 struct iphdr *iph;1515 u16 iphlen;1516 u16 pktsize;1517 u8 *addr = buf->mem.va;1518 1519 iphlen = (buf->ipv4) ? 20 : 40;1520 iph = (struct iphdr *)(addr + buf->maclen);1521 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);1522 pktsize = len + buf->tcphlen + iphlen;1523 iph->tot_len = htons(pktsize);1524 tcph->seq = htonl(seqnum);1525}1526 1527/**1528 * irdma_ieq_update_tcpip_info - update tcpip in the buffer1529 * @buf: puda to update1530 * @len: length of buffer1531 * @seqnum: seq number for tcp1532 */1533void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,1534 u32 seqnum)1535{1536 struct tcphdr *tcph;1537 u8 *addr;1538 1539 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)1540 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);1541 1542 addr = buf->mem.va;1543 tcph = (struct tcphdr *)addr;1544 tcph->seq = htonl(seqnum);1545}1546 1547/**1548 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda1549 * buffer1550 * @info: to get information1551 * @buf: puda buffer1552 */1553static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,1554 struct irdma_puda_buf *buf)1555{1556 struct iphdr *iph;1557 struct ipv6hdr *ip6h;1558 struct tcphdr *tcph;1559 u16 iphlen;1560 u16 pkt_len;1561 u8 *mem = buf->mem.va;1562 struct ethhdr *ethh = buf->mem.va;1563 1564 if (ethh->h_proto == htons(0x8100)) {1565 info->vlan_valid = true;1566 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &1567 VLAN_VID_MASK;1568 }1569 1570 buf->maclen = (info->vlan_valid) ? 18 : 14;1571 iphlen = (info->l3proto) ? 40 : 20;1572 buf->ipv4 = (info->l3proto) ? false : true;1573 buf->iph = mem + buf->maclen;1574 iph = (struct iphdr *)buf->iph;1575 buf->tcph = buf->iph + iphlen;1576 tcph = (struct tcphdr *)buf->tcph;1577 1578 if (buf->ipv4) {1579 pkt_len = ntohs(iph->tot_len);1580 } else {1581 ip6h = (struct ipv6hdr *)buf->iph;1582 pkt_len = ntohs(ip6h->payload_len) + iphlen;1583 }1584 1585 buf->totallen = pkt_len + buf->maclen;1586 1587 if (info->payload_len < buf->totallen) {1588 ibdev_dbg(to_ibdev(buf->vsi->dev),1589 "ERR: payload_len = 0x%x totallen expected0x%x\n",1590 info->payload_len, buf->totallen);1591 return -EINVAL;1592 }1593 1594 buf->tcphlen = tcph->doff << 2;1595 buf->datalen = pkt_len - iphlen - buf->tcphlen;1596 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;1597 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;1598 buf->seqnum = ntohl(tcph->seq);1599 1600 return 0;1601}1602 1603/**1604 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer1605 * @info: to get information1606 * @buf: puda buffer1607 */1608int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,1609 struct irdma_puda_buf *buf)1610{1611 struct tcphdr *tcph;1612 u32 pkt_len;1613 u8 *mem;1614 1615 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)1616 return irdma_gen1_puda_get_tcpip_info(info, buf);1617 1618 mem = buf->mem.va;1619 buf->vlan_valid = info->vlan_valid;1620 if (info->vlan_valid)1621 buf->vlan_id = info->vlan;1622 1623 buf->ipv4 = info->ipv4;1624 if (buf->ipv4)1625 buf->iph = mem + IRDMA_IPV4_PAD;1626 else1627 buf->iph = mem;1628 1629 buf->tcph = mem + IRDMA_TCP_OFFSET;1630 tcph = (struct tcphdr *)buf->tcph;1631 pkt_len = info->payload_len;1632 buf->totallen = pkt_len;1633 buf->tcphlen = tcph->doff << 2;1634 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;1635 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;1636 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;1637 buf->seqnum = ntohl(tcph->seq);1638 1639 if (info->smac_valid) {1640 ether_addr_copy(buf->smac, info->smac);1641 buf->smac_valid = true;1642 }1643 1644 return 0;1645}1646 1647/**1648 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats1649 * @t: timer_list pointer1650 */1651static void irdma_hw_stats_timeout(struct timer_list *t)1652{1653 struct irdma_vsi_pestat *pf_devstat =1654 from_timer(pf_devstat, t, stats_timer);1655 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;1656 1657 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)1658 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);1659 else1660 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);1661 1662 mod_timer(&pf_devstat->stats_timer,1663 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));1664}1665 1666/**1667 * irdma_hw_stats_start_timer - Start periodic stats timer1668 * @vsi: vsi structure pointer1669 */1670void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)1671{1672 struct irdma_vsi_pestat *devstat = vsi->pestat;1673 1674 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);1675 mod_timer(&devstat->stats_timer,1676 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));1677}1678 1679/**1680 * irdma_hw_stats_stop_timer - Delete periodic stats timer1681 * @vsi: pointer to vsi structure1682 */1683void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)1684{1685 struct irdma_vsi_pestat *devstat = vsi->pestat;1686 1687 del_timer_sync(&devstat->stats_timer);1688}1689 1690/**1691 * irdma_process_stats - Checking for wrap and update stats1692 * @pestat: stats structure pointer1693 */1694static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)1695{1696 sc_vsi_update_stats(pestat->vsi);1697}1698 1699/**1700 * irdma_cqp_gather_stats_gen1 - Gather stats1701 * @dev: pointer to device structure1702 * @pestat: statistics structure1703 */1704void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,1705 struct irdma_vsi_pestat *pestat)1706{1707 struct irdma_gather_stats *gather_stats =1708 pestat->gather_info.gather_stats_va;1709 const struct irdma_hw_stat_map *map = dev->hw_stats_map;1710 u16 max_stats_idx = dev->hw_attrs.max_stat_idx;1711 u32 stats_inst_offset_32;1712 u32 stats_inst_offset_64;1713 u64 new_val;1714 u16 i;1715 1716 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?1717 pestat->gather_info.stats_inst_index :1718 pestat->hw->hmc.hmc_fn_id;1719 stats_inst_offset_32 *= 4;1720 stats_inst_offset_64 = stats_inst_offset_32 * 2;1721 1722 for (i = 0; i < max_stats_idx; i++) {1723 if (map[i].bitmask <= IRDMA_MAX_STATS_32)1724 new_val = rd32(dev->hw,1725 dev->hw_stats_regs[i] + stats_inst_offset_32);1726 else1727 new_val = rd64(dev->hw,1728 dev->hw_stats_regs[i] + stats_inst_offset_64);1729 gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val;1730 }1731 1732 irdma_process_stats(pestat);1733}1734 1735/**1736 * irdma_process_cqp_stats - Checking for wrap and update stats1737 * @cqp_request: cqp_request structure pointer1738 */1739static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)1740{1741 struct irdma_vsi_pestat *pestat = cqp_request->param;1742 1743 irdma_process_stats(pestat);1744}1745 1746/**1747 * irdma_cqp_gather_stats_cmd - Gather stats1748 * @dev: pointer to device structure1749 * @pestat: pointer to stats info1750 * @wait: flag to wait or not wait for stats1751 */1752int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,1753 struct irdma_vsi_pestat *pestat, bool wait)1754 1755{1756 struct irdma_pci_f *rf = dev_to_rf(dev);1757 struct irdma_cqp *iwcqp = &rf->cqp;1758 struct irdma_cqp_request *cqp_request;1759 struct cqp_cmds_info *cqp_info;1760 int status;1761 1762 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);1763 if (!cqp_request)1764 return -ENOMEM;1765 1766 cqp_info = &cqp_request->info;1767 memset(cqp_info, 0, sizeof(*cqp_info));1768 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;1769 cqp_info->post_sq = 1;1770 cqp_info->in.u.stats_gather.info = pestat->gather_info;1771 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;1772 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;1773 cqp_request->param = pestat;1774 if (!wait)1775 cqp_request->callback_fcn = irdma_process_cqp_stats;1776 status = irdma_handle_cqp_op(rf, cqp_request);1777 if (wait)1778 irdma_process_stats(pestat);1779 irdma_put_cqp_request(&rf->cqp, cqp_request);1780 1781 return status;1782}1783 1784/**1785 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance1786 * @vsi: pointer to vsi structure1787 * @cmd: command to allocate or free1788 * @stats_info: pointer to allocate stats info1789 */1790int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,1791 struct irdma_stats_inst_info *stats_info)1792{1793 struct irdma_pci_f *rf = dev_to_rf(vsi->dev);1794 struct irdma_cqp *iwcqp = &rf->cqp;1795 struct irdma_cqp_request *cqp_request;1796 struct cqp_cmds_info *cqp_info;1797 int status;1798 bool wait = false;1799 1800 if (cmd == IRDMA_OP_STATS_ALLOCATE)1801 wait = true;1802 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);1803 if (!cqp_request)1804 return -ENOMEM;1805 1806 cqp_info = &cqp_request->info;1807 memset(cqp_info, 0, sizeof(*cqp_info));1808 cqp_info->cqp_cmd = cmd;1809 cqp_info->post_sq = 1;1810 cqp_info->in.u.stats_manage.info = *stats_info;1811 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;1812 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;1813 status = irdma_handle_cqp_op(rf, cqp_request);1814 if (wait)1815 stats_info->stats_idx = cqp_request->compl_info.op_ret_val;1816 irdma_put_cqp_request(iwcqp, cqp_request);1817 1818 return status;1819}1820 1821/**1822 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 01823 * @dev: pointer to device info1824 * @sc_ceq: pointer to ceq structure1825 * @op: Create or Destroy1826 */1827int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,1828 u8 op)1829{1830 struct irdma_cqp_request *cqp_request;1831 struct cqp_cmds_info *cqp_info;1832 struct irdma_pci_f *rf = dev_to_rf(dev);1833 int status;1834 1835 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);1836 if (!cqp_request)1837 return -ENOMEM;1838 1839 cqp_info = &cqp_request->info;1840 cqp_info->post_sq = 1;1841 cqp_info->cqp_cmd = op;1842 cqp_info->in.u.ceq_create.ceq = sc_ceq;1843 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;1844 1845 status = irdma_handle_cqp_op(rf, cqp_request);1846 irdma_put_cqp_request(&rf->cqp, cqp_request);1847 1848 return status;1849}1850 1851/**1852 * irdma_cqp_aeq_cmd - Create/Destroy AEQ1853 * @dev: pointer to device info1854 * @sc_aeq: pointer to aeq structure1855 * @op: Create or Destroy1856 */1857int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,1858 u8 op)1859{1860 struct irdma_cqp_request *cqp_request;1861 struct cqp_cmds_info *cqp_info;1862 struct irdma_pci_f *rf = dev_to_rf(dev);1863 int status;1864 1865 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);1866 if (!cqp_request)1867 return -ENOMEM;1868 1869 cqp_info = &cqp_request->info;1870 cqp_info->post_sq = 1;1871 cqp_info->cqp_cmd = op;1872 cqp_info->in.u.aeq_create.aeq = sc_aeq;1873 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;1874 1875 status = irdma_handle_cqp_op(rf, cqp_request);1876 irdma_put_cqp_request(&rf->cqp, cqp_request);1877 1878 return status;1879}1880 1881/**1882 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node1883 * @dev: pointer to device structure1884 * @cmd: Add, modify or delete1885 * @node_info: pointer to ws node info1886 */1887int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,1888 struct irdma_ws_node_info *node_info)1889{1890 struct irdma_pci_f *rf = dev_to_rf(dev);1891 struct irdma_cqp *iwcqp = &rf->cqp;1892 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;1893 struct irdma_cqp_request *cqp_request;1894 struct cqp_cmds_info *cqp_info;1895 int status;1896 bool poll;1897 1898 if (!rf->sc_dev.ceq_valid)1899 poll = true;1900 else1901 poll = false;1902 1903 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);1904 if (!cqp_request)1905 return -ENOMEM;1906 1907 cqp_info = &cqp_request->info;1908 memset(cqp_info, 0, sizeof(*cqp_info));1909 cqp_info->cqp_cmd = cmd;1910 cqp_info->post_sq = 1;1911 cqp_info->in.u.ws_node.info = *node_info;1912 cqp_info->in.u.ws_node.cqp = cqp;1913 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;1914 status = irdma_handle_cqp_op(rf, cqp_request);1915 if (status)1916 goto exit;1917 1918 if (poll) {1919 struct irdma_ccq_cqe_info compl_info;1920 1921 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,1922 &compl_info);1923 node_info->qs_handle = compl_info.op_ret_val;1924 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",1925 compl_info.op_code, compl_info.op_ret_val);1926 } else {1927 node_info->qs_handle = cqp_request->compl_info.op_ret_val;1928 }1929 1930exit:1931 irdma_put_cqp_request(&rf->cqp, cqp_request);1932 1933 return status;1934}1935 1936/**1937 * irdma_ah_cqp_op - perform an AH cqp operation1938 * @rf: RDMA PCI function1939 * @sc_ah: address handle1940 * @cmd: AH operation1941 * @wait: wait if true1942 * @callback_fcn: Callback function on CQP op completion1943 * @cb_param: parameter for callback function1944 *1945 * returns errno1946 */1947int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,1948 bool wait,1949 void (*callback_fcn)(struct irdma_cqp_request *),1950 void *cb_param)1951{1952 struct irdma_cqp_request *cqp_request;1953 struct cqp_cmds_info *cqp_info;1954 int status;1955 1956 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)1957 return -EINVAL;1958 1959 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);1960 if (!cqp_request)1961 return -ENOMEM;1962 1963 cqp_info = &cqp_request->info;1964 cqp_info->cqp_cmd = cmd;1965 cqp_info->post_sq = 1;1966 if (cmd == IRDMA_OP_AH_CREATE) {1967 cqp_info->in.u.ah_create.info = sc_ah->ah_info;1968 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;1969 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;1970 } else if (cmd == IRDMA_OP_AH_DESTROY) {1971 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;1972 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;1973 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;1974 }1975 1976 if (!wait) {1977 cqp_request->callback_fcn = callback_fcn;1978 cqp_request->param = cb_param;1979 }1980 status = irdma_handle_cqp_op(rf, cqp_request);1981 irdma_put_cqp_request(&rf->cqp, cqp_request);1982 1983 if (status)1984 return -ENOMEM;1985 1986 if (wait)1987 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);1988 1989 return 0;1990}1991 1992/**1993 * irdma_ieq_ah_cb - callback after creation of AH for IEQ1994 * @cqp_request: pointer to cqp_request of create AH1995 */1996static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)1997{1998 struct irdma_sc_qp *qp = cqp_request->param;1999 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;2000 unsigned long flags;2001 2002 spin_lock_irqsave(&qp->pfpdu.lock, flags);2003 if (!cqp_request->compl_info.op_ret_val) {2004 sc_ah->ah_info.ah_valid = true;2005 irdma_ieq_process_fpdus(qp, qp->vsi->ieq);2006 } else {2007 sc_ah->ah_info.ah_valid = false;2008 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);2009 }2010 spin_unlock_irqrestore(&qp->pfpdu.lock, flags);2011}2012 2013/**2014 * irdma_ilq_ah_cb - callback after creation of AH for ILQ2015 * @cqp_request: pointer to cqp_request of create AH2016 */2017static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)2018{2019 struct irdma_cm_node *cm_node = cqp_request->param;2020 struct irdma_sc_ah *sc_ah = cm_node->ah;2021 2022 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;2023 irdma_add_conn_est_qh(cm_node);2024}2025 2026/**2027 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's2028 * @dev: device pointer2029 * @ah_info: Address handle info2030 * @wait: When true will wait for operation to complete2031 * @type: ILQ/IEQ2032 * @cb_param: Callback param when not waiting2033 * @ah_ret: Returned pointer to address handle if created2034 *2035 */2036int irdma_puda_create_ah(struct irdma_sc_dev *dev,2037 struct irdma_ah_info *ah_info, bool wait,2038 enum puda_rsrc_type type, void *cb_param,2039 struct irdma_sc_ah **ah_ret)2040{2041 struct irdma_sc_ah *ah;2042 struct irdma_pci_f *rf = dev_to_rf(dev);2043 int err;2044 2045 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);2046 *ah_ret = ah;2047 if (!ah)2048 return -ENOMEM;2049 2050 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,2051 &ah_info->ah_idx, &rf->next_ah);2052 if (err)2053 goto err_free;2054 2055 ah->dev = dev;2056 ah->ah_info = *ah_info;2057 2058 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)2059 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,2060 irdma_ilq_ah_cb, cb_param);2061 else2062 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,2063 irdma_ieq_ah_cb, cb_param);2064 2065 if (err)2066 goto error;2067 return 0;2068 2069error:2070 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);2071err_free:2072 kfree(ah);2073 *ah_ret = NULL;2074 return -ENOMEM;2075}2076 2077/**2078 * irdma_puda_free_ah - free a puda address handle2079 * @dev: device pointer2080 * @ah: The address handle to free2081 */2082void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)2083{2084 struct irdma_pci_f *rf = dev_to_rf(dev);2085 2086 if (!ah)2087 return;2088 2089 if (ah->ah_info.ah_valid) {2090 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);2091 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);2092 }2093 2094 kfree(ah);2095}2096 2097/**2098 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP2099 * @cqp_request: pointer to cqp_request of create AH2100 */2101void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)2102{2103 struct irdma_sc_ah *sc_ah = cqp_request->param;2104 2105 if (!cqp_request->compl_info.op_ret_val)2106 sc_ah->ah_info.ah_valid = true;2107 else2108 sc_ah->ah_info.ah_valid = false;2109}2110 2111/**2112 * irdma_prm_add_pble_mem - add moemory to pble resources2113 * @pprm: pble resource manager2114 * @pchunk: chunk of memory to add2115 */2116int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,2117 struct irdma_chunk *pchunk)2118{2119 u64 sizeofbitmap;2120 2121 if (pchunk->size & 0xfff)2122 return -EINVAL;2123 2124 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;2125 2126 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);2127 if (!pchunk->bitmapbuf)2128 return -ENOMEM;2129 2130 pchunk->sizeofbitmap = sizeofbitmap;2131 /* each pble is 8 bytes hence shift by 3 */2132 pprm->total_pble_alloc += pchunk->size >> 3;2133 pprm->free_pble_cnt += pchunk->size >> 3;2134 2135 return 0;2136}2137 2138/**2139 * irdma_prm_get_pbles - get pble's from prm2140 * @pprm: pble resource manager2141 * @chunkinfo: nformation about chunk where pble's were acquired2142 * @mem_size: size of pble memory needed2143 * @vaddr: returns virtual address of pble memory2144 * @fpm_addr: returns fpm address of pble memory2145 */2146int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,2147 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,2148 u64 **vaddr, u64 *fpm_addr)2149{2150 u64 bits_needed;2151 u64 bit_idx = PBLE_INVALID_IDX;2152 struct irdma_chunk *pchunk = NULL;2153 struct list_head *chunk_entry = pprm->clist.next;2154 u32 offset;2155 unsigned long flags;2156 *vaddr = NULL;2157 *fpm_addr = 0;2158 2159 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));2160 2161 spin_lock_irqsave(&pprm->prm_lock, flags);2162 while (chunk_entry != &pprm->clist) {2163 pchunk = (struct irdma_chunk *)chunk_entry;2164 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,2165 pchunk->sizeofbitmap, 0,2166 bits_needed, 0);2167 if (bit_idx < pchunk->sizeofbitmap)2168 break;2169 2170 /* list.next used macro */2171 chunk_entry = pchunk->list.next;2172 }2173 2174 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {2175 spin_unlock_irqrestore(&pprm->prm_lock, flags);2176 return -ENOMEM;2177 }2178 2179 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);2180 offset = bit_idx << pprm->pble_shift;2181 *vaddr = pchunk->vaddr + offset;2182 *fpm_addr = pchunk->fpm_addr + offset;2183 2184 chunkinfo->pchunk = pchunk;2185 chunkinfo->bit_idx = bit_idx;2186 chunkinfo->bits_used = bits_needed;2187 /* 3 is sizeof pble divide */2188 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);2189 spin_unlock_irqrestore(&pprm->prm_lock, flags);2190 2191 return 0;2192}2193 2194/**2195 * irdma_prm_return_pbles - return pbles back to prm2196 * @pprm: pble resource manager2197 * @chunkinfo: chunk where pble's were acquired and to be freed2198 */2199void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,2200 struct irdma_pble_chunkinfo *chunkinfo)2201{2202 unsigned long flags;2203 2204 spin_lock_irqsave(&pprm->prm_lock, flags);2205 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);2206 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,2207 chunkinfo->bits_used);2208 spin_unlock_irqrestore(&pprm->prm_lock, flags);2209}2210 2211int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,2212 u32 pg_cnt)2213{2214 struct page *vm_page;2215 int i;2216 u8 *addr;2217 2218 addr = (u8 *)(uintptr_t)va;2219 for (i = 0; i < pg_cnt; i++) {2220 vm_page = vmalloc_to_page(addr);2221 if (!vm_page)2222 goto err;2223 2224 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE,2225 DMA_BIDIRECTIONAL);2226 if (dma_mapping_error(hw->device, pg_dma[i]))2227 goto err;2228 2229 addr += PAGE_SIZE;2230 }2231 2232 return 0;2233 2234err:2235 irdma_unmap_vm_page_list(hw, pg_dma, i);2236 return -ENOMEM;2237}2238 2239void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)2240{2241 int i;2242 2243 for (i = 0; i < pg_cnt; i++)2244 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);2245}2246 2247/**2248 * irdma_pble_free_paged_mem - free virtual paged memory2249 * @chunk: chunk to free with paged memory2250 */2251void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)2252{2253 if (!chunk->pg_cnt)2254 goto done;2255 2256 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,2257 chunk->pg_cnt);2258 2259done:2260 kfree(chunk->dmainfo.dmaaddrs);2261 chunk->dmainfo.dmaaddrs = NULL;2262 vfree(chunk->vaddr);2263 chunk->vaddr = NULL;2264 chunk->type = 0;2265}2266 2267/**2268 * irdma_pble_get_paged_mem -allocate paged memory for pbles2269 * @chunk: chunk to add for paged memory2270 * @pg_cnt: number of pages needed2271 */2272int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)2273{2274 u32 size;2275 void *va;2276 2277 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);2278 if (!chunk->dmainfo.dmaaddrs)2279 return -ENOMEM;2280 2281 size = PAGE_SIZE * pg_cnt;2282 va = vmalloc(size);2283 if (!va)2284 goto err;2285 2286 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,2287 pg_cnt)) {2288 vfree(va);2289 goto err;2290 }2291 chunk->vaddr = va;2292 chunk->size = size;2293 chunk->pg_cnt = pg_cnt;2294 chunk->type = PBLE_SD_PAGED;2295 2296 return 0;2297err:2298 kfree(chunk->dmainfo.dmaaddrs);2299 chunk->dmainfo.dmaaddrs = NULL;2300 2301 return -ENOMEM;2302}2303 2304/**2305 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID2306 * @dev: device pointer2307 */2308u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)2309{2310 struct irdma_pci_f *rf = dev_to_rf(dev);2311 u32 next = 1;2312 u32 node_id;2313 2314 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,2315 &node_id, &next))2316 return IRDMA_WS_NODE_INVALID;2317 2318 return (u16)node_id;2319}2320 2321/**2322 * irdma_free_ws_node_id - Free a tx scheduler node ID2323 * @dev: device pointer2324 * @node_id: Work scheduler node ID2325 */2326void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)2327{2328 struct irdma_pci_f *rf = dev_to_rf(dev);2329 2330 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);2331}2332 2333/**2334 * irdma_modify_qp_to_err - Modify a QP to error2335 * @sc_qp: qp structure2336 */2337void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)2338{2339 struct irdma_qp *qp = sc_qp->qp_uk.back_qp;2340 struct ib_qp_attr attr;2341 2342 if (qp->iwdev->rf->reset)2343 return;2344 attr.qp_state = IB_QPS_ERR;2345 2346 if (rdma_protocol_roce(qp->ibqp.device, 1))2347 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);2348 else2349 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);2350}2351 2352void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)2353{2354 struct ib_event ibevent;2355 2356 if (!iwqp->ibqp.event_handler)2357 return;2358 2359 switch (event) {2360 case IRDMA_QP_EVENT_CATASTROPHIC:2361 ibevent.event = IB_EVENT_QP_FATAL;2362 break;2363 case IRDMA_QP_EVENT_ACCESS_ERR:2364 ibevent.event = IB_EVENT_QP_ACCESS_ERR;2365 break;2366 case IRDMA_QP_EVENT_REQ_ERR:2367 ibevent.event = IB_EVENT_QP_REQ_ERR;2368 break;2369 }2370 ibevent.device = iwqp->ibqp.device;2371 ibevent.element.qp = &iwqp->ibqp;2372 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);2373}2374 2375bool irdma_cq_empty(struct irdma_cq *iwcq)2376{2377 struct irdma_cq_uk *ukcq;2378 u64 qword3;2379 __le64 *cqe;2380 u8 polarity;2381 2382 ukcq = &iwcq->sc_cq.cq_uk;2383 cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);2384 get_64bit_val(cqe, 24, &qword3);2385 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);2386 2387 return polarity != ukcq->polarity;2388}2389 2390void irdma_remove_cmpls_list(struct irdma_cq *iwcq)2391{2392 struct irdma_cmpl_gen *cmpl_node;2393 struct list_head *tmp_node, *list_node;2394 2395 list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) {2396 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);2397 list_del(&cmpl_node->list);2398 kfree(cmpl_node);2399 }2400}2401 2402int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)2403{2404 struct irdma_cmpl_gen *cmpl;2405 2406 if (list_empty(&iwcq->cmpl_generated))2407 return -ENOENT;2408 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);2409 list_del(&cmpl->list);2410 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info));2411 kfree(cmpl);2412 2413 ibdev_dbg(iwcq->ibcq.device,2414 "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n",2415 __func__, cq_poll_info->qp_id, cq_poll_info->op_type,2416 cq_poll_info->wr_id);2417 2418 return 0;2419}2420 2421/**2422 * irdma_set_cpi_common_values - fill in values for polling info struct2423 * @cpi: resulting structure of cq_poll_info type2424 * @qp: QPair2425 * @qp_num: id of the QP2426 */2427static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,2428 struct irdma_qp_uk *qp, u32 qp_num)2429{2430 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED;2431 cpi->error = true;2432 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR;2433 cpi->minor_err = FLUSH_GENERAL_ERR;2434 cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp;2435 cpi->qp_id = qp_num;2436}2437 2438static inline void irdma_comp_handler(struct irdma_cq *cq)2439{2440 if (!cq->ibcq.comp_handler)2441 return;2442 if (atomic_cmpxchg(&cq->armed, 1, 0))2443 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);2444}2445 2446void irdma_generate_flush_completions(struct irdma_qp *iwqp)2447{2448 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;2449 struct irdma_ring *sq_ring = &qp->sq_ring;2450 struct irdma_ring *rq_ring = &qp->rq_ring;2451 struct irdma_cmpl_gen *cmpl;2452 __le64 *sw_wqe;2453 u64 wqe_qword;2454 u32 wqe_idx;2455 bool compl_generated = false;2456 unsigned long flags1;2457 2458 spin_lock_irqsave(&iwqp->iwscq->lock, flags1);2459 if (irdma_cq_empty(iwqp->iwscq)) {2460 unsigned long flags2;2461 2462 spin_lock_irqsave(&iwqp->lock, flags2);2463 while (IRDMA_RING_MORE_WORK(*sq_ring)) {2464 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);2465 if (!cmpl) {2466 spin_unlock_irqrestore(&iwqp->lock, flags2);2467 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);2468 return;2469 }2470 2471 wqe_idx = sq_ring->tail;2472 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);2473 2474 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;2475 sw_wqe = qp->sq_base[wqe_idx].elem;2476 get_64bit_val(sw_wqe, 24, &wqe_qword);2477 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);2478 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;2479 /* remove the SQ WR by moving SQ tail*/2480 IRDMA_RING_SET_TAIL(*sq_ring,2481 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);2482 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) {2483 kfree(cmpl);2484 continue;2485 }2486 ibdev_dbg(iwqp->iwscq->ibcq.device,2487 "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n",2488 __func__, cmpl->cpi.wr_id, qp->qp_id);2489 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated);2490 compl_generated = true;2491 }2492 spin_unlock_irqrestore(&iwqp->lock, flags2);2493 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);2494 if (compl_generated)2495 irdma_comp_handler(iwqp->iwscq);2496 } else {2497 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);2498 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,2499 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));2500 }2501 2502 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1);2503 if (irdma_cq_empty(iwqp->iwrcq)) {2504 unsigned long flags2;2505 2506 spin_lock_irqsave(&iwqp->lock, flags2);2507 while (IRDMA_RING_MORE_WORK(*rq_ring)) {2508 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);2509 if (!cmpl) {2510 spin_unlock_irqrestore(&iwqp->lock, flags2);2511 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);2512 return;2513 }2514 2515 wqe_idx = rq_ring->tail;2516 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);2517 2518 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];2519 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;2520 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;2521 /* remove the RQ WR by moving RQ tail */2522 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);2523 ibdev_dbg(iwqp->iwrcq->ibcq.device,2524 "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n",2525 __func__, cmpl->cpi.wr_id, qp->qp_id,2526 wqe_idx);2527 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated);2528 2529 compl_generated = true;2530 }2531 spin_unlock_irqrestore(&iwqp->lock, flags2);2532 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);2533 if (compl_generated)2534 irdma_comp_handler(iwqp->iwrcq);2535 } else {2536 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);2537 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,2538 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));2539 }2540}2541