1783 lines · c
1/*2 * Copyright (c) 2012, 2013 Intel Corporation. All rights reserved.3 * Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.4 * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.5 *6 * This software is available to you under a choice of one of two7 * licenses. You may choose to be licensed under the terms of the GNU8 * General Public License (GPL) Version 2, available from the file9 * COPYING in the main directory of this source tree, or the10 * OpenIB.org BSD license below:11 *12 * Redistribution and use in source and binary forms, with or13 * without modification, are permitted provided that the following14 * conditions are met:15 *16 * - Redistributions of source code must retain the above17 * copyright notice, this list of conditions and the following18 * disclaimer.19 *20 * - Redistributions in binary form must reproduce the above21 * copyright notice, this list of conditions and the following22 * disclaimer in the documentation and/or other materials23 * provided with the distribution.24 *25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE32 * SOFTWARE.33 */34 35#include <linux/pci.h>36#include <linux/netdevice.h>37#include <linux/vmalloc.h>38#include <linux/delay.h>39#include <linux/module.h>40#include <linux/printk.h>41#ifdef CONFIG_INFINIBAND_QIB_DCA42#include <linux/dca.h>43#endif44#include <rdma/rdma_vt.h>45 46#include "qib.h"47#include "qib_common.h"48#include "qib_mad.h"49#ifdef CONFIG_DEBUG_FS50#include "qib_debugfs.h"51#include "qib_verbs.h"52#endif53 54#undef pr_fmt55#define pr_fmt(fmt) QIB_DRV_NAME ": " fmt56 57/*58 * min buffers we want to have per context, after driver59 */60#define QIB_MIN_USER_CTXT_BUFCNT 761 62#define QLOGIC_IB_R_SOFTWARE_MASK 0xFF63#define QLOGIC_IB_R_SOFTWARE_SHIFT 2464#define QLOGIC_IB_R_EMULATOR_MASK (1ULL<<62)65 66/*67 * Number of ctxts we are configured to use (to allow for more pio68 * buffers per ctxt, etc.) Zero means use chip value.69 */70ushort qib_cfgctxts;71module_param_named(cfgctxts, qib_cfgctxts, ushort, S_IRUGO);72MODULE_PARM_DESC(cfgctxts, "Set max number of contexts to use");73 74unsigned qib_numa_aware;75module_param_named(numa_aware, qib_numa_aware, uint, S_IRUGO);76MODULE_PARM_DESC(numa_aware,77 "0 -> PSM allocation close to HCA, 1 -> PSM allocation local to process");78 79/*80 * If set, do not write to any regs if avoidable, hack to allow81 * check for deranged default register values.82 */83ushort qib_mini_init;84module_param_named(mini_init, qib_mini_init, ushort, S_IRUGO);85MODULE_PARM_DESC(mini_init, "If set, do minimal diag init");86 87unsigned qib_n_krcv_queues;88module_param_named(krcvqs, qib_n_krcv_queues, uint, S_IRUGO);89MODULE_PARM_DESC(krcvqs, "number of kernel receive queues per IB port");90 91unsigned qib_cc_table_size;92module_param_named(cc_table_size, qib_cc_table_size, uint, S_IRUGO);93MODULE_PARM_DESC(cc_table_size, "Congestion control table entries 0 (CCA disabled - default), min = 128, max = 1984");94 95static void verify_interrupt(struct timer_list *);96 97DEFINE_XARRAY_FLAGS(qib_dev_table, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ);98u32 qib_cpulist_count;99unsigned long *qib_cpulist;100 101/* set number of contexts we'll actually use */102void qib_set_ctxtcnt(struct qib_devdata *dd)103{104 if (!qib_cfgctxts) {105 dd->cfgctxts = dd->first_user_ctxt + num_online_cpus();106 if (dd->cfgctxts > dd->ctxtcnt)107 dd->cfgctxts = dd->ctxtcnt;108 } else if (qib_cfgctxts < dd->num_pports)109 dd->cfgctxts = dd->ctxtcnt;110 else if (qib_cfgctxts <= dd->ctxtcnt)111 dd->cfgctxts = qib_cfgctxts;112 else113 dd->cfgctxts = dd->ctxtcnt;114 dd->freectxts = (dd->first_user_ctxt > dd->cfgctxts) ? 0 :115 dd->cfgctxts - dd->first_user_ctxt;116}117 118/*119 * Common code for creating the receive context array.120 */121int qib_create_ctxts(struct qib_devdata *dd)122{123 unsigned i;124 int local_node_id = pcibus_to_node(dd->pcidev->bus);125 126 if (local_node_id < 0)127 local_node_id = numa_node_id();128 dd->assigned_node_id = local_node_id;129 130 /*131 * Allocate full ctxtcnt array, rather than just cfgctxts, because132 * cleanup iterates across all possible ctxts.133 */134 dd->rcd = kcalloc(dd->ctxtcnt, sizeof(*dd->rcd), GFP_KERNEL);135 if (!dd->rcd)136 return -ENOMEM;137 138 /* create (one or more) kctxt */139 for (i = 0; i < dd->first_user_ctxt; ++i) {140 struct qib_pportdata *ppd;141 struct qib_ctxtdata *rcd;142 143 if (dd->skip_kctxt_mask & (1 << i))144 continue;145 146 ppd = dd->pport + (i % dd->num_pports);147 148 rcd = qib_create_ctxtdata(ppd, i, dd->assigned_node_id);149 if (!rcd) {150 qib_dev_err(dd,151 "Unable to allocate ctxtdata for Kernel ctxt, failing\n");152 kfree(dd->rcd);153 dd->rcd = NULL;154 return -ENOMEM;155 }156 rcd->pkeys[0] = QIB_DEFAULT_P_KEY;157 rcd->seq_cnt = 1;158 }159 return 0;160}161 162/*163 * Common code for user and kernel context setup.164 */165struct qib_ctxtdata *qib_create_ctxtdata(struct qib_pportdata *ppd, u32 ctxt,166 int node_id)167{168 struct qib_devdata *dd = ppd->dd;169 struct qib_ctxtdata *rcd;170 171 rcd = kzalloc_node(sizeof(*rcd), GFP_KERNEL, node_id);172 if (rcd) {173 INIT_LIST_HEAD(&rcd->qp_wait_list);174 rcd->node_id = node_id;175 rcd->ppd = ppd;176 rcd->dd = dd;177 rcd->cnt = 1;178 rcd->ctxt = ctxt;179 dd->rcd[ctxt] = rcd;180#ifdef CONFIG_DEBUG_FS181 if (ctxt < dd->first_user_ctxt) { /* N/A for PSM contexts */182 rcd->opstats = kzalloc_node(sizeof(*rcd->opstats),183 GFP_KERNEL, node_id);184 if (!rcd->opstats) {185 kfree(rcd);186 qib_dev_err(dd,187 "Unable to allocate per ctxt stats buffer\n");188 return NULL;189 }190 }191#endif192 dd->f_init_ctxt(rcd);193 194 /*195 * To avoid wasting a lot of memory, we allocate 32KB chunks196 * of physically contiguous memory, advance through it until197 * used up and then allocate more. Of course, we need198 * memory to store those extra pointers, now. 32KB seems to199 * be the most that is "safe" under memory pressure200 * (creating large files and then copying them over201 * NFS while doing lots of MPI jobs). The OOM killer can202 * get invoked, even though we say we can sleep and this can203 * cause significant system problems....204 */205 rcd->rcvegrbuf_size = 0x8000;206 rcd->rcvegrbufs_perchunk =207 rcd->rcvegrbuf_size / dd->rcvegrbufsize;208 rcd->rcvegrbuf_chunks = (rcd->rcvegrcnt +209 rcd->rcvegrbufs_perchunk - 1) /210 rcd->rcvegrbufs_perchunk;211 rcd->rcvegrbufs_perchunk_shift =212 ilog2(rcd->rcvegrbufs_perchunk);213 }214 return rcd;215}216 217/*218 * Common code for initializing the physical port structure.219 */220int qib_init_pportdata(struct qib_pportdata *ppd, struct qib_devdata *dd,221 u8 hw_pidx, u8 port)222{223 int size;224 225 ppd->dd = dd;226 ppd->hw_pidx = hw_pidx;227 ppd->port = port; /* IB port number, not index */228 229 spin_lock_init(&ppd->sdma_lock);230 spin_lock_init(&ppd->lflags_lock);231 spin_lock_init(&ppd->cc_shadow_lock);232 init_waitqueue_head(&ppd->state_wait);233 234 timer_setup(&ppd->symerr_clear_timer, qib_clear_symerror_on_linkup, 0);235 236 ppd->qib_wq = NULL;237 ppd->ibport_data.pmastats =238 alloc_percpu(struct qib_pma_counters);239 if (!ppd->ibport_data.pmastats)240 return -ENOMEM;241 ppd->ibport_data.rvp.rc_acks = alloc_percpu(u64);242 ppd->ibport_data.rvp.rc_qacks = alloc_percpu(u64);243 ppd->ibport_data.rvp.rc_delayed_comp = alloc_percpu(u64);244 if (!(ppd->ibport_data.rvp.rc_acks) ||245 !(ppd->ibport_data.rvp.rc_qacks) ||246 !(ppd->ibport_data.rvp.rc_delayed_comp))247 return -ENOMEM;248 249 if (qib_cc_table_size < IB_CCT_MIN_ENTRIES)250 goto bail;251 252 ppd->cc_supported_table_entries = min(max_t(int, qib_cc_table_size,253 IB_CCT_MIN_ENTRIES), IB_CCT_ENTRIES*IB_CC_TABLE_CAP_DEFAULT);254 255 ppd->cc_max_table_entries =256 ppd->cc_supported_table_entries/IB_CCT_ENTRIES;257 258 size = IB_CC_TABLE_CAP_DEFAULT * sizeof(struct ib_cc_table_entry)259 * IB_CCT_ENTRIES;260 ppd->ccti_entries = kzalloc(size, GFP_KERNEL);261 if (!ppd->ccti_entries)262 goto bail;263 264 size = IB_CC_CCS_ENTRIES * sizeof(struct ib_cc_congestion_entry);265 ppd->congestion_entries = kzalloc(size, GFP_KERNEL);266 if (!ppd->congestion_entries)267 goto bail_1;268 269 size = sizeof(struct cc_table_shadow);270 ppd->ccti_entries_shadow = kzalloc(size, GFP_KERNEL);271 if (!ppd->ccti_entries_shadow)272 goto bail_2;273 274 size = sizeof(struct ib_cc_congestion_setting_attr);275 ppd->congestion_entries_shadow = kzalloc(size, GFP_KERNEL);276 if (!ppd->congestion_entries_shadow)277 goto bail_3;278 279 return 0;280 281bail_3:282 kfree(ppd->ccti_entries_shadow);283 ppd->ccti_entries_shadow = NULL;284bail_2:285 kfree(ppd->congestion_entries);286 ppd->congestion_entries = NULL;287bail_1:288 kfree(ppd->ccti_entries);289 ppd->ccti_entries = NULL;290bail:291 /* User is intentionally disabling the congestion control agent */292 if (!qib_cc_table_size)293 return 0;294 295 if (qib_cc_table_size < IB_CCT_MIN_ENTRIES) {296 qib_cc_table_size = 0;297 qib_dev_err(dd,298 "Congestion Control table size %d less than minimum %d for port %d\n",299 qib_cc_table_size, IB_CCT_MIN_ENTRIES, port);300 }301 302 qib_dev_err(dd, "Congestion Control Agent disabled for port %d\n",303 port);304 return 0;305}306 307static int init_pioavailregs(struct qib_devdata *dd)308{309 int ret, pidx;310 u64 *status_page;311 312 dd->pioavailregs_dma = dma_alloc_coherent(313 &dd->pcidev->dev, PAGE_SIZE, &dd->pioavailregs_phys,314 GFP_KERNEL);315 if (!dd->pioavailregs_dma) {316 qib_dev_err(dd,317 "failed to allocate PIOavail reg area in memory\n");318 ret = -ENOMEM;319 goto done;320 }321 322 /*323 * We really want L2 cache aligned, but for current CPUs of324 * interest, they are the same.325 */326 status_page = (u64 *)327 ((char *) dd->pioavailregs_dma +328 ((2 * L1_CACHE_BYTES +329 dd->pioavregs * sizeof(u64)) & ~L1_CACHE_BYTES));330 /* device status comes first, for backwards compatibility */331 dd->devstatusp = status_page;332 *status_page++ = 0;333 for (pidx = 0; pidx < dd->num_pports; ++pidx) {334 dd->pport[pidx].statusp = status_page;335 *status_page++ = 0;336 }337 338 /*339 * Setup buffer to hold freeze and other messages, accessible to340 * apps, following statusp. This is per-unit, not per port.341 */342 dd->freezemsg = (char *) status_page;343 *dd->freezemsg = 0;344 /* length of msg buffer is "whatever is left" */345 ret = (char *) status_page - (char *) dd->pioavailregs_dma;346 dd->freezelen = PAGE_SIZE - ret;347 348 ret = 0;349 350done:351 return ret;352}353 354/**355 * init_shadow_tids - allocate the shadow TID array356 * @dd: the qlogic_ib device357 *358 * allocate the shadow TID array, so we can qib_munlock previous359 * entries. It may make more sense to move the pageshadow to the360 * ctxt data structure, so we only allocate memory for ctxts actually361 * in use, since we at 8k per ctxt, now.362 * We don't want failures here to prevent use of the driver/chip,363 * so no return value.364 */365static void init_shadow_tids(struct qib_devdata *dd)366{367 struct page **pages;368 dma_addr_t *addrs;369 370 pages = vzalloc(array_size(sizeof(struct page *),371 dd->cfgctxts * dd->rcvtidcnt));372 if (!pages)373 goto bail;374 375 addrs = vzalloc(array_size(sizeof(dma_addr_t),376 dd->cfgctxts * dd->rcvtidcnt));377 if (!addrs)378 goto bail_free;379 380 dd->pageshadow = pages;381 dd->physshadow = addrs;382 return;383 384bail_free:385 vfree(pages);386bail:387 dd->pageshadow = NULL;388}389 390/*391 * Do initialization for device that is only needed on392 * first detect, not on resets.393 */394static int loadtime_init(struct qib_devdata *dd)395{396 int ret = 0;397 398 if (((dd->revision >> QLOGIC_IB_R_SOFTWARE_SHIFT) &399 QLOGIC_IB_R_SOFTWARE_MASK) != QIB_CHIP_SWVERSION) {400 qib_dev_err(dd,401 "Driver only handles version %d, chip swversion is %d (%llx), failing\n",402 QIB_CHIP_SWVERSION,403 (int)(dd->revision >>404 QLOGIC_IB_R_SOFTWARE_SHIFT) &405 QLOGIC_IB_R_SOFTWARE_MASK,406 (unsigned long long) dd->revision);407 ret = -ENOSYS;408 goto done;409 }410 411 if (dd->revision & QLOGIC_IB_R_EMULATOR_MASK)412 qib_devinfo(dd->pcidev, "%s", dd->boardversion);413 414 spin_lock_init(&dd->pioavail_lock);415 spin_lock_init(&dd->sendctrl_lock);416 spin_lock_init(&dd->uctxt_lock);417 spin_lock_init(&dd->qib_diag_trans_lock);418 spin_lock_init(&dd->eep_st_lock);419 mutex_init(&dd->eep_lock);420 421 if (qib_mini_init)422 goto done;423 424 ret = init_pioavailregs(dd);425 init_shadow_tids(dd);426 427 qib_get_eeprom_info(dd);428 429 /* setup time (don't start yet) to verify we got interrupt */430 timer_setup(&dd->intrchk_timer, verify_interrupt, 0);431done:432 return ret;433}434 435/**436 * init_after_reset - re-initialize after a reset437 * @dd: the qlogic_ib device438 *439 * sanity check at least some of the values after reset, and440 * ensure no receive or transmit (explicitly, in case reset441 * failed442 */443static int init_after_reset(struct qib_devdata *dd)444{445 int i;446 447 /*448 * Ensure chip does no sends or receives, tail updates, or449 * pioavail updates while we re-initialize. This is mostly450 * for the driver data structures, not chip registers.451 */452 for (i = 0; i < dd->num_pports; ++i) {453 /*454 * ctxt == -1 means "all contexts". Only really safe for455 * _dis_abling things, as here.456 */457 dd->f_rcvctrl(dd->pport + i, QIB_RCVCTRL_CTXT_DIS |458 QIB_RCVCTRL_INTRAVAIL_DIS |459 QIB_RCVCTRL_TAILUPD_DIS, -1);460 /* Redundant across ports for some, but no big deal. */461 dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_DIS |462 QIB_SENDCTRL_AVAIL_DIS);463 }464 465 return 0;466}467 468static void enable_chip(struct qib_devdata *dd)469{470 u64 rcvmask;471 int i;472 473 /*474 * Enable PIO send, and update of PIOavail regs to memory.475 */476 for (i = 0; i < dd->num_pports; ++i)477 dd->f_sendctrl(dd->pport + i, QIB_SENDCTRL_SEND_ENB |478 QIB_SENDCTRL_AVAIL_ENB);479 /*480 * Enable kernel ctxts' receive and receive interrupt.481 * Other ctxts done as user opens and inits them.482 */483 rcvmask = QIB_RCVCTRL_CTXT_ENB | QIB_RCVCTRL_INTRAVAIL_ENB;484 rcvmask |= (dd->flags & QIB_NODMA_RTAIL) ?485 QIB_RCVCTRL_TAILUPD_DIS : QIB_RCVCTRL_TAILUPD_ENB;486 for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) {487 struct qib_ctxtdata *rcd = dd->rcd[i];488 489 if (rcd)490 dd->f_rcvctrl(rcd->ppd, rcvmask, i);491 }492}493 494static void verify_interrupt(struct timer_list *t)495{496 struct qib_devdata *dd = from_timer(dd, t, intrchk_timer);497 u64 int_counter;498 499 if (!dd)500 return; /* being torn down */501 502 /*503 * If we don't have a lid or any interrupts, let the user know and504 * don't bother checking again.505 */506 int_counter = qib_int_counter(dd) - dd->z_int_counter;507 if (int_counter == 0) {508 if (!dd->f_intr_fallback(dd))509 dev_err(&dd->pcidev->dev,510 "No interrupts detected, not usable.\n");511 else /* re-arm the timer to see if fallback works */512 mod_timer(&dd->intrchk_timer, jiffies + HZ/2);513 }514}515 516static void init_piobuf_state(struct qib_devdata *dd)517{518 int i, pidx;519 u32 uctxts;520 521 /*522 * Ensure all buffers are free, and fifos empty. Buffers523 * are common, so only do once for port 0.524 *525 * After enable and qib_chg_pioavailkernel so we can safely526 * enable pioavail updates and PIOENABLE. After this, packets527 * are ready and able to go out.528 */529 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_ALL);530 for (pidx = 0; pidx < dd->num_pports; ++pidx)531 dd->f_sendctrl(dd->pport + pidx, QIB_SENDCTRL_FLUSH);532 533 /*534 * If not all sendbufs are used, add the one to each of the lower535 * numbered contexts. pbufsctxt and lastctxt_piobuf are536 * calculated in chip-specific code because it may cause some537 * chip-specific adjustments to be made.538 */539 uctxts = dd->cfgctxts - dd->first_user_ctxt;540 dd->ctxts_extrabuf = dd->pbufsctxt ?541 dd->lastctxt_piobuf - (dd->pbufsctxt * uctxts) : 0;542 543 /*544 * Set up the shadow copies of the piobufavail registers,545 * which we compare against the chip registers for now, and546 * the in memory DMA'ed copies of the registers.547 * By now pioavail updates to memory should have occurred, so548 * copy them into our working/shadow registers; this is in549 * case something went wrong with abort, but mostly to get the550 * initial values of the generation bit correct.551 */552 for (i = 0; i < dd->pioavregs; i++) {553 __le64 tmp;554 555 tmp = dd->pioavailregs_dma[i];556 /*557 * Don't need to worry about pioavailkernel here558 * because we will call qib_chg_pioavailkernel() later559 * in initialization, to busy out buffers as needed.560 */561 dd->pioavailshadow[i] = le64_to_cpu(tmp);562 }563 while (i < ARRAY_SIZE(dd->pioavailshadow))564 dd->pioavailshadow[i++] = 0; /* for debugging sanity */565 566 /* after pioavailshadow is setup */567 qib_chg_pioavailkernel(dd, 0, dd->piobcnt2k + dd->piobcnt4k,568 TXCHK_CHG_TYPE_KERN, NULL);569 dd->f_initvl15_bufs(dd);570}571 572/**573 * qib_create_workqueues - create per port workqueues574 * @dd: the qlogic_ib device575 */576static int qib_create_workqueues(struct qib_devdata *dd)577{578 int pidx;579 struct qib_pportdata *ppd;580 581 for (pidx = 0; pidx < dd->num_pports; ++pidx) {582 ppd = dd->pport + pidx;583 if (!ppd->qib_wq) {584 ppd->qib_wq = alloc_ordered_workqueue("qib%d_%d",585 WQ_MEM_RECLAIM,586 dd->unit, pidx);587 if (!ppd->qib_wq)588 goto wq_error;589 }590 }591 return 0;592wq_error:593 pr_err("create_singlethread_workqueue failed for port %d\n",594 pidx + 1);595 for (pidx = 0; pidx < dd->num_pports; ++pidx) {596 ppd = dd->pport + pidx;597 if (ppd->qib_wq) {598 destroy_workqueue(ppd->qib_wq);599 ppd->qib_wq = NULL;600 }601 }602 return -ENOMEM;603}604 605static void qib_free_pportdata(struct qib_pportdata *ppd)606{607 free_percpu(ppd->ibport_data.pmastats);608 free_percpu(ppd->ibport_data.rvp.rc_acks);609 free_percpu(ppd->ibport_data.rvp.rc_qacks);610 free_percpu(ppd->ibport_data.rvp.rc_delayed_comp);611 ppd->ibport_data.pmastats = NULL;612}613 614/**615 * qib_init - do the actual initialization sequence on the chip616 * @dd: the qlogic_ib device617 * @reinit: reinitializing, so don't allocate new memory618 *619 * Do the actual initialization sequence on the chip. This is done620 * both from the init routine called from the PCI infrastructure, and621 * when we reset the chip, or detect that it was reset internally,622 * or it's administratively re-enabled.623 *624 * Memory allocation here and in called routines is only done in625 * the first case (reinit == 0). We have to be careful, because even626 * without memory allocation, we need to re-write all the chip registers627 * TIDs, etc. after the reset or enable has completed.628 */629int qib_init(struct qib_devdata *dd, int reinit)630{631 int ret = 0, pidx, lastfail = 0;632 u32 portok = 0;633 unsigned i;634 struct qib_ctxtdata *rcd;635 struct qib_pportdata *ppd;636 unsigned long flags;637 638 /* Set linkstate to unknown, so we can watch for a transition. */639 for (pidx = 0; pidx < dd->num_pports; ++pidx) {640 ppd = dd->pport + pidx;641 spin_lock_irqsave(&ppd->lflags_lock, flags);642 ppd->lflags &= ~(QIBL_LINKACTIVE | QIBL_LINKARMED |643 QIBL_LINKDOWN | QIBL_LINKINIT |644 QIBL_LINKV);645 spin_unlock_irqrestore(&ppd->lflags_lock, flags);646 }647 648 if (reinit)649 ret = init_after_reset(dd);650 else651 ret = loadtime_init(dd);652 if (ret)653 goto done;654 655 /* Bypass most chip-init, to get to device creation */656 if (qib_mini_init)657 return 0;658 659 ret = dd->f_late_initreg(dd);660 if (ret)661 goto done;662 663 /* dd->rcd can be NULL if early init failed */664 for (i = 0; dd->rcd && i < dd->first_user_ctxt; ++i) {665 /*666 * Set up the (kernel) rcvhdr queue and egr TIDs. If doing667 * re-init, the simplest way to handle this is to free668 * existing, and re-allocate.669 * Need to re-create rest of ctxt 0 ctxtdata as well.670 */671 rcd = dd->rcd[i];672 if (!rcd)673 continue;674 675 lastfail = qib_create_rcvhdrq(dd, rcd);676 if (!lastfail)677 lastfail = qib_setup_eagerbufs(rcd);678 if (lastfail)679 qib_dev_err(dd,680 "failed to allocate kernel ctxt's rcvhdrq and/or egr bufs\n");681 }682 683 for (pidx = 0; pidx < dd->num_pports; ++pidx) {684 int mtu;685 686 if (lastfail)687 ret = lastfail;688 ppd = dd->pport + pidx;689 mtu = ib_mtu_enum_to_int(qib_ibmtu);690 if (mtu == -1) {691 mtu = QIB_DEFAULT_MTU;692 qib_ibmtu = 0; /* don't leave invalid value */693 }694 /* set max we can ever have for this driver load */695 ppd->init_ibmaxlen = min(mtu > 2048 ?696 dd->piosize4k : dd->piosize2k,697 dd->rcvegrbufsize +698 (dd->rcvhdrentsize << 2));699 /*700 * Have to initialize ibmaxlen, but this will normally701 * change immediately in qib_set_mtu().702 */703 ppd->ibmaxlen = ppd->init_ibmaxlen;704 qib_set_mtu(ppd, mtu);705 706 spin_lock_irqsave(&ppd->lflags_lock, flags);707 ppd->lflags |= QIBL_IB_LINK_DISABLED;708 spin_unlock_irqrestore(&ppd->lflags_lock, flags);709 710 lastfail = dd->f_bringup_serdes(ppd);711 if (lastfail) {712 qib_devinfo(dd->pcidev,713 "Failed to bringup IB port %u\n", ppd->port);714 lastfail = -ENETDOWN;715 continue;716 }717 718 portok++;719 }720 721 if (!portok) {722 /* none of the ports initialized */723 if (!ret && lastfail)724 ret = lastfail;725 else if (!ret)726 ret = -ENETDOWN;727 /* but continue on, so we can debug cause */728 }729 730 enable_chip(dd);731 732 init_piobuf_state(dd);733 734done:735 if (!ret) {736 /* chip is OK for user apps; mark it as initialized */737 for (pidx = 0; pidx < dd->num_pports; ++pidx) {738 ppd = dd->pport + pidx;739 /*740 * Set status even if port serdes is not initialized741 * so that diags will work.742 */743 *ppd->statusp |= QIB_STATUS_CHIP_PRESENT |744 QIB_STATUS_INITTED;745 if (!ppd->link_speed_enabled)746 continue;747 if (dd->flags & QIB_HAS_SEND_DMA)748 ret = qib_setup_sdma(ppd);749 timer_setup(&ppd->hol_timer, qib_hol_event, 0);750 ppd->hol_state = QIB_HOL_UP;751 }752 753 /* now we can enable all interrupts from the chip */754 dd->f_set_intr_state(dd, 1);755 756 /*757 * Setup to verify we get an interrupt, and fallback758 * to an alternate if necessary and possible.759 */760 mod_timer(&dd->intrchk_timer, jiffies + HZ/2);761 /* start stats retrieval timer */762 mod_timer(&dd->stats_timer, jiffies + HZ * ACTIVITY_TIMER);763 }764 765 /* if ret is non-zero, we probably should do some cleanup here... */766 return ret;767}768 769/*770 * These next two routines are placeholders in case we don't have per-arch771 * code for controlling write combining. If explicit control of write772 * combining is not available, performance will probably be awful.773 */774 775int __attribute__((weak)) qib_enable_wc(struct qib_devdata *dd)776{777 return -EOPNOTSUPP;778}779 780void __attribute__((weak)) qib_disable_wc(struct qib_devdata *dd)781{782}783 784struct qib_devdata *qib_lookup(int unit)785{786 return xa_load(&qib_dev_table, unit);787}788 789/*790 * Stop the timers during unit shutdown, or after an error late791 * in initialization.792 */793static void qib_stop_timers(struct qib_devdata *dd)794{795 struct qib_pportdata *ppd;796 int pidx;797 798 if (dd->stats_timer.function)799 del_timer_sync(&dd->stats_timer);800 if (dd->intrchk_timer.function)801 del_timer_sync(&dd->intrchk_timer);802 for (pidx = 0; pidx < dd->num_pports; ++pidx) {803 ppd = dd->pport + pidx;804 if (ppd->hol_timer.function)805 del_timer_sync(&ppd->hol_timer);806 if (ppd->led_override_timer.function) {807 del_timer_sync(&ppd->led_override_timer);808 atomic_set(&ppd->led_override_timer_active, 0);809 }810 if (ppd->symerr_clear_timer.function)811 del_timer_sync(&ppd->symerr_clear_timer);812 }813}814 815/**816 * qib_shutdown_device - shut down a device817 * @dd: the qlogic_ib device818 *819 * This is called to make the device quiet when we are about to820 * unload the driver, and also when the device is administratively821 * disabled. It does not free any data structures.822 * Everything it does has to be setup again by qib_init(dd, 1)823 */824static void qib_shutdown_device(struct qib_devdata *dd)825{826 struct qib_pportdata *ppd;827 unsigned pidx;828 829 if (dd->flags & QIB_SHUTDOWN)830 return;831 dd->flags |= QIB_SHUTDOWN;832 833 for (pidx = 0; pidx < dd->num_pports; ++pidx) {834 ppd = dd->pport + pidx;835 836 spin_lock_irq(&ppd->lflags_lock);837 ppd->lflags &= ~(QIBL_LINKDOWN | QIBL_LINKINIT |838 QIBL_LINKARMED | QIBL_LINKACTIVE |839 QIBL_LINKV);840 spin_unlock_irq(&ppd->lflags_lock);841 *ppd->statusp &= ~(QIB_STATUS_IB_CONF | QIB_STATUS_IB_READY);842 }843 dd->flags &= ~QIB_INITTED;844 845 /* mask interrupts, but not errors */846 dd->f_set_intr_state(dd, 0);847 848 for (pidx = 0; pidx < dd->num_pports; ++pidx) {849 ppd = dd->pport + pidx;850 dd->f_rcvctrl(ppd, QIB_RCVCTRL_TAILUPD_DIS |851 QIB_RCVCTRL_CTXT_DIS |852 QIB_RCVCTRL_INTRAVAIL_DIS |853 QIB_RCVCTRL_PKEY_ENB, -1);854 /*855 * Gracefully stop all sends allowing any in progress to856 * trickle out first.857 */858 dd->f_sendctrl(ppd, QIB_SENDCTRL_CLEAR);859 }860 861 /*862 * Enough for anything that's going to trickle out to have actually863 * done so.864 */865 udelay(20);866 867 for (pidx = 0; pidx < dd->num_pports; ++pidx) {868 ppd = dd->pport + pidx;869 dd->f_setextled(ppd, 0); /* make sure LEDs are off */870 871 if (dd->flags & QIB_HAS_SEND_DMA)872 qib_teardown_sdma(ppd);873 874 dd->f_sendctrl(ppd, QIB_SENDCTRL_AVAIL_DIS |875 QIB_SENDCTRL_SEND_DIS);876 /*877 * Clear SerdesEnable.878 * We can't count on interrupts since we are stopping.879 */880 dd->f_quiet_serdes(ppd);881 882 if (ppd->qib_wq) {883 destroy_workqueue(ppd->qib_wq);884 ppd->qib_wq = NULL;885 }886 qib_free_pportdata(ppd);887 }888 889}890 891/**892 * qib_free_ctxtdata - free a context's allocated data893 * @dd: the qlogic_ib device894 * @rcd: the ctxtdata structure895 *896 * free up any allocated data for a context897 * This should not touch anything that would affect a simultaneous898 * re-allocation of context data, because it is called after qib_mutex899 * is released (and can be called from reinit as well).900 * It should never change any chip state, or global driver state.901 */902void qib_free_ctxtdata(struct qib_devdata *dd, struct qib_ctxtdata *rcd)903{904 if (!rcd)905 return;906 907 if (rcd->rcvhdrq) {908 dma_free_coherent(&dd->pcidev->dev, rcd->rcvhdrq_size,909 rcd->rcvhdrq, rcd->rcvhdrq_phys);910 rcd->rcvhdrq = NULL;911 if (rcd->rcvhdrtail_kvaddr) {912 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,913 rcd->rcvhdrtail_kvaddr,914 rcd->rcvhdrqtailaddr_phys);915 rcd->rcvhdrtail_kvaddr = NULL;916 }917 }918 if (rcd->rcvegrbuf) {919 unsigned e;920 921 for (e = 0; e < rcd->rcvegrbuf_chunks; e++) {922 void *base = rcd->rcvegrbuf[e];923 size_t size = rcd->rcvegrbuf_size;924 925 dma_free_coherent(&dd->pcidev->dev, size,926 base, rcd->rcvegrbuf_phys[e]);927 }928 kfree(rcd->rcvegrbuf);929 rcd->rcvegrbuf = NULL;930 kfree(rcd->rcvegrbuf_phys);931 rcd->rcvegrbuf_phys = NULL;932 rcd->rcvegrbuf_chunks = 0;933 }934 935 kfree(rcd->tid_pg_list);936 vfree(rcd->user_event_mask);937 vfree(rcd->subctxt_uregbase);938 vfree(rcd->subctxt_rcvegrbuf);939 vfree(rcd->subctxt_rcvhdr_base);940#ifdef CONFIG_DEBUG_FS941 kfree(rcd->opstats);942 rcd->opstats = NULL;943#endif944 kfree(rcd);945}946 947/*948 * Perform a PIO buffer bandwidth write test, to verify proper system949 * configuration. Even when all the setup calls work, occasionally950 * BIOS or other issues can prevent write combining from working, or951 * can cause other bandwidth problems to the chip.952 *953 * This test simply writes the same buffer over and over again, and954 * measures close to the peak bandwidth to the chip (not testing955 * data bandwidth to the wire). On chips that use an address-based956 * trigger to send packets to the wire, this is easy. On chips that957 * use a count to trigger, we want to make sure that the packet doesn't958 * go out on the wire, or trigger flow control checks.959 */960static void qib_verify_pioperf(struct qib_devdata *dd)961{962 u32 pbnum, cnt, lcnt;963 u32 __iomem *piobuf;964 u32 *addr;965 u64 msecs, emsecs;966 967 piobuf = dd->f_getsendbuf(dd->pport, 0ULL, &pbnum);968 if (!piobuf) {969 qib_devinfo(dd->pcidev,970 "No PIObufs for checking perf, skipping\n");971 return;972 }973 974 /*975 * Enough to give us a reasonable test, less than piobuf size, and976 * likely multiple of store buffer length.977 */978 cnt = 1024;979 980 addr = vmalloc(cnt);981 if (!addr)982 goto done;983 984 preempt_disable(); /* we want reasonably accurate elapsed time */985 msecs = 1 + jiffies_to_msecs(jiffies);986 for (lcnt = 0; lcnt < 10000U; lcnt++) {987 /* wait until we cross msec boundary */988 if (jiffies_to_msecs(jiffies) >= msecs)989 break;990 udelay(1);991 }992 993 dd->f_set_armlaunch(dd, 0);994 995 /*996 * length 0, no dwords actually sent997 */998 writeq(0, piobuf);999 qib_flush_wc();1000 1001 /*1002 * This is only roughly accurate, since even with preempt we1003 * still take interrupts that could take a while. Running for1004 * >= 5 msec seems to get us "close enough" to accurate values.1005 */1006 msecs = jiffies_to_msecs(jiffies);1007 for (emsecs = lcnt = 0; emsecs <= 5UL; lcnt++) {1008 qib_pio_copy(piobuf + 64, addr, cnt >> 2);1009 emsecs = jiffies_to_msecs(jiffies) - msecs;1010 }1011 1012 /* 1 GiB/sec, slightly over IB SDR line rate */1013 if (lcnt < (emsecs * 1024U))1014 qib_dev_err(dd,1015 "Performance problem: bandwidth to PIO buffers is only %u MiB/sec\n",1016 lcnt / (u32) emsecs);1017 1018 preempt_enable();1019 1020 vfree(addr);1021 1022done:1023 /* disarm piobuf, so it's available again */1024 dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_BUF(pbnum));1025 qib_sendbuf_done(dd, pbnum);1026 dd->f_set_armlaunch(dd, 1);1027}1028 1029void qib_free_devdata(struct qib_devdata *dd)1030{1031 unsigned long flags;1032 1033 xa_lock_irqsave(&qib_dev_table, flags);1034 __xa_erase(&qib_dev_table, dd->unit);1035 xa_unlock_irqrestore(&qib_dev_table, flags);1036 1037#ifdef CONFIG_DEBUG_FS1038 qib_dbg_ibdev_exit(&dd->verbs_dev);1039#endif1040 free_percpu(dd->int_counter);1041 rvt_dealloc_device(&dd->verbs_dev.rdi);1042}1043 1044u64 qib_int_counter(struct qib_devdata *dd)1045{1046 int cpu;1047 u64 int_counter = 0;1048 1049 for_each_possible_cpu(cpu)1050 int_counter += *per_cpu_ptr(dd->int_counter, cpu);1051 return int_counter;1052}1053 1054u64 qib_sps_ints(void)1055{1056 unsigned long index, flags;1057 struct qib_devdata *dd;1058 u64 sps_ints = 0;1059 1060 xa_lock_irqsave(&qib_dev_table, flags);1061 xa_for_each(&qib_dev_table, index, dd) {1062 sps_ints += qib_int_counter(dd);1063 }1064 xa_unlock_irqrestore(&qib_dev_table, flags);1065 return sps_ints;1066}1067 1068/*1069 * Allocate our primary per-unit data structure. Must be done via verbs1070 * allocator, because the verbs cleanup process both does cleanup and1071 * free of the data structure.1072 * "extra" is for chip-specific data.1073 */1074struct qib_devdata *qib_alloc_devdata(struct pci_dev *pdev, size_t extra)1075{1076 struct qib_devdata *dd;1077 int ret, nports;1078 1079 /* extra is * number of ports */1080 nports = extra / sizeof(struct qib_pportdata);1081 dd = (struct qib_devdata *)rvt_alloc_device(sizeof(*dd) + extra,1082 nports);1083 if (!dd)1084 return ERR_PTR(-ENOMEM);1085 1086 ret = xa_alloc_irq(&qib_dev_table, &dd->unit, dd, xa_limit_32b,1087 GFP_KERNEL);1088 if (ret < 0) {1089 qib_early_err(&pdev->dev,1090 "Could not allocate unit ID: error %d\n", -ret);1091 goto bail;1092 }1093 rvt_set_ibdev_name(&dd->verbs_dev.rdi, "%s%d", "qib", dd->unit);1094 1095 dd->int_counter = alloc_percpu(u64);1096 if (!dd->int_counter) {1097 ret = -ENOMEM;1098 qib_early_err(&pdev->dev,1099 "Could not allocate per-cpu int_counter\n");1100 goto bail;1101 }1102 1103 if (!qib_cpulist_count) {1104 u32 count = num_online_cpus();1105 1106 qib_cpulist = bitmap_zalloc(count, GFP_KERNEL);1107 if (qib_cpulist)1108 qib_cpulist_count = count;1109 }1110#ifdef CONFIG_DEBUG_FS1111 qib_dbg_ibdev_init(&dd->verbs_dev);1112#endif1113 return dd;1114bail:1115 if (!list_empty(&dd->list))1116 list_del_init(&dd->list);1117 rvt_dealloc_device(&dd->verbs_dev.rdi);1118 return ERR_PTR(ret);1119}1120 1121/*1122 * Called from freeze mode handlers, and from PCI error1123 * reporting code. Should be paranoid about state of1124 * system and data structures.1125 */1126void qib_disable_after_error(struct qib_devdata *dd)1127{1128 if (dd->flags & QIB_INITTED) {1129 u32 pidx;1130 1131 dd->flags &= ~QIB_INITTED;1132 if (dd->pport)1133 for (pidx = 0; pidx < dd->num_pports; ++pidx) {1134 struct qib_pportdata *ppd;1135 1136 ppd = dd->pport + pidx;1137 if (dd->flags & QIB_PRESENT) {1138 qib_set_linkstate(ppd,1139 QIB_IB_LINKDOWN_DISABLE);1140 dd->f_setextled(ppd, 0);1141 }1142 *ppd->statusp &= ~QIB_STATUS_IB_READY;1143 }1144 }1145 1146 /*1147 * Mark as having had an error for driver, and also1148 * for /sys and status word mapped to user programs.1149 * This marks unit as not usable, until reset.1150 */1151 if (dd->devstatusp)1152 *dd->devstatusp |= QIB_STATUS_HWERROR;1153}1154 1155static void qib_remove_one(struct pci_dev *);1156static int qib_init_one(struct pci_dev *, const struct pci_device_id *);1157static void qib_shutdown_one(struct pci_dev *);1158 1159#define DRIVER_LOAD_MSG "Intel " QIB_DRV_NAME " loaded: "1160#define PFX QIB_DRV_NAME ": "1161 1162static const struct pci_device_id qib_pci_tbl[] = {1163 { PCI_DEVICE(PCI_VENDOR_ID_PATHSCALE, PCI_DEVICE_ID_QLOGIC_IB_6120) },1164 { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7220) },1165 { PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, PCI_DEVICE_ID_QLOGIC_IB_7322) },1166 { 0, }1167};1168 1169MODULE_DEVICE_TABLE(pci, qib_pci_tbl);1170 1171static struct pci_driver qib_driver = {1172 .name = QIB_DRV_NAME,1173 .probe = qib_init_one,1174 .remove = qib_remove_one,1175 .shutdown = qib_shutdown_one,1176 .id_table = qib_pci_tbl,1177 .err_handler = &qib_pci_err_handler,1178};1179 1180#ifdef CONFIG_INFINIBAND_QIB_DCA1181 1182static int qib_notify_dca(struct notifier_block *, unsigned long, void *);1183static struct notifier_block dca_notifier = {1184 .notifier_call = qib_notify_dca,1185 .next = NULL,1186 .priority = 01187};1188 1189static int qib_notify_dca_device(struct device *device, void *data)1190{1191 struct qib_devdata *dd = dev_get_drvdata(device);1192 unsigned long event = *(unsigned long *)data;1193 1194 return dd->f_notify_dca(dd, event);1195}1196 1197static int qib_notify_dca(struct notifier_block *nb, unsigned long event,1198 void *p)1199{1200 int rval;1201 1202 rval = driver_for_each_device(&qib_driver.driver, NULL,1203 &event, qib_notify_dca_device);1204 return rval ? NOTIFY_BAD : NOTIFY_DONE;1205}1206 1207#endif1208 1209/*1210 * Do all the generic driver unit- and chip-independent memory1211 * allocation and initialization.1212 */1213static int __init qib_ib_init(void)1214{1215 int ret;1216 1217 ret = qib_dev_init();1218 if (ret)1219 goto bail;1220 1221 /*1222 * These must be called before the driver is registered with1223 * the PCI subsystem.1224 */1225#ifdef CONFIG_INFINIBAND_QIB_DCA1226 dca_register_notify(&dca_notifier);1227#endif1228#ifdef CONFIG_DEBUG_FS1229 qib_dbg_init();1230#endif1231 ret = pci_register_driver(&qib_driver);1232 if (ret < 0) {1233 pr_err("Unable to register driver: error %d\n", -ret);1234 goto bail_dev;1235 }1236 1237 /* not fatal if it doesn't work */1238 if (qib_init_qibfs())1239 pr_err("Unable to register ipathfs\n");1240 goto bail; /* all OK */1241 1242bail_dev:1243#ifdef CONFIG_INFINIBAND_QIB_DCA1244 dca_unregister_notify(&dca_notifier);1245#endif1246#ifdef CONFIG_DEBUG_FS1247 qib_dbg_exit();1248#endif1249 qib_dev_cleanup();1250bail:1251 return ret;1252}1253 1254module_init(qib_ib_init);1255 1256/*1257 * Do the non-unit driver cleanup, memory free, etc. at unload.1258 */1259static void __exit qib_ib_cleanup(void)1260{1261 int ret;1262 1263 ret = qib_exit_qibfs();1264 if (ret)1265 pr_err(1266 "Unable to cleanup counter filesystem: error %d\n",1267 -ret);1268 1269#ifdef CONFIG_INFINIBAND_QIB_DCA1270 dca_unregister_notify(&dca_notifier);1271#endif1272 pci_unregister_driver(&qib_driver);1273#ifdef CONFIG_DEBUG_FS1274 qib_dbg_exit();1275#endif1276 1277 qib_cpulist_count = 0;1278 bitmap_free(qib_cpulist);1279 1280 WARN_ON(!xa_empty(&qib_dev_table));1281 qib_dev_cleanup();1282}1283 1284module_exit(qib_ib_cleanup);1285 1286/* this can only be called after a successful initialization */1287static void cleanup_device_data(struct qib_devdata *dd)1288{1289 int ctxt;1290 int pidx;1291 struct qib_ctxtdata **tmp;1292 unsigned long flags;1293 1294 /* users can't do anything more with chip */1295 for (pidx = 0; pidx < dd->num_pports; ++pidx) {1296 if (dd->pport[pidx].statusp)1297 *dd->pport[pidx].statusp &= ~QIB_STATUS_CHIP_PRESENT;1298 1299 spin_lock(&dd->pport[pidx].cc_shadow_lock);1300 1301 kfree(dd->pport[pidx].congestion_entries);1302 dd->pport[pidx].congestion_entries = NULL;1303 kfree(dd->pport[pidx].ccti_entries);1304 dd->pport[pidx].ccti_entries = NULL;1305 kfree(dd->pport[pidx].ccti_entries_shadow);1306 dd->pport[pidx].ccti_entries_shadow = NULL;1307 kfree(dd->pport[pidx].congestion_entries_shadow);1308 dd->pport[pidx].congestion_entries_shadow = NULL;1309 1310 spin_unlock(&dd->pport[pidx].cc_shadow_lock);1311 }1312 1313 qib_disable_wc(dd);1314 1315 if (dd->pioavailregs_dma) {1316 dma_free_coherent(&dd->pcidev->dev, PAGE_SIZE,1317 (void *) dd->pioavailregs_dma,1318 dd->pioavailregs_phys);1319 dd->pioavailregs_dma = NULL;1320 }1321 1322 if (dd->pageshadow) {1323 struct page **tmpp = dd->pageshadow;1324 dma_addr_t *tmpd = dd->physshadow;1325 int i;1326 1327 for (ctxt = 0; ctxt < dd->cfgctxts; ctxt++) {1328 int ctxt_tidbase = ctxt * dd->rcvtidcnt;1329 int maxtid = ctxt_tidbase + dd->rcvtidcnt;1330 1331 for (i = ctxt_tidbase; i < maxtid; i++) {1332 if (!tmpp[i])1333 continue;1334 dma_unmap_page(&dd->pcidev->dev, tmpd[i],1335 PAGE_SIZE, DMA_FROM_DEVICE);1336 qib_release_user_pages(&tmpp[i], 1);1337 tmpp[i] = NULL;1338 }1339 }1340 1341 dd->pageshadow = NULL;1342 vfree(tmpp);1343 dd->physshadow = NULL;1344 vfree(tmpd);1345 }1346 1347 /*1348 * Free any resources still in use (usually just kernel contexts)1349 * at unload; we do for ctxtcnt, because that's what we allocate.1350 * We acquire lock to be really paranoid that rcd isn't being1351 * accessed from some interrupt-related code (that should not happen,1352 * but best to be sure).1353 */1354 spin_lock_irqsave(&dd->uctxt_lock, flags);1355 tmp = dd->rcd;1356 dd->rcd = NULL;1357 spin_unlock_irqrestore(&dd->uctxt_lock, flags);1358 for (ctxt = 0; tmp && ctxt < dd->ctxtcnt; ctxt++) {1359 struct qib_ctxtdata *rcd = tmp[ctxt];1360 1361 tmp[ctxt] = NULL; /* debugging paranoia */1362 qib_free_ctxtdata(dd, rcd);1363 }1364 kfree(tmp);1365}1366 1367/*1368 * Clean up on unit shutdown, or error during unit load after1369 * successful initialization.1370 */1371static void qib_postinit_cleanup(struct qib_devdata *dd)1372{1373 /*1374 * Clean up chip-specific stuff.1375 * We check for NULL here, because it's outside1376 * the kregbase check, and we need to call it1377 * after the free_irq. Thus it's possible that1378 * the function pointers were never initialized.1379 */1380 if (dd->f_cleanup)1381 dd->f_cleanup(dd);1382 1383 qib_pcie_ddcleanup(dd);1384 1385 cleanup_device_data(dd);1386 1387 qib_free_devdata(dd);1388}1389 1390static int qib_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)1391{1392 int ret, j, pidx, initfail;1393 struct qib_devdata *dd = NULL;1394 1395 ret = qib_pcie_init(pdev, ent);1396 if (ret)1397 goto bail;1398 1399 /*1400 * Do device-specific initialiation, function table setup, dd1401 * allocation, etc.1402 */1403 switch (ent->device) {1404 case PCI_DEVICE_ID_QLOGIC_IB_6120:1405#ifdef CONFIG_PCI_MSI1406 dd = qib_init_iba6120_funcs(pdev, ent);1407#else1408 qib_early_err(&pdev->dev,1409 "Intel PCIE device 0x%x cannot work if CONFIG_PCI_MSI is not enabled\n",1410 ent->device);1411 dd = ERR_PTR(-ENODEV);1412#endif1413 break;1414 1415 case PCI_DEVICE_ID_QLOGIC_IB_7220:1416 dd = qib_init_iba7220_funcs(pdev, ent);1417 break;1418 1419 case PCI_DEVICE_ID_QLOGIC_IB_7322:1420 dd = qib_init_iba7322_funcs(pdev, ent);1421 break;1422 1423 default:1424 qib_early_err(&pdev->dev,1425 "Failing on unknown Intel deviceid 0x%x\n",1426 ent->device);1427 ret = -ENODEV;1428 }1429 1430 if (IS_ERR(dd))1431 ret = PTR_ERR(dd);1432 if (ret)1433 goto bail; /* error already printed */1434 1435 ret = qib_create_workqueues(dd);1436 if (ret)1437 goto bail;1438 1439 /* do the generic initialization */1440 initfail = qib_init(dd, 0);1441 1442 ret = qib_register_ib_device(dd);1443 1444 /*1445 * Now ready for use. this should be cleared whenever we1446 * detect a reset, or initiate one. If earlier failure,1447 * we still create devices, so diags, etc. can be used1448 * to determine cause of problem.1449 */1450 if (!qib_mini_init && !initfail && !ret)1451 dd->flags |= QIB_INITTED;1452 1453 j = qib_device_create(dd);1454 if (j)1455 qib_dev_err(dd, "Failed to create /dev devices: %d\n", -j);1456 j = qibfs_add(dd);1457 if (j)1458 qib_dev_err(dd, "Failed filesystem setup for counters: %d\n",1459 -j);1460 1461 if (qib_mini_init || initfail || ret) {1462 qib_stop_timers(dd);1463 flush_workqueue(ib_wq);1464 for (pidx = 0; pidx < dd->num_pports; ++pidx)1465 dd->f_quiet_serdes(dd->pport + pidx);1466 if (qib_mini_init)1467 goto bail;1468 if (!j) {1469 (void) qibfs_remove(dd);1470 qib_device_remove(dd);1471 }1472 if (!ret)1473 qib_unregister_ib_device(dd);1474 qib_postinit_cleanup(dd);1475 if (initfail)1476 ret = initfail;1477 goto bail;1478 }1479 1480 ret = qib_enable_wc(dd);1481 if (ret) {1482 qib_dev_err(dd,1483 "Write combining not enabled (err %d): performance may be poor\n",1484 -ret);1485 ret = 0;1486 }1487 1488 qib_verify_pioperf(dd);1489bail:1490 return ret;1491}1492 1493static void qib_remove_one(struct pci_dev *pdev)1494{1495 struct qib_devdata *dd = pci_get_drvdata(pdev);1496 int ret;1497 1498 /* unregister from IB core */1499 qib_unregister_ib_device(dd);1500 1501 /*1502 * Disable the IB link, disable interrupts on the device,1503 * clear dma engines, etc.1504 */1505 if (!qib_mini_init)1506 qib_shutdown_device(dd);1507 1508 qib_stop_timers(dd);1509 1510 /* wait until all of our (qsfp) queue_work() calls complete */1511 flush_workqueue(ib_wq);1512 1513 ret = qibfs_remove(dd);1514 if (ret)1515 qib_dev_err(dd, "Failed counters filesystem cleanup: %d\n",1516 -ret);1517 1518 qib_device_remove(dd);1519 1520 qib_postinit_cleanup(dd);1521}1522 1523static void qib_shutdown_one(struct pci_dev *pdev)1524{1525 struct qib_devdata *dd = pci_get_drvdata(pdev);1526 1527 qib_shutdown_device(dd);1528}1529 1530/**1531 * qib_create_rcvhdrq - create a receive header queue1532 * @dd: the qlogic_ib device1533 * @rcd: the context data1534 *1535 * This must be contiguous memory (from an i/o perspective), and must be1536 * DMA'able (which means for some systems, it will go through an IOMMU,1537 * or be forced into a low address range).1538 */1539int qib_create_rcvhdrq(struct qib_devdata *dd, struct qib_ctxtdata *rcd)1540{1541 unsigned amt;1542 int old_node_id;1543 1544 if (!rcd->rcvhdrq) {1545 dma_addr_t phys_hdrqtail;1546 1547 amt = ALIGN(dd->rcvhdrcnt * dd->rcvhdrentsize *1548 sizeof(u32), PAGE_SIZE);1549 1550 old_node_id = dev_to_node(&dd->pcidev->dev);1551 set_dev_node(&dd->pcidev->dev, rcd->node_id);1552 rcd->rcvhdrq = dma_alloc_coherent(&dd->pcidev->dev, amt,1553 &rcd->rcvhdrq_phys, GFP_KERNEL);1554 set_dev_node(&dd->pcidev->dev, old_node_id);1555 1556 if (!rcd->rcvhdrq) {1557 qib_dev_err(dd,1558 "attempt to allocate %d bytes for ctxt %u rcvhdrq failed\n",1559 amt, rcd->ctxt);1560 goto bail;1561 }1562 1563 if (rcd->ctxt >= dd->first_user_ctxt) {1564 rcd->user_event_mask = vmalloc_user(PAGE_SIZE);1565 if (!rcd->user_event_mask)1566 goto bail_free_hdrq;1567 }1568 1569 if (!(dd->flags & QIB_NODMA_RTAIL)) {1570 set_dev_node(&dd->pcidev->dev, rcd->node_id);1571 rcd->rcvhdrtail_kvaddr = dma_alloc_coherent(1572 &dd->pcidev->dev, PAGE_SIZE, &phys_hdrqtail,1573 GFP_KERNEL);1574 set_dev_node(&dd->pcidev->dev, old_node_id);1575 if (!rcd->rcvhdrtail_kvaddr)1576 goto bail_free;1577 rcd->rcvhdrqtailaddr_phys = phys_hdrqtail;1578 }1579 1580 rcd->rcvhdrq_size = amt;1581 }1582 1583 /* clear for security and sanity on each use */1584 memset(rcd->rcvhdrq, 0, rcd->rcvhdrq_size);1585 if (rcd->rcvhdrtail_kvaddr)1586 memset(rcd->rcvhdrtail_kvaddr, 0, PAGE_SIZE);1587 return 0;1588 1589bail_free:1590 qib_dev_err(dd,1591 "attempt to allocate 1 page for ctxt %u rcvhdrqtailaddr failed\n",1592 rcd->ctxt);1593 vfree(rcd->user_event_mask);1594 rcd->user_event_mask = NULL;1595bail_free_hdrq:1596 dma_free_coherent(&dd->pcidev->dev, amt, rcd->rcvhdrq,1597 rcd->rcvhdrq_phys);1598 rcd->rcvhdrq = NULL;1599bail:1600 return -ENOMEM;1601}1602 1603/**1604 * qib_setup_eagerbufs - allocate eager buffers, both kernel and user contexts.1605 * @rcd: the context we are setting up.1606 *1607 * Allocate the eager TID buffers and program them into hip.1608 * They are no longer completely contiguous, we do multiple allocation1609 * calls. Otherwise we get the OOM code involved, by asking for too1610 * much per call, with disastrous results on some kernels.1611 */1612int qib_setup_eagerbufs(struct qib_ctxtdata *rcd)1613{1614 struct qib_devdata *dd = rcd->dd;1615 unsigned e, egrcnt, egrperchunk, chunk, egrsize, egroff;1616 size_t size;1617 int old_node_id;1618 1619 egrcnt = rcd->rcvegrcnt;1620 egroff = rcd->rcvegr_tid_base;1621 egrsize = dd->rcvegrbufsize;1622 1623 chunk = rcd->rcvegrbuf_chunks;1624 egrperchunk = rcd->rcvegrbufs_perchunk;1625 size = rcd->rcvegrbuf_size;1626 if (!rcd->rcvegrbuf) {1627 rcd->rcvegrbuf =1628 kcalloc_node(chunk, sizeof(rcd->rcvegrbuf[0]),1629 GFP_KERNEL, rcd->node_id);1630 if (!rcd->rcvegrbuf)1631 goto bail;1632 }1633 if (!rcd->rcvegrbuf_phys) {1634 rcd->rcvegrbuf_phys =1635 kmalloc_array_node(chunk,1636 sizeof(rcd->rcvegrbuf_phys[0]),1637 GFP_KERNEL, rcd->node_id);1638 if (!rcd->rcvegrbuf_phys)1639 goto bail_rcvegrbuf;1640 }1641 for (e = 0; e < rcd->rcvegrbuf_chunks; e++) {1642 if (rcd->rcvegrbuf[e])1643 continue;1644 1645 old_node_id = dev_to_node(&dd->pcidev->dev);1646 set_dev_node(&dd->pcidev->dev, rcd->node_id);1647 rcd->rcvegrbuf[e] =1648 dma_alloc_coherent(&dd->pcidev->dev, size,1649 &rcd->rcvegrbuf_phys[e],1650 GFP_KERNEL);1651 set_dev_node(&dd->pcidev->dev, old_node_id);1652 if (!rcd->rcvegrbuf[e])1653 goto bail_rcvegrbuf_phys;1654 }1655 1656 rcd->rcvegr_phys = rcd->rcvegrbuf_phys[0];1657 1658 for (e = chunk = 0; chunk < rcd->rcvegrbuf_chunks; chunk++) {1659 dma_addr_t pa = rcd->rcvegrbuf_phys[chunk];1660 unsigned i;1661 1662 /* clear for security and sanity on each use */1663 memset(rcd->rcvegrbuf[chunk], 0, size);1664 1665 for (i = 0; e < egrcnt && i < egrperchunk; e++, i++) {1666 dd->f_put_tid(dd, e + egroff +1667 (u64 __iomem *)1668 ((char __iomem *)1669 dd->kregbase +1670 dd->rcvegrbase),1671 RCVHQ_RCV_TYPE_EAGER, pa);1672 pa += egrsize;1673 }1674 cond_resched(); /* don't hog the cpu */1675 }1676 1677 return 0;1678 1679bail_rcvegrbuf_phys:1680 for (e = 0; e < rcd->rcvegrbuf_chunks && rcd->rcvegrbuf[e]; e++)1681 dma_free_coherent(&dd->pcidev->dev, size,1682 rcd->rcvegrbuf[e], rcd->rcvegrbuf_phys[e]);1683 kfree(rcd->rcvegrbuf_phys);1684 rcd->rcvegrbuf_phys = NULL;1685bail_rcvegrbuf:1686 kfree(rcd->rcvegrbuf);1687 rcd->rcvegrbuf = NULL;1688bail:1689 return -ENOMEM;1690}1691 1692/*1693 * Note: Changes to this routine should be mirrored1694 * for the diagnostics routine qib_remap_ioaddr32().1695 * There is also related code for VL15 buffers in qib_init_7322_variables().1696 * The teardown code that unmaps is in qib_pcie_ddcleanup()1697 */1698int init_chip_wc_pat(struct qib_devdata *dd, u32 vl15buflen)1699{1700 u64 __iomem *qib_kregbase = NULL;1701 void __iomem *qib_piobase = NULL;1702 u64 __iomem *qib_userbase = NULL;1703 u64 qib_kreglen;1704 u64 qib_pio2koffset = dd->piobufbase & 0xffffffff;1705 u64 qib_pio4koffset = dd->piobufbase >> 32;1706 u64 qib_pio2klen = dd->piobcnt2k * dd->palign;1707 u64 qib_pio4klen = dd->piobcnt4k * dd->align4k;1708 u64 qib_physaddr = dd->physaddr;1709 u64 qib_piolen;1710 u64 qib_userlen = 0;1711 1712 /*1713 * Free the old mapping because the kernel will try to reuse the1714 * old mapping and not create a new mapping with the1715 * write combining attribute.1716 */1717 iounmap(dd->kregbase);1718 dd->kregbase = NULL;1719 1720 /*1721 * Assumes chip address space looks like:1722 * - kregs + sregs + cregs + uregs (in any order)1723 * - piobufs (2K and 4K bufs in either order)1724 * or:1725 * - kregs + sregs + cregs (in any order)1726 * - piobufs (2K and 4K bufs in either order)1727 * - uregs1728 */1729 if (dd->piobcnt4k == 0) {1730 qib_kreglen = qib_pio2koffset;1731 qib_piolen = qib_pio2klen;1732 } else if (qib_pio2koffset < qib_pio4koffset) {1733 qib_kreglen = qib_pio2koffset;1734 qib_piolen = qib_pio4koffset + qib_pio4klen - qib_kreglen;1735 } else {1736 qib_kreglen = qib_pio4koffset;1737 qib_piolen = qib_pio2koffset + qib_pio2klen - qib_kreglen;1738 }1739 qib_piolen += vl15buflen;1740 /* Map just the configured ports (not all hw ports) */1741 if (dd->uregbase > qib_kreglen)1742 qib_userlen = dd->ureg_align * dd->cfgctxts;1743 1744 /* Sanity checks passed, now create the new mappings */1745 qib_kregbase = ioremap(qib_physaddr, qib_kreglen);1746 if (!qib_kregbase)1747 goto bail;1748 1749 qib_piobase = ioremap_wc(qib_physaddr + qib_kreglen, qib_piolen);1750 if (!qib_piobase)1751 goto bail_kregbase;1752 1753 if (qib_userlen) {1754 qib_userbase = ioremap(qib_physaddr + dd->uregbase,1755 qib_userlen);1756 if (!qib_userbase)1757 goto bail_piobase;1758 }1759 1760 dd->kregbase = qib_kregbase;1761 dd->kregend = (u64 __iomem *)1762 ((char __iomem *) qib_kregbase + qib_kreglen);1763 dd->piobase = qib_piobase;1764 dd->pio2kbase = (void __iomem *)1765 (((char __iomem *) dd->piobase) +1766 qib_pio2koffset - qib_kreglen);1767 if (dd->piobcnt4k)1768 dd->pio4kbase = (void __iomem *)1769 (((char __iomem *) dd->piobase) +1770 qib_pio4koffset - qib_kreglen);1771 if (qib_userlen)1772 /* ureg will now be accessed relative to dd->userbase */1773 dd->userbase = qib_userbase;1774 return 0;1775 1776bail_piobase:1777 iounmap(qib_piobase);1778bail_kregbase:1779 iounmap(qib_kregbase);1780bail:1781 return -ENOMEM;1782}1783