907 lines · c
1/*2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.4 *5 * This software is available to you under a choice of one of two6 * licenses. You may choose to be licensed under the terms of the GNU7 * General Public License (GPL) Version 2, available from the file8 * COPYING in the main directory of this source tree, or the9 * OpenIB.org BSD license below:10 *11 * Redistribution and use in source and binary forms, with or12 * without modification, are permitted provided that the following13 * conditions are met:14 *15 * - Redistributions of source code must retain the above16 * copyright notice, this list of conditions and the following17 * disclaimer.18 *19 * - Redistributions in binary form must reproduce the above20 * copyright notice, this list of conditions and the following21 * disclaimer in the documentation and/or other materials22 * provided with the distribution.23 *24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE31 * SOFTWARE.32 */33 34#include <linux/errno.h>35#include <linux/interrupt.h>36#include <linux/pci.h>37#include <linux/slab.h>38 39#include "mthca_dev.h"40#include "mthca_cmd.h"41#include "mthca_config_reg.h"42 43enum {44 MTHCA_NUM_ASYNC_EQE = 0x80,45 MTHCA_NUM_CMD_EQE = 0x80,46 MTHCA_NUM_SPARE_EQE = 0x80,47 MTHCA_EQ_ENTRY_SIZE = 0x2048};49 50/*51 * Must be packed because start is 64 bits but only aligned to 32 bits.52 */53struct mthca_eq_context {54 __be32 flags;55 __be64 start;56 __be32 logsize_usrpage;57 __be32 tavor_pd; /* reserved for Arbel */58 u8 reserved1[3];59 u8 intr;60 __be32 arbel_pd; /* lost_count for Tavor */61 __be32 lkey;62 u32 reserved2[2];63 __be32 consumer_index;64 __be32 producer_index;65 u32 reserved3[4];66} __packed;67 68#define MTHCA_EQ_STATUS_OK ( 0 << 28)69#define MTHCA_EQ_STATUS_OVERFLOW ( 9 << 28)70#define MTHCA_EQ_STATUS_WRITE_FAIL (10 << 28)71#define MTHCA_EQ_OWNER_SW ( 0 << 24)72#define MTHCA_EQ_OWNER_HW ( 1 << 24)73#define MTHCA_EQ_FLAG_TR ( 1 << 18)74#define MTHCA_EQ_FLAG_OI ( 1 << 17)75#define MTHCA_EQ_STATE_ARMED ( 1 << 8)76#define MTHCA_EQ_STATE_FIRED ( 2 << 8)77#define MTHCA_EQ_STATE_ALWAYS_ARMED ( 3 << 8)78#define MTHCA_EQ_STATE_ARBEL ( 8 << 8)79 80enum {81 MTHCA_EVENT_TYPE_COMP = 0x00,82 MTHCA_EVENT_TYPE_PATH_MIG = 0x01,83 MTHCA_EVENT_TYPE_COMM_EST = 0x02,84 MTHCA_EVENT_TYPE_SQ_DRAINED = 0x03,85 MTHCA_EVENT_TYPE_SRQ_QP_LAST_WQE = 0x13,86 MTHCA_EVENT_TYPE_SRQ_LIMIT = 0x14,87 MTHCA_EVENT_TYPE_CQ_ERROR = 0x04,88 MTHCA_EVENT_TYPE_WQ_CATAS_ERROR = 0x05,89 MTHCA_EVENT_TYPE_EEC_CATAS_ERROR = 0x06,90 MTHCA_EVENT_TYPE_PATH_MIG_FAILED = 0x07,91 MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10,92 MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11,93 MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12,94 MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR = 0x08,95 MTHCA_EVENT_TYPE_PORT_CHANGE = 0x09,96 MTHCA_EVENT_TYPE_EQ_OVERFLOW = 0x0f,97 MTHCA_EVENT_TYPE_ECC_DETECT = 0x0e,98 MTHCA_EVENT_TYPE_CMD = 0x0a99};100 101#define MTHCA_ASYNC_EVENT_MASK ((1ULL << MTHCA_EVENT_TYPE_PATH_MIG) | \102 (1ULL << MTHCA_EVENT_TYPE_COMM_EST) | \103 (1ULL << MTHCA_EVENT_TYPE_SQ_DRAINED) | \104 (1ULL << MTHCA_EVENT_TYPE_CQ_ERROR) | \105 (1ULL << MTHCA_EVENT_TYPE_WQ_CATAS_ERROR) | \106 (1ULL << MTHCA_EVENT_TYPE_EEC_CATAS_ERROR) | \107 (1ULL << MTHCA_EVENT_TYPE_PATH_MIG_FAILED) | \108 (1ULL << MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \109 (1ULL << MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR) | \110 (1ULL << MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR) | \111 (1ULL << MTHCA_EVENT_TYPE_PORT_CHANGE) | \112 (1ULL << MTHCA_EVENT_TYPE_ECC_DETECT))113#define MTHCA_SRQ_EVENT_MASK ((1ULL << MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR) | \114 (1ULL << MTHCA_EVENT_TYPE_SRQ_QP_LAST_WQE) | \115 (1ULL << MTHCA_EVENT_TYPE_SRQ_LIMIT))116#define MTHCA_CMD_EVENT_MASK (1ULL << MTHCA_EVENT_TYPE_CMD)117 118#define MTHCA_EQ_DB_INC_CI (1 << 24)119#define MTHCA_EQ_DB_REQ_NOT (2 << 24)120#define MTHCA_EQ_DB_DISARM_CQ (3 << 24)121#define MTHCA_EQ_DB_SET_CI (4 << 24)122#define MTHCA_EQ_DB_ALWAYS_ARM (5 << 24)123 124struct mthca_eqe {125 u8 reserved1;126 u8 type;127 u8 reserved2;128 u8 subtype;129 union {130 u32 raw[6];131 struct {132 __be32 cqn;133 } __packed comp;134 struct {135 u16 reserved1;136 __be16 token;137 u32 reserved2;138 u8 reserved3[3];139 u8 status;140 __be64 out_param;141 } __packed cmd;142 struct {143 __be32 qpn;144 } __packed qp;145 struct {146 __be32 srqn;147 } __packed srq;148 struct {149 __be32 cqn;150 u32 reserved1;151 u8 reserved2[3];152 u8 syndrome;153 } __packed cq_err;154 struct {155 u32 reserved1[2];156 __be32 port;157 } __packed port_change;158 } event;159 u8 reserved3[3];160 u8 owner;161} __packed;162 163#define MTHCA_EQ_ENTRY_OWNER_SW (0 << 7)164#define MTHCA_EQ_ENTRY_OWNER_HW (1 << 7)165 166static inline u64 async_mask(struct mthca_dev *dev)167{168 return dev->mthca_flags & MTHCA_FLAG_SRQ ?169 MTHCA_ASYNC_EVENT_MASK | MTHCA_SRQ_EVENT_MASK :170 MTHCA_ASYNC_EVENT_MASK;171}172 173static inline void tavor_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)174{175 /*176 * This barrier makes sure that all updates to ownership bits177 * done by set_eqe_hw() hit memory before the consumer index178 * is updated. set_eq_ci() allows the HCA to possibly write179 * more EQ entries, and we want to avoid the exceedingly180 * unlikely possibility of the HCA writing an entry and then181 * having set_eqe_hw() overwrite the owner field.182 */183 wmb();184 mthca_write64(MTHCA_EQ_DB_SET_CI | eq->eqn, ci & (eq->nent - 1),185 dev->kar + MTHCA_EQ_DOORBELL,186 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));187}188 189static inline void arbel_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)190{191 /* See comment in tavor_set_eq_ci() above. */192 wmb();193 __raw_writel((__force u32) cpu_to_be32(ci),194 dev->eq_regs.arbel.eq_set_ci_base + eq->eqn * 8);195 /* We still want ordering, just not swabbing, so add a barrier */196 mb();197}198 199static inline void set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci)200{201 if (mthca_is_memfree(dev))202 arbel_set_eq_ci(dev, eq, ci);203 else204 tavor_set_eq_ci(dev, eq, ci);205}206 207static inline void tavor_eq_req_not(struct mthca_dev *dev, int eqn)208{209 mthca_write64(MTHCA_EQ_DB_REQ_NOT | eqn, 0,210 dev->kar + MTHCA_EQ_DOORBELL,211 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));212}213 214static inline void arbel_eq_req_not(struct mthca_dev *dev, u32 eqn_mask)215{216 writel(eqn_mask, dev->eq_regs.arbel.eq_arm);217}218 219static inline void disarm_cq(struct mthca_dev *dev, int eqn, int cqn)220{221 if (!mthca_is_memfree(dev)) {222 mthca_write64(MTHCA_EQ_DB_DISARM_CQ | eqn, cqn,223 dev->kar + MTHCA_EQ_DOORBELL,224 MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock));225 }226}227 228static inline struct mthca_eqe *get_eqe(struct mthca_eq *eq, u32 entry)229{230 unsigned long off = (entry & (eq->nent - 1)) * MTHCA_EQ_ENTRY_SIZE;231 return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE;232}233 234static inline struct mthca_eqe *next_eqe_sw(struct mthca_eq *eq)235{236 struct mthca_eqe *eqe;237 eqe = get_eqe(eq, eq->cons_index);238 return (MTHCA_EQ_ENTRY_OWNER_HW & eqe->owner) ? NULL : eqe;239}240 241static inline void set_eqe_hw(struct mthca_eqe *eqe)242{243 eqe->owner = MTHCA_EQ_ENTRY_OWNER_HW;244}245 246static void port_change(struct mthca_dev *dev, int port, int active)247{248 struct ib_event record;249 250 mthca_dbg(dev, "Port change to %s for port %d\n",251 active ? "active" : "down", port);252 253 record.device = &dev->ib_dev;254 record.event = active ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;255 record.element.port_num = port;256 257 ib_dispatch_event(&record);258}259 260static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq)261{262 struct mthca_eqe *eqe;263 int disarm_cqn;264 int eqes_found = 0;265 int set_ci = 0;266 267 while ((eqe = next_eqe_sw(eq))) {268 /*269 * Make sure we read EQ entry contents after we've270 * checked the ownership bit.271 */272 rmb();273 274 switch (eqe->type) {275 case MTHCA_EVENT_TYPE_COMP:276 disarm_cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff;277 disarm_cq(dev, eq->eqn, disarm_cqn);278 mthca_cq_completion(dev, disarm_cqn);279 break;280 281 case MTHCA_EVENT_TYPE_PATH_MIG:282 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,283 IB_EVENT_PATH_MIG);284 break;285 286 case MTHCA_EVENT_TYPE_COMM_EST:287 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,288 IB_EVENT_COMM_EST);289 break;290 291 case MTHCA_EVENT_TYPE_SQ_DRAINED:292 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,293 IB_EVENT_SQ_DRAINED);294 break;295 296 case MTHCA_EVENT_TYPE_SRQ_QP_LAST_WQE:297 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,298 IB_EVENT_QP_LAST_WQE_REACHED);299 break;300 301 case MTHCA_EVENT_TYPE_SRQ_LIMIT:302 mthca_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & 0xffffff,303 IB_EVENT_SRQ_LIMIT_REACHED);304 break;305 306 case MTHCA_EVENT_TYPE_WQ_CATAS_ERROR:307 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,308 IB_EVENT_QP_FATAL);309 break;310 311 case MTHCA_EVENT_TYPE_PATH_MIG_FAILED:312 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,313 IB_EVENT_PATH_MIG_ERR);314 break;315 316 case MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR:317 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,318 IB_EVENT_QP_REQ_ERR);319 break;320 321 case MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR:322 mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff,323 IB_EVENT_QP_ACCESS_ERR);324 break;325 326 case MTHCA_EVENT_TYPE_CMD:327 mthca_cmd_event(dev,328 be16_to_cpu(eqe->event.cmd.token),329 eqe->event.cmd.status,330 be64_to_cpu(eqe->event.cmd.out_param));331 break;332 333 case MTHCA_EVENT_TYPE_PORT_CHANGE:334 port_change(dev,335 (be32_to_cpu(eqe->event.port_change.port) >> 28) & 3,336 eqe->subtype == 0x4);337 break;338 339 case MTHCA_EVENT_TYPE_CQ_ERROR:340 mthca_warn(dev, "CQ %s on CQN %06x\n",341 eqe->event.cq_err.syndrome == 1 ?342 "overrun" : "access violation",343 be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff);344 mthca_cq_event(dev, be32_to_cpu(eqe->event.cq_err.cqn),345 IB_EVENT_CQ_ERR);346 break;347 348 case MTHCA_EVENT_TYPE_EQ_OVERFLOW:349 mthca_warn(dev, "EQ overrun on EQN %d\n", eq->eqn);350 break;351 352 case MTHCA_EVENT_TYPE_EEC_CATAS_ERROR:353 case MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR:354 case MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR:355 case MTHCA_EVENT_TYPE_ECC_DETECT:356 default:357 mthca_warn(dev, "Unhandled event %02x(%02x) on EQ %d\n",358 eqe->type, eqe->subtype, eq->eqn);359 break;360 }361 362 set_eqe_hw(eqe);363 ++eq->cons_index;364 eqes_found = 1;365 ++set_ci;366 367 /*368 * The HCA will think the queue has overflowed if we369 * don't tell it we've been processing events. We370 * create our EQs with MTHCA_NUM_SPARE_EQE extra371 * entries, so we must update our consumer index at372 * least that often.373 */374 if (unlikely(set_ci >= MTHCA_NUM_SPARE_EQE)) {375 /*376 * Conditional on hca_type is OK here because377 * this is a rare case, not the fast path.378 */379 set_eq_ci(dev, eq, eq->cons_index);380 set_ci = 0;381 }382 }383 384 /*385 * Rely on caller to set consumer index so that we don't have386 * to test hca_type in our interrupt handling fast path.387 */388 return eqes_found;389}390 391static irqreturn_t mthca_tavor_interrupt(int irq, void *dev_ptr)392{393 struct mthca_dev *dev = dev_ptr;394 u32 ecr;395 int i;396 397 if (dev->eq_table.clr_mask)398 writel(dev->eq_table.clr_mask, dev->eq_table.clr_int);399 400 ecr = readl(dev->eq_regs.tavor.ecr_base + 4);401 if (!ecr)402 return IRQ_NONE;403 404 writel(ecr, dev->eq_regs.tavor.ecr_base +405 MTHCA_ECR_CLR_BASE - MTHCA_ECR_BASE + 4);406 407 for (i = 0; i < MTHCA_NUM_EQ; ++i)408 if (ecr & dev->eq_table.eq[i].eqn_mask) {409 if (mthca_eq_int(dev, &dev->eq_table.eq[i]))410 tavor_set_eq_ci(dev, &dev->eq_table.eq[i],411 dev->eq_table.eq[i].cons_index);412 tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn);413 }414 415 return IRQ_HANDLED;416}417 418static irqreturn_t mthca_tavor_msi_x_interrupt(int irq, void *eq_ptr)419{420 struct mthca_eq *eq = eq_ptr;421 struct mthca_dev *dev = eq->dev;422 423 mthca_eq_int(dev, eq);424 tavor_set_eq_ci(dev, eq, eq->cons_index);425 tavor_eq_req_not(dev, eq->eqn);426 427 /* MSI-X vectors always belong to us */428 return IRQ_HANDLED;429}430 431static irqreturn_t mthca_arbel_interrupt(int irq, void *dev_ptr)432{433 struct mthca_dev *dev = dev_ptr;434 int work = 0;435 int i;436 437 if (dev->eq_table.clr_mask)438 writel(dev->eq_table.clr_mask, dev->eq_table.clr_int);439 440 for (i = 0; i < MTHCA_NUM_EQ; ++i)441 if (mthca_eq_int(dev, &dev->eq_table.eq[i])) {442 work = 1;443 arbel_set_eq_ci(dev, &dev->eq_table.eq[i],444 dev->eq_table.eq[i].cons_index);445 }446 447 arbel_eq_req_not(dev, dev->eq_table.arm_mask);448 449 return IRQ_RETVAL(work);450}451 452static irqreturn_t mthca_arbel_msi_x_interrupt(int irq, void *eq_ptr)453{454 struct mthca_eq *eq = eq_ptr;455 struct mthca_dev *dev = eq->dev;456 457 mthca_eq_int(dev, eq);458 arbel_set_eq_ci(dev, eq, eq->cons_index);459 arbel_eq_req_not(dev, eq->eqn_mask);460 461 /* MSI-X vectors always belong to us */462 return IRQ_HANDLED;463}464 465static int mthca_create_eq(struct mthca_dev *dev,466 int nent,467 u8 intr,468 struct mthca_eq *eq)469{470 int npages;471 u64 *dma_list = NULL;472 dma_addr_t t;473 struct mthca_mailbox *mailbox;474 struct mthca_eq_context *eq_context;475 int err = -ENOMEM;476 int i;477 478 eq->dev = dev;479 eq->nent = roundup_pow_of_two(max(nent, 2));480 npages = ALIGN(eq->nent * MTHCA_EQ_ENTRY_SIZE, PAGE_SIZE) / PAGE_SIZE;481 482 eq->page_list = kmalloc_array(npages, sizeof(*eq->page_list),483 GFP_KERNEL);484 if (!eq->page_list)485 goto err_out;486 487 for (i = 0; i < npages; ++i)488 eq->page_list[i].buf = NULL;489 490 dma_list = kmalloc_array(npages, sizeof(*dma_list), GFP_KERNEL);491 if (!dma_list)492 goto err_out_free;493 494 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);495 if (IS_ERR(mailbox))496 goto err_out_free;497 eq_context = mailbox->buf;498 499 for (i = 0; i < npages; ++i) {500 eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev,501 PAGE_SIZE, &t, GFP_KERNEL);502 if (!eq->page_list[i].buf)503 goto err_out_free_pages;504 505 dma_list[i] = t;506 dma_unmap_addr_set(&eq->page_list[i], mapping, t);507 508 clear_page(eq->page_list[i].buf);509 }510 511 for (i = 0; i < eq->nent; ++i)512 set_eqe_hw(get_eqe(eq, i));513 514 eq->eqn = mthca_alloc(&dev->eq_table.alloc);515 if (eq->eqn == -1)516 goto err_out_free_pages;517 518 err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num,519 dma_list, PAGE_SHIFT, npages,520 0, npages * PAGE_SIZE,521 MTHCA_MPT_FLAG_LOCAL_WRITE |522 MTHCA_MPT_FLAG_LOCAL_READ,523 &eq->mr);524 if (err)525 goto err_out_free_eq;526 527 memset(eq_context, 0, sizeof *eq_context);528 eq_context->flags = cpu_to_be32(MTHCA_EQ_STATUS_OK |529 MTHCA_EQ_OWNER_HW |530 MTHCA_EQ_STATE_ARMED |531 MTHCA_EQ_FLAG_TR);532 if (mthca_is_memfree(dev))533 eq_context->flags |= cpu_to_be32(MTHCA_EQ_STATE_ARBEL);534 535 eq_context->logsize_usrpage = cpu_to_be32((ffs(eq->nent) - 1) << 24);536 if (mthca_is_memfree(dev)) {537 eq_context->arbel_pd = cpu_to_be32(dev->driver_pd.pd_num);538 } else {539 eq_context->logsize_usrpage |= cpu_to_be32(dev->driver_uar.index);540 eq_context->tavor_pd = cpu_to_be32(dev->driver_pd.pd_num);541 }542 eq_context->intr = intr;543 eq_context->lkey = cpu_to_be32(eq->mr.ibmr.lkey);544 545 err = mthca_SW2HW_EQ(dev, mailbox, eq->eqn);546 if (err) {547 mthca_warn(dev, "SW2HW_EQ returned %d\n", err);548 goto err_out_free_mr;549 }550 551 kfree(dma_list);552 mthca_free_mailbox(dev, mailbox);553 554 eq->eqn_mask = swab32(1 << eq->eqn);555 eq->cons_index = 0;556 557 dev->eq_table.arm_mask |= eq->eqn_mask;558 559 mthca_dbg(dev, "Allocated EQ %d with %d entries\n",560 eq->eqn, eq->nent);561 562 return err;563 564 err_out_free_mr:565 mthca_free_mr(dev, &eq->mr);566 567 err_out_free_eq:568 mthca_free(&dev->eq_table.alloc, eq->eqn);569 570 err_out_free_pages:571 for (i = 0; i < npages; ++i)572 if (eq->page_list[i].buf)573 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,574 eq->page_list[i].buf,575 dma_unmap_addr(&eq->page_list[i],576 mapping));577 578 mthca_free_mailbox(dev, mailbox);579 580 err_out_free:581 kfree(eq->page_list);582 kfree(dma_list);583 584 err_out:585 return err;586}587 588static void mthca_free_eq(struct mthca_dev *dev,589 struct mthca_eq *eq)590{591 struct mthca_mailbox *mailbox;592 int err;593 int npages = (eq->nent * MTHCA_EQ_ENTRY_SIZE + PAGE_SIZE - 1) /594 PAGE_SIZE;595 int i;596 597 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL);598 if (IS_ERR(mailbox))599 return;600 601 err = mthca_HW2SW_EQ(dev, mailbox, eq->eqn);602 if (err)603 mthca_warn(dev, "HW2SW_EQ returned %d\n", err);604 605 dev->eq_table.arm_mask &= ~eq->eqn_mask;606 607 if (0) {608 mthca_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn);609 for (i = 0; i < sizeof (struct mthca_eq_context) / 4; ++i) {610 if (i % 4 == 0)611 printk("[%02x] ", i * 4);612 printk(" %08x", be32_to_cpup(mailbox->buf + i * 4));613 if ((i + 1) % 4 == 0)614 printk("\n");615 }616 }617 618 mthca_free_mr(dev, &eq->mr);619 for (i = 0; i < npages; ++i)620 dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,621 eq->page_list[i].buf,622 dma_unmap_addr(&eq->page_list[i], mapping));623 624 kfree(eq->page_list);625 mthca_free_mailbox(dev, mailbox);626}627 628static void mthca_free_irqs(struct mthca_dev *dev)629{630 int i;631 632 if (dev->eq_table.have_irq)633 free_irq(dev->pdev->irq, dev);634 for (i = 0; i < MTHCA_NUM_EQ; ++i)635 if (dev->eq_table.eq[i].have_irq) {636 free_irq(dev->eq_table.eq[i].msi_x_vector,637 dev->eq_table.eq + i);638 dev->eq_table.eq[i].have_irq = 0;639 }640}641 642static int mthca_map_reg(struct mthca_dev *dev,643 unsigned long offset, unsigned long size,644 void __iomem **map)645{646 phys_addr_t base = pci_resource_start(dev->pdev, 0);647 648 *map = ioremap(base + offset, size);649 if (!*map)650 return -ENOMEM;651 652 return 0;653}654 655static int mthca_map_eq_regs(struct mthca_dev *dev)656{657 if (mthca_is_memfree(dev)) {658 /*659 * We assume that the EQ arm and EQ set CI registers660 * fall within the first BAR. We can't trust the661 * values firmware gives us, since those addresses are662 * valid on the HCA's side of the PCI bus but not663 * necessarily the host side.664 */665 if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &666 dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE,667 &dev->clr_base)) {668 mthca_err(dev, "Couldn't map interrupt clear register, "669 "aborting.\n");670 return -ENOMEM;671 }672 673 /*674 * Add 4 because we limit ourselves to EQs 0 ... 31,675 * so we only need the low word of the register.676 */677 if (mthca_map_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) &678 dev->fw.arbel.eq_arm_base) + 4, 4,679 &dev->eq_regs.arbel.eq_arm)) {680 mthca_err(dev, "Couldn't map EQ arm register, aborting.\n");681 iounmap(dev->clr_base);682 return -ENOMEM;683 }684 685 if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) &686 dev->fw.arbel.eq_set_ci_base,687 MTHCA_EQ_SET_CI_SIZE,688 &dev->eq_regs.arbel.eq_set_ci_base)) {689 mthca_err(dev, "Couldn't map EQ CI register, aborting.\n");690 iounmap(dev->eq_regs.arbel.eq_arm);691 iounmap(dev->clr_base);692 return -ENOMEM;693 }694 } else {695 if (mthca_map_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE,696 &dev->clr_base)) {697 mthca_err(dev, "Couldn't map interrupt clear register, "698 "aborting.\n");699 return -ENOMEM;700 }701 702 if (mthca_map_reg(dev, MTHCA_ECR_BASE,703 MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE,704 &dev->eq_regs.tavor.ecr_base)) {705 mthca_err(dev, "Couldn't map ecr register, "706 "aborting.\n");707 iounmap(dev->clr_base);708 return -ENOMEM;709 }710 }711 712 return 0;713 714}715 716static void mthca_unmap_eq_regs(struct mthca_dev *dev)717{718 if (mthca_is_memfree(dev)) {719 iounmap(dev->eq_regs.arbel.eq_set_ci_base);720 iounmap(dev->eq_regs.arbel.eq_arm);721 iounmap(dev->clr_base);722 } else {723 iounmap(dev->eq_regs.tavor.ecr_base);724 iounmap(dev->clr_base);725 }726}727 728int mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt)729{730 int ret;731 732 /*733 * We assume that mapping one page is enough for the whole EQ734 * context table. This is fine with all current HCAs, because735 * we only use 32 EQs and each EQ uses 32 bytes of context736 * memory, or 1 KB total.737 */738 dev->eq_table.icm_virt = icm_virt;739 dev->eq_table.icm_page = alloc_page(GFP_HIGHUSER);740 if (!dev->eq_table.icm_page)741 return -ENOMEM;742 dev->eq_table.icm_dma =743 dma_map_page(&dev->pdev->dev, dev->eq_table.icm_page, 0,744 PAGE_SIZE, DMA_BIDIRECTIONAL);745 if (dma_mapping_error(&dev->pdev->dev, dev->eq_table.icm_dma)) {746 __free_page(dev->eq_table.icm_page);747 return -ENOMEM;748 }749 750 ret = mthca_MAP_ICM_page(dev, dev->eq_table.icm_dma, icm_virt);751 if (ret) {752 dma_unmap_page(&dev->pdev->dev, dev->eq_table.icm_dma,753 PAGE_SIZE, DMA_BIDIRECTIONAL);754 __free_page(dev->eq_table.icm_page);755 }756 757 return ret;758}759 760void mthca_unmap_eq_icm(struct mthca_dev *dev)761{762 mthca_UNMAP_ICM(dev, dev->eq_table.icm_virt, 1);763 dma_unmap_page(&dev->pdev->dev, dev->eq_table.icm_dma, PAGE_SIZE,764 DMA_BIDIRECTIONAL);765 __free_page(dev->eq_table.icm_page);766}767 768int mthca_init_eq_table(struct mthca_dev *dev)769{770 int err;771 u8 intr;772 int i;773 774 err = mthca_alloc_init(&dev->eq_table.alloc,775 dev->limits.num_eqs,776 dev->limits.num_eqs - 1,777 dev->limits.reserved_eqs);778 if (err)779 return err;780 781 err = mthca_map_eq_regs(dev);782 if (err)783 goto err_out_free;784 785 if (dev->mthca_flags & MTHCA_FLAG_MSI_X) {786 dev->eq_table.clr_mask = 0;787 } else {788 dev->eq_table.clr_mask =789 swab32(1 << (dev->eq_table.inta_pin & 31));790 dev->eq_table.clr_int = dev->clr_base +791 (dev->eq_table.inta_pin < 32 ? 4 : 0);792 }793 794 dev->eq_table.arm_mask = 0;795 796 intr = dev->eq_table.inta_pin;797 798 err = mthca_create_eq(dev, dev->limits.num_cqs + MTHCA_NUM_SPARE_EQE,799 (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 128 : intr,800 &dev->eq_table.eq[MTHCA_EQ_COMP]);801 if (err)802 goto err_out_unmap;803 804 err = mthca_create_eq(dev, MTHCA_NUM_ASYNC_EQE + MTHCA_NUM_SPARE_EQE,805 (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 129 : intr,806 &dev->eq_table.eq[MTHCA_EQ_ASYNC]);807 if (err)808 goto err_out_comp;809 810 err = mthca_create_eq(dev, MTHCA_NUM_CMD_EQE + MTHCA_NUM_SPARE_EQE,811 (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 130 : intr,812 &dev->eq_table.eq[MTHCA_EQ_CMD]);813 if (err)814 goto err_out_async;815 816 if (dev->mthca_flags & MTHCA_FLAG_MSI_X) {817 static const char *eq_name[] = {818 [MTHCA_EQ_COMP] = DRV_NAME "-comp",819 [MTHCA_EQ_ASYNC] = DRV_NAME "-async",820 [MTHCA_EQ_CMD] = DRV_NAME "-cmd"821 };822 823 for (i = 0; i < MTHCA_NUM_EQ; ++i) {824 snprintf(dev->eq_table.eq[i].irq_name,825 IB_DEVICE_NAME_MAX,826 "%s@pci:%s", eq_name[i],827 pci_name(dev->pdev));828 err = request_irq(dev->eq_table.eq[i].msi_x_vector,829 mthca_is_memfree(dev) ?830 mthca_arbel_msi_x_interrupt :831 mthca_tavor_msi_x_interrupt,832 0, dev->eq_table.eq[i].irq_name,833 dev->eq_table.eq + i);834 if (err)835 goto err_out_cmd;836 dev->eq_table.eq[i].have_irq = 1;837 }838 } else {839 snprintf(dev->eq_table.eq[0].irq_name, IB_DEVICE_NAME_MAX,840 DRV_NAME "@pci:%s", pci_name(dev->pdev));841 err = request_irq(dev->pdev->irq,842 mthca_is_memfree(dev) ?843 mthca_arbel_interrupt :844 mthca_tavor_interrupt,845 IRQF_SHARED, dev->eq_table.eq[0].irq_name, dev);846 if (err)847 goto err_out_cmd;848 dev->eq_table.have_irq = 1;849 }850 851 err = mthca_MAP_EQ(dev, async_mask(dev),852 0, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn);853 if (err)854 mthca_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",855 dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, err);856 857 err = mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK,858 0, dev->eq_table.eq[MTHCA_EQ_CMD].eqn);859 if (err)860 mthca_warn(dev, "MAP_EQ for cmd EQ %d failed (%d)\n",861 dev->eq_table.eq[MTHCA_EQ_CMD].eqn, err);862 863 for (i = 0; i < MTHCA_NUM_EQ; ++i)864 if (mthca_is_memfree(dev))865 arbel_eq_req_not(dev, dev->eq_table.eq[i].eqn_mask);866 else867 tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn);868 869 return 0;870 871err_out_cmd:872 mthca_free_irqs(dev);873 mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_CMD]);874 875err_out_async:876 mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_ASYNC]);877 878err_out_comp:879 mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_COMP]);880 881err_out_unmap:882 mthca_unmap_eq_regs(dev);883 884err_out_free:885 mthca_alloc_cleanup(&dev->eq_table.alloc);886 return err;887}888 889void mthca_cleanup_eq_table(struct mthca_dev *dev)890{891 int i;892 893 mthca_free_irqs(dev);894 895 mthca_MAP_EQ(dev, async_mask(dev),896 1, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn);897 mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK,898 1, dev->eq_table.eq[MTHCA_EQ_CMD].eqn);899 900 for (i = 0; i < MTHCA_NUM_EQ; ++i)901 mthca_free_eq(dev, &dev->eq_table.eq[i]);902 903 mthca_unmap_eq_regs(dev);904 905 mthca_alloc_cleanup(&dev->eq_table.alloc);906}907