1377 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 *4 * Bluetooth support for Intel PCIe devices5 *6 * Copyright (C) 2024 Intel Corporation7 */8 9#include <linux/kernel.h>10#include <linux/module.h>11#include <linux/firmware.h>12#include <linux/pci.h>13#include <linux/wait.h>14#include <linux/delay.h>15#include <linux/interrupt.h>16 17#include <linux/unaligned.h>18 19#include <net/bluetooth/bluetooth.h>20#include <net/bluetooth/hci_core.h>21 22#include "btintel.h"23#include "btintel_pcie.h"24 25#define VERSION "0.1"26 27#define BTINTEL_PCI_DEVICE(dev, subdev) \28 .vendor = PCI_VENDOR_ID_INTEL, \29 .device = (dev), \30 .subvendor = PCI_ANY_ID, \31 .subdevice = (subdev), \32 .driver_data = 033 34#define POLL_INTERVAL_US 1035 36/* Intel Bluetooth PCIe device id table */37static const struct pci_device_id btintel_pcie_table[] = {38 { BTINTEL_PCI_DEVICE(0xA876, PCI_ANY_ID) },39 { 0 }40};41MODULE_DEVICE_TABLE(pci, btintel_pcie_table);42 43/* Intel PCIe uses 4 bytes of HCI type instead of 1 byte BT SIG HCI type */44#define BTINTEL_PCIE_HCI_TYPE_LEN 445#define BTINTEL_PCIE_HCI_CMD_PKT 0x0000000146#define BTINTEL_PCIE_HCI_ACL_PKT 0x0000000247#define BTINTEL_PCIE_HCI_SCO_PKT 0x0000000348#define BTINTEL_PCIE_HCI_EVT_PKT 0x0000000449#define BTINTEL_PCIE_HCI_ISO_PKT 0x0000000550 51static inline void ipc_print_ia_ring(struct hci_dev *hdev, struct ia *ia,52 u16 queue_num)53{54 bt_dev_dbg(hdev, "IA: %s: tr-h:%02u tr-t:%02u cr-h:%02u cr-t:%02u",55 queue_num == BTINTEL_PCIE_TXQ_NUM ? "TXQ" : "RXQ",56 ia->tr_hia[queue_num], ia->tr_tia[queue_num],57 ia->cr_hia[queue_num], ia->cr_tia[queue_num]);58}59 60static inline void ipc_print_urbd1(struct hci_dev *hdev, struct urbd1 *urbd1,61 u16 index)62{63 bt_dev_dbg(hdev, "RXQ:urbd1(%u) frbd_tag:%u status: 0x%x fixed:0x%x",64 index, urbd1->frbd_tag, urbd1->status, urbd1->fixed);65}66 67static int btintel_pcie_poll_bit(struct btintel_pcie_data *data, u32 offset,68 u32 bits, u32 mask, int timeout_us)69{70 int t = 0;71 u32 reg;72 73 do {74 reg = btintel_pcie_rd_reg32(data, offset);75 76 if ((reg & mask) == (bits & mask))77 return t;78 udelay(POLL_INTERVAL_US);79 t += POLL_INTERVAL_US;80 } while (t < timeout_us);81 82 return -ETIMEDOUT;83}84 85static struct btintel_pcie_data *btintel_pcie_get_data(struct msix_entry *entry)86{87 u8 queue = entry->entry;88 struct msix_entry *entries = entry - queue;89 90 return container_of(entries, struct btintel_pcie_data, msix_entries[0]);91}92 93/* Set the doorbell for TXQ to notify the device that @index (actually index-1)94 * of the TFD is updated and ready to transmit.95 */96static void btintel_pcie_set_tx_db(struct btintel_pcie_data *data, u16 index)97{98 u32 val;99 100 val = index;101 val |= (BTINTEL_PCIE_TX_DB_VEC << 16);102 103 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val);104}105 106/* Copy the data to next(@tfd_index) data buffer and update the TFD(transfer107 * descriptor) with the data length and the DMA address of the data buffer.108 */109static void btintel_pcie_prepare_tx(struct txq *txq, u16 tfd_index,110 struct sk_buff *skb)111{112 struct data_buf *buf;113 struct tfd *tfd;114 115 tfd = &txq->tfds[tfd_index];116 memset(tfd, 0, sizeof(*tfd));117 118 buf = &txq->bufs[tfd_index];119 120 tfd->size = skb->len;121 tfd->addr = buf->data_p_addr;122 123 /* Copy the outgoing data to DMA buffer */124 memcpy(buf->data, skb->data, tfd->size);125}126 127static int btintel_pcie_send_sync(struct btintel_pcie_data *data,128 struct sk_buff *skb)129{130 int ret;131 u16 tfd_index;132 struct txq *txq = &data->txq;133 134 tfd_index = data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM];135 136 if (tfd_index > txq->count)137 return -ERANGE;138 139 /* Prepare for TX. It updates the TFD with the length of data and140 * address of the DMA buffer, and copy the data to the DMA buffer141 */142 btintel_pcie_prepare_tx(txq, tfd_index, skb);143 144 tfd_index = (tfd_index + 1) % txq->count;145 data->ia.tr_hia[BTINTEL_PCIE_TXQ_NUM] = tfd_index;146 147 /* Arm wait event condition */148 data->tx_wait_done = false;149 150 /* Set the doorbell to notify the device */151 btintel_pcie_set_tx_db(data, tfd_index);152 153 /* Wait for the complete interrupt - URBD0 */154 ret = wait_event_timeout(data->tx_wait_q, data->tx_wait_done,155 msecs_to_jiffies(BTINTEL_PCIE_TX_WAIT_TIMEOUT_MS));156 if (!ret)157 return -ETIME;158 159 return 0;160}161 162/* Set the doorbell for RXQ to notify the device that @index (actually index-1)163 * is available to receive the data164 */165static void btintel_pcie_set_rx_db(struct btintel_pcie_data *data, u16 index)166{167 u32 val;168 169 val = index;170 val |= (BTINTEL_PCIE_RX_DB_VEC << 16);171 172 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_HBUS_TARG_WRPTR, val);173}174 175/* Update the FRBD (free buffer descriptor) with the @frbd_index and the176 * DMA address of the free buffer.177 */178static void btintel_pcie_prepare_rx(struct rxq *rxq, u16 frbd_index)179{180 struct data_buf *buf;181 struct frbd *frbd;182 183 /* Get the buffer of the FRBD for DMA */184 buf = &rxq->bufs[frbd_index];185 186 frbd = &rxq->frbds[frbd_index];187 memset(frbd, 0, sizeof(*frbd));188 189 /* Update FRBD */190 frbd->tag = frbd_index;191 frbd->addr = buf->data_p_addr;192}193 194static int btintel_pcie_submit_rx(struct btintel_pcie_data *data)195{196 u16 frbd_index;197 struct rxq *rxq = &data->rxq;198 199 frbd_index = data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM];200 201 if (frbd_index > rxq->count)202 return -ERANGE;203 204 /* Prepare for RX submit. It updates the FRBD with the address of DMA205 * buffer206 */207 btintel_pcie_prepare_rx(rxq, frbd_index);208 209 frbd_index = (frbd_index + 1) % rxq->count;210 data->ia.tr_hia[BTINTEL_PCIE_RXQ_NUM] = frbd_index;211 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM);212 213 /* Set the doorbell to notify the device */214 btintel_pcie_set_rx_db(data, frbd_index);215 216 return 0;217}218 219static int btintel_pcie_start_rx(struct btintel_pcie_data *data)220{221 int i, ret;222 223 for (i = 0; i < BTINTEL_PCIE_RX_MAX_QUEUE; i++) {224 ret = btintel_pcie_submit_rx(data);225 if (ret)226 return ret;227 }228 229 return 0;230}231 232static void btintel_pcie_reset_ia(struct btintel_pcie_data *data)233{234 memset(data->ia.tr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);235 memset(data->ia.tr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);236 memset(data->ia.cr_hia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);237 memset(data->ia.cr_tia, 0, sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES);238}239 240static void btintel_pcie_reset_bt(struct btintel_pcie_data *data)241{242 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG,243 BTINTEL_PCIE_CSR_FUNC_CTRL_SW_RESET);244}245 246/* This function enables BT function by setting BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT bit in247 * BTINTEL_PCIE_CSR_FUNC_CTRL_REG register and wait for MSI-X with248 * BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0.249 * Then the host reads firmware version from BTINTEL_CSR_F2D_MBX and the boot stage250 * from BTINTEL_PCIE_CSR_BOOT_STAGE_REG.251 */252static int btintel_pcie_enable_bt(struct btintel_pcie_data *data)253{254 int err;255 256 data->gp0_received = false;257 258 /* Update the DMA address of CI struct to CSR */259 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_LSB_REG,260 data->ci_p_addr & 0xffffffff);261 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_CI_ADDR_MSB_REG,262 (u64)data->ci_p_addr >> 32);263 264 /* Reset the cached value of boot stage. it is updated by the MSI-X265 * gp0 interrupt handler.266 */267 data->boot_stage_cache = 0x0;268 269 /* Set MAC_INIT bit to start primary bootloader */270 btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);271 272 btintel_pcie_set_reg_bits(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG,273 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_INIT);274 275 /* Wait until MAC_ACCESS is granted */276 err = btintel_pcie_poll_bit(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG,277 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS,278 BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS,279 BTINTEL_DEFAULT_MAC_ACCESS_TIMEOUT_US);280 if (err < 0)281 return -ENODEV;282 283 /* MAC is ready. Enable BT FUNC */284 btintel_pcie_set_reg_bits(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG,285 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_ENA |286 BTINTEL_PCIE_CSR_FUNC_CTRL_FUNC_INIT);287 288 btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);289 290 /* wait for interrupt from the device after booting up to primary291 * bootloader.292 */293 err = wait_event_timeout(data->gp0_wait_q, data->gp0_received,294 msecs_to_jiffies(BTINTEL_DEFAULT_INTR_TIMEOUT));295 if (!err)296 return -ETIME;297 298 /* Check cached boot stage is BTINTEL_PCIE_CSR_BOOT_STAGE_ROM(BIT(0)) */299 if (~data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_ROM)300 return -ENODEV;301 302 return 0;303}304 305/* This function handles the MSI-X interrupt for gp0 cause (bit 0 in306 * BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES) which is sent for boot stage and image response.307 */308static void btintel_pcie_msix_gp0_handler(struct btintel_pcie_data *data)309{310 u32 reg;311 312 /* This interrupt is for three different causes and it is not easy to313 * know what causes the interrupt. So, it compares each register value314 * with cached value and update it before it wake up the queue.315 */316 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_BOOT_STAGE_REG);317 if (reg != data->boot_stage_cache)318 data->boot_stage_cache = reg;319 320 reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_IMG_RESPONSE_REG);321 if (reg != data->img_resp_cache)322 data->img_resp_cache = reg;323 324 data->gp0_received = true;325 326 /* If the boot stage is OP or IML, reset IA and start RX again */327 if (data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_OPFW ||328 data->boot_stage_cache & BTINTEL_PCIE_CSR_BOOT_STAGE_IML) {329 btintel_pcie_reset_ia(data);330 btintel_pcie_start_rx(data);331 }332 333 wake_up(&data->gp0_wait_q);334}335 336/* This function handles the MSX-X interrupt for rx queue 0 which is for TX337 */338static void btintel_pcie_msix_tx_handle(struct btintel_pcie_data *data)339{340 u16 cr_tia, cr_hia;341 struct txq *txq;342 struct urbd0 *urbd0;343 344 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM];345 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_TXQ_NUM];346 347 if (cr_tia == cr_hia)348 return;349 350 txq = &data->txq;351 352 while (cr_tia != cr_hia) {353 data->tx_wait_done = true;354 wake_up(&data->tx_wait_q);355 356 urbd0 = &txq->urbd0s[cr_tia];357 358 if (urbd0->tfd_index > txq->count)359 return;360 361 cr_tia = (cr_tia + 1) % txq->count;362 data->ia.cr_tia[BTINTEL_PCIE_TXQ_NUM] = cr_tia;363 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_TXQ_NUM);364 }365}366 367/* Process the received rx data368 * It check the frame header to identify the data type and create skb369 * and calling HCI API370 */371static int btintel_pcie_recv_frame(struct btintel_pcie_data *data,372 struct sk_buff *skb)373{374 int ret;375 u8 pkt_type;376 u16 plen;377 u32 pcie_pkt_type;378 struct sk_buff *new_skb;379 void *pdata;380 struct hci_dev *hdev = data->hdev;381 382 spin_lock(&data->hci_rx_lock);383 384 /* The first 4 bytes indicates the Intel PCIe specific packet type */385 pdata = skb_pull_data(skb, BTINTEL_PCIE_HCI_TYPE_LEN);386 if (!pdata) {387 bt_dev_err(hdev, "Corrupted packet received");388 ret = -EILSEQ;389 goto exit_error;390 }391 392 pcie_pkt_type = get_unaligned_le32(pdata);393 394 switch (pcie_pkt_type) {395 case BTINTEL_PCIE_HCI_ACL_PKT:396 if (skb->len >= HCI_ACL_HDR_SIZE) {397 plen = HCI_ACL_HDR_SIZE + __le16_to_cpu(hci_acl_hdr(skb)->dlen);398 pkt_type = HCI_ACLDATA_PKT;399 } else {400 bt_dev_err(hdev, "ACL packet is too short");401 ret = -EILSEQ;402 goto exit_error;403 }404 break;405 406 case BTINTEL_PCIE_HCI_SCO_PKT:407 if (skb->len >= HCI_SCO_HDR_SIZE) {408 plen = HCI_SCO_HDR_SIZE + hci_sco_hdr(skb)->dlen;409 pkt_type = HCI_SCODATA_PKT;410 } else {411 bt_dev_err(hdev, "SCO packet is too short");412 ret = -EILSEQ;413 goto exit_error;414 }415 break;416 417 case BTINTEL_PCIE_HCI_EVT_PKT:418 if (skb->len >= HCI_EVENT_HDR_SIZE) {419 plen = HCI_EVENT_HDR_SIZE + hci_event_hdr(skb)->plen;420 pkt_type = HCI_EVENT_PKT;421 } else {422 bt_dev_err(hdev, "Event packet is too short");423 ret = -EILSEQ;424 goto exit_error;425 }426 break;427 428 case BTINTEL_PCIE_HCI_ISO_PKT:429 if (skb->len >= HCI_ISO_HDR_SIZE) {430 plen = HCI_ISO_HDR_SIZE + __le16_to_cpu(hci_iso_hdr(skb)->dlen);431 pkt_type = HCI_ISODATA_PKT;432 } else {433 bt_dev_err(hdev, "ISO packet is too short");434 ret = -EILSEQ;435 goto exit_error;436 }437 break;438 439 default:440 bt_dev_err(hdev, "Invalid packet type received: 0x%4.4x",441 pcie_pkt_type);442 ret = -EINVAL;443 goto exit_error;444 }445 446 if (skb->len < plen) {447 bt_dev_err(hdev, "Received corrupted packet. type: 0x%2.2x",448 pkt_type);449 ret = -EILSEQ;450 goto exit_error;451 }452 453 bt_dev_dbg(hdev, "pkt_type: 0x%2.2x len: %u", pkt_type, plen);454 455 new_skb = bt_skb_alloc(plen, GFP_ATOMIC);456 if (!new_skb) {457 bt_dev_err(hdev, "Failed to allocate memory for skb of len: %u",458 skb->len);459 ret = -ENOMEM;460 goto exit_error;461 }462 463 hci_skb_pkt_type(new_skb) = pkt_type;464 skb_put_data(new_skb, skb->data, plen);465 hdev->stat.byte_rx += plen;466 467 if (pcie_pkt_type == BTINTEL_PCIE_HCI_EVT_PKT)468 ret = btintel_recv_event(hdev, new_skb);469 else470 ret = hci_recv_frame(hdev, new_skb);471 472exit_error:473 if (ret)474 hdev->stat.err_rx++;475 476 spin_unlock(&data->hci_rx_lock);477 478 return ret;479}480 481static void btintel_pcie_rx_work(struct work_struct *work)482{483 struct btintel_pcie_data *data = container_of(work,484 struct btintel_pcie_data, rx_work);485 struct sk_buff *skb;486 int err;487 struct hci_dev *hdev = data->hdev;488 489 /* Process the sk_buf in queue and send to the HCI layer */490 while ((skb = skb_dequeue(&data->rx_skb_q))) {491 err = btintel_pcie_recv_frame(data, skb);492 if (err)493 bt_dev_err(hdev, "Failed to send received frame: %d",494 err);495 kfree_skb(skb);496 }497}498 499/* create sk_buff with data and save it to queue and start RX work */500static int btintel_pcie_submit_rx_work(struct btintel_pcie_data *data, u8 status,501 void *buf)502{503 int ret, len;504 struct rfh_hdr *rfh_hdr;505 struct sk_buff *skb;506 507 rfh_hdr = buf;508 509 len = rfh_hdr->packet_len;510 if (len <= 0) {511 ret = -EINVAL;512 goto resubmit;513 }514 515 /* Remove RFH header */516 buf += sizeof(*rfh_hdr);517 518 skb = alloc_skb(len, GFP_ATOMIC);519 if (!skb) {520 ret = -ENOMEM;521 goto resubmit;522 }523 524 skb_put_data(skb, buf, len);525 skb_queue_tail(&data->rx_skb_q, skb);526 queue_work(data->workqueue, &data->rx_work);527 528resubmit:529 ret = btintel_pcie_submit_rx(data);530 531 return ret;532}533 534/* Handles the MSI-X interrupt for rx queue 1 which is for RX */535static void btintel_pcie_msix_rx_handle(struct btintel_pcie_data *data)536{537 u16 cr_hia, cr_tia;538 struct rxq *rxq;539 struct urbd1 *urbd1;540 struct data_buf *buf;541 int ret;542 struct hci_dev *hdev = data->hdev;543 544 cr_hia = data->ia.cr_hia[BTINTEL_PCIE_RXQ_NUM];545 cr_tia = data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM];546 547 bt_dev_dbg(hdev, "RXQ: cr_hia: %u cr_tia: %u", cr_hia, cr_tia);548 549 /* Check CR_TIA and CR_HIA for change */550 if (cr_tia == cr_hia) {551 bt_dev_warn(hdev, "RXQ: no new CD found");552 return;553 }554 555 rxq = &data->rxq;556 557 /* The firmware sends multiple CD in a single MSI-X and it needs to558 * process all received CDs in this interrupt.559 */560 while (cr_tia != cr_hia) {561 urbd1 = &rxq->urbd1s[cr_tia];562 ipc_print_urbd1(data->hdev, urbd1, cr_tia);563 564 buf = &rxq->bufs[urbd1->frbd_tag];565 if (!buf) {566 bt_dev_err(hdev, "RXQ: failed to get the DMA buffer for %d",567 urbd1->frbd_tag);568 return;569 }570 571 ret = btintel_pcie_submit_rx_work(data, urbd1->status,572 buf->data);573 if (ret) {574 bt_dev_err(hdev, "RXQ: failed to submit rx request");575 return;576 }577 578 cr_tia = (cr_tia + 1) % rxq->count;579 data->ia.cr_tia[BTINTEL_PCIE_RXQ_NUM] = cr_tia;580 ipc_print_ia_ring(data->hdev, &data->ia, BTINTEL_PCIE_RXQ_NUM);581 }582}583 584static irqreturn_t btintel_pcie_msix_isr(int irq, void *data)585{586 return IRQ_WAKE_THREAD;587}588 589static irqreturn_t btintel_pcie_irq_msix_handler(int irq, void *dev_id)590{591 struct msix_entry *entry = dev_id;592 struct btintel_pcie_data *data = btintel_pcie_get_data(entry);593 u32 intr_fh, intr_hw;594 595 spin_lock(&data->irq_lock);596 intr_fh = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES);597 intr_hw = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES);598 599 /* Clear causes registers to avoid being handling the same cause */600 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_CAUSES, intr_fh);601 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_CAUSES, intr_hw);602 spin_unlock(&data->irq_lock);603 604 if (unlikely(!(intr_fh | intr_hw))) {605 /* Ignore interrupt, inta == 0 */606 return IRQ_NONE;607 }608 609 /* This interrupt is triggered by the firmware after updating610 * boot_stage register and image_response register611 */612 if (intr_hw & BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0)613 btintel_pcie_msix_gp0_handler(data);614 615 /* For TX */616 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0)617 btintel_pcie_msix_tx_handle(data);618 619 /* For RX */620 if (intr_fh & BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1)621 btintel_pcie_msix_rx_handle(data);622 623 /*624 * Before sending the interrupt the HW disables it to prevent a nested625 * interrupt. This is done by writing 1 to the corresponding bit in626 * the mask register. After handling the interrupt, it should be627 * re-enabled by clearing this bit. This register is defined as write 1628 * clear (W1C) register, meaning that it's cleared by writing 1629 * to the bit.630 */631 btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_MSIX_AUTOMASK_ST,632 BIT(entry->entry));633 634 return IRQ_HANDLED;635}636 637/* This function requests the irq for MSI-X and registers the handlers per irq.638 * Currently, it requests only 1 irq for all interrupt causes.639 */640static int btintel_pcie_setup_irq(struct btintel_pcie_data *data)641{642 int err;643 int num_irqs, i;644 645 for (i = 0; i < BTINTEL_PCIE_MSIX_VEC_MAX; i++)646 data->msix_entries[i].entry = i;647 648 num_irqs = pci_alloc_irq_vectors(data->pdev, BTINTEL_PCIE_MSIX_VEC_MIN,649 BTINTEL_PCIE_MSIX_VEC_MAX, PCI_IRQ_MSIX);650 if (num_irqs < 0)651 return num_irqs;652 653 data->alloc_vecs = num_irqs;654 data->msix_enabled = 1;655 data->def_irq = 0;656 657 /* setup irq handler */658 for (i = 0; i < data->alloc_vecs; i++) {659 struct msix_entry *msix_entry;660 661 msix_entry = &data->msix_entries[i];662 msix_entry->vector = pci_irq_vector(data->pdev, i);663 664 err = devm_request_threaded_irq(&data->pdev->dev,665 msix_entry->vector,666 btintel_pcie_msix_isr,667 btintel_pcie_irq_msix_handler,668 IRQF_SHARED,669 KBUILD_MODNAME,670 msix_entry);671 if (err) {672 pci_free_irq_vectors(data->pdev);673 data->alloc_vecs = 0;674 return err;675 }676 }677 return 0;678}679 680struct btintel_pcie_causes_list {681 u32 cause;682 u32 mask_reg;683 u8 cause_num;684};685 686static struct btintel_pcie_causes_list causes_list[] = {687 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_0, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x00 },688 { BTINTEL_PCIE_MSIX_FH_INT_CAUSES_1, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK, 0x01 },689 { BTINTEL_PCIE_MSIX_HW_INT_CAUSES_GP0, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK, 0x20 },690};691 692/* This function configures the interrupt masks for both HW_INT_CAUSES and693 * FH_INT_CAUSES which are meaningful to us.694 *695 * After resetting BT function via PCIE FLR or FUNC_CTRL reset, the driver696 * need to call this function again to configure since the masks697 * are reset to 0xFFFFFFFF after reset.698 */699static void btintel_pcie_config_msix(struct btintel_pcie_data *data)700{701 int i;702 int val = data->def_irq | BTINTEL_PCIE_MSIX_NON_AUTO_CLEAR_CAUSE;703 704 /* Set Non Auto Clear Cause */705 for (i = 0; i < ARRAY_SIZE(causes_list); i++) {706 btintel_pcie_wr_reg8(data,707 BTINTEL_PCIE_CSR_MSIX_IVAR(causes_list[i].cause_num),708 val);709 btintel_pcie_clr_reg_bits(data,710 causes_list[i].mask_reg,711 causes_list[i].cause);712 }713 714 /* Save the initial interrupt mask */715 data->fh_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_FH_INT_MASK);716 data->hw_init_mask = ~btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_MSIX_HW_INT_MASK);717}718 719static int btintel_pcie_config_pcie(struct pci_dev *pdev,720 struct btintel_pcie_data *data)721{722 int err;723 724 err = pcim_enable_device(pdev);725 if (err)726 return err;727 728 pci_set_master(pdev);729 730 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));731 if (err) {732 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));733 if (err)734 return err;735 }736 737 err = pcim_iomap_regions(pdev, BIT(0), KBUILD_MODNAME);738 if (err)739 return err;740 741 data->base_addr = pcim_iomap_table(pdev)[0];742 if (!data->base_addr)743 return -ENODEV;744 745 err = btintel_pcie_setup_irq(data);746 if (err)747 return err;748 749 /* Configure MSI-X with causes list */750 btintel_pcie_config_msix(data);751 752 return 0;753}754 755static void btintel_pcie_init_ci(struct btintel_pcie_data *data,756 struct ctx_info *ci)757{758 ci->version = 0x1;759 ci->size = sizeof(*ci);760 ci->config = 0x0000;761 ci->addr_cr_hia = data->ia.cr_hia_p_addr;762 ci->addr_tr_tia = data->ia.tr_tia_p_addr;763 ci->addr_cr_tia = data->ia.cr_tia_p_addr;764 ci->addr_tr_hia = data->ia.tr_hia_p_addr;765 ci->num_cr_ia = BTINTEL_PCIE_NUM_QUEUES;766 ci->num_tr_ia = BTINTEL_PCIE_NUM_QUEUES;767 ci->addr_urbdq0 = data->txq.urbd0s_p_addr;768 ci->addr_tfdq = data->txq.tfds_p_addr;769 ci->num_tfdq = data->txq.count;770 ci->num_urbdq0 = data->txq.count;771 ci->tfdq_db_vec = BTINTEL_PCIE_TXQ_NUM;772 ci->urbdq0_db_vec = BTINTEL_PCIE_TXQ_NUM;773 ci->rbd_size = BTINTEL_PCIE_RBD_SIZE_4K;774 ci->addr_frbdq = data->rxq.frbds_p_addr;775 ci->num_frbdq = data->rxq.count;776 ci->frbdq_db_vec = BTINTEL_PCIE_RXQ_NUM;777 ci->addr_urbdq1 = data->rxq.urbd1s_p_addr;778 ci->num_urbdq1 = data->rxq.count;779 ci->urbdq_db_vec = BTINTEL_PCIE_RXQ_NUM;780}781 782static void btintel_pcie_free_txq_bufs(struct btintel_pcie_data *data,783 struct txq *txq)784{785 /* Free data buffers first */786 dma_free_coherent(&data->pdev->dev, txq->count * BTINTEL_PCIE_BUFFER_SIZE,787 txq->buf_v_addr, txq->buf_p_addr);788 kfree(txq->bufs);789}790 791static int btintel_pcie_setup_txq_bufs(struct btintel_pcie_data *data,792 struct txq *txq)793{794 int i;795 struct data_buf *buf;796 797 /* Allocate the same number of buffers as the descriptor */798 txq->bufs = kmalloc_array(txq->count, sizeof(*buf), GFP_KERNEL);799 if (!txq->bufs)800 return -ENOMEM;801 802 /* Allocate full chunk of data buffer for DMA first and do indexing and803 * initialization next, so it can be freed easily804 */805 txq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev,806 txq->count * BTINTEL_PCIE_BUFFER_SIZE,807 &txq->buf_p_addr,808 GFP_KERNEL | __GFP_NOWARN);809 if (!txq->buf_v_addr) {810 kfree(txq->bufs);811 return -ENOMEM;812 }813 814 /* Setup the allocated DMA buffer to bufs. Each data_buf should815 * have virtual address and physical address816 */817 for (i = 0; i < txq->count; i++) {818 buf = &txq->bufs[i];819 buf->data_p_addr = txq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);820 buf->data = txq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);821 }822 823 return 0;824}825 826static void btintel_pcie_free_rxq_bufs(struct btintel_pcie_data *data,827 struct rxq *rxq)828{829 /* Free data buffers first */830 dma_free_coherent(&data->pdev->dev, rxq->count * BTINTEL_PCIE_BUFFER_SIZE,831 rxq->buf_v_addr, rxq->buf_p_addr);832 kfree(rxq->bufs);833}834 835static int btintel_pcie_setup_rxq_bufs(struct btintel_pcie_data *data,836 struct rxq *rxq)837{838 int i;839 struct data_buf *buf;840 841 /* Allocate the same number of buffers as the descriptor */842 rxq->bufs = kmalloc_array(rxq->count, sizeof(*buf), GFP_KERNEL);843 if (!rxq->bufs)844 return -ENOMEM;845 846 /* Allocate full chunk of data buffer for DMA first and do indexing and847 * initialization next, so it can be freed easily848 */849 rxq->buf_v_addr = dma_alloc_coherent(&data->pdev->dev,850 rxq->count * BTINTEL_PCIE_BUFFER_SIZE,851 &rxq->buf_p_addr,852 GFP_KERNEL | __GFP_NOWARN);853 if (!rxq->buf_v_addr) {854 kfree(rxq->bufs);855 return -ENOMEM;856 }857 858 /* Setup the allocated DMA buffer to bufs. Each data_buf should859 * have virtual address and physical address860 */861 for (i = 0; i < rxq->count; i++) {862 buf = &rxq->bufs[i];863 buf->data_p_addr = rxq->buf_p_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);864 buf->data = rxq->buf_v_addr + (i * BTINTEL_PCIE_BUFFER_SIZE);865 }866 867 return 0;868}869 870static void btintel_pcie_setup_ia(struct btintel_pcie_data *data,871 dma_addr_t p_addr, void *v_addr,872 struct ia *ia)873{874 /* TR Head Index Array */875 ia->tr_hia_p_addr = p_addr;876 ia->tr_hia = v_addr;877 878 /* TR Tail Index Array */879 ia->tr_tia_p_addr = p_addr + sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES;880 ia->tr_tia = v_addr + sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES;881 882 /* CR Head index Array */883 ia->cr_hia_p_addr = p_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 2);884 ia->cr_hia = v_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 2);885 886 /* CR Tail Index Array */887 ia->cr_tia_p_addr = p_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 3);888 ia->cr_tia = v_addr + (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 3);889}890 891static void btintel_pcie_free(struct btintel_pcie_data *data)892{893 btintel_pcie_free_rxq_bufs(data, &data->rxq);894 btintel_pcie_free_txq_bufs(data, &data->txq);895 896 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr);897 dma_pool_destroy(data->dma_pool);898}899 900/* Allocate tx and rx queues, any related data structures and buffers.901 */902static int btintel_pcie_alloc(struct btintel_pcie_data *data)903{904 int err = 0;905 size_t total;906 dma_addr_t p_addr;907 void *v_addr;908 909 /* Allocate the chunk of DMA memory for descriptors, index array, and910 * context information, instead of allocating individually.911 * The DMA memory for data buffer is allocated while setting up the912 * each queue.913 *914 * Total size is sum of the following915 * + size of TFD * Number of descriptors in queue916 * + size of URBD0 * Number of descriptors in queue917 * + size of FRBD * Number of descriptors in queue918 * + size of URBD1 * Number of descriptors in queue919 * + size of index * Number of queues(2) * type of index array(4)920 * + size of context information921 */922 total = (sizeof(struct tfd) + sizeof(struct urbd0) + sizeof(struct frbd)923 + sizeof(struct urbd1)) * BTINTEL_DESCS_COUNT;924 925 /* Add the sum of size of index array and size of ci struct */926 total += (sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 4) + sizeof(struct ctx_info);927 928 /* Allocate DMA Pool */929 data->dma_pool = dma_pool_create(KBUILD_MODNAME, &data->pdev->dev,930 total, BTINTEL_PCIE_DMA_POOL_ALIGNMENT, 0);931 if (!data->dma_pool) {932 err = -ENOMEM;933 goto exit_error;934 }935 936 v_addr = dma_pool_zalloc(data->dma_pool, GFP_KERNEL | __GFP_NOWARN,937 &p_addr);938 if (!v_addr) {939 dma_pool_destroy(data->dma_pool);940 err = -ENOMEM;941 goto exit_error;942 }943 944 data->dma_p_addr = p_addr;945 data->dma_v_addr = v_addr;946 947 /* Setup descriptor count */948 data->txq.count = BTINTEL_DESCS_COUNT;949 data->rxq.count = BTINTEL_DESCS_COUNT;950 951 /* Setup tfds */952 data->txq.tfds_p_addr = p_addr;953 data->txq.tfds = v_addr;954 955 p_addr += (sizeof(struct tfd) * BTINTEL_DESCS_COUNT);956 v_addr += (sizeof(struct tfd) * BTINTEL_DESCS_COUNT);957 958 /* Setup urbd0 */959 data->txq.urbd0s_p_addr = p_addr;960 data->txq.urbd0s = v_addr;961 962 p_addr += (sizeof(struct urbd0) * BTINTEL_DESCS_COUNT);963 v_addr += (sizeof(struct urbd0) * BTINTEL_DESCS_COUNT);964 965 /* Setup FRBD*/966 data->rxq.frbds_p_addr = p_addr;967 data->rxq.frbds = v_addr;968 969 p_addr += (sizeof(struct frbd) * BTINTEL_DESCS_COUNT);970 v_addr += (sizeof(struct frbd) * BTINTEL_DESCS_COUNT);971 972 /* Setup urbd1 */973 data->rxq.urbd1s_p_addr = p_addr;974 data->rxq.urbd1s = v_addr;975 976 p_addr += (sizeof(struct urbd1) * BTINTEL_DESCS_COUNT);977 v_addr += (sizeof(struct urbd1) * BTINTEL_DESCS_COUNT);978 979 /* Setup data buffers for txq */980 err = btintel_pcie_setup_txq_bufs(data, &data->txq);981 if (err)982 goto exit_error_pool;983 984 /* Setup data buffers for rxq */985 err = btintel_pcie_setup_rxq_bufs(data, &data->rxq);986 if (err)987 goto exit_error_txq;988 989 /* Setup Index Array */990 btintel_pcie_setup_ia(data, p_addr, v_addr, &data->ia);991 992 /* Setup Context Information */993 p_addr += sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 4;994 v_addr += sizeof(u16) * BTINTEL_PCIE_NUM_QUEUES * 4;995 996 data->ci = v_addr;997 data->ci_p_addr = p_addr;998 999 /* Initialize the CI */1000 btintel_pcie_init_ci(data, data->ci);1001 1002 return 0;1003 1004exit_error_txq:1005 btintel_pcie_free_txq_bufs(data, &data->txq);1006exit_error_pool:1007 dma_pool_free(data->dma_pool, data->dma_v_addr, data->dma_p_addr);1008 dma_pool_destroy(data->dma_pool);1009exit_error:1010 return err;1011}1012 1013static int btintel_pcie_open(struct hci_dev *hdev)1014{1015 bt_dev_dbg(hdev, "");1016 1017 return 0;1018}1019 1020static int btintel_pcie_close(struct hci_dev *hdev)1021{1022 bt_dev_dbg(hdev, "");1023 1024 return 0;1025}1026 1027static int btintel_pcie_inject_cmd_complete(struct hci_dev *hdev, __u16 opcode)1028{1029 struct sk_buff *skb;1030 struct hci_event_hdr *hdr;1031 struct hci_ev_cmd_complete *evt;1032 1033 skb = bt_skb_alloc(sizeof(*hdr) + sizeof(*evt) + 1, GFP_KERNEL);1034 if (!skb)1035 return -ENOMEM;1036 1037 hdr = (struct hci_event_hdr *)skb_put(skb, sizeof(*hdr));1038 hdr->evt = HCI_EV_CMD_COMPLETE;1039 hdr->plen = sizeof(*evt) + 1;1040 1041 evt = (struct hci_ev_cmd_complete *)skb_put(skb, sizeof(*evt));1042 evt->ncmd = 0x01;1043 evt->opcode = cpu_to_le16(opcode);1044 1045 *(u8 *)skb_put(skb, 1) = 0x00;1046 1047 hci_skb_pkt_type(skb) = HCI_EVENT_PKT;1048 1049 return hci_recv_frame(hdev, skb);1050}1051 1052static int btintel_pcie_send_frame(struct hci_dev *hdev,1053 struct sk_buff *skb)1054{1055 struct btintel_pcie_data *data = hci_get_drvdata(hdev);1056 int ret;1057 u32 type;1058 1059 /* Due to the fw limitation, the type header of the packet should be1060 * 4 bytes unlike 1 byte for UART. In UART, the firmware can read1061 * the first byte to get the packet type and redirect the rest of data1062 * packet to the right handler.1063 *1064 * But for PCIe, THF(Transfer Flow Handler) fetches the 4 bytes of data1065 * from DMA memory and by the time it reads the first 4 bytes, it has1066 * already consumed some part of packet. Thus the packet type indicator1067 * for iBT PCIe is 4 bytes.1068 *1069 * Luckily, when HCI core creates the skb, it allocates 8 bytes of1070 * head room for profile and driver use, and before sending the data1071 * to the device, append the iBT PCIe packet type in the front.1072 */1073 switch (hci_skb_pkt_type(skb)) {1074 case HCI_COMMAND_PKT:1075 type = BTINTEL_PCIE_HCI_CMD_PKT;1076 if (btintel_test_flag(hdev, INTEL_BOOTLOADER)) {1077 struct hci_command_hdr *cmd = (void *)skb->data;1078 __u16 opcode = le16_to_cpu(cmd->opcode);1079 1080 /* When the 0xfc01 command is issued to boot into1081 * the operational firmware, it will actually not1082 * send a command complete event. To keep the flow1083 * control working inject that event here.1084 */1085 if (opcode == 0xfc01)1086 btintel_pcie_inject_cmd_complete(hdev, opcode);1087 }1088 hdev->stat.cmd_tx++;1089 break;1090 case HCI_ACLDATA_PKT:1091 type = BTINTEL_PCIE_HCI_ACL_PKT;1092 hdev->stat.acl_tx++;1093 break;1094 case HCI_SCODATA_PKT:1095 type = BTINTEL_PCIE_HCI_SCO_PKT;1096 hdev->stat.sco_tx++;1097 break;1098 case HCI_ISODATA_PKT:1099 type = BTINTEL_PCIE_HCI_ISO_PKT;1100 break;1101 default:1102 bt_dev_err(hdev, "Unknown HCI packet type");1103 return -EILSEQ;1104 }1105 memcpy(skb_push(skb, BTINTEL_PCIE_HCI_TYPE_LEN), &type,1106 BTINTEL_PCIE_HCI_TYPE_LEN);1107 1108 ret = btintel_pcie_send_sync(data, skb);1109 if (ret) {1110 hdev->stat.err_tx++;1111 bt_dev_err(hdev, "Failed to send frame (%d)", ret);1112 goto exit_error;1113 }1114 hdev->stat.byte_tx += skb->len;1115 kfree_skb(skb);1116 1117exit_error:1118 return ret;1119}1120 1121static void btintel_pcie_release_hdev(struct btintel_pcie_data *data)1122{1123 struct hci_dev *hdev;1124 1125 hdev = data->hdev;1126 hci_unregister_dev(hdev);1127 hci_free_dev(hdev);1128 data->hdev = NULL;1129}1130 1131static int btintel_pcie_setup(struct hci_dev *hdev)1132{1133 const u8 param[1] = { 0xFF };1134 struct intel_version_tlv ver_tlv;1135 struct sk_buff *skb;1136 int err;1137 1138 BT_DBG("%s", hdev->name);1139 1140 skb = __hci_cmd_sync(hdev, 0xfc05, 1, param, HCI_CMD_TIMEOUT);1141 if (IS_ERR(skb)) {1142 bt_dev_err(hdev, "Reading Intel version command failed (%ld)",1143 PTR_ERR(skb));1144 return PTR_ERR(skb);1145 }1146 1147 /* Check the status */1148 if (skb->data[0]) {1149 bt_dev_err(hdev, "Intel Read Version command failed (%02x)",1150 skb->data[0]);1151 err = -EIO;1152 goto exit_error;1153 }1154 1155 /* Apply the common HCI quirks for Intel device */1156 set_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks);1157 set_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, &hdev->quirks);1158 set_bit(HCI_QUIRK_NON_PERSISTENT_DIAG, &hdev->quirks);1159 1160 /* Set up the quality report callback for Intel devices */1161 hdev->set_quality_report = btintel_set_quality_report;1162 1163 memset(&ver_tlv, 0, sizeof(ver_tlv));1164 /* For TLV type device, parse the tlv data */1165 err = btintel_parse_version_tlv(hdev, &ver_tlv, skb);1166 if (err) {1167 bt_dev_err(hdev, "Failed to parse TLV version information");1168 goto exit_error;1169 }1170 1171 switch (INTEL_HW_PLATFORM(ver_tlv.cnvi_bt)) {1172 case 0x37:1173 break;1174 default:1175 bt_dev_err(hdev, "Unsupported Intel hardware platform (0x%2x)",1176 INTEL_HW_PLATFORM(ver_tlv.cnvi_bt));1177 err = -EINVAL;1178 goto exit_error;1179 }1180 1181 /* Check for supported iBT hardware variants of this firmware1182 * loading method.1183 *1184 * This check has been put in place to ensure correct forward1185 * compatibility options when newer hardware variants come1186 * along.1187 */1188 switch (INTEL_HW_VARIANT(ver_tlv.cnvi_bt)) {1189 case 0x1e: /* BzrI */1190 /* Display version information of TLV type */1191 btintel_version_info_tlv(hdev, &ver_tlv);1192 1193 /* Apply the device specific HCI quirks for TLV based devices1194 *1195 * All TLV based devices support WBS1196 */1197 set_bit(HCI_QUIRK_WIDEBAND_SPEECH_SUPPORTED, &hdev->quirks);1198 1199 /* Setup MSFT Extension support */1200 btintel_set_msft_opcode(hdev,1201 INTEL_HW_VARIANT(ver_tlv.cnvi_bt));1202 1203 err = btintel_bootloader_setup_tlv(hdev, &ver_tlv);1204 if (err)1205 goto exit_error;1206 break;1207 default:1208 bt_dev_err(hdev, "Unsupported Intel hw variant (%u)",1209 INTEL_HW_VARIANT(ver_tlv.cnvi_bt));1210 err = -EINVAL;1211 goto exit_error;1212 break;1213 }1214 1215 btintel_print_fseq_info(hdev);1216exit_error:1217 kfree_skb(skb);1218 1219 return err;1220}1221 1222static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data)1223{1224 int err;1225 struct hci_dev *hdev;1226 1227 hdev = hci_alloc_dev_priv(sizeof(struct btintel_data));1228 if (!hdev)1229 return -ENOMEM;1230 1231 hdev->bus = HCI_PCI;1232 hci_set_drvdata(hdev, data);1233 1234 data->hdev = hdev;1235 SET_HCIDEV_DEV(hdev, &data->pdev->dev);1236 1237 hdev->manufacturer = 2;1238 hdev->open = btintel_pcie_open;1239 hdev->close = btintel_pcie_close;1240 hdev->send = btintel_pcie_send_frame;1241 hdev->setup = btintel_pcie_setup;1242 hdev->shutdown = btintel_shutdown_combined;1243 hdev->hw_error = btintel_hw_error;1244 hdev->set_diag = btintel_set_diag;1245 hdev->set_bdaddr = btintel_set_bdaddr;1246 1247 err = hci_register_dev(hdev);1248 if (err < 0) {1249 BT_ERR("Failed to register to hdev (%d)", err);1250 goto exit_error;1251 }1252 1253 return 0;1254 1255exit_error:1256 hci_free_dev(hdev);1257 return err;1258}1259 1260static int btintel_pcie_probe(struct pci_dev *pdev,1261 const struct pci_device_id *ent)1262{1263 int err;1264 struct btintel_pcie_data *data;1265 1266 if (!pdev)1267 return -ENODEV;1268 1269 data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);1270 if (!data)1271 return -ENOMEM;1272 1273 data->pdev = pdev;1274 1275 spin_lock_init(&data->irq_lock);1276 spin_lock_init(&data->hci_rx_lock);1277 1278 init_waitqueue_head(&data->gp0_wait_q);1279 data->gp0_received = false;1280 1281 init_waitqueue_head(&data->tx_wait_q);1282 data->tx_wait_done = false;1283 1284 data->workqueue = alloc_ordered_workqueue(KBUILD_MODNAME, WQ_HIGHPRI);1285 if (!data->workqueue)1286 return -ENOMEM;1287 1288 skb_queue_head_init(&data->rx_skb_q);1289 INIT_WORK(&data->rx_work, btintel_pcie_rx_work);1290 1291 data->boot_stage_cache = 0x00;1292 data->img_resp_cache = 0x00;1293 1294 err = btintel_pcie_config_pcie(pdev, data);1295 if (err)1296 goto exit_error;1297 1298 pci_set_drvdata(pdev, data);1299 1300 err = btintel_pcie_alloc(data);1301 if (err)1302 goto exit_error;1303 1304 err = btintel_pcie_enable_bt(data);1305 if (err)1306 goto exit_error;1307 1308 /* CNV information (CNVi and CNVr) is in CSR */1309 data->cnvi = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_HW_REV_REG);1310 1311 data->cnvr = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_RF_ID_REG);1312 1313 err = btintel_pcie_start_rx(data);1314 if (err)1315 goto exit_error;1316 1317 err = btintel_pcie_setup_hdev(data);1318 if (err)1319 goto exit_error;1320 1321 bt_dev_dbg(data->hdev, "cnvi: 0x%8.8x cnvr: 0x%8.8x", data->cnvi,1322 data->cnvr);1323 return 0;1324 1325exit_error:1326 /* reset device before exit */1327 btintel_pcie_reset_bt(data);1328 1329 pci_clear_master(pdev);1330 1331 pci_set_drvdata(pdev, NULL);1332 1333 return err;1334}1335 1336static void btintel_pcie_remove(struct pci_dev *pdev)1337{1338 struct btintel_pcie_data *data;1339 1340 data = pci_get_drvdata(pdev);1341 1342 btintel_pcie_reset_bt(data);1343 for (int i = 0; i < data->alloc_vecs; i++) {1344 struct msix_entry *msix_entry;1345 1346 msix_entry = &data->msix_entries[i];1347 free_irq(msix_entry->vector, msix_entry);1348 }1349 1350 pci_free_irq_vectors(pdev);1351 1352 btintel_pcie_release_hdev(data);1353 1354 flush_work(&data->rx_work);1355 1356 destroy_workqueue(data->workqueue);1357 1358 btintel_pcie_free(data);1359 1360 pci_clear_master(pdev);1361 1362 pci_set_drvdata(pdev, NULL);1363}1364 1365static struct pci_driver btintel_pcie_driver = {1366 .name = KBUILD_MODNAME,1367 .id_table = btintel_pcie_table,1368 .probe = btintel_pcie_probe,1369 .remove = btintel_pcie_remove,1370};1371module_pci_driver(btintel_pcie_driver);1372 1373MODULE_AUTHOR("Tedd Ho-Jeong An <tedd.an@intel.com>");1374MODULE_DESCRIPTION("Intel Bluetooth PCIe transport driver ver " VERSION);1375MODULE_VERSION(VERSION);1376MODULE_LICENSE("GPL");1377