brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only2/*3 * QLogic iSCSI Offload Driver4 * Copyright (c) 2016 Cavium Inc.5 */6 7#include <linux/module.h>8#include <linux/pci.h>9#include <linux/kernel.h>10#include <linux/if_arp.h>11#include <scsi/iscsi_if.h>12#include <linux/inet.h>13#include <net/arp.h>14#include <linux/list.h>15#include <linux/kthread.h>16#include <linux/mm.h>17#include <linux/if_vlan.h>18#include <linux/cpu.h>19#include <linux/iscsi_boot_sysfs.h>20 21#include <scsi/scsi_cmnd.h>22#include <scsi/scsi_device.h>23#include <scsi/scsi_eh.h>24#include <scsi/scsi_host.h>25#include <scsi/scsi.h>26 27#include "qedi.h"28#include "qedi_gbl.h"29#include "qedi_iscsi.h"30 31static uint qedi_qed_debug;32module_param(qedi_qed_debug, uint, 0644);33MODULE_PARM_DESC(qedi_qed_debug, " QED debug level 0 (default)");34 35static uint qedi_fw_debug;36module_param(qedi_fw_debug, uint, 0644);37MODULE_PARM_DESC(qedi_fw_debug, " Firmware debug level 0(default) to 3");38 39uint qedi_dbg_log = QEDI_LOG_WARN | QEDI_LOG_SCSI_TM;40module_param(qedi_dbg_log, uint, 0644);41MODULE_PARM_DESC(qedi_dbg_log, " Default debug level");42 43uint qedi_io_tracing;44module_param(qedi_io_tracing, uint, 0644);45MODULE_PARM_DESC(qedi_io_tracing,46		 " Enable logging of SCSI requests/completions into trace buffer. (default off).");47 48static uint qedi_ll2_buf_size = 0x400;49module_param(qedi_ll2_buf_size, uint, 0644);50MODULE_PARM_DESC(qedi_ll2_buf_size,51		 "parameter to set ping packet size, default - 0x400, Jumbo packets - 0x2400.");52 53static uint qedi_flags_override;54module_param(qedi_flags_override, uint, 0644);55MODULE_PARM_DESC(qedi_flags_override, "Disable/Enable MFW error flags bits action.");56 57const struct qed_iscsi_ops *qedi_ops;58static struct scsi_transport_template *qedi_scsi_transport;59static struct pci_driver qedi_pci_driver;60static DEFINE_PER_CPU(struct qedi_percpu_s, qedi_percpu);61static LIST_HEAD(qedi_udev_list);62/* Static function declaration */63static int qedi_alloc_global_queues(struct qedi_ctx *qedi);64static void qedi_free_global_queues(struct qedi_ctx *qedi);65static struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid);66static void qedi_reset_uio_rings(struct qedi_uio_dev *udev);67static void qedi_ll2_free_skbs(struct qedi_ctx *qedi);68static struct nvm_iscsi_block *qedi_get_nvram_block(struct qedi_ctx *qedi);69static void qedi_recovery_handler(struct work_struct *work);70static void qedi_schedule_hw_err_handler(void *dev,71					 enum qed_hw_err_type err_type);72static int qedi_suspend(struct pci_dev *pdev, pm_message_t state);73 74static int qedi_iscsi_event_cb(void *context, u8 fw_event_code, void *fw_handle)75{76	struct qedi_ctx *qedi;77	struct qedi_endpoint *qedi_ep;78	struct iscsi_eqe_data *data;79	int rval = 0;80 81	if (!context || !fw_handle) {82		QEDI_ERR(NULL, "Recv event with ctx NULL\n");83		return -EINVAL;84	}85 86	qedi = (struct qedi_ctx *)context;87	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,88		  "Recv Event %d fw_handle %p\n", fw_event_code, fw_handle);89 90	data = (struct iscsi_eqe_data *)fw_handle;91	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,92		  "icid=0x%x conn_id=0x%x err-code=0x%x error-pdu-opcode-reserved=0x%x\n",93		   data->icid, data->conn_id, data->error_code,94		   data->error_pdu_opcode_reserved);95 96	qedi_ep = qedi->ep_tbl[data->icid];97 98	if (!qedi_ep) {99		QEDI_WARN(&qedi->dbg_ctx,100			  "Cannot process event, ep already disconnected, cid=0x%x\n",101			   data->icid);102		WARN_ON(1);103		return -ENODEV;104	}105 106	switch (fw_event_code) {107	case ISCSI_EVENT_TYPE_ASYN_CONNECT_COMPLETE:108		if (qedi_ep->state == EP_STATE_OFLDCONN_START)109			qedi_ep->state = EP_STATE_OFLDCONN_COMPL;110 111		wake_up_interruptible(&qedi_ep->tcp_ofld_wait);112		break;113	case ISCSI_EVENT_TYPE_ASYN_TERMINATE_DONE:114		qedi_ep->state = EP_STATE_DISCONN_COMPL;115		wake_up_interruptible(&qedi_ep->tcp_ofld_wait);116		break;117	case ISCSI_EVENT_TYPE_ISCSI_CONN_ERROR:118		qedi_process_iscsi_error(qedi_ep, data);119		break;120	case ISCSI_EVENT_TYPE_ASYN_ABORT_RCVD:121	case ISCSI_EVENT_TYPE_ASYN_SYN_RCVD:122	case ISCSI_EVENT_TYPE_ASYN_MAX_RT_TIME:123	case ISCSI_EVENT_TYPE_ASYN_MAX_RT_CNT:124	case ISCSI_EVENT_TYPE_ASYN_MAX_KA_PROBES_CNT:125	case ISCSI_EVENT_TYPE_ASYN_FIN_WAIT2:126	case ISCSI_EVENT_TYPE_TCP_CONN_ERROR:127		qedi_process_tcp_error(qedi_ep, data);128		break;129	default:130		QEDI_ERR(&qedi->dbg_ctx, "Recv Unknown Event %u\n",131			 fw_event_code);132	}133 134	return rval;135}136 137static int qedi_uio_open(struct uio_info *uinfo, struct inode *inode)138{139	struct qedi_uio_dev *udev = uinfo->priv;140	struct qedi_ctx *qedi = udev->qedi;141 142	if (!capable(CAP_NET_ADMIN))143		return -EPERM;144 145	if (udev->uio_dev != -1)146		return -EBUSY;147 148	rtnl_lock();149	udev->uio_dev = iminor(inode);150	qedi_reset_uio_rings(udev);151	set_bit(UIO_DEV_OPENED, &qedi->flags);152	rtnl_unlock();153 154	return 0;155}156 157static int qedi_uio_close(struct uio_info *uinfo, struct inode *inode)158{159	struct qedi_uio_dev *udev = uinfo->priv;160	struct qedi_ctx *qedi = udev->qedi;161 162	udev->uio_dev = -1;163	clear_bit(UIO_DEV_OPENED, &qedi->flags);164	qedi_ll2_free_skbs(qedi);165	return 0;166}167 168static void __qedi_free_uio_rings(struct qedi_uio_dev *udev)169{170	if (udev->uctrl) {171		free_page((unsigned long)udev->uctrl);172		udev->uctrl = NULL;173	}174 175	if (udev->ll2_ring) {176		free_page((unsigned long)udev->ll2_ring);177		udev->ll2_ring = NULL;178	}179 180	if (udev->ll2_buf) {181		free_pages((unsigned long)udev->ll2_buf, 2);182		udev->ll2_buf = NULL;183	}184}185 186static void __qedi_free_uio(struct qedi_uio_dev *udev)187{188	uio_unregister_device(&udev->qedi_uinfo);189 190	__qedi_free_uio_rings(udev);191 192	pci_dev_put(udev->pdev);193	kfree(udev);194}195 196static void qedi_free_uio(struct qedi_uio_dev *udev)197{198	if (!udev)199		return;200 201	list_del_init(&udev->list);202	__qedi_free_uio(udev);203}204 205static void qedi_reset_uio_rings(struct qedi_uio_dev *udev)206{207	struct qedi_ctx *qedi = NULL;208	struct qedi_uio_ctrl *uctrl = NULL;209 210	qedi = udev->qedi;211	uctrl = udev->uctrl;212 213	spin_lock_bh(&qedi->ll2_lock);214	uctrl->host_rx_cons = 0;215	uctrl->hw_rx_prod = 0;216	uctrl->hw_rx_bd_prod = 0;217	uctrl->host_rx_bd_cons = 0;218 219	memset(udev->ll2_ring, 0, udev->ll2_ring_size);220	memset(udev->ll2_buf, 0, udev->ll2_buf_size);221	spin_unlock_bh(&qedi->ll2_lock);222}223 224static int __qedi_alloc_uio_rings(struct qedi_uio_dev *udev)225{226	int rc = 0;227 228	if (udev->ll2_ring || udev->ll2_buf)229		return rc;230 231	/* Memory for control area.  */232	udev->uctrl = (void *)get_zeroed_page(GFP_KERNEL);233	if (!udev->uctrl)234		return -ENOMEM;235 236	/* Allocating memory for LL2 ring  */237	udev->ll2_ring_size = QEDI_PAGE_SIZE;238	udev->ll2_ring = (void *)get_zeroed_page(GFP_KERNEL | __GFP_COMP);239	if (!udev->ll2_ring) {240		rc = -ENOMEM;241		goto exit_alloc_ring;242	}243 244	/* Allocating memory for Tx/Rx pkt buffer */245	udev->ll2_buf_size = TX_RX_RING * qedi_ll2_buf_size;246	udev->ll2_buf_size = QEDI_PAGE_ALIGN(udev->ll2_buf_size);247	udev->ll2_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_COMP |248						 __GFP_ZERO, 2);249	if (!udev->ll2_buf) {250		rc = -ENOMEM;251		goto exit_alloc_buf;252	}253	return rc;254 255exit_alloc_buf:256	free_page((unsigned long)udev->ll2_ring);257	udev->ll2_ring = NULL;258exit_alloc_ring:259	return rc;260}261 262static int qedi_alloc_uio_rings(struct qedi_ctx *qedi)263{264	struct qedi_uio_dev *udev = NULL;265	int rc = 0;266 267	list_for_each_entry(udev, &qedi_udev_list, list) {268		if (udev->pdev == qedi->pdev) {269			udev->qedi = qedi;270			if (__qedi_alloc_uio_rings(udev)) {271				udev->qedi = NULL;272				return -ENOMEM;273			}274			qedi->udev = udev;275			return 0;276		}277	}278 279	udev = kzalloc(sizeof(*udev), GFP_KERNEL);280	if (!udev)281		goto err_udev;282 283	udev->uio_dev = -1;284 285	udev->qedi = qedi;286	udev->pdev = qedi->pdev;287 288	rc = __qedi_alloc_uio_rings(udev);289	if (rc)290		goto err_uctrl;291 292	list_add(&udev->list, &qedi_udev_list);293 294	pci_dev_get(udev->pdev);295	qedi->udev = udev;296 297	udev->tx_pkt = udev->ll2_buf;298	udev->rx_pkt = udev->ll2_buf + qedi_ll2_buf_size;299	return 0;300 301 err_uctrl:302	kfree(udev);303 err_udev:304	return -ENOMEM;305}306 307static int qedi_init_uio(struct qedi_ctx *qedi)308{309	struct qedi_uio_dev *udev = qedi->udev;310	struct uio_info *uinfo;311	int ret = 0;312 313	if (!udev)314		return -ENOMEM;315 316	uinfo = &udev->qedi_uinfo;317 318	uinfo->mem[0].addr = (unsigned long)udev->uctrl;319	uinfo->mem[0].size = sizeof(struct qedi_uio_ctrl);320	uinfo->mem[0].memtype = UIO_MEM_LOGICAL;321 322	uinfo->mem[1].addr = (unsigned long)udev->ll2_ring;323	uinfo->mem[1].size = udev->ll2_ring_size;324	uinfo->mem[1].memtype = UIO_MEM_LOGICAL;325 326	uinfo->mem[2].addr = (unsigned long)udev->ll2_buf;327	uinfo->mem[2].size = udev->ll2_buf_size;328	uinfo->mem[2].memtype = UIO_MEM_LOGICAL;329 330	uinfo->name = "qedi_uio";331	uinfo->version = QEDI_MODULE_VERSION;332	uinfo->irq = UIO_IRQ_CUSTOM;333 334	uinfo->open = qedi_uio_open;335	uinfo->release = qedi_uio_close;336 337	if (udev->uio_dev == -1) {338		if (!uinfo->priv) {339			uinfo->priv = udev;340 341			ret = uio_register_device(&udev->pdev->dev, uinfo);342			if (ret) {343				QEDI_ERR(&qedi->dbg_ctx,344					 "UIO registration failed\n");345			}346		}347	}348 349	return ret;350}351 352static int qedi_alloc_and_init_sb(struct qedi_ctx *qedi,353				  struct qed_sb_info *sb_info, u16 sb_id)354{355	struct status_block *sb_virt;356	dma_addr_t sb_phys;357	int ret;358 359	sb_virt = dma_alloc_coherent(&qedi->pdev->dev,360				     sizeof(struct status_block), &sb_phys,361				     GFP_KERNEL);362	if (!sb_virt) {363		QEDI_ERR(&qedi->dbg_ctx,364			 "Status block allocation failed for id = %d.\n",365			  sb_id);366		return -ENOMEM;367	}368 369	ret = qedi_ops->common->sb_init(qedi->cdev, sb_info, sb_virt, sb_phys,370				       sb_id, QED_SB_TYPE_STORAGE);371	if (ret) {372		QEDI_ERR(&qedi->dbg_ctx,373			 "Status block initialization failed for id = %d.\n",374			  sb_id);375		return ret;376	}377 378	return 0;379}380 381static void qedi_free_sb(struct qedi_ctx *qedi)382{383	struct qed_sb_info *sb_info;384	int id;385 386	for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {387		sb_info = &qedi->sb_array[id];388		if (sb_info->sb_virt)389			dma_free_coherent(&qedi->pdev->dev,390					  sizeof(*sb_info->sb_virt),391					  (void *)sb_info->sb_virt,392					  sb_info->sb_phys);393	}394}395 396static void qedi_free_fp(struct qedi_ctx *qedi)397{398	kfree(qedi->fp_array);399	kfree(qedi->sb_array);400}401 402static void qedi_destroy_fp(struct qedi_ctx *qedi)403{404	qedi_free_sb(qedi);405	qedi_free_fp(qedi);406}407 408static int qedi_alloc_fp(struct qedi_ctx *qedi)409{410	int ret = 0;411 412	qedi->fp_array = kcalloc(MIN_NUM_CPUS_MSIX(qedi),413				 sizeof(struct qedi_fastpath), GFP_KERNEL);414	if (!qedi->fp_array) {415		QEDI_ERR(&qedi->dbg_ctx,416			 "fastpath fp array allocation failed.\n");417		return -ENOMEM;418	}419 420	qedi->sb_array = kcalloc(MIN_NUM_CPUS_MSIX(qedi),421				 sizeof(struct qed_sb_info), GFP_KERNEL);422	if (!qedi->sb_array) {423		QEDI_ERR(&qedi->dbg_ctx,424			 "fastpath sb array allocation failed.\n");425		ret = -ENOMEM;426		goto free_fp;427	}428 429	return ret;430 431free_fp:432	qedi_free_fp(qedi);433	return ret;434}435 436static void qedi_int_fp(struct qedi_ctx *qedi)437{438	struct qedi_fastpath *fp;439	int id;440 441	memset(qedi->fp_array, 0, MIN_NUM_CPUS_MSIX(qedi) *442	       sizeof(*qedi->fp_array));443	memset(qedi->sb_array, 0, MIN_NUM_CPUS_MSIX(qedi) *444	       sizeof(*qedi->sb_array));445 446	for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {447		fp = &qedi->fp_array[id];448		fp->sb_info = &qedi->sb_array[id];449		fp->sb_id = id;450		fp->qedi = qedi;451		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",452			 "qedi", id);453 454		/* fp_array[i] ---- irq cookie455		 * So init data which is needed in int ctx456		 */457	}458}459 460static int qedi_prepare_fp(struct qedi_ctx *qedi)461{462	struct qedi_fastpath *fp;463	int id, ret = 0;464 465	ret = qedi_alloc_fp(qedi);466	if (ret)467		goto err;468 469	qedi_int_fp(qedi);470 471	for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {472		fp = &qedi->fp_array[id];473		ret = qedi_alloc_and_init_sb(qedi, fp->sb_info, fp->sb_id);474		if (ret) {475			QEDI_ERR(&qedi->dbg_ctx,476				 "SB allocation and initialization failed.\n");477			ret = -EIO;478			goto err_init;479		}480	}481 482	return 0;483 484err_init:485	qedi_free_sb(qedi);486	qedi_free_fp(qedi);487err:488	return ret;489}490 491static int qedi_setup_cid_que(struct qedi_ctx *qedi)492{493	int i;494 495	qedi->cid_que.cid_que_base = kmalloc_array(qedi->max_active_conns,496						   sizeof(u32), GFP_KERNEL);497	if (!qedi->cid_que.cid_que_base)498		return -ENOMEM;499 500	qedi->cid_que.conn_cid_tbl = kmalloc_array(qedi->max_active_conns,501						   sizeof(struct qedi_conn *),502						   GFP_KERNEL);503	if (!qedi->cid_que.conn_cid_tbl) {504		kfree(qedi->cid_que.cid_que_base);505		qedi->cid_que.cid_que_base = NULL;506		return -ENOMEM;507	}508 509	qedi->cid_que.cid_que = (u32 *)qedi->cid_que.cid_que_base;510	qedi->cid_que.cid_q_prod_idx = 0;511	qedi->cid_que.cid_q_cons_idx = 0;512	qedi->cid_que.cid_q_max_idx = qedi->max_active_conns;513	qedi->cid_que.cid_free_cnt = qedi->max_active_conns;514 515	for (i = 0; i < qedi->max_active_conns; i++) {516		qedi->cid_que.cid_que[i] = i;517		qedi->cid_que.conn_cid_tbl[i] = NULL;518	}519 520	return 0;521}522 523static void qedi_release_cid_que(struct qedi_ctx *qedi)524{525	kfree(qedi->cid_que.cid_que_base);526	qedi->cid_que.cid_que_base = NULL;527 528	kfree(qedi->cid_que.conn_cid_tbl);529	qedi->cid_que.conn_cid_tbl = NULL;530}531 532static int qedi_init_id_tbl(struct qedi_portid_tbl *id_tbl, u16 size,533			    u16 start_id, u16 next)534{535	id_tbl->start = start_id;536	id_tbl->max = size;537	id_tbl->next = next;538	spin_lock_init(&id_tbl->lock);539	id_tbl->table = kcalloc(BITS_TO_LONGS(size), sizeof(long), GFP_KERNEL);540	if (!id_tbl->table)541		return -ENOMEM;542 543	return 0;544}545 546static void qedi_free_id_tbl(struct qedi_portid_tbl *id_tbl)547{548	kfree(id_tbl->table);549	id_tbl->table = NULL;550}551 552int qedi_alloc_id(struct qedi_portid_tbl *id_tbl, u16 id)553{554	int ret = -1;555 556	id -= id_tbl->start;557	if (id >= id_tbl->max)558		return ret;559 560	spin_lock(&id_tbl->lock);561	if (!test_bit(id, id_tbl->table)) {562		set_bit(id, id_tbl->table);563		ret = 0;564	}565	spin_unlock(&id_tbl->lock);566	return ret;567}568 569u16 qedi_alloc_new_id(struct qedi_portid_tbl *id_tbl)570{571	u16 id;572 573	spin_lock(&id_tbl->lock);574	id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next);575	if (id >= id_tbl->max) {576		id = QEDI_LOCAL_PORT_INVALID;577		if (id_tbl->next != 0) {578			id = find_first_zero_bit(id_tbl->table, id_tbl->next);579			if (id >= id_tbl->next)580				id = QEDI_LOCAL_PORT_INVALID;581		}582	}583 584	if (id < id_tbl->max) {585		set_bit(id, id_tbl->table);586		id_tbl->next = (id + 1) & (id_tbl->max - 1);587		id += id_tbl->start;588	}589 590	spin_unlock(&id_tbl->lock);591 592	return id;593}594 595void qedi_free_id(struct qedi_portid_tbl *id_tbl, u16 id)596{597	if (id == QEDI_LOCAL_PORT_INVALID)598		return;599 600	id -= id_tbl->start;601	if (id >= id_tbl->max)602		return;603 604	clear_bit(id, id_tbl->table);605}606 607static void qedi_cm_free_mem(struct qedi_ctx *qedi)608{609	kfree(qedi->ep_tbl);610	qedi->ep_tbl = NULL;611	qedi_free_id_tbl(&qedi->lcl_port_tbl);612}613 614static int qedi_cm_alloc_mem(struct qedi_ctx *qedi)615{616	u16 port_id;617 618	qedi->ep_tbl = kzalloc((qedi->max_active_conns *619				sizeof(struct qedi_endpoint *)), GFP_KERNEL);620	if (!qedi->ep_tbl)621		return -ENOMEM;622	port_id = get_random_u32_below(QEDI_LOCAL_PORT_RANGE);623	if (qedi_init_id_tbl(&qedi->lcl_port_tbl, QEDI_LOCAL_PORT_RANGE,624			     QEDI_LOCAL_PORT_MIN, port_id)) {625		qedi_cm_free_mem(qedi);626		return -ENOMEM;627	}628 629	return 0;630}631 632static struct qedi_ctx *qedi_host_alloc(struct pci_dev *pdev)633{634	struct Scsi_Host *shost;635	struct qedi_ctx *qedi = NULL;636 637	shost = iscsi_host_alloc(&qedi_host_template,638				 sizeof(struct qedi_ctx), 0);639	if (!shost) {640		QEDI_ERR(NULL, "Could not allocate shost\n");641		goto exit_setup_shost;642	}643 644	shost->max_id = QEDI_MAX_ISCSI_CONNS_PER_HBA - 1;645	shost->max_channel = 0;646	shost->max_lun = ~0;647	shost->max_cmd_len = 16;648	shost->transportt = qedi_scsi_transport;649 650	qedi = iscsi_host_priv(shost);651	memset(qedi, 0, sizeof(*qedi));652	qedi->shost = shost;653	qedi->dbg_ctx.host_no = shost->host_no;654	qedi->pdev = pdev;655	qedi->dbg_ctx.pdev = pdev;656	qedi->max_active_conns = ISCSI_MAX_SESS_PER_HBA;657	qedi->max_sqes = QEDI_SQ_SIZE;658 659	shost->nr_hw_queues = MIN_NUM_CPUS_MSIX(qedi);660 661	pci_set_drvdata(pdev, qedi);662 663exit_setup_shost:664	return qedi;665}666 667static int qedi_ll2_rx(void *cookie, struct sk_buff *skb, u32 arg1, u32 arg2)668{669	struct qedi_ctx *qedi = (struct qedi_ctx *)cookie;670	struct skb_work_list *work;671	struct ethhdr *eh;672 673	if (!qedi) {674		QEDI_ERR(NULL, "qedi is NULL\n");675		return -1;676	}677 678	if (!test_bit(UIO_DEV_OPENED, &qedi->flags)) {679		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_UIO,680			  "UIO DEV is not opened\n");681		kfree_skb(skb);682		return 0;683	}684 685	eh = (struct ethhdr *)skb->data;686	/* Undo VLAN encapsulation */687	if (eh->h_proto == htons(ETH_P_8021Q)) {688		memmove((u8 *)eh + VLAN_HLEN, eh, ETH_ALEN * 2);689		eh = (struct ethhdr *)skb_pull(skb, VLAN_HLEN);690		skb_reset_mac_header(skb);691	}692 693	/* Filter out non FIP/FCoE frames here to free them faster */694	if (eh->h_proto != htons(ETH_P_ARP) &&695	    eh->h_proto != htons(ETH_P_IP) &&696	    eh->h_proto != htons(ETH_P_IPV6)) {697		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,698			  "Dropping frame ethertype [0x%x] len [0x%x].\n",699			  eh->h_proto, skb->len);700		kfree_skb(skb);701		return 0;702	}703 704	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,705		  "Allowed frame ethertype [0x%x] len [0x%x].\n",706		  eh->h_proto, skb->len);707 708	work = kzalloc(sizeof(*work), GFP_ATOMIC);709	if (!work) {710		QEDI_WARN(&qedi->dbg_ctx,711			  "Could not allocate work so dropping frame.\n");712		kfree_skb(skb);713		return 0;714	}715 716	INIT_LIST_HEAD(&work->list);717	work->skb = skb;718 719	if (skb_vlan_tag_present(skb))720		work->vlan_id = skb_vlan_tag_get(skb);721 722	if (work->vlan_id)723		__vlan_insert_tag(work->skb, htons(ETH_P_8021Q), work->vlan_id);724 725	spin_lock_bh(&qedi->ll2_lock);726	list_add_tail(&work->list, &qedi->ll2_skb_list);727	spin_unlock_bh(&qedi->ll2_lock);728 729	wake_up_process(qedi->ll2_recv_thread);730 731	return 0;732}733 734/* map this skb to iscsiuio mmaped region */735static int qedi_ll2_process_skb(struct qedi_ctx *qedi, struct sk_buff *skb,736				u16 vlan_id)737{738	struct qedi_uio_dev *udev = NULL;739	struct qedi_uio_ctrl *uctrl = NULL;740	struct qedi_rx_bd rxbd;741	struct qedi_rx_bd *p_rxbd;742	u32 rx_bd_prod;743	void *pkt;744	int len = 0;745	u32 prod;746 747	if (!qedi) {748		QEDI_ERR(NULL, "qedi is NULL\n");749		return -1;750	}751 752	udev = qedi->udev;753	uctrl = udev->uctrl;754 755	++uctrl->hw_rx_prod_cnt;756	prod = (uctrl->hw_rx_prod + 1) % RX_RING;757 758	pkt = udev->rx_pkt + (prod * qedi_ll2_buf_size);759	len = min_t(u32, skb->len, (u32)qedi_ll2_buf_size);760	memcpy(pkt, skb->data, len);761 762	memset(&rxbd, 0, sizeof(rxbd));763	rxbd.rx_pkt_index = prod;764	rxbd.rx_pkt_len = len;765	rxbd.vlan_id = vlan_id;766 767	uctrl->hw_rx_bd_prod = (uctrl->hw_rx_bd_prod + 1) % QEDI_NUM_RX_BD;768	rx_bd_prod = uctrl->hw_rx_bd_prod;769	p_rxbd = (struct qedi_rx_bd *)udev->ll2_ring;770	p_rxbd += rx_bd_prod;771 772	memcpy(p_rxbd, &rxbd, sizeof(rxbd));773 774	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,775		  "hw_rx_prod [%d] prod [%d] hw_rx_bd_prod [%d] rx_pkt_idx [%d] rx_len [%d].\n",776		  uctrl->hw_rx_prod, prod, uctrl->hw_rx_bd_prod,777		  rxbd.rx_pkt_index, rxbd.rx_pkt_len);778	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,779		  "host_rx_cons [%d] hw_rx_bd_cons [%d].\n",780		  uctrl->host_rx_cons, uctrl->host_rx_bd_cons);781 782	uctrl->hw_rx_prod = prod;783 784	/* notify the iscsiuio about new packet */785	uio_event_notify(&udev->qedi_uinfo);786 787	return 0;788}789 790static void qedi_ll2_free_skbs(struct qedi_ctx *qedi)791{792	struct skb_work_list *work, *work_tmp;793 794	spin_lock_bh(&qedi->ll2_lock);795	list_for_each_entry_safe(work, work_tmp, &qedi->ll2_skb_list, list) {796		list_del(&work->list);797		kfree_skb(work->skb);798		kfree(work);799	}800	spin_unlock_bh(&qedi->ll2_lock);801}802 803static int qedi_ll2_recv_thread(void *arg)804{805	struct qedi_ctx *qedi = (struct qedi_ctx *)arg;806	struct skb_work_list *work, *work_tmp;807 808	set_user_nice(current, -20);809 810	while (!kthread_should_stop()) {811		spin_lock_bh(&qedi->ll2_lock);812		list_for_each_entry_safe(work, work_tmp, &qedi->ll2_skb_list,813					 list) {814			list_del(&work->list);815			qedi_ll2_process_skb(qedi, work->skb, work->vlan_id);816			kfree_skb(work->skb);817			kfree(work);818		}819		set_current_state(TASK_INTERRUPTIBLE);820		spin_unlock_bh(&qedi->ll2_lock);821		schedule();822	}823 824	__set_current_state(TASK_RUNNING);825	return 0;826}827 828static int qedi_set_iscsi_pf_param(struct qedi_ctx *qedi)829{830	u8 num_sq_pages;831	u32 log_page_size;832	int rval = 0;833 834 835	num_sq_pages = (MAX_OUTSTANDING_TASKS_PER_CON * 8) / QEDI_PAGE_SIZE;836 837	qedi->num_queues = MIN_NUM_CPUS_MSIX(qedi);838 839	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,840		  "Number of CQ count is %d\n", qedi->num_queues);841 842	memset(&qedi->pf_params.iscsi_pf_params, 0,843	       sizeof(qedi->pf_params.iscsi_pf_params));844 845	qedi->p_cpuq = dma_alloc_coherent(&qedi->pdev->dev,846			qedi->num_queues * sizeof(struct qedi_glbl_q_params),847			&qedi->hw_p_cpuq, GFP_KERNEL);848	if (!qedi->p_cpuq) {849		QEDI_ERR(&qedi->dbg_ctx, "dma_alloc_coherent fail\n");850		rval = -1;851		goto err_alloc_mem;852	}853 854	rval = qedi_alloc_global_queues(qedi);855	if (rval) {856		QEDI_ERR(&qedi->dbg_ctx, "Global queue allocation failed.\n");857		rval = -1;858		goto err_alloc_mem;859	}860 861	qedi->pf_params.iscsi_pf_params.num_cons = QEDI_MAX_ISCSI_CONNS_PER_HBA;862	qedi->pf_params.iscsi_pf_params.num_tasks = QEDI_MAX_ISCSI_TASK;863	qedi->pf_params.iscsi_pf_params.half_way_close_timeout = 10;864	qedi->pf_params.iscsi_pf_params.num_sq_pages_in_ring = num_sq_pages;865	qedi->pf_params.iscsi_pf_params.num_r2tq_pages_in_ring = num_sq_pages;866	qedi->pf_params.iscsi_pf_params.num_uhq_pages_in_ring = num_sq_pages;867	qedi->pf_params.iscsi_pf_params.num_queues = qedi->num_queues;868	qedi->pf_params.iscsi_pf_params.debug_mode = qedi_fw_debug;869	qedi->pf_params.iscsi_pf_params.two_msl_timer = QED_TWO_MSL_TIMER_DFLT;870	qedi->pf_params.iscsi_pf_params.tx_sws_timer = QED_TX_SWS_TIMER_DFLT;871	qedi->pf_params.iscsi_pf_params.max_fin_rt = 2;872 873	for (log_page_size = 0 ; log_page_size < 32 ; log_page_size++) {874		if ((1 << log_page_size) == QEDI_PAGE_SIZE)875			break;876	}877	qedi->pf_params.iscsi_pf_params.log_page_size = log_page_size;878 879	qedi->pf_params.iscsi_pf_params.glbl_q_params_addr =880							   (u64)qedi->hw_p_cpuq;881 882	/* RQ BDQ initializations.883	 * rq_num_entries: suggested value for Initiator is 16 (4KB RQ)884	 * rqe_log_size: 8 for 256B RQE885	 */886	qedi->pf_params.iscsi_pf_params.rqe_log_size = 8;887	/* BDQ address and size */888	qedi->pf_params.iscsi_pf_params.bdq_pbl_base_addr[BDQ_ID_RQ] =889							qedi->bdq_pbl_list_dma;890	qedi->pf_params.iscsi_pf_params.bdq_pbl_num_entries[BDQ_ID_RQ] =891						qedi->bdq_pbl_list_num_entries;892	qedi->pf_params.iscsi_pf_params.rq_buffer_size = QEDI_BDQ_BUF_SIZE;893 894	/* cq_num_entries: num_tasks + rq_num_entries */895	qedi->pf_params.iscsi_pf_params.cq_num_entries = 2048;896 897	qedi->pf_params.iscsi_pf_params.gl_rq_pi = QEDI_PROTO_CQ_PROD_IDX;898	qedi->pf_params.iscsi_pf_params.gl_cmd_pi = 1;899 900err_alloc_mem:901	return rval;902}903 904/* Free DMA coherent memory for array of queue pointers we pass to qed */905static void qedi_free_iscsi_pf_param(struct qedi_ctx *qedi)906{907	size_t size = 0;908 909	if (qedi->p_cpuq) {910		size = qedi->num_queues * sizeof(struct qedi_glbl_q_params);911		dma_free_coherent(&qedi->pdev->dev, size, qedi->p_cpuq,912				    qedi->hw_p_cpuq);913	}914 915	qedi_free_global_queues(qedi);916 917	kfree(qedi->global_queues);918}919 920static void qedi_get_boot_tgt_info(struct nvm_iscsi_block *block,921				   struct qedi_boot_target *tgt, u8 index)922{923	u32 ipv6_en;924 925	ipv6_en = !!(block->generic.ctrl_flags &926		     NVM_ISCSI_CFG_GEN_IPV6_ENABLED);927 928	snprintf(tgt->iscsi_name, sizeof(tgt->iscsi_name), "%s",929		 block->target[index].target_name.byte);930 931	tgt->ipv6_en = ipv6_en;932 933	if (ipv6_en)934		snprintf(tgt->ip_addr, IPV6_LEN, "%pI6\n",935			 block->target[index].ipv6_addr.byte);936	else937		snprintf(tgt->ip_addr, IPV4_LEN, "%pI4\n",938			 block->target[index].ipv4_addr.byte);939}940 941static int qedi_find_boot_info(struct qedi_ctx *qedi,942			       struct qed_mfw_tlv_iscsi *iscsi,943			       struct nvm_iscsi_block *block)944{945	struct qedi_boot_target *pri_tgt = NULL, *sec_tgt = NULL;946	u32 pri_ctrl_flags = 0, sec_ctrl_flags = 0, found = 0;947	struct iscsi_cls_session *cls_sess;948	struct iscsi_cls_conn *cls_conn;949	struct qedi_conn *qedi_conn;950	struct iscsi_session *sess;951	struct iscsi_conn *conn;952	char ep_ip_addr[64];953	int i, ret = 0;954 955	pri_ctrl_flags = !!(block->target[0].ctrl_flags &956					NVM_ISCSI_CFG_TARGET_ENABLED);957	if (pri_ctrl_flags) {958		pri_tgt = kzalloc(sizeof(*pri_tgt), GFP_KERNEL);959		if (!pri_tgt)960			return -1;961		qedi_get_boot_tgt_info(block, pri_tgt, 0);962	}963 964	sec_ctrl_flags = !!(block->target[1].ctrl_flags &965					NVM_ISCSI_CFG_TARGET_ENABLED);966	if (sec_ctrl_flags) {967		sec_tgt = kzalloc(sizeof(*sec_tgt), GFP_KERNEL);968		if (!sec_tgt) {969			ret = -1;970			goto free_tgt;971		}972		qedi_get_boot_tgt_info(block, sec_tgt, 1);973	}974 975	for (i = 0; i < qedi->max_active_conns; i++) {976		qedi_conn = qedi_get_conn_from_id(qedi, i);977		if (!qedi_conn)978			continue;979 980		if (qedi_conn->ep->ip_type == TCP_IPV4)981			snprintf(ep_ip_addr, IPV4_LEN, "%pI4\n",982				 qedi_conn->ep->dst_addr);983		else984			snprintf(ep_ip_addr, IPV6_LEN, "%pI6\n",985				 qedi_conn->ep->dst_addr);986 987		cls_conn = qedi_conn->cls_conn;988		conn = cls_conn->dd_data;989		cls_sess = iscsi_conn_to_session(cls_conn);990		sess = cls_sess->dd_data;991 992		if (!iscsi_is_session_online(cls_sess))993			continue;994 995		if (!sess->targetname)996			continue;997 998		if (pri_ctrl_flags) {999			if (!strcmp(pri_tgt->iscsi_name, sess->targetname) &&1000			    !strcmp(pri_tgt->ip_addr, ep_ip_addr)) {1001				found = 1;1002				break;1003			}1004		}1005 1006		if (sec_ctrl_flags) {1007			if (!strcmp(sec_tgt->iscsi_name, sess->targetname) &&1008			    !strcmp(sec_tgt->ip_addr, ep_ip_addr)) {1009				found = 1;1010				break;1011			}1012		}1013	}1014 1015	if (found) {1016		if (conn->hdrdgst_en) {1017			iscsi->header_digest_set = true;1018			iscsi->header_digest = 1;1019		}1020 1021		if (conn->datadgst_en) {1022			iscsi->data_digest_set = true;1023			iscsi->data_digest = 1;1024		}1025		iscsi->boot_taget_portal_set = true;1026		iscsi->boot_taget_portal = sess->tpgt;1027 1028	} else {1029		ret = -1;1030	}1031 1032	if (sec_ctrl_flags)1033		kfree(sec_tgt);1034free_tgt:1035	if (pri_ctrl_flags)1036		kfree(pri_tgt);1037 1038	return ret;1039}1040 1041static void qedi_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)1042{1043	struct qedi_ctx *qedi;1044 1045	if (!dev) {1046		QEDI_INFO(NULL, QEDI_LOG_EVT,1047			  "dev is NULL so ignoring get_generic_tlv_data request.\n");1048		return;1049	}1050	qedi = (struct qedi_ctx *)dev;1051 1052	memset(data, 0, sizeof(struct qed_generic_tlvs));1053	ether_addr_copy(data->mac[0], qedi->mac);1054}1055 1056/*1057 * Protocol TLV handler1058 */1059static void qedi_get_protocol_tlv_data(void *dev, void *data)1060{1061	struct qed_mfw_tlv_iscsi *iscsi = data;1062	struct qed_iscsi_stats *fw_iscsi_stats;1063	struct nvm_iscsi_block *block = NULL;1064	u32 chap_en = 0, mchap_en = 0;1065	struct qedi_ctx *qedi = dev;1066	int rval = 0;1067 1068	fw_iscsi_stats = kmalloc(sizeof(*fw_iscsi_stats), GFP_KERNEL);1069	if (!fw_iscsi_stats) {1070		QEDI_ERR(&qedi->dbg_ctx,1071			 "Could not allocate memory for fw_iscsi_stats.\n");1072		goto exit_get_data;1073	}1074 1075	mutex_lock(&qedi->stats_lock);1076	/* Query firmware for offload stats */1077	qedi_ops->get_stats(qedi->cdev, fw_iscsi_stats);1078	mutex_unlock(&qedi->stats_lock);1079 1080	iscsi->rx_frames_set = true;1081	iscsi->rx_frames = fw_iscsi_stats->iscsi_rx_packet_cnt;1082	iscsi->rx_bytes_set = true;1083	iscsi->rx_bytes = fw_iscsi_stats->iscsi_rx_bytes_cnt;1084	iscsi->tx_frames_set = true;1085	iscsi->tx_frames = fw_iscsi_stats->iscsi_tx_packet_cnt;1086	iscsi->tx_bytes_set = true;1087	iscsi->tx_bytes = fw_iscsi_stats->iscsi_tx_bytes_cnt;1088	iscsi->frame_size_set = true;1089	iscsi->frame_size = qedi->ll2_mtu;1090	block = qedi_get_nvram_block(qedi);1091	if (block) {1092		chap_en = !!(block->generic.ctrl_flags &1093			     NVM_ISCSI_CFG_GEN_CHAP_ENABLED);1094		mchap_en = !!(block->generic.ctrl_flags &1095			      NVM_ISCSI_CFG_GEN_CHAP_MUTUAL_ENABLED);1096 1097		iscsi->auth_method_set = (chap_en || mchap_en) ? true : false;1098		iscsi->auth_method = 1;1099		if (chap_en)1100			iscsi->auth_method = 2;1101		if (mchap_en)1102			iscsi->auth_method = 3;1103 1104		iscsi->tx_desc_size_set = true;1105		iscsi->tx_desc_size = QEDI_SQ_SIZE;1106		iscsi->rx_desc_size_set = true;1107		iscsi->rx_desc_size = QEDI_CQ_SIZE;1108 1109		/* tpgt, hdr digest, data digest */1110		rval = qedi_find_boot_info(qedi, iscsi, block);1111		if (rval)1112			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,1113				  "Boot target not set");1114	}1115 1116	kfree(fw_iscsi_stats);1117exit_get_data:1118	return;1119}1120 1121void qedi_schedule_hw_err_handler(void *dev,1122				  enum qed_hw_err_type err_type)1123{1124	struct qedi_ctx *qedi = (struct qedi_ctx *)dev;1125	unsigned long override_flags = qedi_flags_override;1126 1127	if (override_flags && test_bit(QEDI_ERR_OVERRIDE_EN, &override_flags))1128		qedi->qedi_err_flags = qedi_flags_override;1129 1130	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,1131		  "HW error handler scheduled, err=%d err_flags=0x%x\n",1132		  err_type, qedi->qedi_err_flags);1133 1134	switch (err_type) {1135	case QED_HW_ERR_FAN_FAIL:1136		schedule_delayed_work(&qedi->board_disable_work, 0);1137		break;1138	case QED_HW_ERR_MFW_RESP_FAIL:1139	case QED_HW_ERR_HW_ATTN:1140	case QED_HW_ERR_DMAE_FAIL:1141	case QED_HW_ERR_RAMROD_FAIL:1142	case QED_HW_ERR_FW_ASSERT:1143		/* Prevent HW attentions from being reasserted */1144		if (test_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags))1145			qedi_ops->common->attn_clr_enable(qedi->cdev, true);1146 1147		if (err_type == QED_HW_ERR_RAMROD_FAIL &&1148		    test_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags))1149			qedi_ops->common->recovery_process(qedi->cdev);1150 1151		break;1152	default:1153		break;1154	}1155}1156 1157static void qedi_schedule_recovery_handler(void *dev)1158{1159	struct qedi_ctx *qedi = dev;1160 1161	QEDI_ERR(&qedi->dbg_ctx, "Recovery handler scheduled.\n");1162 1163	if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags))1164		return;1165 1166	atomic_set(&qedi->link_state, QEDI_LINK_DOWN);1167 1168	schedule_delayed_work(&qedi->recovery_work, 0);1169}1170 1171static void qedi_set_conn_recovery(struct iscsi_cls_session *cls_session)1172{1173	struct iscsi_session *session = cls_session->dd_data;1174	struct iscsi_conn *conn = session->leadconn;1175	struct qedi_conn *qedi_conn = conn->dd_data;1176 1177	qedi_start_conn_recovery(qedi_conn->qedi, qedi_conn);1178}1179 1180static void qedi_link_update(void *dev, struct qed_link_output *link)1181{1182	struct qedi_ctx *qedi = (struct qedi_ctx *)dev;1183 1184	if (link->link_up) {1185		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, "Link Up event.\n");1186		atomic_set(&qedi->link_state, QEDI_LINK_UP);1187	} else {1188		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,1189			  "Link Down event.\n");1190		atomic_set(&qedi->link_state, QEDI_LINK_DOWN);1191		iscsi_host_for_each_session(qedi->shost, qedi_set_conn_recovery);1192	}1193}1194 1195static struct qed_iscsi_cb_ops qedi_cb_ops = {1196	{1197		.link_update =		qedi_link_update,1198		.schedule_recovery_handler = qedi_schedule_recovery_handler,1199		.schedule_hw_err_handler = qedi_schedule_hw_err_handler,1200		.get_protocol_tlv_data = qedi_get_protocol_tlv_data,1201		.get_generic_tlv_data = qedi_get_generic_tlv_data,1202	}1203};1204 1205static int qedi_queue_cqe(struct qedi_ctx *qedi, union iscsi_cqe *cqe,1206			  u16 que_idx, struct qedi_percpu_s *p)1207{1208	struct qedi_work *qedi_work;1209	struct qedi_conn *q_conn;1210	struct qedi_cmd *qedi_cmd;1211	u32 iscsi_cid;1212	int rc = 0;1213 1214	iscsi_cid  = cqe->cqe_common.conn_id;1215	q_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];1216	if (!q_conn) {1217		QEDI_WARN(&qedi->dbg_ctx,1218			  "Session no longer exists for cid=0x%x!!\n",1219			  iscsi_cid);1220		return -1;1221	}1222 1223	switch (cqe->cqe_common.cqe_type) {1224	case ISCSI_CQE_TYPE_SOLICITED:1225	case ISCSI_CQE_TYPE_SOLICITED_WITH_SENSE:1226		qedi_cmd = qedi_get_cmd_from_tid(qedi, cqe->cqe_solicited.itid);1227		if (!qedi_cmd) {1228			rc = -1;1229			break;1230		}1231		INIT_LIST_HEAD(&qedi_cmd->cqe_work.list);1232		qedi_cmd->cqe_work.qedi = qedi;1233		memcpy(&qedi_cmd->cqe_work.cqe, cqe, sizeof(union iscsi_cqe));1234		qedi_cmd->cqe_work.que_idx = que_idx;1235		qedi_cmd->cqe_work.is_solicited = true;1236		list_add_tail(&qedi_cmd->cqe_work.list, &p->work_list);1237		break;1238	case ISCSI_CQE_TYPE_UNSOLICITED:1239	case ISCSI_CQE_TYPE_DUMMY:1240	case ISCSI_CQE_TYPE_TASK_CLEANUP:1241		qedi_work = kzalloc(sizeof(*qedi_work), GFP_ATOMIC);1242		if (!qedi_work) {1243			rc = -1;1244			break;1245		}1246		INIT_LIST_HEAD(&qedi_work->list);1247		qedi_work->qedi = qedi;1248		memcpy(&qedi_work->cqe, cqe, sizeof(union iscsi_cqe));1249		qedi_work->que_idx = que_idx;1250		qedi_work->is_solicited = false;1251		list_add_tail(&qedi_work->list, &p->work_list);1252		break;1253	default:1254		rc = -1;1255		QEDI_ERR(&qedi->dbg_ctx, "FW Error cqe.\n");1256	}1257	return rc;1258}1259 1260static bool qedi_process_completions(struct qedi_fastpath *fp)1261{1262	struct qedi_ctx *qedi = fp->qedi;1263	struct qed_sb_info *sb_info = fp->sb_info;1264	struct status_block *sb = sb_info->sb_virt;1265	struct qedi_percpu_s *p = NULL;1266	struct global_queue *que;1267	u16 prod_idx;1268	unsigned long flags;1269	union iscsi_cqe *cqe;1270	int cpu;1271	int ret;1272 1273	/* Get the current firmware producer index */1274	prod_idx = sb->pi_array[QEDI_PROTO_CQ_PROD_IDX];1275 1276	if (prod_idx >= QEDI_CQ_SIZE)1277		prod_idx = prod_idx % QEDI_CQ_SIZE;1278 1279	que = qedi->global_queues[fp->sb_id];1280	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,1281		  "Before: global queue=%p prod_idx=%d cons_idx=%d, sb_id=%d\n",1282		  que, prod_idx, que->cq_cons_idx, fp->sb_id);1283 1284	qedi->intr_cpu = fp->sb_id;1285	cpu = smp_processor_id();1286	p = &per_cpu(qedi_percpu, cpu);1287 1288	if (unlikely(!p->iothread))1289		WARN_ON(1);1290 1291	spin_lock_irqsave(&p->p_work_lock, flags);1292	while (que->cq_cons_idx != prod_idx) {1293		cqe = &que->cq[que->cq_cons_idx];1294 1295		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,1296			  "cqe=%p prod_idx=%d cons_idx=%d.\n",1297			  cqe, prod_idx, que->cq_cons_idx);1298 1299		ret = qedi_queue_cqe(qedi, cqe, fp->sb_id, p);1300		if (ret)1301			QEDI_WARN(&qedi->dbg_ctx,1302				  "Dropping CQE 0x%x for cid=0x%x.\n",1303				  que->cq_cons_idx, cqe->cqe_common.conn_id);1304 1305		que->cq_cons_idx++;1306		if (que->cq_cons_idx == QEDI_CQ_SIZE)1307			que->cq_cons_idx = 0;1308	}1309	wake_up_process(p->iothread);1310	spin_unlock_irqrestore(&p->p_work_lock, flags);1311 1312	return true;1313}1314 1315static bool qedi_fp_has_work(struct qedi_fastpath *fp)1316{1317	struct qedi_ctx *qedi = fp->qedi;1318	struct global_queue *que;1319	struct qed_sb_info *sb_info = fp->sb_info;1320	struct status_block *sb = sb_info->sb_virt;1321	u16 prod_idx;1322 1323	barrier();1324 1325	/* Get the current firmware producer index */1326	prod_idx = sb->pi_array[QEDI_PROTO_CQ_PROD_IDX];1327 1328	/* Get the pointer to the global CQ this completion is on */1329	que = qedi->global_queues[fp->sb_id];1330 1331	/* prod idx wrap around uint16 */1332	if (prod_idx >= QEDI_CQ_SIZE)1333		prod_idx = prod_idx % QEDI_CQ_SIZE;1334 1335	return (que->cq_cons_idx != prod_idx);1336}1337 1338/* MSI-X fastpath handler code */1339static irqreturn_t qedi_msix_handler(int irq, void *dev_id)1340{1341	struct qedi_fastpath *fp = dev_id;1342	struct qedi_ctx *qedi = fp->qedi;1343	bool wake_io_thread = true;1344 1345	qed_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);1346 1347process_again:1348	wake_io_thread = qedi_process_completions(fp);1349	if (wake_io_thread) {1350		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,1351			  "process already running\n");1352	}1353 1354	if (!qedi_fp_has_work(fp))1355		qed_sb_update_sb_idx(fp->sb_info);1356 1357	/* Check for more work */1358	rmb();1359 1360	if (!qedi_fp_has_work(fp))1361		qed_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);1362	else1363		goto process_again;1364 1365	return IRQ_HANDLED;1366}1367 1368/* simd handler for MSI/INTa */1369static void qedi_simd_int_handler(void *cookie)1370{1371	/* Cookie is qedi_ctx struct */1372	struct qedi_ctx *qedi = (struct qedi_ctx *)cookie;1373 1374	QEDI_WARN(&qedi->dbg_ctx, "qedi=%p.\n", qedi);1375}1376 1377#define QEDI_SIMD_HANDLER_NUM		01378static void qedi_sync_free_irqs(struct qedi_ctx *qedi)1379{1380	int i;1381	u16 idx;1382 1383	if (qedi->int_info.msix_cnt) {1384		for (i = 0; i < qedi->int_info.used_cnt; i++) {1385			idx = i * qedi->dev_info.common.num_hwfns +1386			qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);1387 1388			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,1389				  "Freeing IRQ #%d vector_idx=%d.\n", i, idx);1390 1391			synchronize_irq(qedi->int_info.msix[idx].vector);1392			irq_set_affinity_hint(qedi->int_info.msix[idx].vector,1393					      NULL);1394			free_irq(qedi->int_info.msix[idx].vector,1395				 &qedi->fp_array[i]);1396		}1397	} else {1398		qedi_ops->common->simd_handler_clean(qedi->cdev,1399						     QEDI_SIMD_HANDLER_NUM);1400	}1401 1402	qedi->int_info.used_cnt = 0;1403	qedi_ops->common->set_fp_int(qedi->cdev, 0);1404}1405 1406static int qedi_request_msix_irq(struct qedi_ctx *qedi)1407{1408	int i, rc, cpu;1409	u16 idx;1410 1411	cpu = cpumask_first(cpu_online_mask);1412	for (i = 0; i < qedi->msix_count; i++) {1413		idx = i * qedi->dev_info.common.num_hwfns +1414			  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);1415 1416		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,1417			  "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",1418			  qedi->dev_info.common.num_hwfns,1419			  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));1420 1421		rc = request_irq(qedi->int_info.msix[idx].vector,1422				 qedi_msix_handler, 0, "qedi",1423				 &qedi->fp_array[i]);1424		if (rc) {1425			QEDI_WARN(&qedi->dbg_ctx, "request_irq failed.\n");1426			qedi_sync_free_irqs(qedi);1427			return rc;1428		}1429		qedi->int_info.used_cnt++;1430		rc = irq_set_affinity_hint(qedi->int_info.msix[idx].vector,1431					   get_cpu_mask(cpu));1432		cpu = cpumask_next(cpu, cpu_online_mask);1433	}1434 1435	return 0;1436}1437 1438static int qedi_setup_int(struct qedi_ctx *qedi)1439{1440	int rc = 0;1441 1442	rc = qedi_ops->common->set_fp_int(qedi->cdev, qedi->num_queues);1443	if (rc < 0)1444		goto exit_setup_int;1445 1446	qedi->msix_count = rc;1447 1448	rc = qedi_ops->common->get_fp_int(qedi->cdev, &qedi->int_info);1449	if (rc)1450		goto exit_setup_int;1451 1452	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,1453		  "Number of msix_cnt = 0x%x num of cpus = 0x%x\n",1454		   qedi->int_info.msix_cnt, num_online_cpus());1455 1456	if (qedi->int_info.msix_cnt) {1457		rc = qedi_request_msix_irq(qedi);1458		goto exit_setup_int;1459	} else {1460		qedi_ops->common->simd_handler_config(qedi->cdev, &qedi,1461						      QEDI_SIMD_HANDLER_NUM,1462						      qedi_simd_int_handler);1463		qedi->int_info.used_cnt = 1;1464	}1465 1466exit_setup_int:1467	return rc;1468}1469 1470static void qedi_free_nvm_iscsi_cfg(struct qedi_ctx *qedi)1471{1472	if (qedi->iscsi_image)1473		dma_free_coherent(&qedi->pdev->dev,1474				  sizeof(struct qedi_nvm_iscsi_image),1475				  qedi->iscsi_image, qedi->nvm_buf_dma);1476}1477 1478static int qedi_alloc_nvm_iscsi_cfg(struct qedi_ctx *qedi)1479{1480	qedi->iscsi_image = dma_alloc_coherent(&qedi->pdev->dev,1481					       sizeof(struct qedi_nvm_iscsi_image),1482					       &qedi->nvm_buf_dma, GFP_KERNEL);1483	if (!qedi->iscsi_image) {1484		QEDI_ERR(&qedi->dbg_ctx, "Could not allocate NVM BUF.\n");1485		return -ENOMEM;1486	}1487	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,1488		  "NVM BUF addr=0x%p dma=0x%llx.\n", qedi->iscsi_image,1489		  qedi->nvm_buf_dma);1490 1491	return 0;1492}1493 1494static void qedi_free_bdq(struct qedi_ctx *qedi)1495{1496	int i;1497 1498	if (qedi->bdq_pbl_list)1499		dma_free_coherent(&qedi->pdev->dev, QEDI_PAGE_SIZE,1500				  qedi->bdq_pbl_list, qedi->bdq_pbl_list_dma);1501 1502	if (qedi->bdq_pbl)1503		dma_free_coherent(&qedi->pdev->dev, qedi->bdq_pbl_mem_size,1504				  qedi->bdq_pbl, qedi->bdq_pbl_dma);1505 1506	for (i = 0; i < QEDI_BDQ_NUM; i++) {1507		if (qedi->bdq[i].buf_addr) {1508			dma_free_coherent(&qedi->pdev->dev, QEDI_BDQ_BUF_SIZE,1509					  qedi->bdq[i].buf_addr,1510					  qedi->bdq[i].buf_dma);1511		}1512	}1513}1514 1515static void qedi_free_global_queues(struct qedi_ctx *qedi)1516{1517	int i;1518	struct global_queue **gl = qedi->global_queues;1519 1520	for (i = 0; i < qedi->num_queues; i++) {1521		if (!gl[i])1522			continue;1523 1524		if (gl[i]->cq)1525			dma_free_coherent(&qedi->pdev->dev, gl[i]->cq_mem_size,1526					  gl[i]->cq, gl[i]->cq_dma);1527		if (gl[i]->cq_pbl)1528			dma_free_coherent(&qedi->pdev->dev, gl[i]->cq_pbl_size,1529					  gl[i]->cq_pbl, gl[i]->cq_pbl_dma);1530 1531		kfree(gl[i]);1532	}1533	qedi_free_bdq(qedi);1534	qedi_free_nvm_iscsi_cfg(qedi);1535}1536 1537static int qedi_alloc_bdq(struct qedi_ctx *qedi)1538{1539	int i;1540	struct scsi_bd *pbl;1541	u64 *list;1542 1543	/* Alloc dma memory for BDQ buffers */1544	for (i = 0; i < QEDI_BDQ_NUM; i++) {1545		qedi->bdq[i].buf_addr =1546				dma_alloc_coherent(&qedi->pdev->dev,1547						   QEDI_BDQ_BUF_SIZE,1548						   &qedi->bdq[i].buf_dma,1549						   GFP_KERNEL);1550		if (!qedi->bdq[i].buf_addr) {1551			QEDI_ERR(&qedi->dbg_ctx,1552				 "Could not allocate BDQ buffer %d.\n", i);1553			return -ENOMEM;1554		}1555	}1556 1557	/* Alloc dma memory for BDQ page buffer list */1558	qedi->bdq_pbl_mem_size = QEDI_BDQ_NUM * sizeof(struct scsi_bd);1559	qedi->bdq_pbl_mem_size = ALIGN(qedi->bdq_pbl_mem_size, QEDI_PAGE_SIZE);1560	qedi->rq_num_entries = qedi->bdq_pbl_mem_size / sizeof(struct scsi_bd);1561 1562	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, "rq_num_entries = %d.\n",1563		  qedi->rq_num_entries);1564 1565	qedi->bdq_pbl = dma_alloc_coherent(&qedi->pdev->dev,1566					   qedi->bdq_pbl_mem_size,1567					   &qedi->bdq_pbl_dma, GFP_KERNEL);1568	if (!qedi->bdq_pbl) {1569		QEDI_ERR(&qedi->dbg_ctx, "Could not allocate BDQ PBL.\n");1570		return -ENOMEM;1571	}1572 1573	/*1574	 * Populate BDQ PBL with physical and virtual address of individual1575	 * BDQ buffers1576	 */1577	pbl = (struct scsi_bd  *)qedi->bdq_pbl;1578	for (i = 0; i < QEDI_BDQ_NUM; i++) {1579		pbl->address.hi =1580				cpu_to_le32(QEDI_U64_HI(qedi->bdq[i].buf_dma));1581		pbl->address.lo =1582				cpu_to_le32(QEDI_U64_LO(qedi->bdq[i].buf_dma));1583		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,1584			  "pbl [0x%p] pbl->address hi [0x%llx] lo [0x%llx], idx [%d]\n",1585			  pbl, pbl->address.hi, pbl->address.lo, i);1586		pbl->opaque.iscsi_opaque.reserved_zero[0] = 0;1587		pbl->opaque.iscsi_opaque.reserved_zero[1] = 0;1588		pbl->opaque.iscsi_opaque.reserved_zero[2] = 0;1589		pbl->opaque.iscsi_opaque.opaque = cpu_to_le16(i);1590		pbl++;1591	}1592 1593	/* Allocate list of PBL pages */1594	qedi->bdq_pbl_list = dma_alloc_coherent(&qedi->pdev->dev,1595						QEDI_PAGE_SIZE,1596						&qedi->bdq_pbl_list_dma,1597						GFP_KERNEL);1598	if (!qedi->bdq_pbl_list) {1599		QEDI_ERR(&qedi->dbg_ctx,1600			 "Could not allocate list of PBL pages.\n");1601		return -ENOMEM;1602	}1603 1604	/*1605	 * Now populate PBL list with pages that contain pointers to the1606	 * individual buffers.1607	 */1608	qedi->bdq_pbl_list_num_entries = qedi->bdq_pbl_mem_size /1609					 QEDI_PAGE_SIZE;1610	list = (u64 *)qedi->bdq_pbl_list;1611	for (i = 0; i < qedi->bdq_pbl_list_num_entries; i++) {1612		*list = qedi->bdq_pbl_dma;1613		list++;1614	}1615 1616	return 0;1617}1618 1619static int qedi_alloc_global_queues(struct qedi_ctx *qedi)1620{1621	u32 *list;1622	int i;1623	int status;1624	u32 *pbl;1625	dma_addr_t page;1626	int num_pages;1627 1628	/*1629	 * Number of global queues (CQ / RQ). This should1630	 * be <= number of available MSIX vectors for the PF1631	 */1632	if (!qedi->num_queues) {1633		QEDI_ERR(&qedi->dbg_ctx, "No MSI-X vectors available!\n");1634		return -ENOMEM;1635	}1636 1637	/* Make sure we allocated the PBL that will contain the physical1638	 * addresses of our queues1639	 */1640	if (!qedi->p_cpuq) {1641		status = -EINVAL;1642		goto mem_alloc_failure;1643	}1644 1645	qedi->global_queues = kzalloc((sizeof(struct global_queue *) *1646				       qedi->num_queues), GFP_KERNEL);1647	if (!qedi->global_queues) {1648		QEDI_ERR(&qedi->dbg_ctx,1649			 "Unable to allocate global queues array ptr memory\n");1650		return -ENOMEM;1651	}1652	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,1653		  "qedi->global_queues=%p.\n", qedi->global_queues);1654 1655	/* Allocate DMA coherent buffers for BDQ */1656	status = qedi_alloc_bdq(qedi);1657	if (status)1658		goto mem_alloc_failure;1659 1660	/* Allocate DMA coherent buffers for NVM_ISCSI_CFG */1661	status = qedi_alloc_nvm_iscsi_cfg(qedi);1662	if (status)1663		goto mem_alloc_failure;1664 1665	/* Allocate a CQ and an associated PBL for each MSI-X1666	 * vector.1667	 */1668	for (i = 0; i < qedi->num_queues; i++) {1669		qedi->global_queues[i] =1670					kzalloc(sizeof(*qedi->global_queues[0]),1671						GFP_KERNEL);1672		if (!qedi->global_queues[i]) {1673			QEDI_ERR(&qedi->dbg_ctx,1674				 "Unable to allocation global queue %d.\n", i);1675			status = -ENOMEM;1676			goto mem_alloc_failure;1677		}1678 1679		qedi->global_queues[i]->cq_mem_size =1680		    (QEDI_CQ_SIZE + 8) * sizeof(union iscsi_cqe);1681		qedi->global_queues[i]->cq_mem_size =1682		    (qedi->global_queues[i]->cq_mem_size +1683		    (QEDI_PAGE_SIZE - 1));1684 1685		qedi->global_queues[i]->cq_pbl_size =1686		    (qedi->global_queues[i]->cq_mem_size /1687		    QEDI_PAGE_SIZE) * sizeof(void *);1688		qedi->global_queues[i]->cq_pbl_size =1689		    (qedi->global_queues[i]->cq_pbl_size +1690		    (QEDI_PAGE_SIZE - 1));1691 1692		qedi->global_queues[i]->cq = dma_alloc_coherent(&qedi->pdev->dev,1693								qedi->global_queues[i]->cq_mem_size,1694								&qedi->global_queues[i]->cq_dma,1695								GFP_KERNEL);1696 1697		if (!qedi->global_queues[i]->cq) {1698			QEDI_WARN(&qedi->dbg_ctx,1699				  "Could not allocate cq.\n");1700			status = -ENOMEM;1701			goto mem_alloc_failure;1702		}1703		qedi->global_queues[i]->cq_pbl = dma_alloc_coherent(&qedi->pdev->dev,1704								    qedi->global_queues[i]->cq_pbl_size,1705								    &qedi->global_queues[i]->cq_pbl_dma,1706								    GFP_KERNEL);1707 1708		if (!qedi->global_queues[i]->cq_pbl) {1709			QEDI_WARN(&qedi->dbg_ctx,1710				  "Could not allocate cq PBL.\n");1711			status = -ENOMEM;1712			goto mem_alloc_failure;1713		}1714 1715		/* Create PBL */1716		num_pages = qedi->global_queues[i]->cq_mem_size /1717		    QEDI_PAGE_SIZE;1718		page = qedi->global_queues[i]->cq_dma;1719		pbl = (u32 *)qedi->global_queues[i]->cq_pbl;1720 1721		while (num_pages--) {1722			*pbl = (u32)page;1723			pbl++;1724			*pbl = (u32)((u64)page >> 32);1725			pbl++;1726			page += QEDI_PAGE_SIZE;1727		}1728	}1729 1730	list = (u32 *)qedi->p_cpuq;1731 1732	/*1733	 * The list is built as follows: CQ#0 PBL pointer, RQ#0 PBL pointer,1734	 * CQ#1 PBL pointer, RQ#1 PBL pointer, etc.  Each PBL pointer points1735	 * to the physical address which contains an array of pointers to the1736	 * physical addresses of the specific queue pages.1737	 */1738	for (i = 0; i < qedi->num_queues; i++) {1739		*list = (u32)qedi->global_queues[i]->cq_pbl_dma;1740		list++;1741		*list = (u32)((u64)qedi->global_queues[i]->cq_pbl_dma >> 32);1742		list++;1743 1744		*list = (u32)0;1745		list++;1746		*list = (u32)((u64)0 >> 32);1747		list++;1748	}1749 1750	return 0;1751 1752mem_alloc_failure:1753	qedi_free_global_queues(qedi);1754	return status;1755}1756 1757int qedi_alloc_sq(struct qedi_ctx *qedi, struct qedi_endpoint *ep)1758{1759	int rval = 0;1760	u32 *pbl;1761	dma_addr_t page;1762	int num_pages;1763 1764	if (!ep)1765		return -EIO;1766 1767	/* Calculate appropriate queue and PBL sizes */1768	ep->sq_mem_size = QEDI_SQ_SIZE * sizeof(struct iscsi_wqe);1769	ep->sq_mem_size += QEDI_PAGE_SIZE - 1;1770 1771	ep->sq_pbl_size = (ep->sq_mem_size / QEDI_PAGE_SIZE) * sizeof(void *);1772	ep->sq_pbl_size = ep->sq_pbl_size + QEDI_PAGE_SIZE;1773 1774	ep->sq = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_mem_size,1775				    &ep->sq_dma, GFP_KERNEL);1776	if (!ep->sq) {1777		QEDI_WARN(&qedi->dbg_ctx,1778			  "Could not allocate send queue.\n");1779		rval = -ENOMEM;1780		goto out;1781	}1782	ep->sq_pbl = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_pbl_size,1783					&ep->sq_pbl_dma, GFP_KERNEL);1784	if (!ep->sq_pbl) {1785		QEDI_WARN(&qedi->dbg_ctx,1786			  "Could not allocate send queue PBL.\n");1787		rval = -ENOMEM;1788		goto out_free_sq;1789	}1790 1791	/* Create PBL */1792	num_pages = ep->sq_mem_size / QEDI_PAGE_SIZE;1793	page = ep->sq_dma;1794	pbl = (u32 *)ep->sq_pbl;1795 1796	while (num_pages--) {1797		*pbl = (u32)page;1798		pbl++;1799		*pbl = (u32)((u64)page >> 32);1800		pbl++;1801		page += QEDI_PAGE_SIZE;1802	}1803 1804	return rval;1805 1806out_free_sq:1807	dma_free_coherent(&qedi->pdev->dev, ep->sq_mem_size, ep->sq,1808			  ep->sq_dma);1809out:1810	return rval;1811}1812 1813void qedi_free_sq(struct qedi_ctx *qedi, struct qedi_endpoint *ep)1814{1815	if (ep->sq_pbl)1816		dma_free_coherent(&qedi->pdev->dev, ep->sq_pbl_size, ep->sq_pbl,1817				  ep->sq_pbl_dma);1818	if (ep->sq)1819		dma_free_coherent(&qedi->pdev->dev, ep->sq_mem_size, ep->sq,1820				  ep->sq_dma);1821}1822 1823int qedi_get_task_idx(struct qedi_ctx *qedi)1824{1825	s16 tmp_idx;1826 1827again:1828	tmp_idx = find_first_zero_bit(qedi->task_idx_map,1829				      MAX_ISCSI_TASK_ENTRIES);1830 1831	if (tmp_idx >= MAX_ISCSI_TASK_ENTRIES) {1832		QEDI_ERR(&qedi->dbg_ctx, "FW task context pool is full.\n");1833		tmp_idx = -1;1834		goto err_idx;1835	}1836 1837	if (test_and_set_bit(tmp_idx, qedi->task_idx_map))1838		goto again;1839 1840err_idx:1841	return tmp_idx;1842}1843 1844void qedi_clear_task_idx(struct qedi_ctx *qedi, int idx)1845{1846	if (!test_and_clear_bit(idx, qedi->task_idx_map))1847		QEDI_ERR(&qedi->dbg_ctx,1848			 "FW task context, already cleared, tid=0x%x\n", idx);1849}1850 1851void qedi_update_itt_map(struct qedi_ctx *qedi, u32 tid, u32 proto_itt,1852			 struct qedi_cmd *cmd)1853{1854	qedi->itt_map[tid].itt = proto_itt;1855	qedi->itt_map[tid].p_cmd = cmd;1856 1857	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,1858		  "update itt map tid=0x%x, with proto itt=0x%x\n", tid,1859		  qedi->itt_map[tid].itt);1860}1861 1862void qedi_get_task_tid(struct qedi_ctx *qedi, u32 itt, s16 *tid)1863{1864	u16 i;1865 1866	for (i = 0; i < MAX_ISCSI_TASK_ENTRIES; i++) {1867		if (qedi->itt_map[i].itt == itt) {1868			*tid = i;1869			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,1870				  "Ref itt=0x%x, found at tid=0x%x\n",1871				  itt, *tid);1872			return;1873		}1874	}1875 1876	WARN_ON(1);1877}1878 1879void qedi_get_proto_itt(struct qedi_ctx *qedi, u32 tid, u32 *proto_itt)1880{1881	*proto_itt = qedi->itt_map[tid].itt;1882	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,1883		  "Get itt map tid [0x%x with proto itt[0x%x]",1884		  tid, *proto_itt);1885}1886 1887struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid)1888{1889	struct qedi_cmd *cmd = NULL;1890 1891	if (tid >= MAX_ISCSI_TASK_ENTRIES)1892		return NULL;1893 1894	cmd = qedi->itt_map[tid].p_cmd;1895	if (cmd->task_id != tid)1896		return NULL;1897 1898	qedi->itt_map[tid].p_cmd = NULL;1899 1900	return cmd;1901}1902 1903static int qedi_alloc_itt(struct qedi_ctx *qedi)1904{1905	qedi->itt_map = kcalloc(MAX_ISCSI_TASK_ENTRIES,1906				sizeof(struct qedi_itt_map), GFP_KERNEL);1907	if (!qedi->itt_map) {1908		QEDI_ERR(&qedi->dbg_ctx,1909			 "Unable to allocate itt map array memory\n");1910		return -ENOMEM;1911	}1912	return 0;1913}1914 1915static void qedi_free_itt(struct qedi_ctx *qedi)1916{1917	kfree(qedi->itt_map);1918}1919 1920static struct qed_ll2_cb_ops qedi_ll2_cb_ops = {1921	.rx_cb = qedi_ll2_rx,1922	.tx_cb = NULL,1923};1924 1925static int qedi_percpu_io_thread(void *arg)1926{1927	struct qedi_percpu_s *p = arg;1928	struct qedi_work *work, *tmp;1929	unsigned long flags;1930	LIST_HEAD(work_list);1931 1932	set_user_nice(current, -20);1933 1934	while (!kthread_should_stop()) {1935		spin_lock_irqsave(&p->p_work_lock, flags);1936		while (!list_empty(&p->work_list)) {1937			list_splice_init(&p->work_list, &work_list);1938			spin_unlock_irqrestore(&p->p_work_lock, flags);1939 1940			list_for_each_entry_safe(work, tmp, &work_list, list) {1941				list_del_init(&work->list);1942				qedi_fp_process_cqes(work);1943				if (!work->is_solicited)1944					kfree(work);1945			}1946			cond_resched();1947			spin_lock_irqsave(&p->p_work_lock, flags);1948		}1949		set_current_state(TASK_INTERRUPTIBLE);1950		spin_unlock_irqrestore(&p->p_work_lock, flags);1951		schedule();1952	}1953	__set_current_state(TASK_RUNNING);1954 1955	return 0;1956}1957 1958static int qedi_cpu_online(unsigned int cpu)1959{1960	struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu);1961	struct task_struct *thread;1962 1963	thread = kthread_create_on_node(qedi_percpu_io_thread, (void *)p,1964					cpu_to_node(cpu),1965					"qedi_thread/%d", cpu);1966	if (IS_ERR(thread))1967		return PTR_ERR(thread);1968 1969	kthread_bind(thread, cpu);1970	p->iothread = thread;1971	wake_up_process(thread);1972	return 0;1973}1974 1975static int qedi_cpu_offline(unsigned int cpu)1976{1977	struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu);1978	struct qedi_work *work, *tmp;1979	struct task_struct *thread;1980	unsigned long flags;1981 1982	spin_lock_irqsave(&p->p_work_lock, flags);1983	thread = p->iothread;1984	p->iothread = NULL;1985 1986	list_for_each_entry_safe(work, tmp, &p->work_list, list) {1987		list_del_init(&work->list);1988		qedi_fp_process_cqes(work);1989		if (!work->is_solicited)1990			kfree(work);1991	}1992 1993	spin_unlock_irqrestore(&p->p_work_lock, flags);1994	if (thread)1995		kthread_stop(thread);1996	return 0;1997}1998 1999void qedi_reset_host_mtu(struct qedi_ctx *qedi, u16 mtu)2000{2001	struct qed_ll2_params params;2002 2003	qedi_recover_all_conns(qedi);2004 2005	qedi_ops->ll2->stop(qedi->cdev);2006	qedi_ll2_free_skbs(qedi);2007 2008	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, "old MTU %u, new MTU %u\n",2009		  qedi->ll2_mtu, mtu);2010	memset(&params, 0, sizeof(params));2011	qedi->ll2_mtu = mtu;2012	params.mtu = qedi->ll2_mtu + IPV6_HDR_LEN + TCP_HDR_LEN;2013	params.drop_ttl0_packets = 0;2014	params.rx_vlan_stripping = 1;2015	ether_addr_copy(params.ll2_mac_address, qedi->dev_info.common.hw_mac);2016	qedi_ops->ll2->start(qedi->cdev, &params);2017}2018 2019/*2020 * qedi_get_nvram_block: - Scan through the iSCSI NVRAM block (while accounting2021 * for gaps) for the matching absolute-pf-id of the QEDI device.2022 */2023static struct nvm_iscsi_block *2024qedi_get_nvram_block(struct qedi_ctx *qedi)2025{2026	int i;2027	u8 pf;2028	u32 flags;2029	struct nvm_iscsi_block *block;2030 2031	pf = qedi->dev_info.common.abs_pf_id;2032	block = &qedi->iscsi_image->iscsi_cfg.block[0];2033	for (i = 0; i < NUM_OF_ISCSI_PF_SUPPORTED; i++, block++) {2034		flags = ((block->id) & NVM_ISCSI_CFG_BLK_CTRL_FLAG_MASK) >>2035			NVM_ISCSI_CFG_BLK_CTRL_FLAG_OFFSET;2036		if (flags & (NVM_ISCSI_CFG_BLK_CTRL_FLAG_IS_NOT_EMPTY |2037				NVM_ISCSI_CFG_BLK_CTRL_FLAG_PF_MAPPED) &&2038			(pf == (block->id & NVM_ISCSI_CFG_BLK_MAPPED_PF_ID_MASK)2039				>> NVM_ISCSI_CFG_BLK_MAPPED_PF_ID_OFFSET))2040			return block;2041	}2042	return NULL;2043}2044 2045static ssize_t qedi_show_boot_eth_info(void *data, int type, char *buf)2046{2047	struct qedi_ctx *qedi = data;2048	struct nvm_iscsi_initiator *initiator;2049	int rc = 1;2050	u32 ipv6_en, dhcp_en, ip_len;2051	struct nvm_iscsi_block *block;2052	char *fmt, *ip, *sub, *gw;2053 2054	block = qedi_get_nvram_block(qedi);2055	if (!block)2056		return 0;2057 2058	initiator = &block->initiator;2059	ipv6_en = block->generic.ctrl_flags &2060		  NVM_ISCSI_CFG_GEN_IPV6_ENABLED;2061	dhcp_en = block->generic.ctrl_flags &2062		  NVM_ISCSI_CFG_GEN_DHCP_TCPIP_CONFIG_ENABLED;2063	/* Static IP assignments. */2064	fmt = ipv6_en ? "%pI6\n" : "%pI4\n";2065	ip = ipv6_en ? initiator->ipv6.addr.byte : initiator->ipv4.addr.byte;2066	ip_len = ipv6_en ? IPV6_LEN : IPV4_LEN;2067	sub = ipv6_en ? initiator->ipv6.subnet_mask.byte :2068	      initiator->ipv4.subnet_mask.byte;2069	gw = ipv6_en ? initiator->ipv6.gateway.byte :2070	     initiator->ipv4.gateway.byte;2071	/* DHCP IP adjustments. */2072	fmt = dhcp_en ? "%s\n" : fmt;2073	if (dhcp_en) {2074		ip = ipv6_en ? "0::0" : "0.0.0.0";2075		sub = ip;2076		gw = ip;2077		ip_len = ipv6_en ? 5 : 8;2078	}2079 2080	switch (type) {2081	case ISCSI_BOOT_ETH_IP_ADDR:2082		rc = snprintf(buf, ip_len, fmt, ip);2083		break;2084	case ISCSI_BOOT_ETH_SUBNET_MASK:2085		rc = snprintf(buf, ip_len, fmt, sub);2086		break;2087	case ISCSI_BOOT_ETH_GATEWAY:2088		rc = snprintf(buf, ip_len, fmt, gw);2089		break;2090	case ISCSI_BOOT_ETH_FLAGS:2091		rc = snprintf(buf, 3, "%d\n", (char)SYSFS_FLAG_FW_SEL_BOOT);2092		break;2093	case ISCSI_BOOT_ETH_INDEX:2094		rc = snprintf(buf, 3, "0\n");2095		break;2096	case ISCSI_BOOT_ETH_MAC:2097		rc = sysfs_format_mac(buf, qedi->mac, ETH_ALEN);2098		break;2099	case ISCSI_BOOT_ETH_VLAN:2100		rc = snprintf(buf, 12, "%d\n",2101			      GET_FIELD2(initiator->generic_cont0,2102					 NVM_ISCSI_CFG_INITIATOR_VLAN));2103		break;2104	case ISCSI_BOOT_ETH_ORIGIN:2105		if (dhcp_en)2106			rc = snprintf(buf, 3, "3\n");2107		break;2108	default:2109		rc = 0;2110		break;2111	}2112 2113	return rc;2114}2115 2116static umode_t qedi_eth_get_attr_visibility(void *data, int type)2117{2118	int rc = 1;2119 2120	switch (type) {2121	case ISCSI_BOOT_ETH_FLAGS:2122	case ISCSI_BOOT_ETH_MAC:2123	case ISCSI_BOOT_ETH_INDEX:2124	case ISCSI_BOOT_ETH_IP_ADDR:2125	case ISCSI_BOOT_ETH_SUBNET_MASK:2126	case ISCSI_BOOT_ETH_GATEWAY:2127	case ISCSI_BOOT_ETH_ORIGIN:2128	case ISCSI_BOOT_ETH_VLAN:2129		rc = 0444;2130		break;2131	default:2132		rc = 0;2133		break;2134	}2135	return rc;2136}2137 2138static ssize_t qedi_show_boot_ini_info(void *data, int type, char *buf)2139{2140	struct qedi_ctx *qedi = data;2141	struct nvm_iscsi_initiator *initiator;2142	int rc;2143	struct nvm_iscsi_block *block;2144 2145	block = qedi_get_nvram_block(qedi);2146	if (!block)2147		return 0;2148 2149	initiator = &block->initiator;2150 2151	switch (type) {2152	case ISCSI_BOOT_INI_INITIATOR_NAME:2153		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_ISCSI_NAME_MAX_LEN,2154			     initiator->initiator_name.byte);2155		break;2156	default:2157		rc = 0;2158		break;2159	}2160	return rc;2161}2162 2163static umode_t qedi_ini_get_attr_visibility(void *data, int type)2164{2165	int rc;2166 2167	switch (type) {2168	case ISCSI_BOOT_INI_INITIATOR_NAME:2169		rc = 0444;2170		break;2171	default:2172		rc = 0;2173		break;2174	}2175	return rc;2176}2177 2178static ssize_t2179qedi_show_boot_tgt_info(struct qedi_ctx *qedi, int type,2180			char *buf, enum qedi_nvm_tgts idx)2181{2182	int rc = 1;2183	u32 ctrl_flags, ipv6_en, chap_en, mchap_en, ip_len;2184	struct nvm_iscsi_block *block;2185	char *chap_name, *chap_secret;2186	char *mchap_name, *mchap_secret;2187 2188	block = qedi_get_nvram_block(qedi);2189	if (!block)2190		goto exit_show_tgt_info;2191 2192	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_EVT,2193		  "Port:%d, tgt_idx:%d\n",2194		  GET_FIELD2(block->id, NVM_ISCSI_CFG_BLK_MAPPED_PF_ID), idx);2195 2196	ctrl_flags = block->target[idx].ctrl_flags &2197		     NVM_ISCSI_CFG_TARGET_ENABLED;2198 2199	if (!ctrl_flags) {2200		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_EVT,2201			  "Target disabled\n");2202		goto exit_show_tgt_info;2203	}2204 2205	ipv6_en = block->generic.ctrl_flags &2206		  NVM_ISCSI_CFG_GEN_IPV6_ENABLED;2207	ip_len = ipv6_en ? IPV6_LEN : IPV4_LEN;2208	chap_en = block->generic.ctrl_flags &2209		  NVM_ISCSI_CFG_GEN_CHAP_ENABLED;2210	chap_name = chap_en ? block->initiator.chap_name.byte : NULL;2211	chap_secret = chap_en ? block->initiator.chap_password.byte : NULL;2212 2213	mchap_en = block->generic.ctrl_flags &2214		  NVM_ISCSI_CFG_GEN_CHAP_MUTUAL_ENABLED;2215	mchap_name = mchap_en ? block->target[idx].chap_name.byte : NULL;2216	mchap_secret = mchap_en ? block->target[idx].chap_password.byte : NULL;2217 2218	switch (type) {2219	case ISCSI_BOOT_TGT_NAME:2220		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_ISCSI_NAME_MAX_LEN,2221			     block->target[idx].target_name.byte);2222		break;2223	case ISCSI_BOOT_TGT_IP_ADDR:2224		if (ipv6_en)2225			rc = snprintf(buf, ip_len, "%pI6\n",2226				      block->target[idx].ipv6_addr.byte);2227		else2228			rc = snprintf(buf, ip_len, "%pI4\n",2229				      block->target[idx].ipv4_addr.byte);2230		break;2231	case ISCSI_BOOT_TGT_PORT:2232		rc = snprintf(buf, 12, "%d\n",2233			      GET_FIELD2(block->target[idx].generic_cont0,2234					 NVM_ISCSI_CFG_TARGET_TCP_PORT));2235		break;2236	case ISCSI_BOOT_TGT_LUN:2237		rc = snprintf(buf, 22, "%.*d\n",2238			      block->target[idx].lun.value[1],2239			      block->target[idx].lun.value[0]);2240		break;2241	case ISCSI_BOOT_TGT_CHAP_NAME:2242		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,2243			     chap_name);2244		break;2245	case ISCSI_BOOT_TGT_CHAP_SECRET:2246		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,2247			     chap_secret);2248		break;2249	case ISCSI_BOOT_TGT_REV_CHAP_NAME:2250		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,2251			     mchap_name);2252		break;2253	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:2254		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,2255			     mchap_secret);2256		break;2257	case ISCSI_BOOT_TGT_FLAGS:2258		rc = snprintf(buf, 3, "%d\n", (char)SYSFS_FLAG_FW_SEL_BOOT);2259		break;2260	case ISCSI_BOOT_TGT_NIC_ASSOC:2261		rc = snprintf(buf, 3, "0\n");2262		break;2263	default:2264		rc = 0;2265		break;2266	}2267 2268exit_show_tgt_info:2269	return rc;2270}2271 2272static ssize_t qedi_show_boot_tgt_pri_info(void *data, int type, char *buf)2273{2274	struct qedi_ctx *qedi = data;2275 2276	return qedi_show_boot_tgt_info(qedi, type, buf, QEDI_NVM_TGT_PRI);2277}2278 2279static ssize_t qedi_show_boot_tgt_sec_info(void *data, int type, char *buf)2280{2281	struct qedi_ctx *qedi = data;2282 2283	return qedi_show_boot_tgt_info(qedi, type, buf, QEDI_NVM_TGT_SEC);2284}2285 2286static umode_t qedi_tgt_get_attr_visibility(void *data, int type)2287{2288	int rc;2289 2290	switch (type) {2291	case ISCSI_BOOT_TGT_NAME:2292	case ISCSI_BOOT_TGT_IP_ADDR:2293	case ISCSI_BOOT_TGT_PORT:2294	case ISCSI_BOOT_TGT_LUN:2295	case ISCSI_BOOT_TGT_CHAP_NAME:2296	case ISCSI_BOOT_TGT_CHAP_SECRET:2297	case ISCSI_BOOT_TGT_REV_CHAP_NAME:2298	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:2299	case ISCSI_BOOT_TGT_NIC_ASSOC:2300	case ISCSI_BOOT_TGT_FLAGS:2301		rc = 0444;2302		break;2303	default:2304		rc = 0;2305		break;2306	}2307	return rc;2308}2309 2310static void qedi_boot_release(void *data)2311{2312	struct qedi_ctx *qedi = data;2313 2314	scsi_host_put(qedi->shost);2315}2316 2317static int qedi_get_boot_info(struct qedi_ctx *qedi)2318{2319	int ret = 1;2320 2321	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,2322		  "Get NVM iSCSI CFG image\n");2323	ret = qedi_ops->common->nvm_get_image(qedi->cdev,2324					      QED_NVM_IMAGE_ISCSI_CFG,2325					      (char *)qedi->iscsi_image,2326					      sizeof(struct qedi_nvm_iscsi_image));2327	if (ret)2328		QEDI_ERR(&qedi->dbg_ctx,2329			 "Could not get NVM image. ret = %d\n", ret);2330 2331	return ret;2332}2333 2334static int qedi_setup_boot_info(struct qedi_ctx *qedi)2335{2336	struct iscsi_boot_kobj *boot_kobj;2337 2338	if (qedi_get_boot_info(qedi))2339		return -EPERM;2340 2341	qedi->boot_kset = iscsi_boot_create_host_kset(qedi->shost->host_no);2342	if (!qedi->boot_kset)2343		goto kset_free;2344 2345	if (!scsi_host_get(qedi->shost))2346		goto kset_free;2347 2348	boot_kobj = iscsi_boot_create_target(qedi->boot_kset, 0, qedi,2349					     qedi_show_boot_tgt_pri_info,2350					     qedi_tgt_get_attr_visibility,2351					     qedi_boot_release);2352	if (!boot_kobj)2353		goto put_host;2354 2355	if (!scsi_host_get(qedi->shost))2356		goto kset_free;2357 2358	boot_kobj = iscsi_boot_create_target(qedi->boot_kset, 1, qedi,2359					     qedi_show_boot_tgt_sec_info,2360					     qedi_tgt_get_attr_visibility,2361					     qedi_boot_release);2362	if (!boot_kobj)2363		goto put_host;2364 2365	if (!scsi_host_get(qedi->shost))2366		goto kset_free;2367 2368	boot_kobj = iscsi_boot_create_initiator(qedi->boot_kset, 0, qedi,2369						qedi_show_boot_ini_info,2370						qedi_ini_get_attr_visibility,2371						qedi_boot_release);2372	if (!boot_kobj)2373		goto put_host;2374 2375	if (!scsi_host_get(qedi->shost))2376		goto kset_free;2377 2378	boot_kobj = iscsi_boot_create_ethernet(qedi->boot_kset, 0, qedi,2379					       qedi_show_boot_eth_info,2380					       qedi_eth_get_attr_visibility,2381					       qedi_boot_release);2382	if (!boot_kobj)2383		goto put_host;2384 2385	return 0;2386 2387put_host:2388	scsi_host_put(qedi->shost);2389kset_free:2390	iscsi_boot_destroy_kset(qedi->boot_kset);2391	return -ENOMEM;2392}2393 2394static pci_ers_result_t qedi_io_error_detected(struct pci_dev *pdev,2395					       pci_channel_state_t state)2396{2397	struct qedi_ctx *qedi = pci_get_drvdata(pdev);2398 2399	QEDI_ERR(&qedi->dbg_ctx, "%s: PCI error detected [%d]\n",2400		 __func__, state);2401 2402	if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags)) {2403		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,2404			  "Recovery already in progress.\n");2405		return PCI_ERS_RESULT_NONE;2406	}2407 2408	qedi_ops->common->recovery_process(qedi->cdev);2409 2410	return PCI_ERS_RESULT_CAN_RECOVER;2411}2412 2413static void __qedi_remove(struct pci_dev *pdev, int mode)2414{2415	struct qedi_ctx *qedi = pci_get_drvdata(pdev);2416	int rval;2417	u16 retry = 10;2418 2419	if (mode == QEDI_MODE_NORMAL)2420		iscsi_host_remove(qedi->shost, false);2421	else if (mode == QEDI_MODE_SHUTDOWN)2422		iscsi_host_remove(qedi->shost, true);2423 2424	if (mode == QEDI_MODE_NORMAL || mode == QEDI_MODE_SHUTDOWN) {2425		if (qedi->tmf_thread) {2426			destroy_workqueue(qedi->tmf_thread);2427			qedi->tmf_thread = NULL;2428		}2429 2430		if (qedi->offload_thread) {2431			destroy_workqueue(qedi->offload_thread);2432			qedi->offload_thread = NULL;2433		}2434	}2435 2436#ifdef CONFIG_DEBUG_FS2437	qedi_dbg_host_exit(&qedi->dbg_ctx);2438#endif2439	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags))2440		qedi_ops->common->set_power_state(qedi->cdev, PCI_D0);2441 2442	qedi_sync_free_irqs(qedi);2443 2444	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags)) {2445		while (retry--) {2446			rval = qedi_ops->stop(qedi->cdev);2447			if (rval < 0)2448				msleep(1000);2449			else2450				break;2451		}2452		qedi_ops->ll2->stop(qedi->cdev);2453	}2454 2455	cancel_delayed_work_sync(&qedi->recovery_work);2456	cancel_delayed_work_sync(&qedi->board_disable_work);2457 2458	qedi_free_iscsi_pf_param(qedi);2459 2460	rval = qedi_ops->common->update_drv_state(qedi->cdev, false);2461	if (rval)2462		QEDI_ERR(&qedi->dbg_ctx, "Failed to send drv state to MFW\n");2463 2464	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags)) {2465		qedi_ops->common->slowpath_stop(qedi->cdev);2466		qedi_ops->common->remove(qedi->cdev);2467	}2468 2469	qedi_destroy_fp(qedi);2470 2471	if (mode == QEDI_MODE_NORMAL || mode == QEDI_MODE_SHUTDOWN) {2472		qedi_release_cid_que(qedi);2473		qedi_cm_free_mem(qedi);2474		qedi_free_uio(qedi->udev);2475		qedi_free_itt(qedi);2476 2477		if (qedi->ll2_recv_thread) {2478			kthread_stop(qedi->ll2_recv_thread);2479			qedi->ll2_recv_thread = NULL;2480		}2481		qedi_ll2_free_skbs(qedi);2482 2483		if (qedi->boot_kset)2484			iscsi_boot_destroy_kset(qedi->boot_kset);2485 2486		iscsi_host_free(qedi->shost);2487	}2488}2489 2490static void qedi_board_disable_work(struct work_struct *work)2491{2492	struct qedi_ctx *qedi =2493			container_of(work, struct qedi_ctx,2494				     board_disable_work.work);2495 2496	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,2497		  "Fan failure, Unloading firmware context.\n");2498 2499	if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))2500		return;2501 2502	__qedi_remove(qedi->pdev, QEDI_MODE_NORMAL);2503}2504 2505static void qedi_shutdown(struct pci_dev *pdev)2506{2507	struct qedi_ctx *qedi = pci_get_drvdata(pdev);2508 2509	QEDI_ERR(&qedi->dbg_ctx, "%s: Shutdown qedi\n", __func__);2510	if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))2511		return;2512	__qedi_remove(pdev, QEDI_MODE_SHUTDOWN);2513}2514 2515static int qedi_suspend(struct pci_dev *pdev, pm_message_t state)2516{2517	struct qedi_ctx *qedi;2518 2519	if (!pdev) {2520		QEDI_ERR(NULL, "pdev is NULL.\n");2521		return -ENODEV;2522	}2523 2524	qedi = pci_get_drvdata(pdev);2525 2526	QEDI_ERR(&qedi->dbg_ctx, "%s: Device does not support suspend operation\n", __func__);2527 2528	return -EPERM;2529}2530 2531static int __qedi_probe(struct pci_dev *pdev, int mode)2532{2533	struct qedi_ctx *qedi;2534	struct qed_ll2_params params;2535	u8 dp_level = 0;2536	bool is_vf = false;2537	char host_buf[16];2538	struct qed_link_params link_params;2539	struct qed_slowpath_params sp_params;2540	struct qed_probe_params qed_params;2541	void *task_start, *task_end;2542	int rc;2543	u16 retry = 10;2544 2545	if (mode != QEDI_MODE_RECOVERY) {2546		qedi = qedi_host_alloc(pdev);2547		if (!qedi) {2548			rc = -ENOMEM;2549			goto exit_probe;2550		}2551	} else {2552		qedi = pci_get_drvdata(pdev);2553	}2554 2555retry_probe:2556	if (mode == QEDI_MODE_RECOVERY)2557		msleep(2000);2558 2559	memset(&qed_params, 0, sizeof(qed_params));2560	qed_params.protocol = QED_PROTOCOL_ISCSI;2561	qed_params.dp_module = qedi_qed_debug;2562	qed_params.dp_level = dp_level;2563	qed_params.is_vf = is_vf;2564	qedi->cdev = qedi_ops->common->probe(pdev, &qed_params);2565	if (!qedi->cdev) {2566		if (mode == QEDI_MODE_RECOVERY && retry) {2567			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,2568				  "Retry %d initialize hardware\n", retry);2569			retry--;2570			goto retry_probe;2571		}2572 2573		rc = -ENODEV;2574		QEDI_ERR(&qedi->dbg_ctx, "Cannot initialize hardware\n");2575		goto free_host;2576	}2577 2578	set_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags);2579	set_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags);2580	atomic_set(&qedi->link_state, QEDI_LINK_DOWN);2581 2582	rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);2583	if (rc)2584		goto free_host;2585 2586	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,2587		  "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",2588		  qedi->dev_info.common.num_hwfns,2589		  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));2590 2591	rc = qedi_set_iscsi_pf_param(qedi);2592	if (rc) {2593		rc = -ENOMEM;2594		QEDI_ERR(&qedi->dbg_ctx,2595			 "Set iSCSI pf param fail\n");2596		goto free_host;2597	}2598 2599	qedi_ops->common->update_pf_params(qedi->cdev, &qedi->pf_params);2600 2601	rc = qedi_prepare_fp(qedi);2602	if (rc) {2603		QEDI_ERR(&qedi->dbg_ctx, "Cannot start slowpath.\n");2604		goto free_pf_params;2605	}2606 2607	/* Start the Slowpath-process */2608	memset(&sp_params, 0, sizeof(struct qed_slowpath_params));2609	sp_params.int_mode = QED_INT_MODE_MSIX;2610	sp_params.drv_major = QEDI_DRIVER_MAJOR_VER;2611	sp_params.drv_minor = QEDI_DRIVER_MINOR_VER;2612	sp_params.drv_rev = QEDI_DRIVER_REV_VER;2613	sp_params.drv_eng = QEDI_DRIVER_ENG_VER;2614	strscpy(sp_params.name, "qedi iSCSI", QED_DRV_VER_STR_SIZE);2615	rc = qedi_ops->common->slowpath_start(qedi->cdev, &sp_params);2616	if (rc) {2617		QEDI_ERR(&qedi->dbg_ctx, "Cannot start slowpath\n");2618		goto stop_hw;2619	}2620 2621	/* update_pf_params needs to be called before and after slowpath2622	 * start2623	 */2624	qedi_ops->common->update_pf_params(qedi->cdev, &qedi->pf_params);2625 2626	rc = qedi_setup_int(qedi);2627	if (rc)2628		goto stop_iscsi_func;2629 2630	qedi_ops->common->set_power_state(qedi->cdev, PCI_D0);2631 2632	/* Learn information crucial for qedi to progress */2633	rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);2634	if (rc)2635		goto stop_iscsi_func;2636 2637	/* Record BDQ producer doorbell addresses */2638	qedi->bdq_primary_prod = qedi->dev_info.primary_dbq_rq_addr;2639	qedi->bdq_secondary_prod = qedi->dev_info.secondary_bdq_rq_addr;2640	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,2641		  "BDQ primary_prod=%p secondary_prod=%p.\n",2642		  qedi->bdq_primary_prod,2643		  qedi->bdq_secondary_prod);2644 2645	/*2646	 * We need to write the number of BDs in the BDQ we've preallocated so2647	 * the f/w will do a prefetch and we'll get an unsolicited CQE when a2648	 * packet arrives.2649	 */2650	qedi->bdq_prod_idx = QEDI_BDQ_NUM;2651	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,2652		  "Writing %d to primary and secondary BDQ doorbell registers.\n",2653		  qedi->bdq_prod_idx);2654	writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);2655	readw(qedi->bdq_primary_prod);2656	writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);2657	readw(qedi->bdq_secondary_prod);2658 2659	ether_addr_copy(qedi->mac, qedi->dev_info.common.hw_mac);2660	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC, "MAC address is %pM.\n",2661		  qedi->mac);2662 2663	snprintf(host_buf, sizeof(host_buf), "host_%d", qedi->shost->host_no);2664	qedi_ops->common->set_name(qedi->cdev, host_buf);2665 2666	qedi_ops->register_ops(qedi->cdev, &qedi_cb_ops, qedi);2667 2668	memset(&params, 0, sizeof(params));2669	params.mtu = DEF_PATH_MTU + IPV6_HDR_LEN + TCP_HDR_LEN;2670	qedi->ll2_mtu = DEF_PATH_MTU;2671	params.drop_ttl0_packets = 0;2672	params.rx_vlan_stripping = 1;2673	ether_addr_copy(params.ll2_mac_address, qedi->dev_info.common.hw_mac);2674 2675	if (mode != QEDI_MODE_RECOVERY) {2676		/* set up rx path */2677		INIT_LIST_HEAD(&qedi->ll2_skb_list);2678		spin_lock_init(&qedi->ll2_lock);2679		/* start qedi context */2680		spin_lock_init(&qedi->hba_lock);2681		spin_lock_init(&qedi->task_idx_lock);2682		mutex_init(&qedi->stats_lock);2683	}2684	qedi_ops->ll2->register_cb_ops(qedi->cdev, &qedi_ll2_cb_ops, qedi);2685	qedi_ops->ll2->start(qedi->cdev, &params);2686 2687	if (mode != QEDI_MODE_RECOVERY) {2688		qedi->ll2_recv_thread = kthread_run(qedi_ll2_recv_thread,2689						    (void *)qedi,2690						    "qedi_ll2_thread");2691	}2692 2693	rc = qedi_ops->start(qedi->cdev, &qedi->tasks,2694			     qedi, qedi_iscsi_event_cb);2695	if (rc) {2696		rc = -ENODEV;2697		QEDI_ERR(&qedi->dbg_ctx, "Cannot start iSCSI function\n");2698		goto stop_slowpath;2699	}2700 2701	task_start = qedi_get_task_mem(&qedi->tasks, 0);2702	task_end = qedi_get_task_mem(&qedi->tasks, MAX_TID_BLOCKS_ISCSI - 1);2703	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,2704		  "Task context start=%p, end=%p block_size=%u.\n",2705		   task_start, task_end, qedi->tasks.size);2706 2707	memset(&link_params, 0, sizeof(link_params));2708	link_params.link_up = true;2709	rc = qedi_ops->common->set_link(qedi->cdev, &link_params);2710	if (rc) {2711		QEDI_WARN(&qedi->dbg_ctx, "Link set up failed.\n");2712		atomic_set(&qedi->link_state, QEDI_LINK_DOWN);2713	}2714 2715#ifdef CONFIG_DEBUG_FS2716	qedi_dbg_host_init(&qedi->dbg_ctx, qedi_debugfs_ops,2717			   qedi_dbg_fops);2718#endif2719	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,2720		  "QLogic FastLinQ iSCSI Module qedi %s, FW %d.%d.%d.%d\n",2721		  QEDI_MODULE_VERSION, FW_MAJOR_VERSION, FW_MINOR_VERSION,2722		  FW_REVISION_VERSION, FW_ENGINEERING_VERSION);2723 2724	if (mode == QEDI_MODE_NORMAL) {2725		if (iscsi_host_add(qedi->shost, &pdev->dev)) {2726			QEDI_ERR(&qedi->dbg_ctx,2727				 "Could not add iscsi host\n");2728			rc = -ENOMEM;2729			goto remove_host;2730		}2731 2732		/* Allocate uio buffers */2733		rc = qedi_alloc_uio_rings(qedi);2734		if (rc) {2735			QEDI_ERR(&qedi->dbg_ctx,2736				 "UIO alloc ring failed err=%d\n", rc);2737			goto remove_host;2738		}2739 2740		rc = qedi_init_uio(qedi);2741		if (rc) {2742			QEDI_ERR(&qedi->dbg_ctx,2743				 "UIO init failed, err=%d\n", rc);2744			goto free_uio;2745		}2746 2747		/* host the array on iscsi_conn */2748		rc = qedi_setup_cid_que(qedi);2749		if (rc) {2750			QEDI_ERR(&qedi->dbg_ctx,2751				 "Could not setup cid que\n");2752			goto free_uio;2753		}2754 2755		rc = qedi_cm_alloc_mem(qedi);2756		if (rc) {2757			QEDI_ERR(&qedi->dbg_ctx,2758				 "Could not alloc cm memory\n");2759			goto free_cid_que;2760		}2761 2762		rc = qedi_alloc_itt(qedi);2763		if (rc) {2764			QEDI_ERR(&qedi->dbg_ctx,2765				 "Could not alloc itt memory\n");2766			goto free_cid_que;2767		}2768 2769		sprintf(host_buf, "host_%d", qedi->shost->host_no);2770		qedi->tmf_thread =2771			alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM, host_buf);2772		if (!qedi->tmf_thread) {2773			QEDI_ERR(&qedi->dbg_ctx,2774				 "Unable to start tmf thread!\n");2775			rc = -ENODEV;2776			goto free_cid_que;2777		}2778 2779		qedi->offload_thread = alloc_workqueue("qedi_ofld%d",2780						       WQ_MEM_RECLAIM,2781						       1, qedi->shost->host_no);2782		if (!qedi->offload_thread) {2783			QEDI_ERR(&qedi->dbg_ctx,2784				 "Unable to start offload thread!\n");2785			rc = -ENODEV;2786			goto free_tmf_thread;2787		}2788 2789		INIT_DELAYED_WORK(&qedi->recovery_work, qedi_recovery_handler);2790		INIT_DELAYED_WORK(&qedi->board_disable_work,2791				  qedi_board_disable_work);2792 2793		/* F/w needs 1st task context memory entry for performance */2794		set_bit(QEDI_RESERVE_TASK_ID, qedi->task_idx_map);2795		atomic_set(&qedi->num_offloads, 0);2796 2797		if (qedi_setup_boot_info(qedi))2798			QEDI_ERR(&qedi->dbg_ctx,2799				 "No iSCSI boot target configured\n");2800 2801		rc = qedi_ops->common->update_drv_state(qedi->cdev, true);2802		if (rc)2803			QEDI_ERR(&qedi->dbg_ctx,2804				 "Failed to send drv state to MFW\n");2805 2806	}2807 2808	return 0;2809 2810free_tmf_thread:2811	destroy_workqueue(qedi->tmf_thread);2812free_cid_que:2813	qedi_release_cid_que(qedi);2814free_uio:2815	qedi_free_uio(qedi->udev);2816remove_host:2817#ifdef CONFIG_DEBUG_FS2818	qedi_dbg_host_exit(&qedi->dbg_ctx);2819#endif2820	iscsi_host_remove(qedi->shost, false);2821stop_iscsi_func:2822	qedi_ops->stop(qedi->cdev);2823stop_slowpath:2824	qedi_ops->common->slowpath_stop(qedi->cdev);2825stop_hw:2826	qedi_ops->common->remove(qedi->cdev);2827free_pf_params:2828	qedi_free_iscsi_pf_param(qedi);2829free_host:2830	iscsi_host_free(qedi->shost);2831exit_probe:2832	return rc;2833}2834 2835static void qedi_mark_conn_recovery(struct iscsi_cls_session *cls_session)2836{2837	struct iscsi_session *session = cls_session->dd_data;2838	struct iscsi_conn *conn = session->leadconn;2839	struct qedi_conn *qedi_conn = conn->dd_data;2840 2841	iscsi_conn_failure(qedi_conn->cls_conn->dd_data, ISCSI_ERR_CONN_FAILED);2842}2843 2844static void qedi_recovery_handler(struct work_struct *work)2845{2846	struct qedi_ctx *qedi =2847			container_of(work, struct qedi_ctx, recovery_work.work);2848 2849	iscsi_host_for_each_session(qedi->shost, qedi_mark_conn_recovery);2850 2851	/* Call common_ops->recovery_prolog to allow the MFW to quiesce2852	 * any PCI transactions.2853	 */2854	qedi_ops->common->recovery_prolog(qedi->cdev);2855 2856	__qedi_remove(qedi->pdev, QEDI_MODE_RECOVERY);2857	__qedi_probe(qedi->pdev, QEDI_MODE_RECOVERY);2858	clear_bit(QEDI_IN_RECOVERY, &qedi->flags);2859}2860 2861static int qedi_probe(struct pci_dev *pdev, const struct pci_device_id *id)2862{2863	return __qedi_probe(pdev, QEDI_MODE_NORMAL);2864}2865 2866static void qedi_remove(struct pci_dev *pdev)2867{2868	__qedi_remove(pdev, QEDI_MODE_NORMAL);2869}2870 2871static struct pci_device_id qedi_pci_tbl[] = {2872	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x165E) },2873	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x8084) },2874	{ 0 },2875};2876MODULE_DEVICE_TABLE(pci, qedi_pci_tbl);2877 2878static enum cpuhp_state qedi_cpuhp_state;2879 2880static struct pci_error_handlers qedi_err_handler = {2881	.error_detected = qedi_io_error_detected,2882};2883 2884static struct pci_driver qedi_pci_driver = {2885	.name = QEDI_MODULE_NAME,2886	.id_table = qedi_pci_tbl,2887	.probe = qedi_probe,2888	.remove = qedi_remove,2889	.shutdown = qedi_shutdown,2890	.err_handler = &qedi_err_handler,2891	.suspend = qedi_suspend,2892};2893 2894static int __init qedi_init(void)2895{2896	struct qedi_percpu_s *p;2897	int cpu, rc = 0;2898 2899	qedi_ops = qed_get_iscsi_ops();2900	if (!qedi_ops) {2901		QEDI_ERR(NULL, "Failed to get qed iSCSI operations\n");2902		return -EINVAL;2903	}2904 2905#ifdef CONFIG_DEBUG_FS2906	qedi_dbg_init("qedi");2907#endif2908 2909	qedi_scsi_transport = iscsi_register_transport(&qedi_iscsi_transport);2910	if (!qedi_scsi_transport) {2911		QEDI_ERR(NULL, "Could not register qedi transport");2912		rc = -ENOMEM;2913		goto exit_qedi_init_1;2914	}2915 2916	for_each_possible_cpu(cpu) {2917		p = &per_cpu(qedi_percpu, cpu);2918		INIT_LIST_HEAD(&p->work_list);2919		spin_lock_init(&p->p_work_lock);2920		p->iothread = NULL;2921	}2922 2923	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "scsi/qedi:online",2924			       qedi_cpu_online, qedi_cpu_offline);2925	if (rc < 0)2926		goto exit_qedi_init_2;2927	qedi_cpuhp_state = rc;2928 2929	rc = pci_register_driver(&qedi_pci_driver);2930	if (rc) {2931		QEDI_ERR(NULL, "Failed to register driver\n");2932		goto exit_qedi_hp;2933	}2934 2935	return 0;2936 2937exit_qedi_hp:2938	cpuhp_remove_state(qedi_cpuhp_state);2939exit_qedi_init_2:2940	iscsi_unregister_transport(&qedi_iscsi_transport);2941exit_qedi_init_1:2942#ifdef CONFIG_DEBUG_FS2943	qedi_dbg_exit();2944#endif2945	qed_put_iscsi_ops();2946	return rc;2947}2948 2949static void __exit qedi_cleanup(void)2950{2951	pci_unregister_driver(&qedi_pci_driver);2952	cpuhp_remove_state(qedi_cpuhp_state);2953	iscsi_unregister_transport(&qedi_iscsi_transport);2954 2955#ifdef CONFIG_DEBUG_FS2956	qedi_dbg_exit();2957#endif2958	qed_put_iscsi_ops();2959}2960 2961MODULE_DESCRIPTION("QLogic FastLinQ 4xxxx iSCSI Module");2962MODULE_LICENSE("GPL");2963MODULE_AUTHOR("QLogic Corporation");2964MODULE_VERSION(QEDI_MODULE_VERSION);2965module_init(qedi_init);2966module_exit(qedi_cleanup);2967