brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 *  Linux MegaRAID driver for SAS based RAID controllers4 *5 *  Copyright (c) 2009-2013  LSI Corporation6 *  Copyright (c) 2013-2016  Avago Technologies7 *  Copyright (c) 2016-2018  Broadcom Inc.8 *9 *  FILE: megaraid_sas_fusion.c10 *11 *  Authors: Broadcom Inc.12 *           Sumant Patro13 *           Adam Radford14 *           Kashyap Desai <kashyap.desai@broadcom.com>15 *           Sumit Saxena <sumit.saxena@broadcom.com>16 *17 *  Send feedback to: megaraidlinux.pdl@broadcom.com18 */19 20#include <linux/kernel.h>21#include <linux/types.h>22#include <linux/pci.h>23#include <linux/list.h>24#include <linux/moduleparam.h>25#include <linux/module.h>26#include <linux/spinlock.h>27#include <linux/interrupt.h>28#include <linux/delay.h>29#include <linux/uio.h>30#include <linux/uaccess.h>31#include <linux/fs.h>32#include <linux/compat.h>33#include <linux/blkdev.h>34#include <linux/mutex.h>35#include <linux/poll.h>36#include <linux/vmalloc.h>37#include <linux/workqueue.h>38#include <linux/irq_poll.h>39 40#include <scsi/scsi.h>41#include <scsi/scsi_cmnd.h>42#include <scsi/scsi_device.h>43#include <scsi/scsi_host.h>44#include <scsi/scsi_dbg.h>45#include <linux/dmi.h>46 47#include "megaraid_sas_fusion.h"48#include "megaraid_sas.h"49 50 51extern void52megasas_complete_cmd(struct megasas_instance *instance,53		     struct megasas_cmd *cmd, u8 alt_status);54int55wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,56	      int seconds);57 58int59megasas_clear_intr_fusion(struct megasas_instance *instance);60 61int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);62 63extern u32 megasas_dbg_lvl;64int megasas_sriov_start_heartbeat(struct megasas_instance *instance,65				  int initial);66extern struct megasas_mgmt_info megasas_mgmt_info;67extern unsigned int resetwaittime;68extern unsigned int dual_qdepth_disable;69static void megasas_free_rdpq_fusion(struct megasas_instance *instance);70static void megasas_free_reply_fusion(struct megasas_instance *instance);71static inline72void megasas_configure_queue_sizes(struct megasas_instance *instance);73static void megasas_fusion_crash_dump(struct megasas_instance *instance);74 75/**76 * megasas_adp_reset_wait_for_ready -	initiate chip reset and wait for77 *					controller to come to ready state78 * @instance:				adapter's soft state79 * @do_adp_reset:			If true, do a chip reset80 * @ocr_context:			If called from OCR context this will81 *					be set to 1, else 082 *83 * This function initiates a chip reset followed by a wait for controller to84 * transition to ready state.85 * During this, driver will block all access to PCI config space from userspace86 */87int88megasas_adp_reset_wait_for_ready(struct megasas_instance *instance,89				 bool do_adp_reset,90				 int ocr_context)91{92	int ret = FAILED;93 94	/*95	 * Block access to PCI config space from userspace96	 * when diag reset is initiated from driver97	 */98	if (megasas_dbg_lvl & OCR_DEBUG)99		dev_info(&instance->pdev->dev,100			 "Block access to PCI config space %s %d\n",101			 __func__, __LINE__);102 103	pci_cfg_access_lock(instance->pdev);104 105	if (do_adp_reset) {106		if (instance->instancet->adp_reset107			(instance, instance->reg_set))108			goto out;109	}110 111	/* Wait for FW to become ready */112	if (megasas_transition_to_ready(instance, ocr_context)) {113		dev_warn(&instance->pdev->dev,114			 "Failed to transition controller to ready for scsi%d.\n",115			 instance->host->host_no);116		goto out;117	}118 119	ret = SUCCESS;120out:121	if (megasas_dbg_lvl & OCR_DEBUG)122		dev_info(&instance->pdev->dev,123			 "Unlock access to PCI config space %s %d\n",124			 __func__, __LINE__);125 126	pci_cfg_access_unlock(instance->pdev);127 128	return ret;129}130 131/**132 * megasas_check_same_4gb_region -	check if allocation133 *					crosses same 4GB boundary or not134 * @instance:				adapter's soft instance135 * @start_addr:				start address of DMA allocation136 * @size:				size of allocation in bytes137 * @return:				true : allocation does not cross same138 *					4GB boundary139 *					false: allocation crosses same140 *					4GB boundary141 */142static inline bool megasas_check_same_4gb_region143	(struct megasas_instance *instance, dma_addr_t start_addr, size_t size)144{145	dma_addr_t end_addr;146 147	end_addr = start_addr + size;148 149	if (upper_32_bits(start_addr) != upper_32_bits(end_addr)) {150		dev_err(&instance->pdev->dev,151			"Failed to get same 4GB boundary: start_addr: 0x%llx end_addr: 0x%llx\n",152			(unsigned long long)start_addr,153			(unsigned long long)end_addr);154		return false;155	}156 157	return true;158}159 160/**161 * megasas_enable_intr_fusion -	Enables interrupts162 * @instance:	adapter's soft instance163 */164static void165megasas_enable_intr_fusion(struct megasas_instance *instance)166{167	struct megasas_register_set __iomem *regs;168	regs = instance->reg_set;169 170	instance->mask_interrupts = 0;171	/* For Thunderbolt/Invader also clear intr on enable */172	writel(~0, &regs->outbound_intr_status);173	readl(&regs->outbound_intr_status);174 175	writel(~MFI_FUSION_ENABLE_INTERRUPT_MASK, &(regs)->outbound_intr_mask);176 177	/* Dummy readl to force pci flush */178	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",179		 __func__, readl(&regs->outbound_intr_mask));180}181 182/**183 * megasas_disable_intr_fusion - Disables interrupt184 * @instance:	adapter's soft instance185 */186static void187megasas_disable_intr_fusion(struct megasas_instance *instance)188{189	u32 mask = 0xFFFFFFFF;190	struct megasas_register_set __iomem *regs;191	regs = instance->reg_set;192	instance->mask_interrupts = 1;193 194	writel(mask, &regs->outbound_intr_mask);195	/* Dummy readl to force pci flush */196	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",197		 __func__, readl(&regs->outbound_intr_mask));198}199 200int201megasas_clear_intr_fusion(struct megasas_instance *instance)202{203	u32 status;204	struct megasas_register_set __iomem *regs;205	regs = instance->reg_set;206	/*207	 * Check if it is our interrupt208	 */209	status = megasas_readl(instance,210			       &regs->outbound_intr_status);211 212	if (status & 1) {213		writel(status, &regs->outbound_intr_status);214		readl(&regs->outbound_intr_status);215		return 1;216	}217	if (!(status & MFI_FUSION_ENABLE_INTERRUPT_MASK))218		return 0;219 220	return 1;221}222 223static inline void224megasas_sdev_busy_inc(struct megasas_instance *instance,225		      struct scsi_cmnd *scmd)226{227	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {228		struct MR_PRIV_DEVICE *mr_device_priv_data =229			scmd->device->hostdata;230		atomic_inc(&mr_device_priv_data->sdev_priv_busy);231	}232}233 234static inline void235megasas_sdev_busy_dec(struct megasas_instance *instance,236		      struct scsi_cmnd *scmd)237{238	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {239		struct MR_PRIV_DEVICE *mr_device_priv_data =240			scmd->device->hostdata;241		atomic_dec(&mr_device_priv_data->sdev_priv_busy);242	}243}244 245static inline int246megasas_sdev_busy_read(struct megasas_instance *instance,247		       struct scsi_cmnd *scmd)248{249	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {250		struct MR_PRIV_DEVICE *mr_device_priv_data =251			scmd->device->hostdata;252		return atomic_read(&mr_device_priv_data->sdev_priv_busy);253	}254	return 0;255}256 257/**258 * megasas_get_cmd_fusion -	Get a command from the free pool259 * @instance:		Adapter soft state260 * @blk_tag:		Command tag261 *262 * Returns a blk_tag indexed mpt frame263 */264inline struct megasas_cmd_fusion *megasas_get_cmd_fusion(struct megasas_instance265						  *instance, u32 blk_tag)266{267	struct fusion_context *fusion;268 269	fusion = instance->ctrl_context;270	return fusion->cmd_list[blk_tag];271}272 273/**274 * megasas_return_cmd_fusion -	Return a cmd to free command pool275 * @instance:		Adapter soft state276 * @cmd:		Command packet to be returned to free command pool277 */278inline void megasas_return_cmd_fusion(struct megasas_instance *instance,279	struct megasas_cmd_fusion *cmd)280{281	cmd->scmd = NULL;282	memset(cmd->io_request, 0, MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);283	cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;284	cmd->cmd_completed = false;285}286 287/**288 * megasas_write_64bit_req_desc -	PCI writes 64bit request descriptor289 * @instance:				Adapter soft state290 * @req_desc:				64bit Request descriptor291 */292static void293megasas_write_64bit_req_desc(struct megasas_instance *instance,294		union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)295{296#if defined(writeq) && defined(CONFIG_64BIT)297	u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) |298		le32_to_cpu(req_desc->u.low));299	writeq(req_data, &instance->reg_set->inbound_low_queue_port);300#else301	unsigned long flags;302	spin_lock_irqsave(&instance->hba_lock, flags);303	writel(le32_to_cpu(req_desc->u.low),304		&instance->reg_set->inbound_low_queue_port);305	writel(le32_to_cpu(req_desc->u.high),306		&instance->reg_set->inbound_high_queue_port);307	spin_unlock_irqrestore(&instance->hba_lock, flags);308#endif309}310 311/**312 * megasas_fire_cmd_fusion -	Sends command to the FW313 * @instance:			Adapter soft state314 * @req_desc:			32bit or 64bit Request descriptor315 *316 * Perform PCI Write. AERO SERIES supports 32 bit Descriptor.317 * Prior to AERO_SERIES support 64 bit Descriptor.318 */319static void320megasas_fire_cmd_fusion(struct megasas_instance *instance,321		union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)322{323	if (instance->atomic_desc_support)324		writel(le32_to_cpu(req_desc->u.low),325			&instance->reg_set->inbound_single_queue_port);326	else327		megasas_write_64bit_req_desc(instance, req_desc);328}329 330/**331 * megasas_fusion_update_can_queue -	Do all Adapter Queue depth related calculations here332 * @instance:		Adapter soft state333 * @fw_boot_context:	Whether this function called during probe or after OCR334 *335 * This function is only for fusion controllers.336 * Update host can queue, if firmware downgrade max supported firmware commands.337 * Firmware upgrade case will be skipped because underlying firmware has338 * more resource than exposed to the OS.339 *340 */341static void342megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_context)343{344	u16 cur_max_fw_cmds = 0;345	u16 ldio_threshold = 0;346 347	/* ventura FW does not fill outbound_scratch_pad_2 with queue depth */348	if (instance->adapter_type < VENTURA_SERIES)349		cur_max_fw_cmds =350		megasas_readl(instance,351			      &instance->reg_set->outbound_scratch_pad_2) & 0x00FFFF;352 353	if (dual_qdepth_disable || !cur_max_fw_cmds)354		cur_max_fw_cmds = instance->instancet->read_fw_status_reg(instance) & 0x00FFFF;355	else356		ldio_threshold =357			(instance->instancet->read_fw_status_reg(instance) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS;358 359	dev_info(&instance->pdev->dev,360		 "Current firmware supports maximum commands: %d\t LDIO threshold: %d\n",361		 cur_max_fw_cmds, ldio_threshold);362 363	if (fw_boot_context == OCR_CONTEXT) {364		cur_max_fw_cmds = cur_max_fw_cmds - 1;365		if (cur_max_fw_cmds < instance->max_fw_cmds) {366			instance->cur_can_queue =367				cur_max_fw_cmds - (MEGASAS_FUSION_INTERNAL_CMDS +368						MEGASAS_FUSION_IOCTL_CMDS);369			instance->host->can_queue = instance->cur_can_queue;370			instance->ldio_threshold = ldio_threshold;371		}372	} else {373		instance->max_fw_cmds = cur_max_fw_cmds;374		instance->ldio_threshold = ldio_threshold;375 376		if (reset_devices)377			instance->max_fw_cmds = min(instance->max_fw_cmds,378						(u16)MEGASAS_KDUMP_QUEUE_DEPTH);379		/*380		* Reduce the max supported cmds by 1. This is to ensure that the381		* reply_q_sz (1 more than the max cmd that driver may send)382		* does not exceed max cmds that the FW can support383		*/384		instance->max_fw_cmds = instance->max_fw_cmds-1;385	}386}387 388static inline void389megasas_get_msix_index(struct megasas_instance *instance,390		       struct scsi_cmnd *scmd,391		       struct megasas_cmd_fusion *cmd,392		       u8 data_arms)393{394	if (instance->perf_mode == MR_BALANCED_PERF_MODE &&395	    (megasas_sdev_busy_read(instance, scmd) >396	     (data_arms * MR_DEVICE_HIGH_IOPS_DEPTH))) {397		cmd->request_desc->SCSIIO.MSIxIndex =398			mega_mod64((atomic64_add_return(1, &instance->high_iops_outstanding) /399					MR_HIGH_IOPS_BATCH_COUNT), instance->low_latency_index_start);400	} else if (instance->msix_load_balance) {401		cmd->request_desc->SCSIIO.MSIxIndex =402			(mega_mod64(atomic64_add_return(1, &instance->total_io_count),403				instance->msix_vectors));404	} else if (instance->host->nr_hw_queues > 1) {405		u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));406 407		cmd->request_desc->SCSIIO.MSIxIndex = blk_mq_unique_tag_to_hwq(tag) +408			instance->low_latency_index_start;409	} else {410		cmd->request_desc->SCSIIO.MSIxIndex =411			instance->reply_map[raw_smp_processor_id()];412	}413}414 415/**416 * megasas_free_cmds_fusion -	Free all the cmds in the free cmd pool417 * @instance:		Adapter soft state418 */419void420megasas_free_cmds_fusion(struct megasas_instance *instance)421{422	int i;423	struct fusion_context *fusion = instance->ctrl_context;424	struct megasas_cmd_fusion *cmd;425 426	if (fusion->sense)427		dma_pool_free(fusion->sense_dma_pool, fusion->sense,428			      fusion->sense_phys_addr);429 430	/* SG */431	if (fusion->cmd_list) {432		for (i = 0; i < instance->max_mpt_cmds; i++) {433			cmd = fusion->cmd_list[i];434			if (cmd) {435				if (cmd->sg_frame)436					dma_pool_free(fusion->sg_dma_pool,437						      cmd->sg_frame,438						      cmd->sg_frame_phys_addr);439			}440			kfree(cmd);441		}442		kfree(fusion->cmd_list);443	}444 445	if (fusion->sg_dma_pool) {446		dma_pool_destroy(fusion->sg_dma_pool);447		fusion->sg_dma_pool = NULL;448	}449	if (fusion->sense_dma_pool) {450		dma_pool_destroy(fusion->sense_dma_pool);451		fusion->sense_dma_pool = NULL;452	}453 454 455	/* Reply Frame, Desc*/456	if (instance->is_rdpq)457		megasas_free_rdpq_fusion(instance);458	else459		megasas_free_reply_fusion(instance);460 461	/* Request Frame, Desc*/462	if (fusion->req_frames_desc)463		dma_free_coherent(&instance->pdev->dev,464			fusion->request_alloc_sz, fusion->req_frames_desc,465			fusion->req_frames_desc_phys);466	if (fusion->io_request_frames)467		dma_pool_free(fusion->io_request_frames_pool,468			fusion->io_request_frames,469			fusion->io_request_frames_phys);470	if (fusion->io_request_frames_pool) {471		dma_pool_destroy(fusion->io_request_frames_pool);472		fusion->io_request_frames_pool = NULL;473	}474}475 476/**477 * megasas_create_sg_sense_fusion -	Creates DMA pool for cmd frames478 * @instance:			Adapter soft state479 *480 */481static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)482{483	int i;484	u16 max_cmd;485	struct fusion_context *fusion;486	struct megasas_cmd_fusion *cmd;487	int sense_sz;488	u32 offset;489 490	fusion = instance->ctrl_context;491	max_cmd = instance->max_fw_cmds;492	sense_sz = instance->max_mpt_cmds * SCSI_SENSE_BUFFERSIZE;493 494	fusion->sg_dma_pool =495			dma_pool_create("mr_sg", &instance->pdev->dev,496				instance->max_chain_frame_sz,497				MR_DEFAULT_NVME_PAGE_SIZE, 0);498	/* SCSI_SENSE_BUFFERSIZE  = 96 bytes */499	fusion->sense_dma_pool =500			dma_pool_create("mr_sense", &instance->pdev->dev,501				sense_sz, 64, 0);502 503	if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) {504		dev_err(&instance->pdev->dev,505			"Failed from %s %d\n",  __func__, __LINE__);506		return -ENOMEM;507	}508 509	fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,510				       GFP_KERNEL, &fusion->sense_phys_addr);511	if (!fusion->sense) {512		dev_err(&instance->pdev->dev,513			"failed from %s %d\n",  __func__, __LINE__);514		return -ENOMEM;515	}516 517	/* sense buffer, request frame and reply desc pool requires to be in518	 * same 4 gb region. Below function will check this.519	 * In case of failure, new pci pool will be created with updated520	 * alignment.521	 * Older allocation and pool will be destroyed.522	 * Alignment will be used such a way that next allocation if success,523	 * will always meet same 4gb region requirement.524	 * Actual requirement is not alignment, but we need start and end of525	 * DMA address must have same upper 32 bit address.526	 */527 528	if (!megasas_check_same_4gb_region(instance, fusion->sense_phys_addr,529					   sense_sz)) {530		dma_pool_free(fusion->sense_dma_pool, fusion->sense,531			      fusion->sense_phys_addr);532		fusion->sense = NULL;533		dma_pool_destroy(fusion->sense_dma_pool);534 535		fusion->sense_dma_pool =536			dma_pool_create("mr_sense_align", &instance->pdev->dev,537					sense_sz, roundup_pow_of_two(sense_sz),538					0);539		if (!fusion->sense_dma_pool) {540			dev_err(&instance->pdev->dev,541				"Failed from %s %d\n",  __func__, __LINE__);542			return -ENOMEM;543		}544		fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,545					       GFP_KERNEL,546					       &fusion->sense_phys_addr);547		if (!fusion->sense) {548			dev_err(&instance->pdev->dev,549				"failed from %s %d\n",  __func__, __LINE__);550			return -ENOMEM;551		}552	}553 554	/*555	 * Allocate and attach a frame to each of the commands in cmd_list556	 */557	for (i = 0; i < max_cmd; i++) {558		cmd = fusion->cmd_list[i];559		cmd->sg_frame = dma_pool_alloc(fusion->sg_dma_pool,560					GFP_KERNEL, &cmd->sg_frame_phys_addr);561 562		offset = SCSI_SENSE_BUFFERSIZE * i;563		cmd->sense = (u8 *)fusion->sense + offset;564		cmd->sense_phys_addr = fusion->sense_phys_addr + offset;565 566		if (!cmd->sg_frame) {567			dev_err(&instance->pdev->dev,568				"Failed from %s %d\n",  __func__, __LINE__);569			return -ENOMEM;570		}571	}572 573	/* create sense buffer for the raid 1/10 fp */574	for (i = max_cmd; i < instance->max_mpt_cmds; i++) {575		cmd = fusion->cmd_list[i];576		offset = SCSI_SENSE_BUFFERSIZE * i;577		cmd->sense = (u8 *)fusion->sense + offset;578		cmd->sense_phys_addr = fusion->sense_phys_addr + offset;579 580	}581 582	return 0;583}584 585static int586megasas_alloc_cmdlist_fusion(struct megasas_instance *instance)587{588	u32 max_mpt_cmd, i, j;589	struct fusion_context *fusion;590 591	fusion = instance->ctrl_context;592 593	max_mpt_cmd = instance->max_mpt_cmds;594 595	/*596	 * fusion->cmd_list is an array of struct megasas_cmd_fusion pointers.597	 * Allocate the dynamic array first and then allocate individual598	 * commands.599	 */600	fusion->cmd_list =601		kcalloc(max_mpt_cmd, sizeof(struct megasas_cmd_fusion *),602			GFP_KERNEL);603	if (!fusion->cmd_list) {604		dev_err(&instance->pdev->dev,605			"Failed from %s %d\n",  __func__, __LINE__);606		return -ENOMEM;607	}608 609	for (i = 0; i < max_mpt_cmd; i++) {610		fusion->cmd_list[i] = kzalloc(sizeof(struct megasas_cmd_fusion),611					      GFP_KERNEL);612		if (!fusion->cmd_list[i]) {613			for (j = 0; j < i; j++)614				kfree(fusion->cmd_list[j]);615			kfree(fusion->cmd_list);616			dev_err(&instance->pdev->dev,617				"Failed from %s %d\n",  __func__, __LINE__);618			return -ENOMEM;619		}620	}621 622	return 0;623}624 625static int626megasas_alloc_request_fusion(struct megasas_instance *instance)627{628	struct fusion_context *fusion;629 630	fusion = instance->ctrl_context;631 632retry_alloc:633	fusion->io_request_frames_pool =634			dma_pool_create("mr_ioreq", &instance->pdev->dev,635				fusion->io_frames_alloc_sz, 16, 0);636 637	if (!fusion->io_request_frames_pool) {638		dev_err(&instance->pdev->dev,639			"Failed from %s %d\n",  __func__, __LINE__);640		return -ENOMEM;641	}642 643	fusion->io_request_frames =644			dma_pool_alloc(fusion->io_request_frames_pool,645				GFP_KERNEL | __GFP_NOWARN,646				&fusion->io_request_frames_phys);647	if (!fusion->io_request_frames) {648		if (instance->max_fw_cmds >= (MEGASAS_REDUCE_QD_COUNT * 2)) {649			instance->max_fw_cmds -= MEGASAS_REDUCE_QD_COUNT;650			dma_pool_destroy(fusion->io_request_frames_pool);651			megasas_configure_queue_sizes(instance);652			goto retry_alloc;653		} else {654			dev_err(&instance->pdev->dev,655				"Failed from %s %d\n",  __func__, __LINE__);656			return -ENOMEM;657		}658	}659 660	if (!megasas_check_same_4gb_region(instance,661					   fusion->io_request_frames_phys,662					   fusion->io_frames_alloc_sz)) {663		dma_pool_free(fusion->io_request_frames_pool,664			      fusion->io_request_frames,665			      fusion->io_request_frames_phys);666		fusion->io_request_frames = NULL;667		dma_pool_destroy(fusion->io_request_frames_pool);668 669		fusion->io_request_frames_pool =670			dma_pool_create("mr_ioreq_align",671					&instance->pdev->dev,672					fusion->io_frames_alloc_sz,673					roundup_pow_of_two(fusion->io_frames_alloc_sz),674					0);675 676		if (!fusion->io_request_frames_pool) {677			dev_err(&instance->pdev->dev,678				"Failed from %s %d\n",  __func__, __LINE__);679			return -ENOMEM;680		}681 682		fusion->io_request_frames =683			dma_pool_alloc(fusion->io_request_frames_pool,684				       GFP_KERNEL | __GFP_NOWARN,685				       &fusion->io_request_frames_phys);686 687		if (!fusion->io_request_frames) {688			dev_err(&instance->pdev->dev,689				"Failed from %s %d\n",  __func__, __LINE__);690			return -ENOMEM;691		}692	}693 694	fusion->req_frames_desc =695		dma_alloc_coherent(&instance->pdev->dev,696				   fusion->request_alloc_sz,697				   &fusion->req_frames_desc_phys, GFP_KERNEL);698	if (!fusion->req_frames_desc) {699		dev_err(&instance->pdev->dev,700			"Failed from %s %d\n",  __func__, __LINE__);701		return -ENOMEM;702	}703 704	return 0;705}706 707static int708megasas_alloc_reply_fusion(struct megasas_instance *instance)709{710	int i, count;711	struct fusion_context *fusion;712	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;713	fusion = instance->ctrl_context;714 715	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;716	count += instance->iopoll_q_count;717 718	fusion->reply_frames_desc_pool =719			dma_pool_create("mr_reply", &instance->pdev->dev,720				fusion->reply_alloc_sz * count, 16, 0);721 722	if (!fusion->reply_frames_desc_pool) {723		dev_err(&instance->pdev->dev,724			"Failed from %s %d\n",  __func__, __LINE__);725		return -ENOMEM;726	}727 728	fusion->reply_frames_desc[0] =729		dma_pool_alloc(fusion->reply_frames_desc_pool,730			GFP_KERNEL, &fusion->reply_frames_desc_phys[0]);731	if (!fusion->reply_frames_desc[0]) {732		dev_err(&instance->pdev->dev,733			"Failed from %s %d\n",  __func__, __LINE__);734		return -ENOMEM;735	}736 737	if (!megasas_check_same_4gb_region(instance,738					   fusion->reply_frames_desc_phys[0],739					   (fusion->reply_alloc_sz * count))) {740		dma_pool_free(fusion->reply_frames_desc_pool,741			      fusion->reply_frames_desc[0],742			      fusion->reply_frames_desc_phys[0]);743		fusion->reply_frames_desc[0] = NULL;744		dma_pool_destroy(fusion->reply_frames_desc_pool);745 746		fusion->reply_frames_desc_pool =747			dma_pool_create("mr_reply_align",748					&instance->pdev->dev,749					fusion->reply_alloc_sz * count,750					roundup_pow_of_two(fusion->reply_alloc_sz * count),751					0);752 753		if (!fusion->reply_frames_desc_pool) {754			dev_err(&instance->pdev->dev,755				"Failed from %s %d\n",  __func__, __LINE__);756			return -ENOMEM;757		}758 759		fusion->reply_frames_desc[0] =760			dma_pool_alloc(fusion->reply_frames_desc_pool,761				       GFP_KERNEL,762				       &fusion->reply_frames_desc_phys[0]);763 764		if (!fusion->reply_frames_desc[0]) {765			dev_err(&instance->pdev->dev,766				"Failed from %s %d\n",  __func__, __LINE__);767			return -ENOMEM;768		}769	}770 771	reply_desc = fusion->reply_frames_desc[0];772	for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++)773		reply_desc->Words = cpu_to_le64(ULLONG_MAX);774 775	/* This is not a rdpq mode, but driver still populate776	 * reply_frame_desc array to use same msix index in ISR path.777	 */778	for (i = 0; i < (count - 1); i++)779		fusion->reply_frames_desc[i + 1] =780			fusion->reply_frames_desc[i] +781			(fusion->reply_alloc_sz)/sizeof(union MPI2_REPLY_DESCRIPTORS_UNION);782 783	return 0;784}785 786static int787megasas_alloc_rdpq_fusion(struct megasas_instance *instance)788{789	int i, j, k, msix_count;790	struct fusion_context *fusion;791	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;792	union MPI2_REPLY_DESCRIPTORS_UNION *rdpq_chunk_virt[RDPQ_MAX_CHUNK_COUNT];793	dma_addr_t rdpq_chunk_phys[RDPQ_MAX_CHUNK_COUNT];794	u8 dma_alloc_count, abs_index;795	u32 chunk_size, array_size, offset;796 797	fusion = instance->ctrl_context;798	chunk_size = fusion->reply_alloc_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;799	array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) *800		     MAX_MSIX_QUEUES_FUSION;801 802	fusion->rdpq_virt = dma_alloc_coherent(&instance->pdev->dev,803					       array_size, &fusion->rdpq_phys,804					       GFP_KERNEL);805	if (!fusion->rdpq_virt) {806		dev_err(&instance->pdev->dev,807			"Failed from %s %d\n",  __func__, __LINE__);808		return -ENOMEM;809	}810 811	msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;812	msix_count += instance->iopoll_q_count;813 814	fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq",815							 &instance->pdev->dev,816							 chunk_size, 16, 0);817	fusion->reply_frames_desc_pool_align =818				dma_pool_create("mr_rdpq_align",819						&instance->pdev->dev,820						chunk_size,821						roundup_pow_of_two(chunk_size),822						0);823 824	if (!fusion->reply_frames_desc_pool ||825	    !fusion->reply_frames_desc_pool_align) {826		dev_err(&instance->pdev->dev,827			"Failed from %s %d\n",  __func__, __LINE__);828		return -ENOMEM;829	}830 831/*832 * For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and833 * VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should be834 * within 4GB boundary and also reply queues in a set must have same835 * upper 32-bits in their memory address. so here driver is allocating the836 * DMA'able memory for reply queues according. Driver uses limitation of837 * VENTURA_SERIES to manage INVADER_SERIES as well.838 */839	dma_alloc_count = DIV_ROUND_UP(msix_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK);840 841	for (i = 0; i < dma_alloc_count; i++) {842		rdpq_chunk_virt[i] =843			dma_pool_alloc(fusion->reply_frames_desc_pool,844				       GFP_KERNEL, &rdpq_chunk_phys[i]);845		if (!rdpq_chunk_virt[i]) {846			dev_err(&instance->pdev->dev,847				"Failed from %s %d\n",  __func__, __LINE__);848			return -ENOMEM;849		}850		/* reply desc pool requires to be in same 4 gb region.851		 * Below function will check this.852		 * In case of failure, new pci pool will be created with updated853		 * alignment.854		 * For RDPQ buffers, driver always allocate two separate pci pool.855		 * Alignment will be used such a way that next allocation if856		 * success, will always meet same 4gb region requirement.857		 * rdpq_tracker keep track of each buffer's physical,858		 * virtual address and pci pool descriptor. It will help driver859		 * while freeing the resources.860		 *861		 */862		if (!megasas_check_same_4gb_region(instance, rdpq_chunk_phys[i],863						   chunk_size)) {864			dma_pool_free(fusion->reply_frames_desc_pool,865				      rdpq_chunk_virt[i],866				      rdpq_chunk_phys[i]);867 868			rdpq_chunk_virt[i] =869				dma_pool_alloc(fusion->reply_frames_desc_pool_align,870					       GFP_KERNEL, &rdpq_chunk_phys[i]);871			if (!rdpq_chunk_virt[i]) {872				dev_err(&instance->pdev->dev,873					"Failed from %s %d\n",874					__func__, __LINE__);875				return -ENOMEM;876			}877			fusion->rdpq_tracker[i].dma_pool_ptr =878					fusion->reply_frames_desc_pool_align;879		} else {880			fusion->rdpq_tracker[i].dma_pool_ptr =881					fusion->reply_frames_desc_pool;882		}883 884		fusion->rdpq_tracker[i].pool_entry_phys = rdpq_chunk_phys[i];885		fusion->rdpq_tracker[i].pool_entry_virt = rdpq_chunk_virt[i];886	}887 888	for (k = 0; k < dma_alloc_count; k++) {889		for (i = 0; i < RDPQ_MAX_INDEX_IN_ONE_CHUNK; i++) {890			abs_index = (k * RDPQ_MAX_INDEX_IN_ONE_CHUNK) + i;891 892			if (abs_index == msix_count)893				break;894			offset = fusion->reply_alloc_sz * i;895			fusion->rdpq_virt[abs_index].RDPQBaseAddress =896					cpu_to_le64(rdpq_chunk_phys[k] + offset);897			fusion->reply_frames_desc_phys[abs_index] =898					rdpq_chunk_phys[k] + offset;899			fusion->reply_frames_desc[abs_index] =900					(union MPI2_REPLY_DESCRIPTORS_UNION *)((u8 *)rdpq_chunk_virt[k] + offset);901 902			reply_desc = fusion->reply_frames_desc[abs_index];903			for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++)904				reply_desc->Words = ULLONG_MAX;905		}906	}907 908	return 0;909}910 911static void912megasas_free_rdpq_fusion(struct megasas_instance *instance) {913 914	int i;915	struct fusion_context *fusion;916 917	fusion = instance->ctrl_context;918 919	for (i = 0; i < RDPQ_MAX_CHUNK_COUNT; i++) {920		if (fusion->rdpq_tracker[i].pool_entry_virt)921			dma_pool_free(fusion->rdpq_tracker[i].dma_pool_ptr,922				      fusion->rdpq_tracker[i].pool_entry_virt,923				      fusion->rdpq_tracker[i].pool_entry_phys);924 925	}926 927	dma_pool_destroy(fusion->reply_frames_desc_pool);928	dma_pool_destroy(fusion->reply_frames_desc_pool_align);929 930	if (fusion->rdpq_virt)931		dma_free_coherent(&instance->pdev->dev,932			sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION,933			fusion->rdpq_virt, fusion->rdpq_phys);934}935 936static void937megasas_free_reply_fusion(struct megasas_instance *instance) {938 939	struct fusion_context *fusion;940 941	fusion = instance->ctrl_context;942 943	if (fusion->reply_frames_desc[0])944		dma_pool_free(fusion->reply_frames_desc_pool,945			fusion->reply_frames_desc[0],946			fusion->reply_frames_desc_phys[0]);947 948	dma_pool_destroy(fusion->reply_frames_desc_pool);949 950}951 952 953/**954 * megasas_alloc_cmds_fusion -	Allocates the command packets955 * @instance:		Adapter soft state956 *957 *958 * Each frame has a 32-bit field called context. This context is used to get959 * back the megasas_cmd_fusion from the frame when a frame gets completed960 * In this driver, the 32 bit values are the indices into an array cmd_list.961 * This array is used only to look up the megasas_cmd_fusion given the context.962 * The free commands themselves are maintained in a linked list called cmd_pool.963 *964 * cmds are formed in the io_request and sg_frame members of the965 * megasas_cmd_fusion. The context field is used to get a request descriptor966 * and is used as SMID of the cmd.967 * SMID value range is from 1 to max_fw_cmds.968 */969static int970megasas_alloc_cmds_fusion(struct megasas_instance *instance)971{972	int i;973	struct fusion_context *fusion;974	struct megasas_cmd_fusion *cmd;975	u32 offset;976	dma_addr_t io_req_base_phys;977	u8 *io_req_base;978 979 980	fusion = instance->ctrl_context;981 982	if (megasas_alloc_request_fusion(instance))983		goto fail_exit;984 985	if (instance->is_rdpq) {986		if (megasas_alloc_rdpq_fusion(instance))987			goto fail_exit;988	} else989		if (megasas_alloc_reply_fusion(instance))990			goto fail_exit;991 992	if (megasas_alloc_cmdlist_fusion(instance))993		goto fail_exit;994 995	/* The first 256 bytes (SMID 0) is not used. Don't add to the cmd list */996	io_req_base = fusion->io_request_frames + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;997	io_req_base_phys = fusion->io_request_frames_phys + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;998 999	/*1000	 * Add all the commands to command pool (fusion->cmd_pool)1001	 */1002 1003	/* SMID 0 is reserved. Set SMID/index from 1 */1004	for (i = 0; i < instance->max_mpt_cmds; i++) {1005		cmd = fusion->cmd_list[i];1006		offset = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE * i;1007		memset(cmd, 0, sizeof(struct megasas_cmd_fusion));1008		cmd->index = i + 1;1009		cmd->scmd = NULL;1010		cmd->sync_cmd_idx =1011		(i >= instance->max_scsi_cmds && i < instance->max_fw_cmds) ?1012				(i - instance->max_scsi_cmds) :1013				(u32)ULONG_MAX; /* Set to Invalid */1014		cmd->instance = instance;1015		cmd->io_request =1016			(struct MPI2_RAID_SCSI_IO_REQUEST *)1017		  (io_req_base + offset);1018		memset(cmd->io_request, 0,1019		       sizeof(struct MPI2_RAID_SCSI_IO_REQUEST));1020		cmd->io_request_phys_addr = io_req_base_phys + offset;1021		cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;1022	}1023 1024	if (megasas_create_sg_sense_fusion(instance))1025		goto fail_exit;1026 1027	return 0;1028 1029fail_exit:1030	megasas_free_cmds_fusion(instance);1031	return -ENOMEM;1032}1033 1034/**1035 * wait_and_poll -	Issues a polling command1036 * @instance:			Adapter soft state1037 * @cmd:			Command packet to be issued1038 * @seconds:			Maximum poll time1039 *1040 * For polling, MFI requires the cmd_status to be set to 0xFF before posting.1041 */1042int1043wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,1044	int seconds)1045{1046	int i;1047	struct megasas_header *frame_hdr = &cmd->frame->hdr;1048	u32 status_reg;1049 1050	u32 msecs = seconds * 1000;1051 1052	/*1053	 * Wait for cmd_status to change1054	 */1055	for (i = 0; (i < msecs) && (frame_hdr->cmd_status == 0xff); i += 20) {1056		rmb();1057		msleep(20);1058		if (!(i % 5000)) {1059			status_reg = instance->instancet->read_fw_status_reg(instance)1060					& MFI_STATE_MASK;1061			if (status_reg == MFI_STATE_FAULT)1062				break;1063		}1064	}1065 1066	if (frame_hdr->cmd_status == MFI_STAT_INVALID_STATUS)1067		return DCMD_TIMEOUT;1068	else if (frame_hdr->cmd_status == MFI_STAT_OK)1069		return DCMD_SUCCESS;1070	else1071		return DCMD_FAILED;1072}1073 1074/**1075 * megasas_ioc_init_fusion -	Initializes the FW1076 * @instance:		Adapter soft state1077 *1078 * Issues the IOC Init cmd1079 */1080int1081megasas_ioc_init_fusion(struct megasas_instance *instance)1082{1083	struct megasas_init_frame *init_frame;1084	struct MPI2_IOC_INIT_REQUEST *IOCInitMessage = NULL;1085	dma_addr_t	ioc_init_handle;1086	struct megasas_cmd *cmd;1087	u8 ret, cur_rdpq_mode;1088	struct fusion_context *fusion;1089	union MEGASAS_REQUEST_DESCRIPTOR_UNION req_desc;1090	int i;1091	struct megasas_header *frame_hdr;1092	const char *sys_info;1093	MFI_CAPABILITIES *drv_ops;1094	u32 scratch_pad_1;1095	ktime_t time;1096	bool cur_fw_64bit_dma_capable;1097	bool cur_intr_coalescing;1098 1099	fusion = instance->ctrl_context;1100 1101	ioc_init_handle = fusion->ioc_init_request_phys;1102	IOCInitMessage = fusion->ioc_init_request;1103 1104	cmd = fusion->ioc_init_cmd;1105 1106	scratch_pad_1 = megasas_readl1107		(instance, &instance->reg_set->outbound_scratch_pad_1);1108 1109	cur_rdpq_mode = (scratch_pad_1 & MR_RDPQ_MODE_OFFSET) ? 1 : 0;1110 1111	if (instance->adapter_type == INVADER_SERIES) {1112		cur_fw_64bit_dma_capable =1113			(scratch_pad_1 & MR_CAN_HANDLE_64_BIT_DMA_OFFSET) ? true : false;1114 1115		if (instance->consistent_mask_64bit && !cur_fw_64bit_dma_capable) {1116			dev_err(&instance->pdev->dev, "Driver was operating on 64bit "1117				"DMA mask, but upcoming FW does not support 64bit DMA mask\n");1118			megaraid_sas_kill_hba(instance);1119			ret = 1;1120			goto fail_fw_init;1121		}1122	}1123 1124	if (instance->is_rdpq && !cur_rdpq_mode) {1125		dev_err(&instance->pdev->dev, "Firmware downgrade *NOT SUPPORTED*"1126			" from RDPQ mode to non RDPQ mode\n");1127		ret = 1;1128		goto fail_fw_init;1129	}1130 1131	cur_intr_coalescing = (scratch_pad_1 & MR_INTR_COALESCING_SUPPORT_OFFSET) ?1132							true : false;1133 1134	if ((instance->low_latency_index_start ==1135		MR_HIGH_IOPS_QUEUE_COUNT) && cur_intr_coalescing)1136		instance->perf_mode = MR_BALANCED_PERF_MODE;1137 1138	dev_info(&instance->pdev->dev, "Performance mode :%s (latency index = %d)\n",1139		MEGASAS_PERF_MODE_2STR(instance->perf_mode),1140		instance->low_latency_index_start);1141 1142	instance->fw_sync_cache_support = (scratch_pad_1 &1143		MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;1144	dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n",1145		 instance->fw_sync_cache_support ? "Yes" : "No");1146 1147	memset(IOCInitMessage, 0, sizeof(struct MPI2_IOC_INIT_REQUEST));1148 1149	IOCInitMessage->Function = MPI2_FUNCTION_IOC_INIT;1150	IOCInitMessage->WhoInit	= MPI2_WHOINIT_HOST_DRIVER;1151	IOCInitMessage->MsgVersion = cpu_to_le16(MPI2_VERSION);1152	IOCInitMessage->HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);1153	IOCInitMessage->SystemRequestFrameSize = cpu_to_le16(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE / 4);1154 1155	IOCInitMessage->ReplyDescriptorPostQueueDepth = cpu_to_le16(fusion->reply_q_depth);1156	IOCInitMessage->ReplyDescriptorPostQueueAddress = instance->is_rdpq ?1157			cpu_to_le64(fusion->rdpq_phys) :1158			cpu_to_le64(fusion->reply_frames_desc_phys[0]);1159	IOCInitMessage->MsgFlags = instance->is_rdpq ?1160			MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE : 0;1161	IOCInitMessage->SystemRequestFrameBaseAddress = cpu_to_le64(fusion->io_request_frames_phys);1162	IOCInitMessage->SenseBufferAddressHigh = cpu_to_le32(upper_32_bits(fusion->sense_phys_addr));1163	IOCInitMessage->HostMSIxVectors = instance->msix_vectors + instance->iopoll_q_count;1164	IOCInitMessage->HostPageSize = MR_DEFAULT_NVME_PAGE_SHIFT;1165 1166	time = ktime_get_real();1167	/* Convert to milliseconds as per FW requirement */1168	IOCInitMessage->TimeStamp = cpu_to_le64(ktime_to_ms(time));1169 1170	init_frame = (struct megasas_init_frame *)cmd->frame;1171	memset(init_frame, 0, IOC_INIT_FRAME_SIZE);1172 1173	frame_hdr = &cmd->frame->hdr;1174	frame_hdr->cmd_status = 0xFF;1175	frame_hdr->flags |= cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE);1176 1177	init_frame->cmd	= MFI_CMD_INIT;1178	init_frame->cmd_status = 0xFF;1179 1180	drv_ops = (MFI_CAPABILITIES *) &(init_frame->driver_operations);1181 1182	/* driver support Extended MSIX */1183	if (instance->adapter_type >= INVADER_SERIES)1184		drv_ops->mfi_capabilities.support_additional_msix = 1;1185	/* driver supports HA / Remote LUN over Fast Path interface */1186	drv_ops->mfi_capabilities.support_fp_remote_lun = 1;1187 1188	drv_ops->mfi_capabilities.support_max_255lds = 1;1189	drv_ops->mfi_capabilities.support_ndrive_r1_lb = 1;1190	drv_ops->mfi_capabilities.security_protocol_cmds_fw = 1;1191 1192	if (instance->max_chain_frame_sz > MEGASAS_CHAIN_FRAME_SZ_MIN)1193		drv_ops->mfi_capabilities.support_ext_io_size = 1;1194 1195	drv_ops->mfi_capabilities.support_fp_rlbypass = 1;1196	if (!dual_qdepth_disable)1197		drv_ops->mfi_capabilities.support_ext_queue_depth = 1;1198 1199	drv_ops->mfi_capabilities.support_qd_throttling = 1;1200	drv_ops->mfi_capabilities.support_pd_map_target_id = 1;1201	drv_ops->mfi_capabilities.support_nvme_passthru = 1;1202	drv_ops->mfi_capabilities.support_fw_exposed_dev_list = 1;1203 1204	if (reset_devices)1205		drv_ops->mfi_capabilities.support_memdump = 1;1206 1207	if (instance->consistent_mask_64bit)1208		drv_ops->mfi_capabilities.support_64bit_mode = 1;1209 1210	/* Convert capability to LE32 */1211	cpu_to_le32s((u32 *)&init_frame->driver_operations.mfi_capabilities);1212 1213	sys_info = dmi_get_system_info(DMI_PRODUCT_UUID);1214	if (instance->system_info_buf && sys_info) {1215		memcpy(instance->system_info_buf->systemId, sys_info,1216			strlen(sys_info) > 64 ? 64 : strlen(sys_info));1217		instance->system_info_buf->systemIdLength =1218			strlen(sys_info) > 64 ? 64 : strlen(sys_info);1219		init_frame->system_info_lo = cpu_to_le32(lower_32_bits(instance->system_info_h));1220		init_frame->system_info_hi = cpu_to_le32(upper_32_bits(instance->system_info_h));1221	}1222 1223	init_frame->queue_info_new_phys_addr_hi =1224		cpu_to_le32(upper_32_bits(ioc_init_handle));1225	init_frame->queue_info_new_phys_addr_lo =1226		cpu_to_le32(lower_32_bits(ioc_init_handle));1227	init_frame->data_xfer_len = cpu_to_le32(sizeof(struct MPI2_IOC_INIT_REQUEST));1228 1229	/*1230	 * Each bit in replyqueue_mask represents one group of MSI-x vectors1231	 * (each group has 8 vectors)1232	 */1233	switch (instance->perf_mode) {1234	case MR_BALANCED_PERF_MODE:1235		init_frame->replyqueue_mask =1236		       cpu_to_le16(~(~0 << instance->low_latency_index_start/8));1237		break;1238	case MR_IOPS_PERF_MODE:1239		init_frame->replyqueue_mask =1240		       cpu_to_le16(~(~0 << instance->msix_vectors/8));1241		break;1242	}1243 1244 1245	req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr));1246	req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr));1247	req_desc.MFAIo.RequestFlags =1248		(MEGASAS_REQ_DESCRIPT_FLAGS_MFA <<1249		MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);1250 1251	/*1252	 * disable the intr before firing the init frame1253	 */1254	instance->instancet->disable_intr(instance);1255 1256	for (i = 0; i < (10 * 1000); i += 20) {1257		if (megasas_readl(instance, &instance->reg_set->doorbell) & 1)1258			msleep(20);1259		else1260			break;1261	}1262 1263	/* For AERO also, IOC_INIT requires 64 bit descriptor write */1264	megasas_write_64bit_req_desc(instance, &req_desc);1265 1266	wait_and_poll(instance, cmd, MFI_IO_TIMEOUT_SECS);1267 1268	frame_hdr = &cmd->frame->hdr;1269	if (frame_hdr->cmd_status != 0) {1270		ret = 1;1271		goto fail_fw_init;1272	}1273 1274	if (instance->adapter_type >= AERO_SERIES) {1275		scratch_pad_1 = megasas_readl1276			(instance, &instance->reg_set->outbound_scratch_pad_1);1277 1278		instance->atomic_desc_support =1279			(scratch_pad_1 & MR_ATOMIC_DESCRIPTOR_SUPPORT_OFFSET) ? 1 : 0;1280 1281		dev_info(&instance->pdev->dev, "FW supports atomic descriptor\t: %s\n",1282			instance->atomic_desc_support ? "Yes" : "No");1283	}1284 1285	return 0;1286 1287fail_fw_init:1288	dev_err(&instance->pdev->dev,1289		"Init cmd return status FAILED for SCSI host %d\n",1290		instance->host->host_no);1291 1292	return ret;1293}1294 1295/**1296 * megasas_sync_pd_seq_num -	JBOD SEQ MAP1297 * @instance:		Adapter soft state1298 * @pend:		set to 1, if it is pended jbod map.1299 *1300 * Issue Jbod map to the firmware. If it is pended command,1301 * issue command and return. If it is first instance of jbod map1302 * issue and receive command.1303 */1304int1305megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) {1306	int ret = 0;1307	size_t pd_seq_map_sz;1308	struct megasas_cmd *cmd;1309	struct megasas_dcmd_frame *dcmd;1310	struct fusion_context *fusion = instance->ctrl_context;1311	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;1312	dma_addr_t pd_seq_h;1313 1314	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)];1315	pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)];1316	pd_seq_map_sz = struct_size(pd_sync, seq, MAX_PHYSICAL_DEVICES);1317 1318	cmd = megasas_get_cmd(instance);1319	if (!cmd) {1320		dev_err(&instance->pdev->dev,1321			"Could not get mfi cmd. Fail from %s %d\n",1322			__func__, __LINE__);1323		return -ENOMEM;1324	}1325 1326	dcmd = &cmd->frame->dcmd;1327 1328	memset(pd_sync, 0, pd_seq_map_sz);1329	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);1330 1331	if (pend) {1332		dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG;1333		dcmd->flags = MFI_FRAME_DIR_WRITE;1334		instance->jbod_seq_cmd = cmd;1335	} else {1336		dcmd->flags = MFI_FRAME_DIR_READ;1337	}1338 1339	dcmd->cmd = MFI_CMD_DCMD;1340	dcmd->cmd_status = 0xFF;1341	dcmd->sge_count = 1;1342	dcmd->timeout = 0;1343	dcmd->pad_0 = 0;1344	dcmd->data_xfer_len = cpu_to_le32(pd_seq_map_sz);1345	dcmd->opcode = cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO);1346 1347	megasas_set_dma_settings(instance, dcmd, pd_seq_h, pd_seq_map_sz);1348 1349	if (pend) {1350		instance->instancet->issue_dcmd(instance, cmd);1351		return 0;1352	}1353 1354	/* Below code is only for non pended DCMD */1355	if (!instance->mask_interrupts)1356		ret = megasas_issue_blocked_cmd(instance, cmd,1357			MFI_IO_TIMEOUT_SECS);1358	else1359		ret = megasas_issue_polled(instance, cmd);1360 1361	if (le32_to_cpu(pd_sync->count) > MAX_PHYSICAL_DEVICES) {1362		dev_warn(&instance->pdev->dev,1363			"driver supports max %d JBOD, but FW reports %d\n",1364			MAX_PHYSICAL_DEVICES, le32_to_cpu(pd_sync->count));1365		ret = -EINVAL;1366	}1367 1368	if (ret == DCMD_TIMEOUT)1369		dev_warn(&instance->pdev->dev,1370			 "%s DCMD timed out, continue without JBOD sequence map\n",1371			 __func__);1372 1373	if (ret == DCMD_SUCCESS)1374		instance->pd_seq_map_id++;1375 1376	megasas_return_cmd(instance, cmd);1377	return ret;1378}1379 1380/*1381 * megasas_get_ld_map_info -	Returns FW's ld_map structure1382 * @instance:				Adapter soft state1383 * @pend:				Pend the command or not1384 * Issues an internal command (DCMD) to get the FW's controller PD1385 * list structure.  This information is mainly used to find out SYSTEM1386 * supported by the FW.1387 * dcmd.mbox value setting for MR_DCMD_LD_MAP_GET_INFO1388 * dcmd.mbox.b[0]	- number of LDs being sync'd1389 * dcmd.mbox.b[1]	- 0 - complete command immediately.1390 *			- 1 - pend till config change1391 * dcmd.mbox.b[2]	- 0 - supports max 64 lds and uses legacy MR_FW_RAID_MAP1392 *			- 1 - supports max MAX_LOGICAL_DRIVES_EXT lds and1393 *				uses extended struct MR_FW_RAID_MAP_EXT1394 */1395static int1396megasas_get_ld_map_info(struct megasas_instance *instance)1397{1398	int ret = 0;1399	struct megasas_cmd *cmd;1400	struct megasas_dcmd_frame *dcmd;1401	void *ci;1402	dma_addr_t ci_h = 0;1403	u32 size_map_info;1404	struct fusion_context *fusion;1405 1406	cmd = megasas_get_cmd(instance);1407 1408	if (!cmd) {1409		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for map info\n");1410		return -ENOMEM;1411	}1412 1413	fusion = instance->ctrl_context;1414 1415	if (!fusion) {1416		megasas_return_cmd(instance, cmd);1417		return -ENXIO;1418	}1419 1420	dcmd = &cmd->frame->dcmd;1421 1422	size_map_info = fusion->current_map_sz;1423 1424	ci = (void *) fusion->ld_map[(instance->map_id & 1)];1425	ci_h = fusion->ld_map_phys[(instance->map_id & 1)];1426 1427	if (!ci) {1428		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to alloc mem for ld_map_info\n");1429		megasas_return_cmd(instance, cmd);1430		return -ENOMEM;1431	}1432 1433	memset(ci, 0, fusion->max_map_sz);1434	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);1435	dcmd->cmd = MFI_CMD_DCMD;1436	dcmd->cmd_status = 0xFF;1437	dcmd->sge_count = 1;1438	dcmd->flags = MFI_FRAME_DIR_READ;1439	dcmd->timeout = 0;1440	dcmd->pad_0 = 0;1441	dcmd->data_xfer_len = cpu_to_le32(size_map_info);1442	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);1443 1444	megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);1445 1446	if (!instance->mask_interrupts)1447		ret = megasas_issue_blocked_cmd(instance, cmd,1448			MFI_IO_TIMEOUT_SECS);1449	else1450		ret = megasas_issue_polled(instance, cmd);1451 1452	if (ret == DCMD_TIMEOUT)1453		dev_warn(&instance->pdev->dev,1454			 "%s DCMD timed out, RAID map is disabled\n",1455			 __func__);1456 1457	megasas_return_cmd(instance, cmd);1458 1459	return ret;1460}1461 1462u81463megasas_get_map_info(struct megasas_instance *instance)1464{1465	struct fusion_context *fusion = instance->ctrl_context;1466 1467	fusion->fast_path_io = 0;1468	if (!megasas_get_ld_map_info(instance)) {1469		if (MR_ValidateMapInfo(instance, instance->map_id)) {1470			fusion->fast_path_io = 1;1471			return 0;1472		}1473	}1474	return 1;1475}1476 1477/*1478 * megasas_sync_map_info -	Returns FW's ld_map structure1479 * @instance:				Adapter soft state1480 *1481 * Issues an internal command (DCMD) to get the FW's controller PD1482 * list structure.  This information is mainly used to find out SYSTEM1483 * supported by the FW.1484 */1485int1486megasas_sync_map_info(struct megasas_instance *instance)1487{1488	int i;1489	struct megasas_cmd *cmd;1490	struct megasas_dcmd_frame *dcmd;1491	u16 num_lds;1492	struct fusion_context *fusion;1493	struct MR_LD_TARGET_SYNC *ci = NULL;1494	struct MR_DRV_RAID_MAP_ALL *map;1495	struct MR_LD_RAID  *raid;1496	struct MR_LD_TARGET_SYNC *ld_sync;1497	dma_addr_t ci_h = 0;1498	u32 size_map_info;1499 1500	cmd = megasas_get_cmd(instance);1501 1502	if (!cmd) {1503		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for sync info\n");1504		return -ENOMEM;1505	}1506 1507	fusion = instance->ctrl_context;1508 1509	if (!fusion) {1510		megasas_return_cmd(instance, cmd);1511		return 1;1512	}1513 1514	map = fusion->ld_drv_map[instance->map_id & 1];1515 1516	num_lds = le16_to_cpu(map->raidMap.ldCount);1517 1518	dcmd = &cmd->frame->dcmd;1519 1520	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);1521 1522	ci = (struct MR_LD_TARGET_SYNC *)1523	  fusion->ld_map[(instance->map_id - 1) & 1];1524	memset(ci, 0, fusion->max_map_sz);1525 1526	ci_h = fusion->ld_map_phys[(instance->map_id - 1) & 1];1527 1528	ld_sync = (struct MR_LD_TARGET_SYNC *)ci;1529 1530	for (i = 0; i < num_lds; i++, ld_sync++) {1531		raid = MR_LdRaidGet(i, map);1532		ld_sync->targetId = MR_GetLDTgtId(i, map);1533		ld_sync->seqNum = raid->seqNum;1534	}1535 1536	size_map_info = fusion->current_map_sz;1537 1538	dcmd->cmd = MFI_CMD_DCMD;1539	dcmd->cmd_status = 0xFF;1540	dcmd->sge_count = 1;1541	dcmd->flags = MFI_FRAME_DIR_WRITE;1542	dcmd->timeout = 0;1543	dcmd->pad_0 = 0;1544	dcmd->data_xfer_len = cpu_to_le32(size_map_info);1545	dcmd->mbox.b[0] = num_lds;1546	dcmd->mbox.b[1] = MEGASAS_DCMD_MBOX_PEND_FLAG;1547	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);1548 1549	megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);1550 1551	instance->map_update_cmd = cmd;1552 1553	instance->instancet->issue_dcmd(instance, cmd);1554 1555	return 0;1556}1557 1558/*1559 * meagasas_display_intel_branding - Display branding string1560 * @instance: per adapter object1561 *1562 * Return nothing.1563 */1564static void1565megasas_display_intel_branding(struct megasas_instance *instance)1566{1567	if (instance->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)1568		return;1569 1570	switch (instance->pdev->device) {1571	case PCI_DEVICE_ID_LSI_INVADER:1572		switch (instance->pdev->subsystem_device) {1573		case MEGARAID_INTEL_RS3DC080_SSDID:1574			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",1575				instance->host->host_no,1576				MEGARAID_INTEL_RS3DC080_BRANDING);1577			break;1578		case MEGARAID_INTEL_RS3DC040_SSDID:1579			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",1580				instance->host->host_no,1581				MEGARAID_INTEL_RS3DC040_BRANDING);1582			break;1583		case MEGARAID_INTEL_RS3SC008_SSDID:1584			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",1585				instance->host->host_no,1586				MEGARAID_INTEL_RS3SC008_BRANDING);1587			break;1588		case MEGARAID_INTEL_RS3MC044_SSDID:1589			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",1590				instance->host->host_no,1591				MEGARAID_INTEL_RS3MC044_BRANDING);1592			break;1593		default:1594			break;1595		}1596		break;1597	case PCI_DEVICE_ID_LSI_FURY:1598		switch (instance->pdev->subsystem_device) {1599		case MEGARAID_INTEL_RS3WC080_SSDID:1600			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",1601				instance->host->host_no,1602				MEGARAID_INTEL_RS3WC080_BRANDING);1603			break;1604		case MEGARAID_INTEL_RS3WC040_SSDID:1605			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",1606				instance->host->host_no,1607				MEGARAID_INTEL_RS3WC040_BRANDING);1608			break;1609		default:1610			break;1611		}1612		break;1613	case PCI_DEVICE_ID_LSI_CUTLASS_52:1614	case PCI_DEVICE_ID_LSI_CUTLASS_53:1615		switch (instance->pdev->subsystem_device) {1616		case MEGARAID_INTEL_RMS3BC160_SSDID:1617			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",1618				instance->host->host_no,1619				MEGARAID_INTEL_RMS3BC160_BRANDING);1620			break;1621		default:1622			break;1623		}1624		break;1625	default:1626		break;1627	}1628}1629 1630/**1631 * megasas_allocate_raid_maps -	Allocate memory for RAID maps1632 * @instance:				Adapter soft state1633 *1634 * return:				if success: return 01635 *					failed:  return -ENOMEM1636 */1637static inline int megasas_allocate_raid_maps(struct megasas_instance *instance)1638{1639	struct fusion_context *fusion;1640	int i = 0;1641 1642	fusion = instance->ctrl_context;1643 1644	fusion->drv_map_pages = get_order(fusion->drv_map_sz);1645 1646	for (i = 0; i < 2; i++) {1647		fusion->ld_map[i] = NULL;1648 1649		fusion->ld_drv_map[i] = (void *)1650			__get_free_pages(__GFP_ZERO | GFP_KERNEL,1651					 fusion->drv_map_pages);1652 1653		if (!fusion->ld_drv_map[i]) {1654			fusion->ld_drv_map[i] = vzalloc(fusion->drv_map_sz);1655 1656			if (!fusion->ld_drv_map[i]) {1657				dev_err(&instance->pdev->dev,1658					"Could not allocate memory for local map"1659					" size requested: %d\n",1660					fusion->drv_map_sz);1661				goto ld_drv_map_alloc_fail;1662			}1663		}1664	}1665 1666	for (i = 0; i < 2; i++) {1667		fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev,1668						       fusion->max_map_sz,1669						       &fusion->ld_map_phys[i],1670						       GFP_KERNEL);1671		if (!fusion->ld_map[i]) {1672			dev_err(&instance->pdev->dev,1673				"Could not allocate memory for map info %s:%d\n",1674				__func__, __LINE__);1675			goto ld_map_alloc_fail;1676		}1677	}1678 1679	return 0;1680 1681ld_map_alloc_fail:1682	for (i = 0; i < 2; i++) {1683		if (fusion->ld_map[i])1684			dma_free_coherent(&instance->pdev->dev,1685					  fusion->max_map_sz,1686					  fusion->ld_map[i],1687					  fusion->ld_map_phys[i]);1688	}1689 1690ld_drv_map_alloc_fail:1691	for (i = 0; i < 2; i++) {1692		if (fusion->ld_drv_map[i]) {1693			if (is_vmalloc_addr(fusion->ld_drv_map[i]))1694				vfree(fusion->ld_drv_map[i]);1695			else1696				free_pages((ulong)fusion->ld_drv_map[i],1697					   fusion->drv_map_pages);1698		}1699	}1700 1701	return -ENOMEM;1702}1703 1704/**1705 * megasas_configure_queue_sizes -	Calculate size of request desc queue,1706 *					reply desc queue,1707 *					IO request frame queue, set can_queue.1708 * @instance:				Adapter soft state1709 * @return:				void1710 */1711static inline1712void megasas_configure_queue_sizes(struct megasas_instance *instance)1713{1714	struct fusion_context *fusion;1715	u16 max_cmd;1716 1717	fusion = instance->ctrl_context;1718	max_cmd = instance->max_fw_cmds;1719 1720	if (instance->adapter_type >= VENTURA_SERIES)1721		instance->max_mpt_cmds = instance->max_fw_cmds * RAID_1_PEER_CMDS;1722	else1723		instance->max_mpt_cmds = instance->max_fw_cmds;1724 1725	instance->max_scsi_cmds = instance->max_fw_cmds - instance->max_mfi_cmds;1726	instance->cur_can_queue = instance->max_scsi_cmds;1727	instance->host->can_queue = instance->cur_can_queue;1728 1729	fusion->reply_q_depth = 2 * ((max_cmd + 1 + 15) / 16) * 16;1730 1731	fusion->request_alloc_sz = sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) *1732					  instance->max_mpt_cmds;1733	fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION) *1734					(fusion->reply_q_depth);1735	fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE +1736		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE1737		 * (instance->max_mpt_cmds + 1)); /* Extra 1 for SMID 0 */1738}1739 1740static int megasas_alloc_ioc_init_frame(struct megasas_instance *instance)1741{1742	struct fusion_context *fusion;1743	struct megasas_cmd *cmd;1744 1745	fusion = instance->ctrl_context;1746 1747	cmd = kzalloc(sizeof(struct megasas_cmd), GFP_KERNEL);1748 1749	if (!cmd) {1750		dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",1751			__func__, __LINE__);1752		return -ENOMEM;1753	}1754 1755	cmd->frame = dma_alloc_coherent(&instance->pdev->dev,1756					IOC_INIT_FRAME_SIZE,1757					&cmd->frame_phys_addr, GFP_KERNEL);1758 1759	if (!cmd->frame) {1760		dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",1761			__func__, __LINE__);1762		kfree(cmd);1763		return -ENOMEM;1764	}1765 1766	fusion->ioc_init_cmd = cmd;1767	return 0;1768}1769 1770/**1771 * megasas_free_ioc_init_cmd -	Free IOC INIT command frame1772 * @instance:		Adapter soft state1773 */1774static inline void megasas_free_ioc_init_cmd(struct megasas_instance *instance)1775{1776	struct fusion_context *fusion;1777 1778	fusion = instance->ctrl_context;1779 1780	if (fusion->ioc_init_cmd && fusion->ioc_init_cmd->frame)1781		dma_free_coherent(&instance->pdev->dev,1782				  IOC_INIT_FRAME_SIZE,1783				  fusion->ioc_init_cmd->frame,1784				  fusion->ioc_init_cmd->frame_phys_addr);1785 1786	kfree(fusion->ioc_init_cmd);1787}1788 1789/**1790 * megasas_init_adapter_fusion -	Initializes the FW1791 * @instance:		Adapter soft state1792 *1793 * This is the main function for initializing firmware.1794 */1795static u321796megasas_init_adapter_fusion(struct megasas_instance *instance)1797{1798	struct fusion_context *fusion;1799	u32 scratch_pad_1;1800	int i = 0, count;1801	u32 status_reg;1802 1803	fusion = instance->ctrl_context;1804 1805	megasas_fusion_update_can_queue(instance, PROBE_CONTEXT);1806 1807	/*1808	 * Only Driver's internal DCMDs and IOCTL DCMDs needs to have MFI frames1809	 */1810	instance->max_mfi_cmds =1811		MEGASAS_FUSION_INTERNAL_CMDS + MEGASAS_FUSION_IOCTL_CMDS;1812 1813	megasas_configure_queue_sizes(instance);1814 1815	scratch_pad_1 = megasas_readl(instance,1816				      &instance->reg_set->outbound_scratch_pad_1);1817	/* If scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK is set,1818	 * Firmware support extended IO chain frame which is 4 times more than1819	 * legacy Firmware.1820	 * Legacy Firmware - Frame size is (8 * 128) = 1K1821	 * 1M IO Firmware  - Frame size is (8 * 128 * 4)  = 4K1822	 */1823	if (scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK)1824		instance->max_chain_frame_sz =1825			((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>1826			MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_1MB_IO;1827	else1828		instance->max_chain_frame_sz =1829			((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>1830			MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_256K_IO;1831 1832	if (instance->max_chain_frame_sz < MEGASAS_CHAIN_FRAME_SZ_MIN) {1833		dev_warn(&instance->pdev->dev, "frame size %d invalid, fall back to legacy max frame size %d\n",1834			instance->max_chain_frame_sz,1835			MEGASAS_CHAIN_FRAME_SZ_MIN);1836		instance->max_chain_frame_sz = MEGASAS_CHAIN_FRAME_SZ_MIN;1837	}1838 1839	fusion->max_sge_in_main_msg =1840		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE1841			- offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL))/16;1842 1843	fusion->max_sge_in_chain =1844		instance->max_chain_frame_sz1845			/ sizeof(union MPI2_SGE_IO_UNION);1846 1847	instance->max_num_sge =1848		rounddown_pow_of_two(fusion->max_sge_in_main_msg1849			+ fusion->max_sge_in_chain - 2);1850 1851	/* Used for pass thru MFI frame (DCMD) */1852	fusion->chain_offset_mfi_pthru =1853		offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL)/16;1854 1855	fusion->chain_offset_io_request =1856		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -1857		 sizeof(union MPI2_SGE_IO_UNION))/16;1858 1859	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;1860	count += instance->iopoll_q_count;1861 1862	for (i = 0 ; i < count; i++)1863		fusion->last_reply_idx[i] = 0;1864 1865	/*1866	 * For fusion adapters, 3 commands for IOCTL and 8 commands1867	 * for driver's internal DCMDs.1868	 */1869	instance->max_scsi_cmds = instance->max_fw_cmds -1870				(MEGASAS_FUSION_INTERNAL_CMDS +1871				MEGASAS_FUSION_IOCTL_CMDS);1872	sema_init(&instance->ioctl_sem, MEGASAS_FUSION_IOCTL_CMDS);1873 1874	for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++)1875		atomic_set(&fusion->busy_mq_poll[i], 0);1876 1877	if (megasas_alloc_ioc_init_frame(instance))1878		return 1;1879 1880	/*1881	 * Allocate memory for descriptors1882	 * Create a pool of commands1883	 */1884	if (megasas_alloc_cmds(instance))1885		goto fail_alloc_mfi_cmds;1886	if (megasas_alloc_cmds_fusion(instance))1887		goto fail_alloc_cmds;1888 1889	if (megasas_ioc_init_fusion(instance)) {1890		status_reg = instance->instancet->read_fw_status_reg(instance);1891		if (((status_reg & MFI_STATE_MASK) == MFI_STATE_FAULT) &&1892		    (status_reg & MFI_RESET_ADAPTER)) {1893			/* Do a chip reset and then retry IOC INIT once */1894			if (megasas_adp_reset_wait_for_ready1895				(instance, true, 0) == FAILED)1896				goto fail_ioc_init;1897 1898			if (megasas_ioc_init_fusion(instance))1899				goto fail_ioc_init;1900		} else {1901			goto fail_ioc_init;1902		}1903	}1904 1905	megasas_display_intel_branding(instance);1906	if (megasas_get_ctrl_info(instance)) {1907		dev_err(&instance->pdev->dev,1908			"Could not get controller info. Fail from %s %d\n",1909			__func__, __LINE__);1910		goto fail_ioc_init;1911	}1912 1913	instance->flag_ieee = 1;1914	instance->r1_ldio_hint_default =  MR_R1_LDIO_PIGGYBACK_DEFAULT;1915	instance->threshold_reply_count = instance->max_fw_cmds / 4;1916	fusion->fast_path_io = 0;1917 1918	if (megasas_allocate_raid_maps(instance))1919		goto fail_ioc_init;1920 1921	if (!megasas_get_map_info(instance))1922		megasas_sync_map_info(instance);1923 1924	return 0;1925 1926fail_ioc_init:1927	megasas_free_cmds_fusion(instance);1928fail_alloc_cmds:1929	megasas_free_cmds(instance);1930fail_alloc_mfi_cmds:1931	megasas_free_ioc_init_cmd(instance);1932	return 1;1933}1934 1935/**1936 * megasas_fault_detect_work	-	Worker function of1937 *					FW fault handling workqueue.1938 * @work:	FW fault work struct1939 */1940static void1941megasas_fault_detect_work(struct work_struct *work)1942{1943	struct megasas_instance *instance =1944		container_of(work, struct megasas_instance,1945			     fw_fault_work.work);1946	u32 fw_state, dma_state, status;1947 1948	/* Check the fw state */1949	fw_state = instance->instancet->read_fw_status_reg(instance) &1950			MFI_STATE_MASK;1951 1952	if (fw_state == MFI_STATE_FAULT) {1953		dma_state = instance->instancet->read_fw_status_reg(instance) &1954				MFI_STATE_DMADONE;1955		/* Start collecting crash, if DMA bit is done */1956		if (instance->crash_dump_drv_support &&1957		    instance->crash_dump_app_support && dma_state) {1958			megasas_fusion_crash_dump(instance);1959		} else {1960			if (instance->unload == 0) {1961				status = megasas_reset_fusion(instance->host, 0);1962				if (status != SUCCESS) {1963					dev_err(&instance->pdev->dev,1964						"Failed from %s %d, do not re-arm timer\n",1965						__func__, __LINE__);1966					return;1967				}1968			}1969		}1970	}1971 1972	if (instance->fw_fault_work_q)1973		queue_delayed_work(instance->fw_fault_work_q,1974			&instance->fw_fault_work,1975			msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));1976}1977 1978int1979megasas_fusion_start_watchdog(struct megasas_instance *instance)1980{1981	/* Check if the Fault WQ is already started */1982	if (instance->fw_fault_work_q)1983		return SUCCESS;1984 1985	INIT_DELAYED_WORK(&instance->fw_fault_work, megasas_fault_detect_work);1986 1987	snprintf(instance->fault_handler_work_q_name,1988		 sizeof(instance->fault_handler_work_q_name),1989		 "poll_megasas%d_status", instance->host->host_no);1990 1991	instance->fw_fault_work_q = alloc_ordered_workqueue(1992		"%s", WQ_MEM_RECLAIM, instance->fault_handler_work_q_name);1993	if (!instance->fw_fault_work_q) {1994		dev_err(&instance->pdev->dev, "Failed from %s %d\n",1995			__func__, __LINE__);1996		return FAILED;1997	}1998 1999	queue_delayed_work(instance->fw_fault_work_q,2000			   &instance->fw_fault_work,2001			   msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));2002 2003	return SUCCESS;2004}2005 2006void2007megasas_fusion_stop_watchdog(struct megasas_instance *instance)2008{2009	struct workqueue_struct *wq;2010 2011	if (instance->fw_fault_work_q) {2012		wq = instance->fw_fault_work_q;2013		instance->fw_fault_work_q = NULL;2014		if (!cancel_delayed_work_sync(&instance->fw_fault_work))2015			flush_workqueue(wq);2016		destroy_workqueue(wq);2017	}2018}2019 2020/**2021 * map_cmd_status -	Maps FW cmd status to OS cmd status2022 * @fusion:		fusion context2023 * @scmd:		Pointer to cmd2024 * @status:		status of cmd returned by FW2025 * @ext_status:		ext status of cmd returned by FW2026 * @data_length:	command data length2027 * @sense:		command sense data2028 */2029static void2030map_cmd_status(struct fusion_context *fusion,2031		struct scsi_cmnd *scmd, u8 status, u8 ext_status,2032		u32 data_length, u8 *sense)2033{2034	u8 cmd_type;2035	int resid;2036 2037	cmd_type = megasas_cmd_type(scmd);2038	switch (status) {2039 2040	case MFI_STAT_OK:2041		scmd->result = DID_OK << 16;2042		break;2043 2044	case MFI_STAT_SCSI_IO_FAILED:2045	case MFI_STAT_LD_INIT_IN_PROGRESS:2046		scmd->result = (DID_ERROR << 16) | ext_status;2047		break;2048 2049	case MFI_STAT_SCSI_DONE_WITH_ERROR:2050 2051		scmd->result = (DID_OK << 16) | ext_status;2052		if (ext_status == SAM_STAT_CHECK_CONDITION) {2053			memcpy(scmd->sense_buffer, sense,2054			       SCSI_SENSE_BUFFERSIZE);2055		}2056 2057		/*2058		 * If the  IO request is partially completed, then MR FW will2059		 * update "io_request->DataLength" field with actual number of2060		 * bytes transferred.Driver will set residual bytes count in2061		 * SCSI command structure.2062		 */2063		resid = (scsi_bufflen(scmd) - data_length);2064		scsi_set_resid(scmd, resid);2065 2066		if (resid &&2067			((cmd_type == READ_WRITE_LDIO) ||2068			(cmd_type == READ_WRITE_SYSPDIO)))2069			scmd_printk(KERN_INFO, scmd, "BRCM Debug mfi stat 0x%x, data len"2070				" requested/completed 0x%x/0x%x\n",2071				status, scsi_bufflen(scmd), data_length);2072		break;2073 2074	case MFI_STAT_LD_OFFLINE:2075	case MFI_STAT_DEVICE_NOT_FOUND:2076		scmd->result = DID_BAD_TARGET << 16;2077		break;2078	case MFI_STAT_CONFIG_SEQ_MISMATCH:2079		scmd->result = DID_IMM_RETRY << 16;2080		break;2081	default:2082		scmd->result = DID_ERROR << 16;2083		break;2084	}2085}2086 2087/**2088 * megasas_is_prp_possible -2089 * Checks if native NVMe PRPs can be built for the IO2090 *2091 * @instance:		Adapter soft state2092 * @scmd:		SCSI command from the mid-layer2093 * @sge_count:		scatter gather element count.2094 *2095 * Returns:		true: PRPs can be built2096 *			false: IEEE SGLs needs to be built2097 */2098static bool2099megasas_is_prp_possible(struct megasas_instance *instance,2100			struct scsi_cmnd *scmd, int sge_count)2101{2102	u32 data_length = 0;2103	struct scatterlist *sg_scmd;2104	bool build_prp = false;2105	u32 mr_nvme_pg_size;2106 2107	mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,2108				MR_DEFAULT_NVME_PAGE_SIZE);2109	data_length = scsi_bufflen(scmd);2110	sg_scmd = scsi_sglist(scmd);2111 2112	/*2113	 * NVMe uses one PRP for each page (or part of a page)2114	 * look at the data length - if 4 pages or less then IEEE is OK2115	 * if  > 5 pages then we need to build a native SGL2116	 * if > 4 and <= 5 pages, then check physical address of 1st SG entry2117	 * if this first size in the page is >= the residual beyond 4 pages2118	 * then use IEEE, otherwise use native SGL2119	 */2120 2121	if (data_length > (mr_nvme_pg_size * 5)) {2122		build_prp = true;2123	} else if ((data_length > (mr_nvme_pg_size * 4)) &&2124			(data_length <= (mr_nvme_pg_size * 5)))  {2125		/* check if 1st SG entry size is < residual beyond 4 pages */2126		if (sg_dma_len(sg_scmd) < (data_length - (mr_nvme_pg_size * 4)))2127			build_prp = true;2128	}2129 2130	return build_prp;2131}2132 2133/**2134 * megasas_make_prp_nvme -2135 * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only2136 *2137 * @instance:		Adapter soft state2138 * @scmd:		SCSI command from the mid-layer2139 * @sgl_ptr:		SGL to be filled in2140 * @cmd:		Fusion command frame2141 * @sge_count:		scatter gather element count.2142 *2143 * Returns:		true: PRPs are built2144 *			false: IEEE SGLs needs to be built2145 */2146static bool2147megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd,2148		      struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,2149		      struct megasas_cmd_fusion *cmd, int sge_count)2150{2151	int sge_len, offset, num_prp_in_chain = 0;2152	struct MPI25_IEEE_SGE_CHAIN64 *main_chain_element, *ptr_first_sgl;2153	u64 *ptr_sgl;2154	dma_addr_t ptr_sgl_phys;2155	u64 sge_addr;2156	u32 page_mask, page_mask_result;2157	struct scatterlist *sg_scmd;2158	u32 first_prp_len;2159	bool build_prp = false;2160	int data_len = scsi_bufflen(scmd);2161	u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,2162					MR_DEFAULT_NVME_PAGE_SIZE);2163 2164	build_prp = megasas_is_prp_possible(instance, scmd, sge_count);2165 2166	if (!build_prp)2167		return false;2168 2169	/*2170	 * Nvme has a very convoluted prp format.  One prp is required2171	 * for each page or partial page. Driver need to split up OS sg_list2172	 * entries if it is longer than one page or cross a page2173	 * boundary.  Driver also have to insert a PRP list pointer entry as2174	 * the last entry in each physical page of the PRP list.2175	 *2176	 * NOTE: The first PRP "entry" is actually placed in the first2177	 * SGL entry in the main message as IEEE 64 format.  The 2nd2178	 * entry in the main message is the chain element, and the rest2179	 * of the PRP entries are built in the contiguous pcie buffer.2180	 */2181	page_mask = mr_nvme_pg_size - 1;2182	ptr_sgl = (u64 *)cmd->sg_frame;2183	ptr_sgl_phys = cmd->sg_frame_phys_addr;2184	memset(ptr_sgl, 0, instance->max_chain_frame_sz);2185 2186	/* Build chain frame element which holds all prps except first*/2187	main_chain_element = (struct MPI25_IEEE_SGE_CHAIN64 *)2188	    ((u8 *)sgl_ptr + sizeof(struct MPI25_IEEE_SGE_CHAIN64));2189 2190	main_chain_element->Address = cpu_to_le64(ptr_sgl_phys);2191	main_chain_element->NextChainOffset = 0;2192	main_chain_element->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |2193					IEEE_SGE_FLAGS_SYSTEM_ADDR |2194					MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;2195 2196	/* Build first prp, sge need not to be page aligned*/2197	ptr_first_sgl = sgl_ptr;2198	sg_scmd = scsi_sglist(scmd);2199	sge_addr = sg_dma_address(sg_scmd);2200	sge_len = sg_dma_len(sg_scmd);2201 2202	offset = (u32)(sge_addr & page_mask);2203	first_prp_len = mr_nvme_pg_size - offset;2204 2205	ptr_first_sgl->Address = cpu_to_le64(sge_addr);2206	ptr_first_sgl->Length = cpu_to_le32(first_prp_len);2207 2208	data_len -= first_prp_len;2209 2210	if (sge_len > first_prp_len) {2211		sge_addr += first_prp_len;2212		sge_len -= first_prp_len;2213	} else if (sge_len == first_prp_len) {2214		sg_scmd = sg_next(sg_scmd);2215		sge_addr = sg_dma_address(sg_scmd);2216		sge_len = sg_dma_len(sg_scmd);2217	}2218 2219	for (;;) {2220		offset = (u32)(sge_addr & page_mask);2221 2222		/* Put PRP pointer due to page boundary*/2223		page_mask_result = (uintptr_t)(ptr_sgl + 1) & page_mask;2224		if (unlikely(!page_mask_result)) {2225			scmd_printk(KERN_NOTICE,2226				    scmd, "page boundary ptr_sgl: 0x%p\n",2227				    ptr_sgl);2228			ptr_sgl_phys += 8;2229			*ptr_sgl = cpu_to_le64(ptr_sgl_phys);2230			ptr_sgl++;2231			num_prp_in_chain++;2232		}2233 2234		*ptr_sgl = cpu_to_le64(sge_addr);2235		ptr_sgl++;2236		ptr_sgl_phys += 8;2237		num_prp_in_chain++;2238 2239		sge_addr += mr_nvme_pg_size;2240		sge_len -= mr_nvme_pg_size;2241		data_len -= mr_nvme_pg_size;2242 2243		if (data_len <= 0)2244			break;2245 2246		if (sge_len > 0)2247			continue;2248 2249		sg_scmd = sg_next(sg_scmd);2250		sge_addr = sg_dma_address(sg_scmd);2251		sge_len = sg_dma_len(sg_scmd);2252	}2253 2254	main_chain_element->Length =2255			cpu_to_le32(num_prp_in_chain * sizeof(u64));2256 2257	return build_prp;2258}2259 2260/**2261 * megasas_make_sgl_fusion -	Prepares 32-bit SGL2262 * @instance:		Adapter soft state2263 * @scp:		SCSI command from the mid-layer2264 * @sgl_ptr:		SGL to be filled in2265 * @cmd:		cmd we are working on2266 * @sge_count:		sge count2267 *2268 */2269static void2270megasas_make_sgl_fusion(struct megasas_instance *instance,2271			struct scsi_cmnd *scp,2272			struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,2273			struct megasas_cmd_fusion *cmd, int sge_count)2274{2275	int i, sg_processed;2276	struct scatterlist *os_sgl;2277	struct fusion_context *fusion;2278 2279	fusion = instance->ctrl_context;2280 2281	if (instance->adapter_type >= INVADER_SERIES) {2282		struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = sgl_ptr;2283		sgl_ptr_end += fusion->max_sge_in_main_msg - 1;2284		sgl_ptr_end->Flags = 0;2285	}2286 2287	scsi_for_each_sg(scp, os_sgl, sge_count, i) {2288		sgl_ptr->Length = cpu_to_le32(sg_dma_len(os_sgl));2289		sgl_ptr->Address = cpu_to_le64(sg_dma_address(os_sgl));2290		sgl_ptr->Flags = 0;2291		if (instance->adapter_type >= INVADER_SERIES)2292			if (i == sge_count - 1)2293				sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST;2294		sgl_ptr++;2295		sg_processed = i + 1;2296 2297		if ((sg_processed ==  (fusion->max_sge_in_main_msg - 1)) &&2298		    (sge_count > fusion->max_sge_in_main_msg)) {2299 2300			struct MPI25_IEEE_SGE_CHAIN64 *sg_chain;2301			if (instance->adapter_type >= INVADER_SERIES) {2302				if ((le16_to_cpu(cmd->io_request->IoFlags) &2303					MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) !=2304					MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)2305					cmd->io_request->ChainOffset =2306						fusion->2307						chain_offset_io_request;2308				else2309					cmd->io_request->ChainOffset = 0;2310			} else2311				cmd->io_request->ChainOffset =2312					fusion->chain_offset_io_request;2313 2314			sg_chain = sgl_ptr;2315			/* Prepare chain element */2316			sg_chain->NextChainOffset = 0;2317			if (instance->adapter_type >= INVADER_SERIES)2318				sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT;2319			else2320				sg_chain->Flags =2321					(IEEE_SGE_FLAGS_CHAIN_ELEMENT |2322					 MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR);2323			sg_chain->Length =  cpu_to_le32((sizeof(union MPI2_SGE_IO_UNION) * (sge_count - sg_processed)));2324			sg_chain->Address = cpu_to_le64(cmd->sg_frame_phys_addr);2325 2326			sgl_ptr =2327			  (struct MPI25_IEEE_SGE_CHAIN64 *)cmd->sg_frame;2328			memset(sgl_ptr, 0, instance->max_chain_frame_sz);2329		}2330	}2331}2332 2333/**2334 * megasas_make_sgl -	Build Scatter Gather List(SGLs)2335 * @scp:		SCSI command pointer2336 * @instance:		Soft instance of controller2337 * @cmd:		Fusion command pointer2338 *2339 * This function will build sgls based on device type.2340 * For nvme drives, there is different way of building sgls in nvme native2341 * format- PRPs(Physical Region Page).2342 *2343 * Returns the number of sg lists actually used, zero if the sg lists2344 * is NULL, or -ENOMEM if the mapping failed2345 */2346static2347int megasas_make_sgl(struct megasas_instance *instance, struct scsi_cmnd *scp,2348		     struct megasas_cmd_fusion *cmd)2349{2350	int sge_count;2351	bool build_prp = false;2352	struct MPI25_IEEE_SGE_CHAIN64 *sgl_chain64;2353 2354	sge_count = scsi_dma_map(scp);2355 2356	if ((sge_count > instance->max_num_sge) || (sge_count <= 0))2357		return sge_count;2358 2359	sgl_chain64 = (struct MPI25_IEEE_SGE_CHAIN64 *)&cmd->io_request->SGL;2360	if ((le16_to_cpu(cmd->io_request->IoFlags) &2361	    MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&2362	    (cmd->pd_interface == NVME_PD))2363		build_prp = megasas_make_prp_nvme(instance, scp, sgl_chain64,2364						  cmd, sge_count);2365 2366	if (!build_prp)2367		megasas_make_sgl_fusion(instance, scp, sgl_chain64,2368					cmd, sge_count);2369 2370	return sge_count;2371}2372 2373/**2374 * megasas_set_pd_lba -	Sets PD LBA2375 * @io_request:		IO request2376 * @cdb_len:		cdb length2377 * @io_info:		IO information2378 * @scp:		SCSI command2379 * @local_map_ptr:	Raid map2380 * @ref_tag:		Primary reference tag2381 *2382 * Used to set the PD LBA in CDB for FP IOs2383 */2384static void2385megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len,2386		   struct IO_REQUEST_INFO *io_info, struct scsi_cmnd *scp,2387		   struct MR_DRV_RAID_MAP_ALL *local_map_ptr, u32 ref_tag)2388{2389	struct MR_LD_RAID *raid;2390	u16 ld;2391	u64 start_blk = io_info->pdBlock;2392	u8 *cdb = io_request->CDB.CDB32;2393	u32 num_blocks = io_info->numBlocks;2394	u8 opcode = 0, flagvals = 0, groupnum = 0, control = 0;2395 2396	/* Check if T10 PI (DIF) is enabled for this LD */2397	ld = MR_TargetIdToLdGet(io_info->ldTgtId, local_map_ptr);2398	raid = MR_LdRaidGet(ld, local_map_ptr);2399	if (raid->capability.ldPiMode == MR_PROT_INFO_TYPE_CONTROLLER) {2400		memset(cdb, 0, sizeof(io_request->CDB.CDB32));2401		cdb[0] =  MEGASAS_SCSI_VARIABLE_LENGTH_CMD;2402		cdb[7] =  MEGASAS_SCSI_ADDL_CDB_LEN;2403 2404		if (scp->sc_data_direction == DMA_FROM_DEVICE)2405			cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_READ32;2406		else2407			cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_WRITE32;2408		cdb[10] = MEGASAS_RD_WR_PROTECT_CHECK_ALL;2409 2410		/* LBA */2411		cdb[12] = (u8)((start_blk >> 56) & 0xff);2412		cdb[13] = (u8)((start_blk >> 48) & 0xff);2413		cdb[14] = (u8)((start_blk >> 40) & 0xff);2414		cdb[15] = (u8)((start_blk >> 32) & 0xff);2415		cdb[16] = (u8)((start_blk >> 24) & 0xff);2416		cdb[17] = (u8)((start_blk >> 16) & 0xff);2417		cdb[18] = (u8)((start_blk >> 8) & 0xff);2418		cdb[19] = (u8)(start_blk & 0xff);2419 2420		/* Logical block reference tag */2421		io_request->CDB.EEDP32.PrimaryReferenceTag =2422			cpu_to_be32(ref_tag);2423		io_request->CDB.EEDP32.PrimaryApplicationTagMask = cpu_to_be16(0xffff);2424		io_request->IoFlags = cpu_to_le16(32); /* Specify 32-byte cdb */2425 2426		/* Transfer length */2427		cdb[28] = (u8)((num_blocks >> 24) & 0xff);2428		cdb[29] = (u8)((num_blocks >> 16) & 0xff);2429		cdb[30] = (u8)((num_blocks >> 8) & 0xff);2430		cdb[31] = (u8)(num_blocks & 0xff);2431 2432		/* set SCSI IO EEDPFlags */2433		if (scp->sc_data_direction == DMA_FROM_DEVICE) {2434			io_request->EEDPFlags = cpu_to_le16(2435				MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG  |2436				MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |2437				MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP |2438				MPI2_SCSIIO_EEDPFLAGS_CHECK_APPTAG |2439				MPI25_SCSIIO_EEDPFLAGS_DO_NOT_DISABLE_MODE |2440				MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);2441		} else {2442			io_request->EEDPFlags = cpu_to_le16(2443				MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |2444				MPI2_SCSIIO_EEDPFLAGS_INSERT_OP);2445		}2446		io_request->Control |= cpu_to_le32((0x4 << 26));2447		io_request->EEDPBlockSize = cpu_to_le32(scp->device->sector_size);2448	} else {2449		/* Some drives don't support 16/12 byte CDB's, convert to 10 */2450		if (((cdb_len == 12) || (cdb_len == 16)) &&2451		    (start_blk <= 0xffffffff)) {2452			if (cdb_len == 16) {2453				opcode = cdb[0] == READ_16 ? READ_10 : WRITE_10;2454				flagvals = cdb[1];2455				groupnum = cdb[14];2456				control = cdb[15];2457			} else {2458				opcode = cdb[0] == READ_12 ? READ_10 : WRITE_10;2459				flagvals = cdb[1];2460				groupnum = cdb[10];2461				control = cdb[11];2462			}2463 2464			memset(cdb, 0, sizeof(io_request->CDB.CDB32));2465 2466			cdb[0] = opcode;2467			cdb[1] = flagvals;2468			cdb[6] = groupnum;2469			cdb[9] = control;2470 2471			/* Transfer length */2472			cdb[8] = (u8)(num_blocks & 0xff);2473			cdb[7] = (u8)((num_blocks >> 8) & 0xff);2474 2475			io_request->IoFlags = cpu_to_le16(10); /* Specify 10-byte cdb */2476			cdb_len = 10;2477		} else if ((cdb_len < 16) && (start_blk > 0xffffffff)) {2478			/* Convert to 16 byte CDB for large LBA's */2479			switch (cdb_len) {2480			case 6:2481				opcode = cdb[0] == READ_6 ? READ_16 : WRITE_16;2482				control = cdb[5];2483				break;2484			case 10:2485				opcode =2486					cdb[0] == READ_10 ? READ_16 : WRITE_16;2487				flagvals = cdb[1];2488				groupnum = cdb[6];2489				control = cdb[9];2490				break;2491			case 12:2492				opcode =2493					cdb[0] == READ_12 ? READ_16 : WRITE_16;2494				flagvals = cdb[1];2495				groupnum = cdb[10];2496				control = cdb[11];2497				break;2498			}2499 2500			memset(cdb, 0, sizeof(io_request->CDB.CDB32));2501 2502			cdb[0] = opcode;2503			cdb[1] = flagvals;2504			cdb[14] = groupnum;2505			cdb[15] = control;2506 2507			/* Transfer length */2508			cdb[13] = (u8)(num_blocks & 0xff);2509			cdb[12] = (u8)((num_blocks >> 8) & 0xff);2510			cdb[11] = (u8)((num_blocks >> 16) & 0xff);2511			cdb[10] = (u8)((num_blocks >> 24) & 0xff);2512 2513			io_request->IoFlags = cpu_to_le16(16); /* Specify 16-byte cdb */2514			cdb_len = 16;2515		}2516 2517		/* Normal case, just load LBA here */2518		switch (cdb_len) {2519		case 6:2520		{2521			u8 val = cdb[1] & 0xE0;2522			cdb[3] = (u8)(start_blk & 0xff);2523			cdb[2] = (u8)((start_blk >> 8) & 0xff);2524			cdb[1] = val | ((u8)(start_blk >> 16) & 0x1f);2525			break;2526		}2527		case 10:2528			cdb[5] = (u8)(start_blk & 0xff);2529			cdb[4] = (u8)((start_blk >> 8) & 0xff);2530			cdb[3] = (u8)((start_blk >> 16) & 0xff);2531			cdb[2] = (u8)((start_blk >> 24) & 0xff);2532			break;2533		case 12:2534			cdb[5]    = (u8)(start_blk & 0xff);2535			cdb[4]    = (u8)((start_blk >> 8) & 0xff);2536			cdb[3]    = (u8)((start_blk >> 16) & 0xff);2537			cdb[2]    = (u8)((start_blk >> 24) & 0xff);2538			break;2539		case 16:2540			cdb[9]    = (u8)(start_blk & 0xff);2541			cdb[8]    = (u8)((start_blk >> 8) & 0xff);2542			cdb[7]    = (u8)((start_blk >> 16) & 0xff);2543			cdb[6]    = (u8)((start_blk >> 24) & 0xff);2544			cdb[5]    = (u8)((start_blk >> 32) & 0xff);2545			cdb[4]    = (u8)((start_blk >> 40) & 0xff);2546			cdb[3]    = (u8)((start_blk >> 48) & 0xff);2547			cdb[2]    = (u8)((start_blk >> 56) & 0xff);2548			break;2549		}2550	}2551}2552 2553/**2554 * megasas_stream_detect -	stream detection on read and and write IOs2555 * @instance:		Adapter soft state2556 * @cmd:		    Command to be prepared2557 * @io_info:		IO Request info2558 *2559 */2560 2561/** stream detection on read and and write IOs */2562static void megasas_stream_detect(struct megasas_instance *instance,2563				  struct megasas_cmd_fusion *cmd,2564				  struct IO_REQUEST_INFO *io_info)2565{2566	struct fusion_context *fusion = instance->ctrl_context;2567	u32 device_id = io_info->ldTgtId;2568	struct LD_STREAM_DETECT *current_ld_sd2569		= fusion->stream_detect_by_ld[device_id];2570	u32 *track_stream = &current_ld_sd->mru_bit_map, stream_num;2571	u32 shifted_values, unshifted_values;2572	u32 index_value_mask, shifted_values_mask;2573	int i;2574	bool is_read_ahead = false;2575	struct STREAM_DETECT *current_sd;2576	/* find possible stream */2577	for (i = 0; i < MAX_STREAMS_TRACKED; ++i) {2578		stream_num = (*track_stream >>2579			(i * BITS_PER_INDEX_STREAM)) &2580			STREAM_MASK;2581		current_sd = &current_ld_sd->stream_track[stream_num];2582		/* if we found a stream, update the raid2583		 *  context and also update the mruBitMap2584		 */2585		/*	boundary condition */2586		if ((current_sd->next_seq_lba) &&2587		    (io_info->ldStartBlock >= current_sd->next_seq_lba) &&2588		    (io_info->ldStartBlock <= (current_sd->next_seq_lba + 32)) &&2589		    (current_sd->is_read == io_info->isRead)) {2590 2591			if ((io_info->ldStartBlock != current_sd->next_seq_lba)	&&2592			    ((!io_info->isRead) || (!is_read_ahead)))2593				/*2594				 * Once the API is available we need to change this.2595				 * At this point we are not allowing any gap2596				 */2597				continue;2598 2599			SET_STREAM_DETECTED(cmd->io_request->RaidContext.raid_context_g35);2600			current_sd->next_seq_lba =2601			io_info->ldStartBlock + io_info->numBlocks;2602			/*2603			 *	update the mruBitMap LRU2604			 */2605			shifted_values_mask =2606				(1 <<  i * BITS_PER_INDEX_STREAM) - 1;2607			shifted_values = ((*track_stream & shifted_values_mask)2608						<< BITS_PER_INDEX_STREAM);2609			index_value_mask =2610				STREAM_MASK << i * BITS_PER_INDEX_STREAM;2611			unshifted_values =2612				*track_stream & ~(shifted_values_mask |2613				index_value_mask);2614			*track_stream =2615				unshifted_values | shifted_values | stream_num;2616			return;2617		}2618	}2619	/*2620	 * if we did not find any stream, create a new one2621	 * from the least recently used2622	 */2623	stream_num = (*track_stream >>2624		((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) &2625		STREAM_MASK;2626	current_sd = &current_ld_sd->stream_track[stream_num];2627	current_sd->is_read = io_info->isRead;2628	current_sd->next_seq_lba = io_info->ldStartBlock + io_info->numBlocks;2629	*track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | stream_num);2630	return;2631}2632 2633/**2634 * megasas_set_raidflag_cpu_affinity - This function sets the cpu2635 * affinity (cpu of the controller) and raid_flags in the raid context2636 * based on IO type.2637 *2638 * @fusion:		Fusion context2639 * @praid_context:	IO RAID context2640 * @raid:		LD raid map2641 * @fp_possible:	Is fast path possible?2642 * @is_read:		Is read IO?2643 * @scsi_buff_len:	SCSI command buffer length2644 *2645 */2646static void2647megasas_set_raidflag_cpu_affinity(struct fusion_context *fusion,2648				union RAID_CONTEXT_UNION *praid_context,2649				struct MR_LD_RAID *raid, bool fp_possible,2650				u8 is_read, u32 scsi_buff_len)2651{2652	u8 cpu_sel = MR_RAID_CTX_CPUSEL_0;2653	struct RAID_CONTEXT_G35 *rctx_g35;2654 2655	rctx_g35 = &praid_context->raid_context_g35;2656	if (fp_possible) {2657		if (is_read) {2658			if ((raid->cpuAffinity.pdRead.cpu0) &&2659			    (raid->cpuAffinity.pdRead.cpu1))2660				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;2661			else if (raid->cpuAffinity.pdRead.cpu1)2662				cpu_sel = MR_RAID_CTX_CPUSEL_1;2663		} else {2664			if ((raid->cpuAffinity.pdWrite.cpu0) &&2665			    (raid->cpuAffinity.pdWrite.cpu1))2666				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;2667			else if (raid->cpuAffinity.pdWrite.cpu1)2668				cpu_sel = MR_RAID_CTX_CPUSEL_1;2669			/* Fast path cache by pass capable R0/R1 VD */2670			if ((raid->level <= 1) &&2671			    (raid->capability.fp_cache_bypass_capable)) {2672				rctx_g35->routing_flags |=2673					(1 << MR_RAID_CTX_ROUTINGFLAGS_SLD_SHIFT);2674				rctx_g35->raid_flags =2675					(MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS2676					<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);2677			}2678		}2679	} else {2680		if (is_read) {2681			if ((raid->cpuAffinity.ldRead.cpu0) &&2682			    (raid->cpuAffinity.ldRead.cpu1))2683				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;2684			else if (raid->cpuAffinity.ldRead.cpu1)2685				cpu_sel = MR_RAID_CTX_CPUSEL_1;2686		} else {2687			if ((raid->cpuAffinity.ldWrite.cpu0) &&2688			    (raid->cpuAffinity.ldWrite.cpu1))2689				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;2690			else if (raid->cpuAffinity.ldWrite.cpu1)2691				cpu_sel = MR_RAID_CTX_CPUSEL_1;2692 2693			if (is_stream_detected(rctx_g35) &&2694			    ((raid->level == 5) || (raid->level == 6)) &&2695			    (raid->writeMode == MR_RL_WRITE_THROUGH_MODE) &&2696			    (cpu_sel == MR_RAID_CTX_CPUSEL_FCFS))2697				cpu_sel = MR_RAID_CTX_CPUSEL_0;2698		}2699	}2700 2701	rctx_g35->routing_flags |=2702		(cpu_sel << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);2703 2704	/* Always give priority to MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT2705	 * vs MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS.2706	 * IO Subtype is not bitmap.2707	 */2708	if ((fusion->pcie_bw_limitation) && (raid->level == 1) && (!is_read) &&2709			(scsi_buff_len > MR_LARGE_IO_MIN_SIZE)) {2710		praid_context->raid_context_g35.raid_flags =2711			(MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT2712			<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);2713	}2714}2715 2716/**2717 * megasas_build_ldio_fusion -	Prepares IOs to devices2718 * @instance:		Adapter soft state2719 * @scp:		SCSI command2720 * @cmd:		Command to be prepared2721 *2722 * Prepares the io_request and chain elements (sg_frame) for IO2723 * The IO can be for PD (Fast Path) or LD2724 */2725static void2726megasas_build_ldio_fusion(struct megasas_instance *instance,2727			  struct scsi_cmnd *scp,2728			  struct megasas_cmd_fusion *cmd)2729{2730	bool fp_possible;2731	u16 ld;2732	u32 start_lba_lo, start_lba_hi, device_id, datalength = 0;2733	u32 scsi_buff_len;2734	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;2735	struct IO_REQUEST_INFO io_info;2736	struct fusion_context *fusion;2737	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;2738	u8 *raidLUN;2739	unsigned long spinlock_flags;2740	struct MR_LD_RAID *raid = NULL;2741	struct MR_PRIV_DEVICE *mrdev_priv;2742	struct RAID_CONTEXT *rctx;2743	struct RAID_CONTEXT_G35 *rctx_g35;2744 2745	device_id = MEGASAS_DEV_INDEX(scp);2746 2747	fusion = instance->ctrl_context;2748 2749	io_request = cmd->io_request;2750	rctx = &io_request->RaidContext.raid_context;2751	rctx_g35 = &io_request->RaidContext.raid_context_g35;2752 2753	rctx->virtual_disk_tgt_id = cpu_to_le16(device_id);2754	rctx->status = 0;2755	rctx->ex_status = 0;2756 2757	start_lba_lo = 0;2758	start_lba_hi = 0;2759	fp_possible = false;2760 2761	/*2762	 * 6-byte READ(0x08) or WRITE(0x0A) cdb2763	 */2764	if (scp->cmd_len == 6) {2765		datalength = (u32) scp->cmnd[4];2766		start_lba_lo = ((u32) scp->cmnd[1] << 16) |2767			((u32) scp->cmnd[2] << 8) | (u32) scp->cmnd[3];2768 2769		start_lba_lo &= 0x1FFFFF;2770	}2771 2772	/*2773	 * 10-byte READ(0x28) or WRITE(0x2A) cdb2774	 */2775	else if (scp->cmd_len == 10) {2776		datalength = (u32) scp->cmnd[8] |2777			((u32) scp->cmnd[7] << 8);2778		start_lba_lo = ((u32) scp->cmnd[2] << 24) |2779			((u32) scp->cmnd[3] << 16) |2780			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];2781	}2782 2783	/*2784	 * 12-byte READ(0xA8) or WRITE(0xAA) cdb2785	 */2786	else if (scp->cmd_len == 12) {2787		datalength = ((u32) scp->cmnd[6] << 24) |2788			((u32) scp->cmnd[7] << 16) |2789			((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];2790		start_lba_lo = ((u32) scp->cmnd[2] << 24) |2791			((u32) scp->cmnd[3] << 16) |2792			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];2793	}2794 2795	/*2796	 * 16-byte READ(0x88) or WRITE(0x8A) cdb2797	 */2798	else if (scp->cmd_len == 16) {2799		datalength = ((u32) scp->cmnd[10] << 24) |2800			((u32) scp->cmnd[11] << 16) |2801			((u32) scp->cmnd[12] << 8) | (u32) scp->cmnd[13];2802		start_lba_lo = ((u32) scp->cmnd[6] << 24) |2803			((u32) scp->cmnd[7] << 16) |2804			((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];2805 2806		start_lba_hi = ((u32) scp->cmnd[2] << 24) |2807			((u32) scp->cmnd[3] << 16) |2808			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];2809	}2810 2811	memset(&io_info, 0, sizeof(struct IO_REQUEST_INFO));2812	io_info.ldStartBlock = ((u64)start_lba_hi << 32) | start_lba_lo;2813	io_info.numBlocks = datalength;2814	io_info.ldTgtId = device_id;2815	io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID;2816	scsi_buff_len = scsi_bufflen(scp);2817	io_request->DataLength = cpu_to_le32(scsi_buff_len);2818	io_info.data_arms = 1;2819 2820	if (scp->sc_data_direction == DMA_FROM_DEVICE)2821		io_info.isRead = 1;2822 2823	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];2824	ld = MR_TargetIdToLdGet(device_id, local_map_ptr);2825 2826	if (ld < instance->fw_supported_vd_count)2827		raid = MR_LdRaidGet(ld, local_map_ptr);2828 2829	if (!raid || (!fusion->fast_path_io)) {2830		rctx->reg_lock_flags  = 0;2831		fp_possible = false;2832	} else {2833		if (MR_BuildRaidContext(instance, &io_info, rctx,2834					local_map_ptr, &raidLUN))2835			fp_possible = (io_info.fpOkForIo > 0) ? true : false;2836	}2837 2838	megasas_get_msix_index(instance, scp, cmd, io_info.data_arms);2839 2840	if (instance->adapter_type >= VENTURA_SERIES) {2841		/* FP for Optimal raid level 1.2842		 * All large RAID-1 writes (> 32 KiB, both WT and WB modes)2843		 * are built by the driver as LD I/Os.2844		 * All small RAID-1 WT writes (<= 32 KiB) are built as FP I/Os2845		 * (there is never a reason to process these as buffered writes)2846		 * All small RAID-1 WB writes (<= 32 KiB) are built as FP I/Os2847		 * with the SLD bit asserted.2848		 */2849		if (io_info.r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {2850			mrdev_priv = scp->device->hostdata;2851 2852			if (atomic_inc_return(&instance->fw_outstanding) >2853				(instance->host->can_queue)) {2854				fp_possible = false;2855				atomic_dec(&instance->fw_outstanding);2856			} else if (fusion->pcie_bw_limitation &&2857				((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) ||2858				   (atomic_dec_if_positive(&mrdev_priv->r1_ldio_hint) > 0))) {2859				fp_possible = false;2860				atomic_dec(&instance->fw_outstanding);2861				if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)2862					atomic_set(&mrdev_priv->r1_ldio_hint,2863						   instance->r1_ldio_hint_default);2864			}2865		}2866 2867		if (!fp_possible ||2868		    (io_info.isRead && io_info.ra_capable)) {2869			spin_lock_irqsave(&instance->stream_lock,2870					  spinlock_flags);2871			megasas_stream_detect(instance, cmd, &io_info);2872			spin_unlock_irqrestore(&instance->stream_lock,2873					       spinlock_flags);2874			/* In ventura if stream detected for a read and it is2875			 * read ahead capable make this IO as LDIO2876			 */2877			if (is_stream_detected(rctx_g35))2878				fp_possible = false;2879		}2880 2881		/* If raid is NULL, set CPU affinity to default CPU0 */2882		if (raid)2883			megasas_set_raidflag_cpu_affinity(fusion, &io_request->RaidContext,2884				raid, fp_possible, io_info.isRead,2885				scsi_buff_len);2886		else2887			rctx_g35->routing_flags |=2888				(MR_RAID_CTX_CPUSEL_0 << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);2889	}2890 2891	if (fp_possible) {2892		megasas_set_pd_lba(io_request, scp->cmd_len, &io_info, scp,2893				   local_map_ptr, start_lba_lo);2894		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;2895		cmd->request_desc->SCSIIO.RequestFlags =2896			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO2897			 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);2898		if (instance->adapter_type == INVADER_SERIES) {2899			rctx->type = MPI2_TYPE_CUDA;2900			rctx->nseg = 0x1;2901			io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);2902			rctx->reg_lock_flags |=2903			  (MR_RL_FLAGS_GRANT_DESTINATION_CUDA |2904			   MR_RL_FLAGS_SEQ_NUM_ENABLE);2905		} else if (instance->adapter_type >= VENTURA_SERIES) {2906			rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);2907			rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);2908			rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);2909			io_request->IoFlags |=2910				cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);2911		}2912		if (fusion->load_balance_info &&2913			(fusion->load_balance_info[device_id].loadBalanceFlag) &&2914			(io_info.isRead)) {2915			io_info.devHandle =2916				get_updated_dev_handle(instance,2917					&fusion->load_balance_info[device_id],2918					&io_info, local_map_ptr);2919			megasas_priv(scp)->status |= MEGASAS_LOAD_BALANCE_FLAG;2920			cmd->pd_r1_lb = io_info.pd_after_lb;2921			if (instance->adapter_type >= VENTURA_SERIES)2922				rctx_g35->span_arm = io_info.span_arm;2923			else2924				rctx->span_arm = io_info.span_arm;2925 2926		} else2927			megasas_priv(scp)->status &= ~MEGASAS_LOAD_BALANCE_FLAG;2928 2929		if (instance->adapter_type >= VENTURA_SERIES)2930			cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle;2931		else2932			cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;2933 2934		if ((raidLUN[0] == 1) &&2935			(local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].validHandles > 1)) {2936			instance->dev_handle = !(instance->dev_handle);2937			io_info.devHandle =2938				local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].devHandle[instance->dev_handle];2939		}2940 2941		cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;2942		io_request->DevHandle = io_info.devHandle;2943		cmd->pd_interface = io_info.pd_interface;2944		/* populate the LUN field */2945		memcpy(io_request->LUN, raidLUN, 8);2946	} else {2947		rctx->timeout_value =2948			cpu_to_le16(local_map_ptr->raidMap.fpPdIoTimeoutSec);2949		cmd->request_desc->SCSIIO.RequestFlags =2950			(MEGASAS_REQ_DESCRIPT_FLAGS_LD_IO2951			 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);2952		if (instance->adapter_type == INVADER_SERIES) {2953			if (io_info.do_fp_rlbypass ||2954			(rctx->reg_lock_flags == REGION_TYPE_UNUSED))2955				cmd->request_desc->SCSIIO.RequestFlags =2956					(MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<2957					MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);2958			rctx->type = MPI2_TYPE_CUDA;2959			rctx->reg_lock_flags |=2960				(MR_RL_FLAGS_GRANT_DESTINATION_CPU0 |2961					MR_RL_FLAGS_SEQ_NUM_ENABLE);2962			rctx->nseg = 0x1;2963		} else if (instance->adapter_type >= VENTURA_SERIES) {2964			rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);2965			rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);2966			rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);2967		}2968		io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;2969		io_request->DevHandle = cpu_to_le16(device_id);2970 2971	} /* Not FP */2972}2973 2974/**2975 * megasas_build_ld_nonrw_fusion - prepares non rw ios for virtual disk2976 * @instance:		Adapter soft state2977 * @scmd:		SCSI command2978 * @cmd:		Command to be prepared2979 *2980 * Prepares the io_request frame for non-rw io cmds for vd.2981 */2982static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance,2983			  struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd)2984{2985	u32 device_id;2986	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;2987	u16 ld;2988	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;2989	struct fusion_context *fusion = instance->ctrl_context;2990	u8                          span, physArm;2991	__le16                      devHandle;2992	u32                         arRef, pd;2993	struct MR_LD_RAID                  *raid;2994	struct RAID_CONTEXT                *pRAID_Context;2995	u8 fp_possible = 1;2996 2997	io_request = cmd->io_request;2998	device_id = MEGASAS_DEV_INDEX(scmd);2999	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];3000	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));3001	/* get RAID_Context pointer */3002	pRAID_Context = &io_request->RaidContext.raid_context;3003	/* Check with FW team */3004	pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);3005	pRAID_Context->reg_lock_row_lba    = 0;3006	pRAID_Context->reg_lock_length    = 0;3007 3008	if (fusion->fast_path_io && (3009		device_id < instance->fw_supported_vd_count)) {3010 3011		ld = MR_TargetIdToLdGet(device_id, local_map_ptr);3012		if (ld >= instance->fw_supported_vd_count - 1)3013			fp_possible = 0;3014		else {3015			raid = MR_LdRaidGet(ld, local_map_ptr);3016			if (!(raid->capability.fpNonRWCapable))3017				fp_possible = 0;3018		}3019	} else3020		fp_possible = 0;3021 3022	if (!fp_possible) {3023		io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;3024		io_request->DevHandle = cpu_to_le16(device_id);3025		io_request->LUN[1] = scmd->device->lun;3026		pRAID_Context->timeout_value =3027			cpu_to_le16(scsi_cmd_to_rq(scmd)->timeout / HZ);3028		cmd->request_desc->SCSIIO.RequestFlags =3029			(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<3030			MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);3031	} else {3032 3033		/* set RAID context values */3034		pRAID_Context->config_seq_num = raid->seqNum;3035		if (instance->adapter_type < VENTURA_SERIES)3036			pRAID_Context->reg_lock_flags = REGION_TYPE_SHARED_READ;3037		pRAID_Context->timeout_value =3038			cpu_to_le16(raid->fpIoTimeoutForLd);3039 3040		/* get the DevHandle for the PD (since this is3041		   fpNonRWCapable, this is a single disk RAID0) */3042		span = physArm = 0;3043		arRef = MR_LdSpanArrayGet(ld, span, local_map_ptr);3044		pd = MR_ArPdGet(arRef, physArm, local_map_ptr);3045		devHandle = MR_PdDevHandleGet(pd, local_map_ptr);3046 3047		/* build request descriptor */3048		cmd->request_desc->SCSIIO.RequestFlags =3049			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<3050			MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);3051		cmd->request_desc->SCSIIO.DevHandle = devHandle;3052 3053		/* populate the LUN field */3054		memcpy(io_request->LUN, raid->LUN, 8);3055 3056		/* build the raidScsiIO structure */3057		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;3058		io_request->DevHandle = devHandle;3059	}3060}3061 3062/**3063 * megasas_build_syspd_fusion - prepares rw/non-rw ios for syspd3064 * @instance:		Adapter soft state3065 * @scmd:		SCSI command3066 * @cmd:		Command to be prepared3067 * @fp_possible:	parameter to detect fast path or firmware path io.3068 *3069 * Prepares the io_request frame for rw/non-rw io cmds for syspds3070 */3071static void3072megasas_build_syspd_fusion(struct megasas_instance *instance,3073	struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd,3074	bool fp_possible)3075{3076	u32 device_id;3077	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;3078	u16 pd_index = 0;3079	u16 os_timeout_value;3080	u16 timeout_limit;3081	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;3082	struct RAID_CONTEXT	*pRAID_Context;3083	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;3084	struct MR_PRIV_DEVICE *mr_device_priv_data;3085	struct fusion_context *fusion = instance->ctrl_context;3086	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id - 1) & 1];3087 3088	device_id = MEGASAS_DEV_INDEX(scmd);3089	pd_index = MEGASAS_PD_INDEX(scmd);3090	os_timeout_value = scsi_cmd_to_rq(scmd)->timeout / HZ;3091	mr_device_priv_data = scmd->device->hostdata;3092	cmd->pd_interface = mr_device_priv_data->interface_type;3093 3094	io_request = cmd->io_request;3095	/* get RAID_Context pointer */3096	pRAID_Context = &io_request->RaidContext.raid_context;3097	pRAID_Context->reg_lock_flags = 0;3098	pRAID_Context->reg_lock_row_lba = 0;3099	pRAID_Context->reg_lock_length = 0;3100	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));3101	io_request->LUN[1] = scmd->device->lun;3102	pRAID_Context->raid_flags = MR_RAID_FLAGS_IO_SUB_TYPE_SYSTEM_PD3103		<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT;3104 3105	/* If FW supports PD sequence number */3106	if (instance->support_seqnum_jbod_fp) {3107		if (instance->use_seqnum_jbod_fp &&3108			instance->pd_list[pd_index].driveType == TYPE_DISK) {3109 3110			/* More than 256 PD/JBOD support for Ventura */3111			if (instance->support_morethan256jbod)3112				pRAID_Context->virtual_disk_tgt_id =3113					pd_sync->seq[pd_index].pd_target_id;3114			else3115				pRAID_Context->virtual_disk_tgt_id =3116					cpu_to_le16(device_id +3117					(MAX_PHYSICAL_DEVICES - 1));3118			pRAID_Context->config_seq_num =3119				pd_sync->seq[pd_index].seqNum;3120			io_request->DevHandle =3121				pd_sync->seq[pd_index].devHandle;3122			if (instance->adapter_type >= VENTURA_SERIES) {3123				io_request->RaidContext.raid_context_g35.routing_flags |=3124					(1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);3125				io_request->RaidContext.raid_context_g35.nseg_type |=3126					(1 << RAID_CONTEXT_NSEG_SHIFT);3127				io_request->RaidContext.raid_context_g35.nseg_type |=3128					(MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);3129			} else {3130				pRAID_Context->type = MPI2_TYPE_CUDA;3131				pRAID_Context->nseg = 0x1;3132				pRAID_Context->reg_lock_flags |=3133					(MR_RL_FLAGS_SEQ_NUM_ENABLE |3134					 MR_RL_FLAGS_GRANT_DESTINATION_CUDA);3135			}3136		} else {3137			pRAID_Context->virtual_disk_tgt_id =3138				cpu_to_le16(device_id +3139				(MAX_PHYSICAL_DEVICES - 1));3140			pRAID_Context->config_seq_num = 0;3141			io_request->DevHandle = cpu_to_le16(0xFFFF);3142		}3143	} else {3144		pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);3145		pRAID_Context->config_seq_num = 0;3146 3147		if (fusion->fast_path_io) {3148			local_map_ptr =3149				fusion->ld_drv_map[(instance->map_id & 1)];3150			io_request->DevHandle =3151				local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl;3152		} else {3153			io_request->DevHandle = cpu_to_le16(0xFFFF);3154		}3155	}3156 3157	cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;3158 3159	megasas_get_msix_index(instance, scmd, cmd, 1);3160 3161	if (!fp_possible) {3162		/* system pd firmware path */3163		io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;3164		cmd->request_desc->SCSIIO.RequestFlags =3165			(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<3166				MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);3167		pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);3168		pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);3169	} else {3170		if (os_timeout_value)3171			os_timeout_value++;3172 3173		/* system pd Fast Path */3174		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;3175		timeout_limit = (scmd->device->type == TYPE_DISK) ?3176				255 : 0xFFFF;3177		pRAID_Context->timeout_value =3178			cpu_to_le16((os_timeout_value > timeout_limit) ?3179			timeout_limit : os_timeout_value);3180		if (instance->adapter_type >= INVADER_SERIES)3181			io_request->IoFlags |=3182				cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);3183 3184		cmd->request_desc->SCSIIO.RequestFlags =3185			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<3186				MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);3187	}3188}3189 3190/**3191 * megasas_build_io_fusion -	Prepares IOs to devices3192 * @instance:		Adapter soft state3193 * @scp:		SCSI command3194 * @cmd:		Command to be prepared3195 *3196 * Invokes helper functions to prepare request frames3197 * and sets flags appropriate for IO/Non-IO cmd3198 */3199static int3200megasas_build_io_fusion(struct megasas_instance *instance,3201			struct scsi_cmnd *scp,3202			struct megasas_cmd_fusion *cmd)3203{3204	int sge_count;3205	u16 pd_index = 0;3206	u8 drive_type = 0;3207	struct MPI2_RAID_SCSI_IO_REQUEST *io_request = cmd->io_request;3208	struct MR_PRIV_DEVICE *mr_device_priv_data;3209	mr_device_priv_data = scp->device->hostdata;3210 3211	/* Zero out some fields so they don't get reused */3212	memset(io_request->LUN, 0x0, 8);3213	io_request->CDB.EEDP32.PrimaryReferenceTag = 0;3214	io_request->CDB.EEDP32.PrimaryApplicationTagMask = 0;3215	io_request->EEDPFlags = 0;3216	io_request->Control = 0;3217	io_request->EEDPBlockSize = 0;3218	io_request->ChainOffset = 0;3219	io_request->RaidContext.raid_context.raid_flags = 0;3220	io_request->RaidContext.raid_context.type = 0;3221	io_request->RaidContext.raid_context.nseg = 0;3222 3223	memcpy(io_request->CDB.CDB32, scp->cmnd, scp->cmd_len);3224	/*3225	 * Just the CDB length,rest of the Flags are zero3226	 * This will be modified for FP in build_ldio_fusion3227	 */3228	io_request->IoFlags = cpu_to_le16(scp->cmd_len);3229 3230	switch (megasas_cmd_type(scp)) {3231	case READ_WRITE_LDIO:3232		megasas_build_ldio_fusion(instance, scp, cmd);3233		break;3234	case NON_READ_WRITE_LDIO:3235		megasas_build_ld_nonrw_fusion(instance, scp, cmd);3236		break;3237	case READ_WRITE_SYSPDIO:3238		megasas_build_syspd_fusion(instance, scp, cmd, true);3239		break;3240	case NON_READ_WRITE_SYSPDIO:3241		pd_index = MEGASAS_PD_INDEX(scp);3242		drive_type = instance->pd_list[pd_index].driveType;3243		if ((instance->secure_jbod_support ||3244		     mr_device_priv_data->is_tm_capable) ||3245		     (instance->adapter_type >= VENTURA_SERIES &&3246		     drive_type == TYPE_ENCLOSURE))3247			megasas_build_syspd_fusion(instance, scp, cmd, false);3248		else3249			megasas_build_syspd_fusion(instance, scp, cmd, true);3250		break;3251	default:3252		break;3253	}3254 3255	/*3256	 * Construct SGL3257	 */3258 3259	sge_count = megasas_make_sgl(instance, scp, cmd);3260 3261	if (sge_count > instance->max_num_sge || (sge_count < 0)) {3262		dev_err(&instance->pdev->dev,3263			"%s %d sge_count (%d) is out of range. Range is:  0-%d\n",3264			__func__, __LINE__, sge_count, instance->max_num_sge);3265		return 1;3266	}3267 3268	if (instance->adapter_type >= VENTURA_SERIES) {3269		set_num_sge(&io_request->RaidContext.raid_context_g35, sge_count);3270		cpu_to_le16s(&io_request->RaidContext.raid_context_g35.routing_flags);3271		cpu_to_le16s(&io_request->RaidContext.raid_context_g35.nseg_type);3272	} else {3273		/* numSGE store lower 8 bit of sge_count.3274		 * numSGEExt store higher 8 bit of sge_count3275		 */3276		io_request->RaidContext.raid_context.num_sge = sge_count;3277		io_request->RaidContext.raid_context.num_sge_ext =3278			(u8)(sge_count >> 8);3279	}3280 3281	io_request->SGLFlags = cpu_to_le16(MPI2_SGE_FLAGS_64_BIT_ADDRESSING);3282 3283	if (scp->sc_data_direction == DMA_TO_DEVICE)3284		io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_WRITE);3285	else if (scp->sc_data_direction == DMA_FROM_DEVICE)3286		io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_READ);3287 3288	io_request->SGLOffset0 =3289		offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL) / 4;3290 3291	io_request->SenseBufferLowAddress =3292		cpu_to_le32(lower_32_bits(cmd->sense_phys_addr));3293	io_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;3294 3295	cmd->scmd = scp;3296	megasas_priv(scp)->cmd_priv = cmd;3297 3298	return 0;3299}3300 3301static union MEGASAS_REQUEST_DESCRIPTOR_UNION *3302megasas_get_request_descriptor(struct megasas_instance *instance, u16 index)3303{3304	u8 *p;3305	struct fusion_context *fusion;3306 3307	fusion = instance->ctrl_context;3308	p = fusion->req_frames_desc +3309		sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * index;3310 3311	return (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)p;3312}3313 3314 3315/* megasas_prepate_secondRaid1_IO3316 *  It prepares the raid 1 second IO3317 */3318static void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance,3319					   struct megasas_cmd_fusion *cmd,3320					   struct megasas_cmd_fusion *r1_cmd)3321{3322	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL;3323	struct fusion_context *fusion;3324	fusion = instance->ctrl_context;3325	req_desc = cmd->request_desc;3326	/* copy the io request frame as well as 8 SGEs data for r1 command*/3327	memcpy(r1_cmd->io_request, cmd->io_request,3328	       (sizeof(struct MPI2_RAID_SCSI_IO_REQUEST)));3329	memcpy(r1_cmd->io_request->SGLs, cmd->io_request->SGLs,3330	       (fusion->max_sge_in_main_msg * sizeof(union MPI2_SGE_IO_UNION)));3331	/*sense buffer is different for r1 command*/3332	r1_cmd->io_request->SenseBufferLowAddress =3333			cpu_to_le32(lower_32_bits(r1_cmd->sense_phys_addr));3334	r1_cmd->scmd = cmd->scmd;3335	req_desc2 = megasas_get_request_descriptor(instance,3336						   (r1_cmd->index - 1));3337	req_desc2->Words = 0;3338	r1_cmd->request_desc = req_desc2;3339	req_desc2->SCSIIO.SMID = cpu_to_le16(r1_cmd->index);3340	req_desc2->SCSIIO.RequestFlags = req_desc->SCSIIO.RequestFlags;3341	r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle;3342	r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle;3343	r1_cmd->r1_alt_dev_handle = cmd->io_request->DevHandle;3344	cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =3345			cpu_to_le16(r1_cmd->index);3346	r1_cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =3347			cpu_to_le16(cmd->index);3348	/*MSIxIndex of both commands request descriptors should be same*/3349	r1_cmd->request_desc->SCSIIO.MSIxIndex =3350			cmd->request_desc->SCSIIO.MSIxIndex;3351	/*span arm is different for r1 cmd*/3352	r1_cmd->io_request->RaidContext.raid_context_g35.span_arm =3353			cmd->io_request->RaidContext.raid_context_g35.span_arm + 1;3354}3355 3356/**3357 * megasas_build_and_issue_cmd_fusion -Main routine for building and3358 *                                     issuing non IOCTL cmd3359 * @instance:			Adapter soft state3360 * @scmd:			pointer to scsi cmd from OS3361 */3362static u323363megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance,3364				   struct scsi_cmnd *scmd)3365{3366	struct megasas_cmd_fusion *cmd, *r1_cmd = NULL;3367	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;3368	u32 index;3369 3370	if ((megasas_cmd_type(scmd) == READ_WRITE_LDIO) &&3371		instance->ldio_threshold &&3372		(atomic_inc_return(&instance->ldio_outstanding) >3373		instance->ldio_threshold)) {3374		atomic_dec(&instance->ldio_outstanding);3375		return SCSI_MLQUEUE_DEVICE_BUSY;3376	}3377 3378	if (atomic_inc_return(&instance->fw_outstanding) >3379			instance->host->can_queue) {3380		atomic_dec(&instance->fw_outstanding);3381		return SCSI_MLQUEUE_HOST_BUSY;3382	}3383 3384	cmd = megasas_get_cmd_fusion(instance, scsi_cmd_to_rq(scmd)->tag);3385 3386	if (!cmd) {3387		atomic_dec(&instance->fw_outstanding);3388		return SCSI_MLQUEUE_HOST_BUSY;3389	}3390 3391	index = cmd->index;3392 3393	req_desc = megasas_get_request_descriptor(instance, index-1);3394 3395	req_desc->Words = 0;3396	cmd->request_desc = req_desc;3397 3398	if (megasas_build_io_fusion(instance, scmd, cmd)) {3399		megasas_return_cmd_fusion(instance, cmd);3400		dev_err(&instance->pdev->dev, "Error building command\n");3401		cmd->request_desc = NULL;3402		atomic_dec(&instance->fw_outstanding);3403		return SCSI_MLQUEUE_HOST_BUSY;3404	}3405 3406	req_desc = cmd->request_desc;3407	req_desc->SCSIIO.SMID = cpu_to_le16(index);3408 3409	if (cmd->io_request->ChainOffset != 0 &&3410	    cmd->io_request->ChainOffset != 0xF)3411		dev_err(&instance->pdev->dev, "The chain offset value is not "3412		       "correct : %x\n", cmd->io_request->ChainOffset);3413	/*3414	 *	if it is raid 1/10 fp write capable.3415	 *	try to get second command from pool and construct it.3416	 *	From FW, it has confirmed that lba values of two PDs3417	 *	corresponds to single R1/10 LD are always same3418	 *3419	 */3420	/*	driver side count always should be less than max_fw_cmds3421	 *	to get new command3422	 */3423	if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {3424		r1_cmd = megasas_get_cmd_fusion(instance,3425				scsi_cmd_to_rq(scmd)->tag + instance->max_fw_cmds);3426		megasas_prepare_secondRaid1_IO(instance, cmd, r1_cmd);3427	}3428 3429 3430	/*3431	 * Issue the command to the FW3432	 */3433 3434	megasas_sdev_busy_inc(instance, scmd);3435	megasas_fire_cmd_fusion(instance, req_desc);3436 3437	if (r1_cmd)3438		megasas_fire_cmd_fusion(instance, r1_cmd->request_desc);3439 3440 3441	return 0;3442}3443 3444/**3445 * megasas_complete_r1_command -3446 * completes R1 FP write commands which has valid peer smid3447 * @instance:			Adapter soft state3448 * @cmd:			MPT command frame3449 *3450 */3451static inline void3452megasas_complete_r1_command(struct megasas_instance *instance,3453			    struct megasas_cmd_fusion *cmd)3454{3455	u8 *sense, status, ex_status;3456	u32 data_length;3457	u16 peer_smid;3458	struct fusion_context *fusion;3459	struct megasas_cmd_fusion *r1_cmd = NULL;3460	struct scsi_cmnd *scmd_local = NULL;3461	struct RAID_CONTEXT_G35 *rctx_g35;3462 3463	rctx_g35 = &cmd->io_request->RaidContext.raid_context_g35;3464	fusion = instance->ctrl_context;3465	peer_smid = le16_to_cpu(rctx_g35->flow_specific.peer_smid);3466 3467	r1_cmd = fusion->cmd_list[peer_smid - 1];3468	scmd_local = cmd->scmd;3469	status = rctx_g35->status;3470	ex_status = rctx_g35->ex_status;3471	data_length = cmd->io_request->DataLength;3472	sense = cmd->sense;3473 3474	cmd->cmd_completed = true;3475 3476	/* Check if peer command is completed or not*/3477	if (r1_cmd->cmd_completed) {3478		rctx_g35 = &r1_cmd->io_request->RaidContext.raid_context_g35;3479		if (rctx_g35->status != MFI_STAT_OK) {3480			status = rctx_g35->status;3481			ex_status = rctx_g35->ex_status;3482			data_length = r1_cmd->io_request->DataLength;3483			sense = r1_cmd->sense;3484		}3485 3486		megasas_return_cmd_fusion(instance, r1_cmd);3487		map_cmd_status(fusion, scmd_local, status, ex_status,3488			       le32_to_cpu(data_length), sense);3489		if (instance->ldio_threshold &&3490		    megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)3491			atomic_dec(&instance->ldio_outstanding);3492		megasas_priv(scmd_local)->cmd_priv = NULL;3493		megasas_return_cmd_fusion(instance, cmd);3494		scsi_dma_unmap(scmd_local);3495		megasas_sdev_busy_dec(instance, scmd_local);3496		scsi_done(scmd_local);3497	}3498}3499 3500/**3501 * access_irq_context:		Access to reply processing3502 * @irq_context:		IRQ context3503 *3504 * Synchronize access to reply processing.3505 *3506 * Return:  true on success, false on failure.3507 */3508static inline3509bool access_irq_context(struct megasas_irq_context  *irq_context)3510{3511	if (!irq_context)3512		return true;3513 3514	if (atomic_add_unless(&irq_context->in_used, 1, 1))3515		return true;3516 3517	return false;3518}3519 3520/**3521 * release_irq_context:		Release reply processing3522 * @irq_context:		IRQ context3523 *3524 * Release access of reply processing.3525 *3526 * Return: Nothing.3527 */3528static inline3529void release_irq_context(struct megasas_irq_context  *irq_context)3530{3531	if (irq_context)3532		atomic_dec(&irq_context->in_used);3533}3534 3535/**3536 * complete_cmd_fusion -	Completes command3537 * @instance:			Adapter soft state3538 * @MSIxIndex:			MSI number3539 * @irq_context:		IRQ context3540 *3541 * Completes all commands that is in reply descriptor queue3542 */3543static int3544complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex,3545		    struct megasas_irq_context *irq_context)3546{3547	union MPI2_REPLY_DESCRIPTORS_UNION *desc;3548	struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc;3549	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;3550	struct fusion_context *fusion;3551	struct megasas_cmd *cmd_mfi;3552	struct megasas_cmd_fusion *cmd_fusion;3553	u16 smid, num_completed;3554	u8 reply_descript_type, *sense, status, extStatus;3555	u32 device_id, data_length;3556	union desc_value d_val;3557	struct LD_LOAD_BALANCE_INFO *lbinfo;3558	int threshold_reply_count = 0;3559	struct scsi_cmnd *scmd_local = NULL;3560	struct MR_TASK_MANAGE_REQUEST *mr_tm_req;3561	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_tm_req;3562 3563	fusion = instance->ctrl_context;3564 3565	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)3566		return IRQ_HANDLED;3567 3568	if (!access_irq_context(irq_context))3569		return 0;3570 3571	desc = fusion->reply_frames_desc[MSIxIndex] +3572				fusion->last_reply_idx[MSIxIndex];3573 3574	reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;3575 3576	d_val.word = desc->Words;3577 3578	reply_descript_type = reply_desc->ReplyFlags &3579		MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;3580 3581	if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {3582		release_irq_context(irq_context);3583		return IRQ_NONE;3584	}3585 3586	num_completed = 0;3587 3588	while (d_val.u.low != cpu_to_le32(UINT_MAX) &&3589	       d_val.u.high != cpu_to_le32(UINT_MAX)) {3590 3591		smid = le16_to_cpu(reply_desc->SMID);3592		cmd_fusion = fusion->cmd_list[smid - 1];3593		scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)3594						cmd_fusion->io_request;3595 3596		scmd_local = cmd_fusion->scmd;3597		status = scsi_io_req->RaidContext.raid_context.status;3598		extStatus = scsi_io_req->RaidContext.raid_context.ex_status;3599		sense = cmd_fusion->sense;3600		data_length = scsi_io_req->DataLength;3601 3602		switch (scsi_io_req->Function) {3603		case MPI2_FUNCTION_SCSI_TASK_MGMT:3604			mr_tm_req = (struct MR_TASK_MANAGE_REQUEST *)3605						cmd_fusion->io_request;3606			mpi_tm_req = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *)3607						&mr_tm_req->TmRequest;3608			dev_dbg(&instance->pdev->dev, "TM completion:"3609				"type: 0x%x TaskMID: 0x%x\n",3610				mpi_tm_req->TaskType, mpi_tm_req->TaskMID);3611			complete(&cmd_fusion->done);3612			break;3613		case MPI2_FUNCTION_SCSI_IO_REQUEST:  /*Fast Path IO.*/3614			/* Update load balancing info */3615			if (fusion->load_balance_info &&3616			    (megasas_priv(cmd_fusion->scmd)->status &3617			    MEGASAS_LOAD_BALANCE_FLAG)) {3618				device_id = MEGASAS_DEV_INDEX(scmd_local);3619				lbinfo = &fusion->load_balance_info[device_id];3620				atomic_dec(&lbinfo->scsi_pending_cmds[cmd_fusion->pd_r1_lb]);3621				megasas_priv(cmd_fusion->scmd)->status &=3622					~MEGASAS_LOAD_BALANCE_FLAG;3623			}3624			fallthrough;	/* and complete IO */3625		case MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST: /* LD-IO Path */3626			atomic_dec(&instance->fw_outstanding);3627			if (cmd_fusion->r1_alt_dev_handle == MR_DEVHANDLE_INVALID) {3628				map_cmd_status(fusion, scmd_local, status,3629					       extStatus, le32_to_cpu(data_length),3630					       sense);3631				if (instance->ldio_threshold &&3632				    (megasas_cmd_type(scmd_local) == READ_WRITE_LDIO))3633					atomic_dec(&instance->ldio_outstanding);3634				megasas_priv(scmd_local)->cmd_priv = NULL;3635				megasas_return_cmd_fusion(instance, cmd_fusion);3636				scsi_dma_unmap(scmd_local);3637				megasas_sdev_busy_dec(instance, scmd_local);3638				scsi_done(scmd_local);3639			} else	/* Optimal VD - R1 FP command completion. */3640				megasas_complete_r1_command(instance, cmd_fusion);3641			break;3642		case MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST: /*MFI command */3643			cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];3644			/* Poll mode. Dummy free.3645			 * In case of Interrupt mode, caller has reverse check.3646			 */3647			if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {3648				cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;3649				megasas_return_cmd(instance, cmd_mfi);3650			} else3651				megasas_complete_cmd(instance, cmd_mfi, DID_OK);3652			break;3653		}3654 3655		fusion->last_reply_idx[MSIxIndex]++;3656		if (fusion->last_reply_idx[MSIxIndex] >=3657		    fusion->reply_q_depth)3658			fusion->last_reply_idx[MSIxIndex] = 0;3659 3660		desc->Words = cpu_to_le64(ULLONG_MAX);3661		num_completed++;3662		threshold_reply_count++;3663 3664		/* Get the next reply descriptor */3665		if (!fusion->last_reply_idx[MSIxIndex])3666			desc = fusion->reply_frames_desc[MSIxIndex];3667		else3668			desc++;3669 3670		reply_desc =3671		  (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;3672 3673		d_val.word = desc->Words;3674 3675		reply_descript_type = reply_desc->ReplyFlags &3676			MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;3677 3678		if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)3679			break;3680		/*3681		 * Write to reply post host index register after completing threshold3682		 * number of reply counts and still there are more replies in reply queue3683		 * pending to be completed3684		 */3685		if (threshold_reply_count >= instance->threshold_reply_count) {3686			if (instance->msix_combined)3687				writel(((MSIxIndex & 0x7) << 24) |3688					fusion->last_reply_idx[MSIxIndex],3689					instance->reply_post_host_index_addr[MSIxIndex/8]);3690			else3691				writel((MSIxIndex << 24) |3692					fusion->last_reply_idx[MSIxIndex],3693					instance->reply_post_host_index_addr[0]);3694			threshold_reply_count = 0;3695			if (irq_context) {3696				if (!irq_context->irq_poll_scheduled) {3697					irq_context->irq_poll_scheduled = true;3698					irq_context->irq_line_enable = true;3699					irq_poll_sched(&irq_context->irqpoll);3700				}3701				release_irq_context(irq_context);3702				return num_completed;3703			}3704		}3705	}3706 3707	if (num_completed) {3708		wmb();3709		if (instance->msix_combined)3710			writel(((MSIxIndex & 0x7) << 24) |3711				fusion->last_reply_idx[MSIxIndex],3712				instance->reply_post_host_index_addr[MSIxIndex/8]);3713		else3714			writel((MSIxIndex << 24) |3715				fusion->last_reply_idx[MSIxIndex],3716				instance->reply_post_host_index_addr[0]);3717		megasas_check_and_restore_queue_depth(instance);3718	}3719 3720	release_irq_context(irq_context);3721 3722	return num_completed;3723}3724 3725int megasas_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)3726{3727 3728	struct megasas_instance *instance;3729	int num_entries = 0;3730	struct fusion_context *fusion;3731 3732	instance = (struct megasas_instance *)shost->hostdata;3733 3734	fusion = instance->ctrl_context;3735 3736	queue_num = queue_num + instance->low_latency_index_start;3737 3738	if (!atomic_add_unless(&fusion->busy_mq_poll[queue_num], 1, 1))3739		return 0;3740 3741	num_entries = complete_cmd_fusion(instance, queue_num, NULL);3742	atomic_dec(&fusion->busy_mq_poll[queue_num]);3743 3744	return num_entries;3745}3746 3747/**3748 * megasas_enable_irq_poll() - enable irqpoll3749 * @instance:			Adapter soft state3750 */3751static void megasas_enable_irq_poll(struct megasas_instance *instance)3752{3753	u32 count, i;3754	struct megasas_irq_context *irq_ctx;3755 3756	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;3757 3758	for (i = 0; i < count; i++) {3759		irq_ctx = &instance->irq_context[i];3760		irq_poll_enable(&irq_ctx->irqpoll);3761	}3762}3763 3764/**3765 * megasas_sync_irqs -	Synchronizes all IRQs owned by adapter3766 * @instance_addr:			Adapter soft state address3767 */3768static void megasas_sync_irqs(unsigned long instance_addr)3769{3770	u32 count, i;3771	struct megasas_instance *instance =3772		(struct megasas_instance *)instance_addr;3773	struct megasas_irq_context *irq_ctx;3774 3775	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;3776 3777	for (i = 0; i < count; i++) {3778		synchronize_irq(pci_irq_vector(instance->pdev, i));3779		irq_ctx = &instance->irq_context[i];3780		irq_poll_disable(&irq_ctx->irqpoll);3781		if (irq_ctx->irq_poll_scheduled) {3782			irq_ctx->irq_poll_scheduled = false;3783			enable_irq(irq_ctx->os_irq);3784			complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);3785		}3786	}3787}3788 3789/**3790 * megasas_irqpoll() - process a queue for completed reply descriptors3791 * @irqpoll:	IRQ poll structure associated with queue to poll.3792 * @budget:	Threshold of reply descriptors to process per poll.3793 *3794 * Return: The number of entries processed.3795 */3796 3797int megasas_irqpoll(struct irq_poll *irqpoll, int budget)3798{3799	struct megasas_irq_context *irq_ctx;3800	struct megasas_instance *instance;3801	int num_entries;3802 3803	irq_ctx = container_of(irqpoll, struct megasas_irq_context, irqpoll);3804	instance = irq_ctx->instance;3805 3806	if (irq_ctx->irq_line_enable) {3807		disable_irq_nosync(irq_ctx->os_irq);3808		irq_ctx->irq_line_enable = false;3809	}3810 3811	num_entries = complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);3812	if (num_entries < budget) {3813		irq_poll_complete(irqpoll);3814		irq_ctx->irq_poll_scheduled = false;3815		enable_irq(irq_ctx->os_irq);3816		complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);3817	}3818 3819	return num_entries;3820}3821 3822/**3823 * megasas_complete_cmd_dpc_fusion -	Completes command3824 * @instance_addr:			Adapter soft state address3825 *3826 * Tasklet to complete cmds3827 */3828static void3829megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)3830{3831	struct megasas_instance *instance =3832		(struct megasas_instance *)instance_addr;3833	struct megasas_irq_context *irq_ctx = NULL;3834	u32 count, MSIxIndex;3835 3836	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;3837 3838	/* If we have already declared adapter dead, donot complete cmds */3839	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)3840		return;3841 3842	for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++) {3843		irq_ctx = &instance->irq_context[MSIxIndex];3844		complete_cmd_fusion(instance, MSIxIndex, irq_ctx);3845	}3846}3847 3848/**3849 * megasas_isr_fusion - isr entry point3850 * @irq:	IRQ number3851 * @devp:	IRQ context3852 */3853static irqreturn_t megasas_isr_fusion(int irq, void *devp)3854{3855	struct megasas_irq_context *irq_context = devp;3856	struct megasas_instance *instance = irq_context->instance;3857	u32 mfiStatus;3858 3859	if (instance->mask_interrupts)3860		return IRQ_NONE;3861 3862	if (irq_context->irq_poll_scheduled)3863		return IRQ_HANDLED;3864 3865	if (!instance->msix_vectors) {3866		mfiStatus = instance->instancet->clear_intr(instance);3867		if (!mfiStatus)3868			return IRQ_NONE;3869	}3870 3871	/* If we are resetting, bail */3872	if (test_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags)) {3873		instance->instancet->clear_intr(instance);3874		return IRQ_HANDLED;3875	}3876 3877	return complete_cmd_fusion(instance, irq_context->MSIxIndex, irq_context)3878			? IRQ_HANDLED : IRQ_NONE;3879}3880 3881/**3882 * build_mpt_mfi_pass_thru - builds a cmd fo MFI Pass thru3883 * @instance:			Adapter soft state3884 * @mfi_cmd:			megasas_cmd pointer3885 *3886 */3887static void3888build_mpt_mfi_pass_thru(struct megasas_instance *instance,3889			struct megasas_cmd *mfi_cmd)3890{3891	struct MPI25_IEEE_SGE_CHAIN64 *mpi25_ieee_chain;3892	struct MPI2_RAID_SCSI_IO_REQUEST *io_req;3893	struct megasas_cmd_fusion *cmd;3894	struct fusion_context *fusion;3895	struct megasas_header *frame_hdr = &mfi_cmd->frame->hdr;3896 3897	fusion = instance->ctrl_context;3898 3899	cmd = megasas_get_cmd_fusion(instance,3900			instance->max_scsi_cmds + mfi_cmd->index);3901 3902	/*  Save the smid. To be used for returning the cmd */3903	mfi_cmd->context.smid = cmd->index;3904 3905	/*3906	 * For cmds where the flag is set, store the flag and check3907	 * on completion. For cmds with this flag, don't call3908	 * megasas_complete_cmd3909	 */3910 3911	if (frame_hdr->flags & cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE))3912		mfi_cmd->flags |= DRV_DCMD_POLLED_MODE;3913 3914	io_req = cmd->io_request;3915 3916	if (instance->adapter_type >= INVADER_SERIES) {3917		struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end =3918			(struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL;3919		sgl_ptr_end += fusion->max_sge_in_main_msg - 1;3920		sgl_ptr_end->Flags = 0;3921	}3922 3923	mpi25_ieee_chain =3924	  (struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL.IeeeChain;3925 3926	io_req->Function    = MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST;3927	io_req->SGLOffset0  = offsetof(struct MPI2_RAID_SCSI_IO_REQUEST,3928				       SGL) / 4;3929	io_req->ChainOffset = fusion->chain_offset_mfi_pthru;3930 3931	mpi25_ieee_chain->Address = cpu_to_le64(mfi_cmd->frame_phys_addr);3932 3933	mpi25_ieee_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |3934		MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR;3935 3936	mpi25_ieee_chain->Length = cpu_to_le32(instance->mfi_frame_size);3937}3938 3939/**3940 * build_mpt_cmd - Calls helper function to build a cmd MFI Pass thru cmd3941 * @instance:			Adapter soft state3942 * @cmd:			mfi cmd to build3943 *3944 */3945static union MEGASAS_REQUEST_DESCRIPTOR_UNION *3946build_mpt_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd)3947{3948	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc = NULL;3949	u16 index;3950 3951	build_mpt_mfi_pass_thru(instance, cmd);3952	index = cmd->context.smid;3953 3954	req_desc = megasas_get_request_descriptor(instance, index - 1);3955 3956	req_desc->Words = 0;3957	req_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<3958					 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);3959 3960	req_desc->SCSIIO.SMID = cpu_to_le16(index);3961 3962	return req_desc;3963}3964 3965/**3966 * megasas_issue_dcmd_fusion - Issues a MFI Pass thru cmd3967 * @instance:			Adapter soft state3968 * @cmd:			mfi cmd pointer3969 *3970 */3971static void3972megasas_issue_dcmd_fusion(struct megasas_instance *instance,3973			  struct megasas_cmd *cmd)3974{3975	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;3976 3977	req_desc = build_mpt_cmd(instance, cmd);3978 3979	megasas_fire_cmd_fusion(instance, req_desc);3980	return;3981}3982 3983/**3984 * megasas_release_fusion -	Reverses the FW initialization3985 * @instance:			Adapter soft state3986 */3987void3988megasas_release_fusion(struct megasas_instance *instance)3989{3990	megasas_free_ioc_init_cmd(instance);3991	megasas_free_cmds(instance);3992	megasas_free_cmds_fusion(instance);3993 3994	iounmap(instance->reg_set);3995 3996	pci_release_selected_regions(instance->pdev, 1<<instance->bar);3997}3998 3999/**4000 * megasas_read_fw_status_reg_fusion - returns the current FW status value4001 * @instance:			Adapter soft state4002 */4003static u324004megasas_read_fw_status_reg_fusion(struct megasas_instance *instance)4005{4006	return megasas_readl(instance, &instance->reg_set->outbound_scratch_pad_0);4007}4008 4009/**4010 * megasas_alloc_host_crash_buffer -	Host buffers for Crash dump collection from Firmware4011 * @instance:				Controller's soft instance4012 * @return:			        Number of allocated host crash buffers4013 */4014static void4015megasas_alloc_host_crash_buffer(struct megasas_instance *instance)4016{4017	unsigned int i;4018 4019	for (i = 0; i < MAX_CRASH_DUMP_SIZE; i++) {4020		instance->crash_buf[i] = vzalloc(CRASH_DMA_BUF_SIZE);4021		if (!instance->crash_buf[i]) {4022			dev_info(&instance->pdev->dev, "Firmware crash dump "4023				"memory allocation failed at index %d\n", i);4024			break;4025		}4026	}4027	instance->drv_buf_alloc = i;4028}4029 4030/**4031 * megasas_free_host_crash_buffer -	Host buffers for Crash dump collection from Firmware4032 * @instance:				Controller's soft instance4033 */4034void4035megasas_free_host_crash_buffer(struct megasas_instance *instance)4036{4037	unsigned int i;4038	for (i = 0; i < instance->drv_buf_alloc; i++) {4039		vfree(instance->crash_buf[i]);4040	}4041	instance->drv_buf_index = 0;4042	instance->drv_buf_alloc = 0;4043	instance->fw_crash_state = UNAVAILABLE;4044	instance->fw_crash_buffer_size = 0;4045}4046 4047/**4048 * megasas_adp_reset_fusion -	For controller reset4049 * @instance:				Controller's soft instance4050 * @regs:				MFI register set4051 */4052static int4053megasas_adp_reset_fusion(struct megasas_instance *instance,4054			 struct megasas_register_set __iomem *regs)4055{4056	u32 host_diag, abs_state, retry;4057 4058	/* Now try to reset the chip */4059	writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);4060	writel(MPI2_WRSEQ_1ST_KEY_VALUE, &instance->reg_set->fusion_seq_offset);4061	writel(MPI2_WRSEQ_2ND_KEY_VALUE, &instance->reg_set->fusion_seq_offset);4062	writel(MPI2_WRSEQ_3RD_KEY_VALUE, &instance->reg_set->fusion_seq_offset);4063	writel(MPI2_WRSEQ_4TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);4064	writel(MPI2_WRSEQ_5TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);4065	writel(MPI2_WRSEQ_6TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);4066 4067	/* Check that the diag write enable (DRWE) bit is on */4068	host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag);4069	retry = 0;4070	while (!(host_diag & HOST_DIAG_WRITE_ENABLE)) {4071		msleep(100);4072		host_diag = megasas_readl(instance,4073					  &instance->reg_set->fusion_host_diag);4074		if (retry++ == 100) {4075			dev_warn(&instance->pdev->dev,4076				"Host diag unlock failed from %s %d\n",4077				__func__, __LINE__);4078			break;4079		}4080	}4081	if (!(host_diag & HOST_DIAG_WRITE_ENABLE))4082		return -1;4083 4084	/* Send chip reset command */4085	writel(host_diag | HOST_DIAG_RESET_ADAPTER,4086		&instance->reg_set->fusion_host_diag);4087	msleep(3000);4088 4089	/* Make sure reset adapter bit is cleared */4090	host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag);4091	retry = 0;4092	while (host_diag & HOST_DIAG_RESET_ADAPTER) {4093		msleep(100);4094		host_diag = megasas_readl(instance,4095					  &instance->reg_set->fusion_host_diag);4096		if (retry++ == 1000) {4097			dev_warn(&instance->pdev->dev,4098				"Diag reset adapter never cleared %s %d\n",4099				__func__, __LINE__);4100			break;4101		}4102	}4103	if (host_diag & HOST_DIAG_RESET_ADAPTER)4104		return -1;4105 4106	abs_state = instance->instancet->read_fw_status_reg(instance)4107			& MFI_STATE_MASK;4108	retry = 0;4109 4110	while ((abs_state <= MFI_STATE_FW_INIT) && (retry++ < 1000)) {4111		msleep(100);4112		abs_state = instance->instancet->4113			read_fw_status_reg(instance) & MFI_STATE_MASK;4114	}4115	if (abs_state <= MFI_STATE_FW_INIT) {4116		dev_warn(&instance->pdev->dev,4117			"fw state < MFI_STATE_FW_INIT, state = 0x%x %s %d\n",4118			abs_state, __func__, __LINE__);4119		return -1;4120	}4121 4122	return 0;4123}4124 4125/**4126 * megasas_check_reset_fusion -	For controller reset check4127 * @instance:				Controller's soft instance4128 * @regs:				MFI register set4129 */4130static int4131megasas_check_reset_fusion(struct megasas_instance *instance,4132			   struct megasas_register_set __iomem *regs)4133{4134	return 0;4135}4136 4137/**4138 * megasas_trigger_snap_dump -	Trigger snap dump in FW4139 * @instance:			Soft instance of adapter4140 */4141static inline void megasas_trigger_snap_dump(struct megasas_instance *instance)4142{4143	int j;4144	u32 fw_state, abs_state;4145 4146	if (!instance->disableOnlineCtrlReset) {4147		dev_info(&instance->pdev->dev, "Trigger snap dump\n");4148		writel(MFI_ADP_TRIGGER_SNAP_DUMP,4149		       &instance->reg_set->doorbell);4150		readl(&instance->reg_set->doorbell);4151	}4152 4153	for (j = 0; j < instance->snapdump_wait_time; j++) {4154		abs_state = instance->instancet->read_fw_status_reg(instance);4155		fw_state = abs_state & MFI_STATE_MASK;4156		if (fw_state == MFI_STATE_FAULT) {4157			dev_printk(KERN_ERR, &instance->pdev->dev,4158				   "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",4159				   abs_state & MFI_STATE_FAULT_CODE,4160				   abs_state & MFI_STATE_FAULT_SUBCODE, __func__);4161			return;4162		}4163		msleep(1000);4164	}4165}4166 4167/* This function waits for outstanding commands on fusion to complete */4168static int4169megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,4170				    int reason, int *convert)4171{4172	int i, outstanding, retval = 0, hb_seconds_missed = 0;4173	u32 fw_state, abs_state;4174	u32 waittime_for_io_completion;4175 4176	waittime_for_io_completion =4177		min_t(u32, resetwaittime,4178			(resetwaittime - instance->snapdump_wait_time));4179 4180	if (reason == MFI_IO_TIMEOUT_OCR) {4181		dev_info(&instance->pdev->dev,4182			"MFI command is timed out\n");4183		megasas_complete_cmd_dpc_fusion((unsigned long)instance);4184		if (instance->snapdump_wait_time)4185			megasas_trigger_snap_dump(instance);4186		retval = 1;4187		goto out;4188	}4189 4190	for (i = 0; i < waittime_for_io_completion; i++) {4191		/* Check if firmware is in fault state */4192		abs_state = instance->instancet->read_fw_status_reg(instance);4193		fw_state = abs_state & MFI_STATE_MASK;4194		if (fw_state == MFI_STATE_FAULT) {4195			dev_printk(KERN_ERR, &instance->pdev->dev,4196				   "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",4197				   abs_state & MFI_STATE_FAULT_CODE,4198				   abs_state & MFI_STATE_FAULT_SUBCODE, __func__);4199			megasas_complete_cmd_dpc_fusion((unsigned long)instance);4200			if (instance->requestorId && reason) {4201				dev_warn(&instance->pdev->dev, "SR-IOV Found FW in FAULT"4202				" state while polling during"4203				" I/O timeout handling for %d\n",4204				instance->host->host_no);4205				*convert = 1;4206			}4207 4208			retval = 1;4209			goto out;4210		}4211 4212 4213		/* If SR-IOV VF mode & heartbeat timeout, don't wait */4214		if (instance->requestorId && !reason) {4215			retval = 1;4216			goto out;4217		}4218 4219		/* If SR-IOV VF mode & I/O timeout, check for HB timeout */4220		if (instance->requestorId && (reason == SCSIIO_TIMEOUT_OCR)) {4221			if (instance->hb_host_mem->HB.fwCounter !=4222			    instance->hb_host_mem->HB.driverCounter) {4223				instance->hb_host_mem->HB.driverCounter =4224					instance->hb_host_mem->HB.fwCounter;4225				hb_seconds_missed = 0;4226			} else {4227				hb_seconds_missed++;4228				if (hb_seconds_missed ==4229				    (MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF/HZ)) {4230					dev_warn(&instance->pdev->dev, "SR-IOV:"4231					       " Heartbeat never completed "4232					       " while polling during I/O "4233					       " timeout handling for "4234					       "scsi%d.\n",4235					       instance->host->host_no);4236					       *convert = 1;4237					       retval = 1;4238					       goto out;4239				}4240			}4241		}4242 4243		megasas_complete_cmd_dpc_fusion((unsigned long)instance);4244		outstanding = atomic_read(&instance->fw_outstanding);4245		if (!outstanding)4246			goto out;4247 4248		if (!(i % MEGASAS_RESET_NOTICE_INTERVAL)) {4249			dev_notice(&instance->pdev->dev, "[%2d]waiting for %d "4250			       "commands to complete for scsi%d\n", i,4251			       outstanding, instance->host->host_no);4252		}4253		msleep(1000);4254	}4255 4256	if (instance->snapdump_wait_time) {4257		megasas_trigger_snap_dump(instance);4258		retval = 1;4259		goto out;4260	}4261 4262	if (atomic_read(&instance->fw_outstanding)) {4263		dev_err(&instance->pdev->dev, "pending commands remain after waiting, "4264		       "will reset adapter scsi%d.\n",4265		       instance->host->host_no);4266		*convert = 1;4267		retval = 1;4268	}4269 4270out:4271	if (!retval && reason == SCSIIO_TIMEOUT_OCR)4272		dev_info(&instance->pdev->dev, "IO is completed, no OCR is required\n");4273 4274	return retval;4275}4276 4277void  megasas_reset_reply_desc(struct megasas_instance *instance)4278{4279	int i, j, count;4280	struct fusion_context *fusion;4281	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;4282 4283	fusion = instance->ctrl_context;4284	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;4285	count += instance->iopoll_q_count;4286 4287	for (i = 0 ; i < count ; i++) {4288		fusion->last_reply_idx[i] = 0;4289		reply_desc = fusion->reply_frames_desc[i];4290		for (j = 0 ; j < fusion->reply_q_depth; j++, reply_desc++)4291			reply_desc->Words = cpu_to_le64(ULLONG_MAX);4292	}4293}4294 4295/*4296 * megasas_refire_mgmt_cmd :	Re-fire management commands4297 * @instance:				Controller's soft instance4298*/4299static void megasas_refire_mgmt_cmd(struct megasas_instance *instance,4300			     bool return_ioctl)4301{4302	int j;4303	struct megasas_cmd_fusion *cmd_fusion;4304	struct fusion_context *fusion;4305	struct megasas_cmd *cmd_mfi;4306	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;4307	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;4308	u16 smid;4309	bool refire_cmd = false;4310	u8 result;4311	u32 opcode = 0;4312 4313	fusion = instance->ctrl_context;4314 4315	/* Re-fire management commands.4316	 * Do not traverse complet MPT frame pool. Start from max_scsi_cmds.4317	 */4318	for (j = instance->max_scsi_cmds ; j < instance->max_fw_cmds; j++) {4319		cmd_fusion = fusion->cmd_list[j];4320		cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];4321		smid = le16_to_cpu(cmd_mfi->context.smid);4322		result = REFIRE_CMD;4323 4324		if (!smid)4325			continue;4326 4327		req_desc = megasas_get_request_descriptor(instance, smid - 1);4328 4329		switch (cmd_mfi->frame->hdr.cmd) {4330		case MFI_CMD_DCMD:4331			opcode = le32_to_cpu(cmd_mfi->frame->dcmd.opcode);4332			 /* Do not refire shutdown command */4333			if (opcode == MR_DCMD_CTRL_SHUTDOWN) {4334				cmd_mfi->frame->dcmd.cmd_status = MFI_STAT_OK;4335				result = COMPLETE_CMD;4336				break;4337			}4338 4339			refire_cmd = ((opcode != MR_DCMD_LD_MAP_GET_INFO)) &&4340				      (opcode != MR_DCMD_SYSTEM_PD_MAP_GET_INFO) &&4341				      !(cmd_mfi->flags & DRV_DCMD_SKIP_REFIRE);4342 4343			if (!refire_cmd)4344				result = RETURN_CMD;4345 4346			break;4347		case MFI_CMD_NVME:4348			if (!instance->support_nvme_passthru) {4349				cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;4350				result = COMPLETE_CMD;4351			}4352 4353			break;4354		case MFI_CMD_TOOLBOX:4355			if (!instance->support_pci_lane_margining) {4356				cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;4357				result = COMPLETE_CMD;4358			}4359 4360			break;4361		default:4362			break;4363		}4364 4365		if (return_ioctl && cmd_mfi->sync_cmd &&4366		    cmd_mfi->frame->hdr.cmd != MFI_CMD_ABORT) {4367			dev_err(&instance->pdev->dev,4368				"return -EBUSY from %s %d cmd 0x%x opcode 0x%x\n",4369				__func__, __LINE__, cmd_mfi->frame->hdr.cmd,4370				le32_to_cpu(cmd_mfi->frame->dcmd.opcode));4371			cmd_mfi->cmd_status_drv = DCMD_BUSY;4372			result = COMPLETE_CMD;4373		}4374 4375		scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)4376				cmd_fusion->io_request;4377		if (scsi_io_req->Function == MPI2_FUNCTION_SCSI_TASK_MGMT)4378			result = RETURN_CMD;4379 4380		switch (result) {4381		case REFIRE_CMD:4382			megasas_fire_cmd_fusion(instance, req_desc);4383			break;4384		case RETURN_CMD:4385			megasas_return_cmd(instance, cmd_mfi);4386			break;4387		case COMPLETE_CMD:4388			megasas_complete_cmd(instance, cmd_mfi, DID_OK);4389			break;4390		}4391	}4392}4393 4394/*4395 * megasas_return_polled_cmds: Return polled mode commands back to the pool4396 *			       before initiating an OCR.4397 * @instance:                  Controller's soft instance4398 */4399static void4400megasas_return_polled_cmds(struct megasas_instance *instance)4401{4402	int i;4403	struct megasas_cmd_fusion *cmd_fusion;4404	struct fusion_context *fusion;4405	struct megasas_cmd *cmd_mfi;4406 4407	fusion = instance->ctrl_context;4408 4409	for (i = instance->max_scsi_cmds; i < instance->max_fw_cmds; i++) {4410		cmd_fusion = fusion->cmd_list[i];4411		cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];4412 4413		if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {4414			if (megasas_dbg_lvl & OCR_DEBUG)4415				dev_info(&instance->pdev->dev,4416					 "%s %d return cmd 0x%x opcode 0x%x\n",4417					 __func__, __LINE__, cmd_mfi->frame->hdr.cmd,4418					 le32_to_cpu(cmd_mfi->frame->dcmd.opcode));4419			cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;4420			megasas_return_cmd(instance, cmd_mfi);4421		}4422	}4423}4424 4425/*4426 * megasas_track_scsiio : Track SCSI IOs outstanding to a SCSI device4427 * @instance: per adapter struct4428 * @channel: the channel assigned by the OS4429 * @id: the id assigned by the OS4430 *4431 * Returns SUCCESS if no IOs pending to SCSI device, else return FAILED4432 */4433 4434static int megasas_track_scsiio(struct megasas_instance *instance,4435		int id, int channel)4436{4437	int i, found = 0;4438	struct megasas_cmd_fusion *cmd_fusion;4439	struct fusion_context *fusion;4440	fusion = instance->ctrl_context;4441 4442	for (i = 0 ; i < instance->max_scsi_cmds; i++) {4443		cmd_fusion = fusion->cmd_list[i];4444		if (cmd_fusion->scmd &&4445			(cmd_fusion->scmd->device->id == id &&4446			cmd_fusion->scmd->device->channel == channel)) {4447			dev_info(&instance->pdev->dev,4448				"SCSI commands pending to target"4449				"channel %d id %d \tSMID: 0x%x\n",4450				channel, id, cmd_fusion->index);4451			scsi_print_command(cmd_fusion->scmd);4452			found = 1;4453			break;4454		}4455	}4456 4457	return found ? FAILED : SUCCESS;4458}4459 4460/**4461 * megasas_tm_response_code - translation of device response code4462 * @instance:	Controller's soft instance4463 * @mpi_reply:	MPI reply returned by firmware4464 *4465 * Return nothing.4466 */4467static void4468megasas_tm_response_code(struct megasas_instance *instance,4469		struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply)4470{4471	char *desc;4472 4473	switch (mpi_reply->ResponseCode) {4474	case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:4475		desc = "task management request completed";4476		break;4477	case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:4478		desc = "invalid frame";4479		break;4480	case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:4481		desc = "task management request not supported";4482		break;4483	case MPI2_SCSITASKMGMT_RSP_TM_FAILED:4484		desc = "task management request failed";4485		break;4486	case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:4487		desc = "task management request succeeded";4488		break;4489	case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:4490		desc = "invalid lun";4491		break;4492	case 0xA:4493		desc = "overlapped tag attempted";4494		break;4495	case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:4496		desc = "task queued, however not sent to target";4497		break;4498	default:4499		desc = "unknown";4500		break;4501	}4502	dev_dbg(&instance->pdev->dev, "response_code(%01x): %s\n",4503		mpi_reply->ResponseCode, desc);4504	dev_dbg(&instance->pdev->dev,4505		"TerminationCount/DevHandle/Function/TaskType/IOCStat/IOCLoginfo"4506		" 0x%x/0x%x/0x%x/0x%x/0x%x/0x%x\n",4507		mpi_reply->TerminationCount, mpi_reply->DevHandle,4508		mpi_reply->Function, mpi_reply->TaskType,4509		mpi_reply->IOCStatus, mpi_reply->IOCLogInfo);4510}4511 4512/**4513 * megasas_issue_tm - main routine for sending tm requests4514 * @instance: per adapter struct4515 * @device_handle: device handle4516 * @channel: the channel assigned by the OS4517 * @id: the id assigned by the OS4518 * @smid_task: smid assigned to the task4519 * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c)4520 * @mr_device_priv_data: private data4521 * Context: user4522 *4523 * MegaRaid use MPT interface for Task Magement request.4524 * A generic API for sending task management requests to firmware.4525 *4526 * Return SUCCESS or FAILED.4527 */4528static int4529megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,4530	uint channel, uint id, u16 smid_task, u8 type,4531	struct MR_PRIV_DEVICE *mr_device_priv_data)4532{4533	struct MR_TASK_MANAGE_REQUEST *mr_request;4534	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_request;4535	unsigned long timeleft;4536	struct megasas_cmd_fusion *cmd_fusion;4537	struct megasas_cmd *cmd_mfi;4538	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;4539	struct fusion_context *fusion = NULL;4540	struct megasas_cmd_fusion *scsi_lookup;4541	int rc;4542	int timeout = MEGASAS_DEFAULT_TM_TIMEOUT;4543	struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply;4544 4545	fusion = instance->ctrl_context;4546 4547	cmd_mfi = megasas_get_cmd(instance);4548 4549	if (!cmd_mfi) {4550		dev_err(&instance->pdev->dev, "Failed from %s %d\n",4551			__func__, __LINE__);4552		return -ENOMEM;4553	}4554 4555	cmd_fusion = megasas_get_cmd_fusion(instance,4556			instance->max_scsi_cmds + cmd_mfi->index);4557 4558	/*  Save the smid. To be used for returning the cmd */4559	cmd_mfi->context.smid = cmd_fusion->index;4560 4561	req_desc = megasas_get_request_descriptor(instance,4562			(cmd_fusion->index - 1));4563 4564	cmd_fusion->request_desc = req_desc;4565	req_desc->Words = 0;4566 4567	mr_request = (struct MR_TASK_MANAGE_REQUEST *) cmd_fusion->io_request;4568	memset(mr_request, 0, sizeof(struct MR_TASK_MANAGE_REQUEST));4569	mpi_request = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest;4570	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;4571	mpi_request->DevHandle = cpu_to_le16(device_handle);4572	mpi_request->TaskType = type;4573	mpi_request->TaskMID = cpu_to_le16(smid_task);4574	mpi_request->LUN[1] = 0;4575 4576 4577	req_desc = cmd_fusion->request_desc;4578	req_desc->HighPriority.SMID = cpu_to_le16(cmd_fusion->index);4579	req_desc->HighPriority.RequestFlags =4580		(MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY <<4581		MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);4582	req_desc->HighPriority.MSIxIndex =  0;4583	req_desc->HighPriority.LMID = 0;4584	req_desc->HighPriority.Reserved1 = 0;4585 4586	if (channel < MEGASAS_MAX_PD_CHANNELS)4587		mr_request->tmReqFlags.isTMForPD = 1;4588	else4589		mr_request->tmReqFlags.isTMForLD = 1;4590 4591	init_completion(&cmd_fusion->done);4592	megasas_fire_cmd_fusion(instance, req_desc);4593 4594	switch (type) {4595	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:4596		timeout = mr_device_priv_data->task_abort_tmo;4597		break;4598	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:4599		timeout = mr_device_priv_data->target_reset_tmo;4600		break;4601	}4602 4603	timeleft = wait_for_completion_timeout(&cmd_fusion->done, timeout * HZ);4604 4605	if (!timeleft) {4606		dev_err(&instance->pdev->dev,4607			"task mgmt type 0x%x timed out\n", type);4608		mutex_unlock(&instance->reset_mutex);4609		rc = megasas_reset_fusion(instance->host, MFI_IO_TIMEOUT_OCR);4610		mutex_lock(&instance->reset_mutex);4611		return rc;4612	}4613 4614	mpi_reply = (struct MPI2_SCSI_TASK_MANAGE_REPLY *) &mr_request->TMReply;4615	megasas_tm_response_code(instance, mpi_reply);4616 4617	megasas_return_cmd(instance, cmd_mfi);4618	rc = SUCCESS;4619	switch (type) {4620	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:4621		scsi_lookup = fusion->cmd_list[smid_task - 1];4622 4623		if (scsi_lookup->scmd == NULL)4624			break;4625		else {4626			instance->instancet->disable_intr(instance);4627			megasas_sync_irqs((unsigned long)instance);4628			instance->instancet->enable_intr(instance);4629			megasas_enable_irq_poll(instance);4630			if (scsi_lookup->scmd == NULL)4631				break;4632		}4633		rc = FAILED;4634		break;4635 4636	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:4637		if ((channel == 0xFFFFFFFF) && (id == 0xFFFFFFFF))4638			break;4639		instance->instancet->disable_intr(instance);4640		megasas_sync_irqs((unsigned long)instance);4641		rc = megasas_track_scsiio(instance, id, channel);4642		instance->instancet->enable_intr(instance);4643		megasas_enable_irq_poll(instance);4644 4645		break;4646	case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:4647	case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:4648		break;4649	default:4650		rc = FAILED;4651		break;4652	}4653 4654	return rc;4655 4656}4657 4658/*4659 * megasas_fusion_smid_lookup : Look for fusion command corresponding to SCSI4660 * @instance: per adapter struct4661 *4662 * Return Non Zero index, if SMID found in outstanding commands4663 */4664static u16 megasas_fusion_smid_lookup(struct scsi_cmnd *scmd)4665{4666	int i, ret = 0;4667	struct megasas_instance *instance;4668	struct megasas_cmd_fusion *cmd_fusion;4669	struct fusion_context *fusion;4670 4671	instance = (struct megasas_instance *)scmd->device->host->hostdata;4672 4673	fusion = instance->ctrl_context;4674 4675	for (i = 0; i < instance->max_scsi_cmds; i++) {4676		cmd_fusion = fusion->cmd_list[i];4677		if (cmd_fusion->scmd && (cmd_fusion->scmd == scmd)) {4678			scmd_printk(KERN_NOTICE, scmd, "Abort request is for"4679				" SMID: %d\n", cmd_fusion->index);4680			ret = cmd_fusion->index;4681			break;4682		}4683	}4684 4685	return ret;4686}4687 4688/*4689* megasas_get_tm_devhandle - Get devhandle for TM request4690* @sdev-		     OS provided scsi device4691*4692* Returns-		     devhandle/targetID of SCSI device4693*/4694static u16 megasas_get_tm_devhandle(struct scsi_device *sdev)4695{4696	u16 pd_index = 0;4697	u32 device_id;4698	struct megasas_instance *instance;4699	struct fusion_context *fusion;4700	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;4701	u16 devhandle = (u16)ULONG_MAX;4702 4703	instance = (struct megasas_instance *)sdev->host->hostdata;4704	fusion = instance->ctrl_context;4705 4706	if (!MEGASAS_IS_LOGICAL(sdev)) {4707		if (instance->use_seqnum_jbod_fp) {4708			pd_index = (sdev->channel * MEGASAS_MAX_DEV_PER_CHANNEL)4709				    + sdev->id;4710			pd_sync = (void *)fusion->pd_seq_sync4711					[(instance->pd_seq_map_id - 1) & 1];4712			devhandle = pd_sync->seq[pd_index].devHandle;4713		} else4714			sdev_printk(KERN_ERR, sdev, "Firmware expose tmCapable"4715				" without JBOD MAP support from %s %d\n", __func__, __LINE__);4716	} else {4717		device_id = ((sdev->channel % 2) * MEGASAS_MAX_DEV_PER_CHANNEL)4718				+ sdev->id;4719		devhandle = device_id;4720	}4721 4722	return devhandle;4723}4724 4725/*4726 * megasas_task_abort_fusion : SCSI task abort function for fusion adapters4727 * @scmd : pointer to scsi command object4728 *4729 * Return SUCCESS, if command aborted else FAILED4730 */4731 4732int megasas_task_abort_fusion(struct scsi_cmnd *scmd)4733{4734	struct megasas_instance *instance;4735	u16 smid, devhandle;4736	int ret;4737	struct MR_PRIV_DEVICE *mr_device_priv_data;4738	mr_device_priv_data = scmd->device->hostdata;4739 4740	instance = (struct megasas_instance *)scmd->device->host->hostdata;4741 4742	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {4743		dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"4744		"SCSI host:%d\n", instance->host->host_no);4745		ret = FAILED;4746		return ret;4747	}4748 4749	if (!mr_device_priv_data) {4750		sdev_printk(KERN_INFO, scmd->device, "device been deleted! "4751			"scmd(%p)\n", scmd);4752		scmd->result = DID_NO_CONNECT << 16;4753		ret = SUCCESS;4754		goto out;4755	}4756 4757	if (!mr_device_priv_data->is_tm_capable) {4758		ret = FAILED;4759		goto out;4760	}4761 4762	mutex_lock(&instance->reset_mutex);4763 4764	smid = megasas_fusion_smid_lookup(scmd);4765 4766	if (!smid) {4767		ret = SUCCESS;4768		scmd_printk(KERN_NOTICE, scmd, "Command for which abort is"4769			" issued is not found in outstanding commands\n");4770		mutex_unlock(&instance->reset_mutex);4771		goto out;4772	}4773 4774	devhandle = megasas_get_tm_devhandle(scmd->device);4775 4776	if (devhandle == (u16)ULONG_MAX) {4777		ret = FAILED;4778		sdev_printk(KERN_INFO, scmd->device,4779			"task abort issued for invalid devhandle\n");4780		mutex_unlock(&instance->reset_mutex);4781		goto out;4782	}4783	sdev_printk(KERN_INFO, scmd->device,4784		"attempting task abort! scmd(0x%p) tm_dev_handle 0x%x\n",4785		scmd, devhandle);4786 4787	mr_device_priv_data->tm_busy = true;4788	ret = megasas_issue_tm(instance, devhandle,4789			scmd->device->channel, scmd->device->id, smid,4790			MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,4791			mr_device_priv_data);4792	mr_device_priv_data->tm_busy = false;4793 4794	mutex_unlock(&instance->reset_mutex);4795	scmd_printk(KERN_INFO, scmd, "task abort %s!! scmd(0x%p)\n",4796			((ret == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);4797out:4798	scsi_print_command(scmd);4799	if (megasas_dbg_lvl & TM_DEBUG)4800		megasas_dump_fusion_io(scmd);4801 4802	return ret;4803}4804 4805/*4806 * megasas_reset_target_fusion : target reset function for fusion adapters4807 * scmd: SCSI command pointer4808 *4809 * Returns SUCCESS if all commands associated with target aborted else FAILED4810 */4811 4812int megasas_reset_target_fusion(struct scsi_cmnd *scmd)4813{4814 4815	struct megasas_instance *instance;4816	int ret = FAILED;4817	u16 devhandle;4818	struct MR_PRIV_DEVICE *mr_device_priv_data;4819	mr_device_priv_data = scmd->device->hostdata;4820 4821	instance = (struct megasas_instance *)scmd->device->host->hostdata;4822 4823	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {4824		dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"4825		"SCSI host:%d\n", instance->host->host_no);4826		ret = FAILED;4827		return ret;4828	}4829 4830	if (!mr_device_priv_data) {4831		sdev_printk(KERN_INFO, scmd->device,4832			    "device been deleted! scmd: (0x%p)\n", scmd);4833		scmd->result = DID_NO_CONNECT << 16;4834		ret = SUCCESS;4835		goto out;4836	}4837 4838	if (!mr_device_priv_data->is_tm_capable) {4839		ret = FAILED;4840		goto out;4841	}4842 4843	mutex_lock(&instance->reset_mutex);4844	devhandle = megasas_get_tm_devhandle(scmd->device);4845 4846	if (devhandle == (u16)ULONG_MAX) {4847		ret = FAILED;4848		sdev_printk(KERN_INFO, scmd->device,4849			"target reset issued for invalid devhandle\n");4850		mutex_unlock(&instance->reset_mutex);4851		goto out;4852	}4853 4854	sdev_printk(KERN_INFO, scmd->device,4855		"attempting target reset! scmd(0x%p) tm_dev_handle: 0x%x\n",4856		scmd, devhandle);4857	mr_device_priv_data->tm_busy = true;4858	ret = megasas_issue_tm(instance, devhandle,4859			scmd->device->channel, scmd->device->id, 0,4860			MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET,4861			mr_device_priv_data);4862	mr_device_priv_data->tm_busy = false;4863	mutex_unlock(&instance->reset_mutex);4864	scmd_printk(KERN_NOTICE, scmd, "target reset %s!!\n",4865		(ret == SUCCESS) ? "SUCCESS" : "FAILED");4866 4867out:4868	return ret;4869}4870 4871/*SRIOV get other instance in cluster if any*/4872static struct4873megasas_instance *megasas_get_peer_instance(struct megasas_instance *instance)4874{4875	int i;4876 4877	for (i = 0; i < MAX_MGMT_ADAPTERS; i++) {4878		if (megasas_mgmt_info.instance[i] &&4879			(megasas_mgmt_info.instance[i] != instance) &&4880			 megasas_mgmt_info.instance[i]->requestorId &&4881			 megasas_mgmt_info.instance[i]->peerIsPresent &&4882			(memcmp((megasas_mgmt_info.instance[i]->clusterId),4883			instance->clusterId, MEGASAS_CLUSTER_ID_SIZE) == 0))4884			return megasas_mgmt_info.instance[i];4885	}4886	return NULL;4887}4888 4889/* Check for a second path that is currently UP */4890int megasas_check_mpio_paths(struct megasas_instance *instance,4891	struct scsi_cmnd *scmd)4892{4893	struct megasas_instance *peer_instance = NULL;4894	int retval = (DID_REQUEUE << 16);4895 4896	if (instance->peerIsPresent) {4897		peer_instance = megasas_get_peer_instance(instance);4898		if ((peer_instance) &&4899			(atomic_read(&peer_instance->adprecovery) ==4900			MEGASAS_HBA_OPERATIONAL))4901			retval = (DID_NO_CONNECT << 16);4902	}4903	return retval;4904}4905 4906/* Core fusion reset function */4907int megasas_reset_fusion(struct Scsi_Host *shost, int reason)4908{4909	int retval = SUCCESS, i, j, convert = 0;4910	struct megasas_instance *instance;4911	struct megasas_cmd_fusion *cmd_fusion, *r1_cmd;4912	struct fusion_context *fusion;4913	u32 abs_state, status_reg, reset_adapter, fpio_count = 0;4914	u32 io_timeout_in_crash_mode = 0;4915	struct scsi_cmnd *scmd_local = NULL;4916	struct scsi_device *sdev;4917	int ret_target_prop = DCMD_FAILED;4918	bool is_target_prop = false;4919	bool do_adp_reset = true;4920	int max_reset_tries = MEGASAS_FUSION_MAX_RESET_TRIES;4921 4922	instance = (struct megasas_instance *)shost->hostdata;4923	fusion = instance->ctrl_context;4924 4925	mutex_lock(&instance->reset_mutex);4926 4927	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) {4928		dev_warn(&instance->pdev->dev, "Hardware critical error, "4929		       "returning FAILED for scsi%d.\n",4930			instance->host->host_no);4931		mutex_unlock(&instance->reset_mutex);4932		return FAILED;4933	}4934	status_reg = instance->instancet->read_fw_status_reg(instance);4935	abs_state = status_reg & MFI_STATE_MASK;4936 4937	/* IO timeout detected, forcibly put FW in FAULT state */4938	if (abs_state != MFI_STATE_FAULT && instance->crash_dump_buf &&4939		instance->crash_dump_app_support && reason) {4940		dev_info(&instance->pdev->dev, "IO/DCMD timeout is detected, "4941			"forcibly FAULT Firmware\n");4942		atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);4943		status_reg = megasas_readl(instance, &instance->reg_set->doorbell);4944		writel(status_reg | MFI_STATE_FORCE_OCR,4945			&instance->reg_set->doorbell);4946		readl(&instance->reg_set->doorbell);4947		mutex_unlock(&instance->reset_mutex);4948		do {4949			ssleep(3);4950			io_timeout_in_crash_mode++;4951			dev_dbg(&instance->pdev->dev, "waiting for [%d] "4952				"seconds for crash dump collection and OCR "4953				"to be done\n", (io_timeout_in_crash_mode * 3));4954		} while ((atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) &&4955			(io_timeout_in_crash_mode < 80));4956 4957		if (atomic_read(&instance->adprecovery) == MEGASAS_HBA_OPERATIONAL) {4958			dev_info(&instance->pdev->dev, "OCR done for IO "4959				"timeout case\n");4960			retval = SUCCESS;4961		} else {4962			dev_info(&instance->pdev->dev, "Controller is not "4963				"operational after 240 seconds wait for IO "4964				"timeout case in FW crash dump mode\n do "4965				"OCR/kill adapter\n");4966			retval = megasas_reset_fusion(shost, 0);4967		}4968		return retval;4969	}4970 4971	if (instance->requestorId && !instance->skip_heartbeat_timer_del)4972		del_timer_sync(&instance->sriov_heartbeat_timer);4973	set_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);4974	set_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags);4975	atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_POLLING);4976	instance->instancet->disable_intr(instance);4977	megasas_sync_irqs((unsigned long)instance);4978 4979	/* First try waiting for commands to complete */4980	if (megasas_wait_for_outstanding_fusion(instance, reason,4981						&convert)) {4982		atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);4983		dev_warn(&instance->pdev->dev, "resetting fusion "4984		       "adapter scsi%d.\n", instance->host->host_no);4985		if (convert)4986			reason = 0;4987 4988		if (megasas_dbg_lvl & OCR_DEBUG)4989			dev_info(&instance->pdev->dev, "\nPending SCSI commands:\n");4990 4991		/* Now return commands back to the OS */4992		for (i = 0 ; i < instance->max_scsi_cmds; i++) {4993			cmd_fusion = fusion->cmd_list[i];4994			/*check for extra commands issued by driver*/4995			if (instance->adapter_type >= VENTURA_SERIES) {4996				r1_cmd = fusion->cmd_list[i + instance->max_fw_cmds];4997				megasas_return_cmd_fusion(instance, r1_cmd);4998			}4999			scmd_local = cmd_fusion->scmd;5000			if (cmd_fusion->scmd) {5001				if (megasas_dbg_lvl & OCR_DEBUG) {5002					sdev_printk(KERN_INFO,5003						cmd_fusion->scmd->device, "SMID: 0x%x\n",5004						cmd_fusion->index);5005					megasas_dump_fusion_io(cmd_fusion->scmd);5006				}5007 5008				if (cmd_fusion->io_request->Function ==5009					MPI2_FUNCTION_SCSI_IO_REQUEST)5010					fpio_count++;5011 5012				scmd_local->result =5013					megasas_check_mpio_paths(instance,5014							scmd_local);5015				if (instance->ldio_threshold &&5016					megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)5017					atomic_dec(&instance->ldio_outstanding);5018				megasas_return_cmd_fusion(instance, cmd_fusion);5019				scsi_dma_unmap(scmd_local);5020				scsi_done(scmd_local);5021			}5022		}5023 5024		dev_info(&instance->pdev->dev, "Outstanding fastpath IOs: %d\n",5025			fpio_count);5026 5027		atomic_set(&instance->fw_outstanding, 0);5028 5029		status_reg = instance->instancet->read_fw_status_reg(instance);5030		abs_state = status_reg & MFI_STATE_MASK;5031		reset_adapter = status_reg & MFI_RESET_ADAPTER;5032		if (instance->disableOnlineCtrlReset ||5033		    (abs_state == MFI_STATE_FAULT && !reset_adapter)) {5034			/* Reset not supported, kill adapter */5035			dev_warn(&instance->pdev->dev, "Reset not supported"5036			       ", killing adapter scsi%d.\n",5037				instance->host->host_no);5038			goto kill_hba;5039		}5040 5041		/* Let SR-IOV VF & PF sync up if there was a HB failure */5042		if (instance->requestorId && !reason) {5043			msleep(MEGASAS_OCR_SETTLE_TIME_VF);5044			do_adp_reset = false;5045			max_reset_tries = MEGASAS_SRIOV_MAX_RESET_TRIES_VF;5046		}5047 5048		/* Now try to reset the chip */5049		for (i = 0; i < max_reset_tries; i++) {5050			/*5051			 * Do adp reset and wait for5052			 * controller to transition to ready5053			 */5054			if (megasas_adp_reset_wait_for_ready(instance,5055				do_adp_reset, 1) == FAILED)5056				continue;5057 5058			/* Wait for FW to become ready */5059			if (megasas_transition_to_ready(instance, 1)) {5060				dev_warn(&instance->pdev->dev,5061					"Failed to transition controller to ready for "5062					"scsi%d.\n", instance->host->host_no);5063				continue;5064			}5065			megasas_reset_reply_desc(instance);5066			megasas_fusion_update_can_queue(instance, OCR_CONTEXT);5067 5068			if (megasas_ioc_init_fusion(instance)) {5069				continue;5070			}5071 5072			if (megasas_get_ctrl_info(instance)) {5073				dev_info(&instance->pdev->dev,5074					"Failed from %s %d\n",5075					__func__, __LINE__);5076				goto kill_hba;5077			}5078 5079			megasas_refire_mgmt_cmd(instance,5080						(i == (MEGASAS_FUSION_MAX_RESET_TRIES - 1)5081							? 1 : 0));5082 5083			/* Reset load balance info */5084			if (fusion->load_balance_info)5085				memset(fusion->load_balance_info, 0,5086				       (sizeof(struct LD_LOAD_BALANCE_INFO) *5087				       MAX_LOGICAL_DRIVES_EXT));5088 5089			if (!megasas_get_map_info(instance)) {5090				megasas_sync_map_info(instance);5091			} else {5092				/*5093				 * Return pending polled mode cmds before5094				 * retrying OCR5095				 */5096				megasas_return_polled_cmds(instance);5097				continue;5098			}5099 5100			megasas_setup_jbod_map(instance);5101 5102			/* reset stream detection array */5103			if (instance->adapter_type >= VENTURA_SERIES) {5104				for (j = 0; j < MAX_LOGICAL_DRIVES_EXT; ++j) {5105					memset(fusion->stream_detect_by_ld[j],5106					       0, sizeof(struct LD_STREAM_DETECT));5107					fusion->stream_detect_by_ld[j]->mru_bit_map5108						= MR_STREAM_BITMAP;5109				}5110			}5111 5112			clear_bit(MEGASAS_FUSION_IN_RESET,5113				  &instance->reset_flags);5114			instance->instancet->enable_intr(instance);5115			megasas_enable_irq_poll(instance);5116			shost_for_each_device(sdev, shost) {5117				if ((instance->tgt_prop) &&5118				    (instance->nvme_page_size))5119					ret_target_prop = megasas_get_target_prop(instance, sdev);5120 5121				is_target_prop = (ret_target_prop == DCMD_SUCCESS) ? true : false;5122				megasas_set_dynamic_target_properties(sdev, NULL,5123						is_target_prop);5124			}5125 5126			status_reg = instance->instancet->read_fw_status_reg5127					(instance);5128			abs_state = status_reg & MFI_STATE_MASK;5129			if (abs_state != MFI_STATE_OPERATIONAL) {5130				dev_info(&instance->pdev->dev,5131					 "Adapter is not OPERATIONAL, state 0x%x for scsi:%d\n",5132					 abs_state, instance->host->host_no);5133				goto out;5134			}5135			atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);5136 5137			dev_info(&instance->pdev->dev,5138				 "Adapter is OPERATIONAL for scsi:%d\n",5139				 instance->host->host_no);5140 5141			/* Restart SR-IOV heartbeat */5142			if (instance->requestorId) {5143				if (!megasas_sriov_start_heartbeat(instance, 0))5144					megasas_start_timer(instance);5145				else5146					instance->skip_heartbeat_timer_del = 1;5147			}5148 5149			if (instance->crash_dump_drv_support &&5150				instance->crash_dump_app_support)5151				megasas_set_crash_dump_params(instance,5152					MR_CRASH_BUF_TURN_ON);5153			else5154				megasas_set_crash_dump_params(instance,5155					MR_CRASH_BUF_TURN_OFF);5156 5157			if (instance->snapdump_wait_time) {5158				megasas_get_snapdump_properties(instance);5159				dev_info(&instance->pdev->dev,5160					 "Snap dump wait time\t: %d\n",5161					 instance->snapdump_wait_time);5162			}5163 5164			retval = SUCCESS;5165 5166			/* Adapter reset completed successfully */5167			dev_warn(&instance->pdev->dev,5168				 "Reset successful for scsi%d.\n",5169				 instance->host->host_no);5170 5171			goto out;5172		}5173		/* Reset failed, kill the adapter */5174		dev_warn(&instance->pdev->dev, "Reset failed, killing "5175		       "adapter scsi%d.\n", instance->host->host_no);5176		goto kill_hba;5177	} else {5178		/* For VF: Restart HB timer if we didn't OCR */5179		if (instance->requestorId) {5180			megasas_start_timer(instance);5181		}5182		clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);5183		instance->instancet->enable_intr(instance);5184		megasas_enable_irq_poll(instance);5185		atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);5186		goto out;5187	}5188kill_hba:5189	megaraid_sas_kill_hba(instance);5190	megasas_enable_irq_poll(instance);5191	instance->skip_heartbeat_timer_del = 1;5192	retval = FAILED;5193out:5194	clear_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags);5195	mutex_unlock(&instance->reset_mutex);5196	return retval;5197}5198 5199/* Fusion Crash dump collection */5200static void  megasas_fusion_crash_dump(struct megasas_instance *instance)5201{5202	u32 status_reg;5203	u8 partial_copy = 0;5204	int wait = 0;5205 5206 5207	status_reg = instance->instancet->read_fw_status_reg(instance);5208 5209	/*5210	 * Allocate host crash buffers to copy data from 1 MB DMA crash buffer5211	 * to host crash buffers5212	 */5213	if (instance->drv_buf_index == 0) {5214		/* Buffer is already allocated for old Crash dump.5215		 * Do OCR and do not wait for crash dump collection5216		 */5217		if (instance->drv_buf_alloc) {5218			dev_info(&instance->pdev->dev, "earlier crash dump is "5219				"not yet copied by application, ignoring this "5220				"crash dump and initiating OCR\n");5221			status_reg |= MFI_STATE_CRASH_DUMP_DONE;5222			writel(status_reg,5223				&instance->reg_set->outbound_scratch_pad_0);5224			readl(&instance->reg_set->outbound_scratch_pad_0);5225			return;5226		}5227		megasas_alloc_host_crash_buffer(instance);5228		dev_info(&instance->pdev->dev, "Number of host crash buffers "5229			"allocated: %d\n", instance->drv_buf_alloc);5230	}5231 5232	while (!(status_reg & MFI_STATE_CRASH_DUMP_DONE) &&5233	       (wait < MEGASAS_WATCHDOG_WAIT_COUNT)) {5234		if (!(status_reg & MFI_STATE_DMADONE)) {5235			/*5236			 * Next crash dump buffer is not yet DMA'd by FW5237			 * Check after 10ms. Wait for 1 second for FW to5238			 * post the next buffer. If not bail out.5239			 */5240			wait++;5241			msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);5242			status_reg = instance->instancet->read_fw_status_reg(5243					instance);5244			continue;5245		}5246 5247		wait = 0;5248		if (instance->drv_buf_index >= instance->drv_buf_alloc) {5249			dev_info(&instance->pdev->dev,5250				 "Driver is done copying the buffer: %d\n",5251				 instance->drv_buf_alloc);5252			status_reg |= MFI_STATE_CRASH_DUMP_DONE;5253			partial_copy = 1;5254			break;5255		} else {5256			memcpy(instance->crash_buf[instance->drv_buf_index],5257			       instance->crash_dump_buf, CRASH_DMA_BUF_SIZE);5258			instance->drv_buf_index++;5259			status_reg &= ~MFI_STATE_DMADONE;5260		}5261 5262		writel(status_reg, &instance->reg_set->outbound_scratch_pad_0);5263		readl(&instance->reg_set->outbound_scratch_pad_0);5264 5265		msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);5266		status_reg = instance->instancet->read_fw_status_reg(instance);5267	}5268 5269	if (status_reg & MFI_STATE_CRASH_DUMP_DONE) {5270		dev_info(&instance->pdev->dev, "Crash Dump is available,number "5271			"of copied buffers: %d\n", instance->drv_buf_index);5272		instance->fw_crash_buffer_size =  instance->drv_buf_index;5273		instance->fw_crash_state = AVAILABLE;5274		instance->drv_buf_index = 0;5275		writel(status_reg, &instance->reg_set->outbound_scratch_pad_0);5276		readl(&instance->reg_set->outbound_scratch_pad_0);5277		if (!partial_copy)5278			megasas_reset_fusion(instance->host, 0);5279	}5280}5281 5282 5283/* Fusion OCR work queue */5284void megasas_fusion_ocr_wq(struct work_struct *work)5285{5286	struct megasas_instance *instance =5287		container_of(work, struct megasas_instance, work_init);5288 5289	megasas_reset_fusion(instance->host, 0);5290}5291 5292/* Allocate fusion context */5293int5294megasas_alloc_fusion_context(struct megasas_instance *instance)5295{5296	struct fusion_context *fusion;5297 5298	instance->ctrl_context = kzalloc(sizeof(struct fusion_context),5299					 GFP_KERNEL);5300	if (!instance->ctrl_context) {5301		dev_err(&instance->pdev->dev, "Failed from %s %d\n",5302			__func__, __LINE__);5303		return -ENOMEM;5304	}5305 5306	fusion = instance->ctrl_context;5307 5308	fusion->log_to_span_pages = get_order(MAX_LOGICAL_DRIVES_EXT *5309					      sizeof(LD_SPAN_INFO));5310	fusion->log_to_span =5311		(PLD_SPAN_INFO)__get_free_pages(GFP_KERNEL | __GFP_ZERO,5312						fusion->log_to_span_pages);5313	if (!fusion->log_to_span) {5314		fusion->log_to_span =5315			vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,5316					   sizeof(LD_SPAN_INFO)));5317		if (!fusion->log_to_span) {5318			dev_err(&instance->pdev->dev, "Failed from %s %d\n",5319				__func__, __LINE__);5320			return -ENOMEM;5321		}5322	}5323 5324	fusion->load_balance_info_pages = get_order(MAX_LOGICAL_DRIVES_EXT *5325		sizeof(struct LD_LOAD_BALANCE_INFO));5326	fusion->load_balance_info =5327		(struct LD_LOAD_BALANCE_INFO *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,5328		fusion->load_balance_info_pages);5329	if (!fusion->load_balance_info) {5330		fusion->load_balance_info =5331			vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,5332					   sizeof(struct LD_LOAD_BALANCE_INFO)));5333		if (!fusion->load_balance_info)5334			dev_err(&instance->pdev->dev, "Failed to allocate load_balance_info, "5335				"continuing without Load Balance support\n");5336	}5337 5338	return 0;5339}5340 5341void5342megasas_free_fusion_context(struct megasas_instance *instance)5343{5344	struct fusion_context *fusion = instance->ctrl_context;5345 5346	if (fusion) {5347		if (fusion->load_balance_info) {5348			if (is_vmalloc_addr(fusion->load_balance_info))5349				vfree(fusion->load_balance_info);5350			else5351				free_pages((ulong)fusion->load_balance_info,5352					fusion->load_balance_info_pages);5353		}5354 5355		if (fusion->log_to_span) {5356			if (is_vmalloc_addr(fusion->log_to_span))5357				vfree(fusion->log_to_span);5358			else5359				free_pages((ulong)fusion->log_to_span,5360					   fusion->log_to_span_pages);5361		}5362 5363		kfree(fusion);5364	}5365}5366 5367struct megasas_instance_template megasas_instance_template_fusion = {5368	.enable_intr = megasas_enable_intr_fusion,5369	.disable_intr = megasas_disable_intr_fusion,5370	.clear_intr = megasas_clear_intr_fusion,5371	.read_fw_status_reg = megasas_read_fw_status_reg_fusion,5372	.adp_reset = megasas_adp_reset_fusion,5373	.check_reset = megasas_check_reset_fusion,5374	.service_isr = megasas_isr_fusion,5375	.tasklet = megasas_complete_cmd_dpc_fusion,5376	.init_adapter = megasas_init_adapter_fusion,5377	.build_and_issue_cmd = megasas_build_and_issue_cmd_fusion,5378	.issue_dcmd = megasas_issue_dcmd_fusion,5379};5380