brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/* Copyright (c) 2019 HiSilicon Limited. */3#include <asm/page.h>4#include <linux/acpi.h>5#include <linux/bitmap.h>6#include <linux/dma-mapping.h>7#include <linux/idr.h>8#include <linux/io.h>9#include <linux/irqreturn.h>10#include <linux/log2.h>11#include <linux/pm_runtime.h>12#include <linux/seq_file.h>13#include <linux/slab.h>14#include <linux/uacce.h>15#include <linux/uaccess.h>16#include <uapi/misc/uacce/hisi_qm.h>17#include <linux/hisi_acc_qm.h>18#include "qm_common.h"19 20/* eq/aeq irq enable */21#define QM_VF_AEQ_INT_SOURCE		0x022#define QM_VF_AEQ_INT_MASK		0x423#define QM_VF_EQ_INT_SOURCE		0x824#define QM_VF_EQ_INT_MASK		0xc25 26#define QM_IRQ_VECTOR_MASK		GENMASK(15, 0)27#define QM_IRQ_TYPE_MASK		GENMASK(15, 0)28#define QM_IRQ_TYPE_SHIFT		1629#define QM_ABN_IRQ_TYPE_MASK		GENMASK(7, 0)30 31/* mailbox */32#define QM_MB_PING_ALL_VFS		0xffff33#define QM_MB_CMD_DATA_SHIFT		3234#define QM_MB_CMD_DATA_MASK		GENMASK(31, 0)35#define QM_MB_STATUS_MASK		GENMASK(12, 9)36 37/* sqc shift */38#define QM_SQ_HOP_NUM_SHIFT		039#define QM_SQ_PAGE_SIZE_SHIFT		440#define QM_SQ_BUF_SIZE_SHIFT		841#define QM_SQ_SQE_SIZE_SHIFT		1242#define QM_SQ_PRIORITY_SHIFT		043#define QM_SQ_ORDERS_SHIFT		444#define QM_SQ_TYPE_SHIFT		845#define QM_QC_PASID_ENABLE		0x146#define QM_QC_PASID_ENABLE_SHIFT	747 48#define QM_SQ_TYPE_MASK			GENMASK(3, 0)49#define QM_SQ_TAIL_IDX(sqc)		((le16_to_cpu((sqc).w11) >> 6) & 0x1)50 51/* cqc shift */52#define QM_CQ_HOP_NUM_SHIFT		053#define QM_CQ_PAGE_SIZE_SHIFT		454#define QM_CQ_BUF_SIZE_SHIFT		855#define QM_CQ_CQE_SIZE_SHIFT		1256#define QM_CQ_PHASE_SHIFT		057#define QM_CQ_FLAG_SHIFT		158 59#define QM_CQE_PHASE(cqe)		(le16_to_cpu((cqe)->w7) & 0x1)60#define QM_QC_CQE_SIZE			461#define QM_CQ_TAIL_IDX(cqc)		((le16_to_cpu((cqc).w11) >> 6) & 0x1)62 63/* eqc shift */64#define QM_EQE_AEQE_SIZE		(2UL << 12)65#define QM_EQC_PHASE_SHIFT		1666 67#define QM_EQE_PHASE(eqe)		((le32_to_cpu((eqe)->dw0) >> 16) & 0x1)68#define QM_EQE_CQN_MASK			GENMASK(15, 0)69 70#define QM_AEQE_PHASE(aeqe)		((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)71#define QM_AEQE_TYPE_SHIFT		1772#define QM_AEQE_TYPE_MASK		0xf73#define QM_AEQE_CQN_MASK		GENMASK(15, 0)74#define QM_CQ_OVERFLOW			075#define QM_EQ_OVERFLOW			176#define QM_CQE_ERROR			277 78#define QM_XQ_DEPTH_SHIFT		1679#define QM_XQ_DEPTH_MASK		GENMASK(15, 0)80 81#define QM_DOORBELL_CMD_SQ		082#define QM_DOORBELL_CMD_CQ		183#define QM_DOORBELL_CMD_EQ		284#define QM_DOORBELL_CMD_AEQ		385 86#define QM_DOORBELL_BASE_V1		0x34087#define QM_DB_CMD_SHIFT_V1		1688#define QM_DB_INDEX_SHIFT_V1		3289#define QM_DB_PRIORITY_SHIFT_V1		4890#define QM_PAGE_SIZE			0x003491#define QM_QP_DB_INTERVAL		0x1000092#define QM_DB_TIMEOUT_CFG		0x10007493#define QM_DB_TIMEOUT_SET		0x1fffff94 95#define QM_MEM_START_INIT		0x10004096#define QM_MEM_INIT_DONE		0x10004497#define QM_VFT_CFG_RDY			0x10006c98#define QM_VFT_CFG_OP_WR		0x10005899#define QM_VFT_CFG_TYPE			0x10005c100#define QM_VFT_CFG			0x100060101#define QM_VFT_CFG_OP_ENABLE		0x100054102#define QM_PM_CTRL			0x100148103#define QM_IDLE_DISABLE			BIT(9)104 105#define QM_VFT_CFG_DATA_L		0x100064106#define QM_VFT_CFG_DATA_H		0x100068107#define QM_SQC_VFT_BUF_SIZE		(7ULL << 8)108#define QM_SQC_VFT_SQC_SIZE		(5ULL << 12)109#define QM_SQC_VFT_INDEX_NUMBER		(1ULL << 16)110#define QM_SQC_VFT_START_SQN_SHIFT	28111#define QM_SQC_VFT_VALID		(1ULL << 44)112#define QM_SQC_VFT_SQN_SHIFT		45113#define QM_CQC_VFT_BUF_SIZE		(7ULL << 8)114#define QM_CQC_VFT_SQC_SIZE		(5ULL << 12)115#define QM_CQC_VFT_INDEX_NUMBER		(1ULL << 16)116#define QM_CQC_VFT_VALID		(1ULL << 28)117 118#define QM_SQC_VFT_BASE_SHIFT_V2	28119#define QM_SQC_VFT_BASE_MASK_V2		GENMASK(15, 0)120#define QM_SQC_VFT_NUM_SHIFT_V2		45121#define QM_SQC_VFT_NUM_MASK_V2		GENMASK(9, 0)122 123#define QM_ABNORMAL_INT_SOURCE		0x100000124#define QM_ABNORMAL_INT_MASK		0x100004125#define QM_ABNORMAL_INT_MASK_VALUE	0x7fff126#define QM_ABNORMAL_INT_STATUS		0x100008127#define QM_ABNORMAL_INT_SET		0x10000c128#define QM_ABNORMAL_INF00		0x100010129#define QM_FIFO_OVERFLOW_TYPE		0xc0130#define QM_FIFO_OVERFLOW_TYPE_SHIFT	6131#define QM_FIFO_OVERFLOW_VF		0x3f132#define QM_FIFO_OVERFLOW_QP_SHIFT	16133#define QM_ABNORMAL_INF01		0x100014134#define QM_DB_TIMEOUT_TYPE		0xc0135#define QM_DB_TIMEOUT_TYPE_SHIFT	6136#define QM_DB_TIMEOUT_VF		0x3f137#define QM_DB_TIMEOUT_QP_SHIFT		16138#define QM_ABNORMAL_INF02		0x100018139#define QM_AXI_POISON_ERR		BIT(22)140#define QM_RAS_CE_ENABLE		0x1000ec141#define QM_RAS_FE_ENABLE		0x1000f0142#define QM_RAS_NFE_ENABLE		0x1000f4143#define QM_RAS_CE_THRESHOLD		0x1000f8144#define QM_RAS_CE_TIMES_PER_IRQ		1145#define QM_OOO_SHUTDOWN_SEL		0x1040f8146#define QM_AXI_RRESP_ERR		BIT(0)147#define QM_ECC_MBIT			BIT(2)148#define QM_DB_TIMEOUT			BIT(10)149#define QM_OF_FIFO_OF			BIT(11)150 151#define QM_RESET_WAIT_TIMEOUT		400152#define QM_PEH_VENDOR_ID		0x1000d8153#define ACC_VENDOR_ID_VALUE		0x5a5a154#define QM_PEH_DFX_INFO0		0x1000fc155#define QM_PEH_DFX_INFO1		0x100100156#define QM_PEH_DFX_MASK			(BIT(0) | BIT(2))157#define QM_PEH_MSI_FINISH_MASK		GENMASK(19, 16)158#define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT	3159#define ACC_PEH_MSI_DISABLE		GENMASK(31, 0)160#define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN	0x1161#define ACC_MASTER_TRANS_RETURN_RW	3162#define ACC_MASTER_TRANS_RETURN		0x300150163#define ACC_MASTER_GLOBAL_CTRL		0x300000164#define ACC_AM_CFG_PORT_WR_EN		0x30001c165#define QM_RAS_NFE_MBIT_DISABLE		~QM_ECC_MBIT166#define ACC_AM_ROB_ECC_INT_STS		0x300104167#define ACC_ROB_ECC_ERR_MULTPL		BIT(1)168#define QM_MSI_CAP_ENABLE		BIT(16)169 170/* interfunction communication */171#define QM_IFC_READY_STATUS		0x100128172#define QM_IFC_INT_SET_P		0x100130173#define QM_IFC_INT_CFG			0x100134174#define QM_IFC_INT_SOURCE_P		0x100138175#define QM_IFC_INT_SOURCE_V		0x0020176#define QM_IFC_INT_MASK			0x0024177#define QM_IFC_INT_STATUS		0x0028178#define QM_IFC_INT_SET_V		0x002C179#define QM_IFC_SEND_ALL_VFS		GENMASK(6, 0)180#define QM_IFC_INT_SOURCE_CLR		GENMASK(63, 0)181#define QM_IFC_INT_SOURCE_MASK		BIT(0)182#define QM_IFC_INT_DISABLE		BIT(0)183#define QM_IFC_INT_STATUS_MASK		BIT(0)184#define QM_IFC_INT_SET_MASK		BIT(0)185#define QM_WAIT_DST_ACK			10186#define QM_MAX_PF_WAIT_COUNT		10187#define QM_MAX_VF_WAIT_COUNT		40188#define QM_VF_RESET_WAIT_US            20000189#define QM_VF_RESET_WAIT_CNT           3000190#define QM_VF_RESET_WAIT_TIMEOUT_US    \191	(QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT)192 193#define POLL_PERIOD			10194#define POLL_TIMEOUT			1000195#define WAIT_PERIOD_US_MAX		200196#define WAIT_PERIOD_US_MIN		100197#define MAX_WAIT_COUNTS			1000198#define QM_CACHE_WB_START		0x204199#define QM_CACHE_WB_DONE		0x208200#define QM_FUNC_CAPS_REG		0x3100201#define QM_CAPBILITY_VERSION		GENMASK(7, 0)202 203#define PCI_BAR_2			2204#define PCI_BAR_4			4205#define QMC_ALIGN(sz)			ALIGN(sz, 32)206 207#define QM_DBG_READ_LEN		256208#define QM_PCI_COMMAND_INVALID		~0209#define QM_RESET_STOP_TX_OFFSET		1210#define QM_RESET_STOP_RX_OFFSET		2211 212#define WAIT_PERIOD			20213#define REMOVE_WAIT_DELAY		10214 215#define QM_QOS_PARAM_NUM		2216#define QM_QOS_MAX_VAL			1000217#define QM_QOS_RATE			100218#define QM_QOS_EXPAND_RATE		1000219#define QM_SHAPER_CIR_B_MASK		GENMASK(7, 0)220#define QM_SHAPER_CIR_U_MASK		GENMASK(10, 8)221#define QM_SHAPER_CIR_S_MASK		GENMASK(14, 11)222#define QM_SHAPER_FACTOR_CIR_U_SHIFT	8223#define QM_SHAPER_FACTOR_CIR_S_SHIFT	11224#define QM_SHAPER_FACTOR_CBS_B_SHIFT	15225#define QM_SHAPER_FACTOR_CBS_S_SHIFT	19226#define QM_SHAPER_CBS_B			1227#define QM_SHAPER_VFT_OFFSET		6228#define QM_QOS_MIN_ERROR_RATE		5229#define QM_SHAPER_MIN_CBS_S		8230#define QM_QOS_TICK			0x300U231#define QM_QOS_DIVISOR_CLK		0x1f40U232#define QM_QOS_MAX_CIR_B		200233#define QM_QOS_MIN_CIR_B		100234#define QM_QOS_MAX_CIR_U		6235#define QM_AUTOSUSPEND_DELAY		3000236 237#define QM_DEV_ALG_MAX_LEN		256238 239 /* abnormal status value for stopping queue */240#define QM_STOP_QUEUE_FAIL		1241#define	QM_DUMP_SQC_FAIL		3242#define	QM_DUMP_CQC_FAIL		4243#define	QM_FINISH_WAIT			5244 245#define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \246	(((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \247	((pg_sz) << QM_CQ_PAGE_SIZE_SHIFT) | \248	((buf_sz) << QM_CQ_BUF_SIZE_SHIFT) | \249	((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))250 251#define QM_MK_CQC_DW3_V2(cqe_sz, cq_depth) \252	((((u32)cq_depth) - 1) | ((cqe_sz) << QM_CQ_CQE_SIZE_SHIFT))253 254#define QM_MK_SQC_W13(priority, orders, alg_type) \255	(((priority) << QM_SQ_PRIORITY_SHIFT) | \256	((orders) << QM_SQ_ORDERS_SHIFT) | \257	(((alg_type) & QM_SQ_TYPE_MASK) << QM_SQ_TYPE_SHIFT))258 259#define QM_MK_SQC_DW3_V1(hop_num, pg_sz, buf_sz, sqe_sz) \260	(((hop_num) << QM_SQ_HOP_NUM_SHIFT) | \261	((pg_sz) << QM_SQ_PAGE_SIZE_SHIFT) | \262	((buf_sz) << QM_SQ_BUF_SIZE_SHIFT) | \263	((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))264 265#define QM_MK_SQC_DW3_V2(sqe_sz, sq_depth) \266	((((u32)sq_depth) - 1) | ((u32)ilog2(sqe_sz) << QM_SQ_SQE_SIZE_SHIFT))267 268enum vft_type {269	SQC_VFT = 0,270	CQC_VFT,271	SHAPER_VFT,272};273 274enum acc_err_result {275	ACC_ERR_NONE,276	ACC_ERR_NEED_RESET,277	ACC_ERR_RECOVERED,278};279 280enum qm_alg_type {281	ALG_TYPE_0,282	ALG_TYPE_1,283};284 285enum qm_mb_cmd {286	QM_PF_FLR_PREPARE = 0x01,287	QM_PF_SRST_PREPARE,288	QM_PF_RESET_DONE,289	QM_VF_PREPARE_DONE,290	QM_VF_PREPARE_FAIL,291	QM_VF_START_DONE,292	QM_VF_START_FAIL,293	QM_PF_SET_QOS,294	QM_VF_GET_QOS,295};296 297enum qm_basic_type {298	QM_TOTAL_QP_NUM_CAP = 0x0,299	QM_FUNC_MAX_QP_CAP,300	QM_XEQ_DEPTH_CAP,301	QM_QP_DEPTH_CAP,302	QM_EQ_IRQ_TYPE_CAP,303	QM_AEQ_IRQ_TYPE_CAP,304	QM_ABN_IRQ_TYPE_CAP,305	QM_PF2VF_IRQ_TYPE_CAP,306	QM_PF_IRQ_NUM_CAP,307	QM_VF_IRQ_NUM_CAP,308};309 310enum qm_pre_store_cap_idx {311	QM_EQ_IRQ_TYPE_CAP_IDX = 0x0,312	QM_AEQ_IRQ_TYPE_CAP_IDX,313	QM_ABN_IRQ_TYPE_CAP_IDX,314	QM_PF2VF_IRQ_TYPE_CAP_IDX,315};316 317static const struct hisi_qm_cap_info qm_cap_info_comm[] = {318	{QM_SUPPORT_DB_ISOLATION, 0x30,   0, BIT(0),  0x0, 0x0, 0x0},319	{QM_SUPPORT_FUNC_QOS,     0x3100, 0, BIT(8),  0x0, 0x0, 0x1},320	{QM_SUPPORT_STOP_QP,      0x3100, 0, BIT(9),  0x0, 0x0, 0x1},321	{QM_SUPPORT_STOP_FUNC,     0x3100, 0, BIT(10), 0x0, 0x0, 0x1},322	{QM_SUPPORT_MB_COMMAND,   0x3100, 0, BIT(11), 0x0, 0x0, 0x1},323	{QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1},324};325 326static const struct hisi_qm_cap_info qm_cap_info_pf[] = {327	{QM_SUPPORT_RPM, 0x3100, 0, BIT(13), 0x0, 0x0, 0x1},328};329 330static const struct hisi_qm_cap_info qm_cap_info_vf[] = {331	{QM_SUPPORT_RPM, 0x3100, 0, BIT(12), 0x0, 0x0, 0x0},332};333 334static const struct hisi_qm_cap_info qm_basic_info[] = {335	{QM_TOTAL_QP_NUM_CAP,   0x100158, 0,  GENMASK(10, 0), 0x1000,    0x400,     0x400},336	{QM_FUNC_MAX_QP_CAP,    0x100158, 11, GENMASK(10, 0), 0x1000,    0x400,     0x400},337	{QM_XEQ_DEPTH_CAP,      0x3104,   0,  GENMASK(31, 0), 0x800,     0x4000800, 0x4000800},338	{QM_QP_DEPTH_CAP,       0x3108,   0,  GENMASK(31, 0), 0x4000400, 0x4000400, 0x4000400},339	{QM_EQ_IRQ_TYPE_CAP,    0x310c,   0,  GENMASK(31, 0), 0x10000,   0x10000,   0x10000},340	{QM_AEQ_IRQ_TYPE_CAP,   0x3110,   0,  GENMASK(31, 0), 0x0,       0x10001,   0x10001},341	{QM_ABN_IRQ_TYPE_CAP,   0x3114,   0,  GENMASK(31, 0), 0x0,       0x10003,   0x10003},342	{QM_PF2VF_IRQ_TYPE_CAP, 0x3118,   0,  GENMASK(31, 0), 0x0,       0x0,       0x10002},343	{QM_PF_IRQ_NUM_CAP,     0x311c,   16, GENMASK(15, 0), 0x1,       0x4,       0x4},344	{QM_VF_IRQ_NUM_CAP,     0x311c,   0,  GENMASK(15, 0), 0x1,       0x2,       0x3},345};346 347static const u32 qm_pre_store_caps[] = {348	QM_EQ_IRQ_TYPE_CAP,349	QM_AEQ_IRQ_TYPE_CAP,350	QM_ABN_IRQ_TYPE_CAP,351	QM_PF2VF_IRQ_TYPE_CAP,352};353 354struct qm_mailbox {355	__le16 w0;356	__le16 queue_num;357	__le32 base_l;358	__le32 base_h;359	__le32 rsvd;360};361 362struct qm_doorbell {363	__le16 queue_num;364	__le16 cmd;365	__le16 index;366	__le16 priority;367};368 369struct hisi_qm_resource {370	struct hisi_qm *qm;371	int distance;372	struct list_head list;373};374 375/**376 * struct qm_hw_err - Structure describing the device errors377 * @list: hardware error list378 * @timestamp: timestamp when the error occurred379 */380struct qm_hw_err {381	struct list_head list;382	unsigned long long timestamp;383};384 385struct hisi_qm_hw_ops {386	int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);387	void (*qm_db)(struct hisi_qm *qm, u16 qn,388		      u8 cmd, u16 index, u8 priority);389	int (*debug_init)(struct hisi_qm *qm);390	void (*hw_error_init)(struct hisi_qm *qm);391	void (*hw_error_uninit)(struct hisi_qm *qm);392	enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);393	int (*set_msi)(struct hisi_qm *qm, bool set);394};395 396struct hisi_qm_hw_error {397	u32 int_msk;398	const char *msg;399};400 401static const struct hisi_qm_hw_error qm_hw_error[] = {402	{ .int_msk = BIT(0), .msg = "qm_axi_rresp" },403	{ .int_msk = BIT(1), .msg = "qm_axi_bresp" },404	{ .int_msk = BIT(2), .msg = "qm_ecc_mbit" },405	{ .int_msk = BIT(3), .msg = "qm_ecc_1bit" },406	{ .int_msk = BIT(4), .msg = "qm_acc_get_task_timeout" },407	{ .int_msk = BIT(5), .msg = "qm_acc_do_task_timeout" },408	{ .int_msk = BIT(6), .msg = "qm_acc_wb_not_ready_timeout" },409	{ .int_msk = BIT(7), .msg = "qm_sq_cq_vf_invalid" },410	{ .int_msk = BIT(8), .msg = "qm_cq_vf_invalid" },411	{ .int_msk = BIT(9), .msg = "qm_sq_vf_invalid" },412	{ .int_msk = BIT(10), .msg = "qm_db_timeout" },413	{ .int_msk = BIT(11), .msg = "qm_of_fifo_of" },414	{ .int_msk = BIT(12), .msg = "qm_db_random_invalid" },415	{ .int_msk = BIT(13), .msg = "qm_mailbox_timeout" },416	{ .int_msk = BIT(14), .msg = "qm_flr_timeout" },417};418 419static const char * const qm_db_timeout[] = {420	"sq", "cq", "eq", "aeq",421};422 423static const char * const qm_fifo_overflow[] = {424	"cq", "eq", "aeq",425};426 427struct qm_typical_qos_table {428	u32 start;429	u32 end;430	u32 val;431};432 433/* the qos step is 100 */434static struct qm_typical_qos_table shaper_cir_s[] = {435	{100, 100, 4},436	{200, 200, 3},437	{300, 500, 2},438	{600, 1000, 1},439	{1100, 100000, 0},440};441 442static struct qm_typical_qos_table shaper_cbs_s[] = {443	{100, 200, 9},444	{300, 500, 11},445	{600, 1000, 12},446	{1100, 10000, 16},447	{10100, 25000, 17},448	{25100, 50000, 18},449	{50100, 100000, 19}450};451 452static void qm_irqs_unregister(struct hisi_qm *qm);453static int qm_reset_device(struct hisi_qm *qm);454 455static u32 qm_get_hw_error_status(struct hisi_qm *qm)456{457	return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);458}459 460static u32 qm_get_dev_err_status(struct hisi_qm *qm)461{462	return qm->err_ini->get_dev_hw_err_status(qm);463}464 465/* Check if the error causes the master ooo block */466static bool qm_check_dev_error(struct hisi_qm *qm)467{468	u32 val, dev_val;469 470	if (qm->fun_type == QM_HW_VF)471		return false;472 473	val = qm_get_hw_error_status(qm) & qm->err_info.qm_shutdown_mask;474	dev_val = qm_get_dev_err_status(qm) & qm->err_info.dev_shutdown_mask;475 476	return val || dev_val;477}478 479static int qm_wait_reset_finish(struct hisi_qm *qm)480{481	int delay = 0;482 483	/* All reset requests need to be queued for processing */484	while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {485		msleep(++delay);486		if (delay > QM_RESET_WAIT_TIMEOUT)487			return -EBUSY;488	}489 490	return 0;491}492 493static int qm_reset_prepare_ready(struct hisi_qm *qm)494{495	struct pci_dev *pdev = qm->pdev;496	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));497 498	/*499	 * PF and VF on host doesnot support resetting at the500	 * same time on Kunpeng920.501	 */502	if (qm->ver < QM_HW_V3)503		return qm_wait_reset_finish(pf_qm);504 505	return qm_wait_reset_finish(qm);506}507 508static void qm_reset_bit_clear(struct hisi_qm *qm)509{510	struct pci_dev *pdev = qm->pdev;511	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));512 513	if (qm->ver < QM_HW_V3)514		clear_bit(QM_RESETTING, &pf_qm->misc_ctl);515 516	clear_bit(QM_RESETTING, &qm->misc_ctl);517}518 519static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd,520			   u64 base, u16 queue, bool op)521{522	mailbox->w0 = cpu_to_le16((cmd) |523		((op) ? 0x1 << QM_MB_OP_SHIFT : 0) |524		(0x1 << QM_MB_BUSY_SHIFT));525	mailbox->queue_num = cpu_to_le16(queue);526	mailbox->base_l = cpu_to_le32(lower_32_bits(base));527	mailbox->base_h = cpu_to_le32(upper_32_bits(base));528	mailbox->rsvd = 0;529}530 531/* return 0 mailbox ready, -ETIMEDOUT hardware timeout */532int hisi_qm_wait_mb_ready(struct hisi_qm *qm)533{534	u32 val;535 536	return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE,537					  val, !((val >> QM_MB_BUSY_SHIFT) &538					  0x1), POLL_PERIOD, POLL_TIMEOUT);539}540EXPORT_SYMBOL_GPL(hisi_qm_wait_mb_ready);541 542/* 128 bit should be written to hardware at one time to trigger a mailbox */543static void qm_mb_write(struct hisi_qm *qm, const void *src)544{545	void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE;546 547#if IS_ENABLED(CONFIG_ARM64)548	unsigned long tmp0 = 0, tmp1 = 0;549#endif550 551	if (!IS_ENABLED(CONFIG_ARM64)) {552		memcpy_toio(fun_base, src, 16);553		dma_wmb();554		return;555	}556 557#if IS_ENABLED(CONFIG_ARM64)558	asm volatile("ldp %0, %1, %3\n"559		     "stp %0, %1, %2\n"560		     "dmb oshst\n"561		     : "=&r" (tmp0),562		       "=&r" (tmp1),563		       "+Q" (*((char __iomem *)fun_base))564		     : "Q" (*((char *)src))565		     : "memory");566#endif567}568 569static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox)570{571	int ret;572	u32 val;573 574	if (unlikely(hisi_qm_wait_mb_ready(qm))) {575		dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n");576		ret = -EBUSY;577		goto mb_busy;578	}579 580	qm_mb_write(qm, mailbox);581 582	if (unlikely(hisi_qm_wait_mb_ready(qm))) {583		dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n");584		ret = -ETIMEDOUT;585		goto mb_busy;586	}587 588	val = readl(qm->io_base + QM_MB_CMD_SEND_BASE);589	if (val & QM_MB_STATUS_MASK) {590		dev_err(&qm->pdev->dev, "QM mailbox operation failed!\n");591		ret = -EIO;592		goto mb_busy;593	}594 595	return 0;596 597mb_busy:598	atomic64_inc(&qm->debug.dfx.mb_err_cnt);599	return ret;600}601 602int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,603	       bool op)604{605	struct qm_mailbox mailbox;606	int ret;607 608	qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op);609 610	mutex_lock(&qm->mailbox_lock);611	ret = qm_mb_nolock(qm, &mailbox);612	mutex_unlock(&qm->mailbox_lock);613 614	return ret;615}616EXPORT_SYMBOL_GPL(hisi_qm_mb);617 618/* op 0: set xqc information to hardware, 1: get xqc information from hardware. */619int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op)620{621	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));622	struct qm_mailbox mailbox;623	dma_addr_t xqc_dma;624	void *tmp_xqc;625	size_t size;626	int ret;627 628	switch (cmd) {629	case QM_MB_CMD_SQC:630		size = sizeof(struct qm_sqc);631		tmp_xqc = qm->xqc_buf.sqc;632		xqc_dma = qm->xqc_buf.sqc_dma;633		break;634	case QM_MB_CMD_CQC:635		size = sizeof(struct qm_cqc);636		tmp_xqc = qm->xqc_buf.cqc;637		xqc_dma = qm->xqc_buf.cqc_dma;638		break;639	case QM_MB_CMD_EQC:640		size = sizeof(struct qm_eqc);641		tmp_xqc = qm->xqc_buf.eqc;642		xqc_dma = qm->xqc_buf.eqc_dma;643		break;644	case QM_MB_CMD_AEQC:645		size = sizeof(struct qm_aeqc);646		tmp_xqc = qm->xqc_buf.aeqc;647		xqc_dma = qm->xqc_buf.aeqc_dma;648		break;649	default:650		dev_err(&qm->pdev->dev, "unknown mailbox cmd %u\n", cmd);651		return -EINVAL;652	}653 654	/* Setting xqc will fail if master OOO is blocked. */655	if (qm_check_dev_error(pf_qm)) {656		dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n");657		return -EIO;658	}659 660	mutex_lock(&qm->mailbox_lock);661	if (!op)662		memcpy(tmp_xqc, xqc, size);663 664	qm_mb_pre_init(&mailbox, cmd, xqc_dma, qp_id, op);665	ret = qm_mb_nolock(qm, &mailbox);666	if (!ret && op)667		memcpy(xqc, tmp_xqc, size);668 669	mutex_unlock(&qm->mailbox_lock);670 671	return ret;672}673 674static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)675{676	u64 doorbell;677 678	doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V1) |679		   ((u64)index << QM_DB_INDEX_SHIFT_V1)  |680		   ((u64)priority << QM_DB_PRIORITY_SHIFT_V1);681 682	writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1);683}684 685static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)686{687	void __iomem *io_base = qm->io_base;688	u16 randata = 0;689	u64 doorbell;690 691	if (cmd == QM_DOORBELL_CMD_SQ || cmd == QM_DOORBELL_CMD_CQ)692		io_base = qm->db_io_base + (u64)qn * qm->db_interval +693			  QM_DOORBELL_SQ_CQ_BASE_V2;694	else695		io_base += QM_DOORBELL_EQ_AEQ_BASE_V2;696 697	doorbell = qn | ((u64)cmd << QM_DB_CMD_SHIFT_V2) |698		   ((u64)randata << QM_DB_RAND_SHIFT_V2) |699		   ((u64)index << QM_DB_INDEX_SHIFT_V2) |700		   ((u64)priority << QM_DB_PRIORITY_SHIFT_V2);701 702	writeq(doorbell, io_base);703}704 705static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)706{707	dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n",708		qn, cmd, index);709 710	qm->ops->qm_db(qm, qn, cmd, index, priority);711}712 713static void qm_disable_clock_gate(struct hisi_qm *qm)714{715	u32 val;716 717	/* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */718	if (qm->ver < QM_HW_V3)719		return;720 721	val = readl(qm->io_base + QM_PM_CTRL);722	val |= QM_IDLE_DISABLE;723	writel(val, qm->io_base +  QM_PM_CTRL);724}725 726static int qm_dev_mem_reset(struct hisi_qm *qm)727{728	u32 val;729 730	writel(0x1, qm->io_base + QM_MEM_START_INIT);731	return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val,732					  val & BIT(0), POLL_PERIOD,733					  POLL_TIMEOUT);734}735 736/**737 * hisi_qm_get_hw_info() - Get device information.738 * @qm: The qm which want to get information.739 * @info_table: Array for storing device information.740 * @index: Index in info_table.741 * @is_read: Whether read from reg, 0: not support read from reg.742 *743 * This function returns device information the caller needs.744 */745u32 hisi_qm_get_hw_info(struct hisi_qm *qm,746			const struct hisi_qm_cap_info *info_table,747			u32 index, bool is_read)748{749	u32 val;750 751	switch (qm->ver) {752	case QM_HW_V1:753		return info_table[index].v1_val;754	case QM_HW_V2:755		return info_table[index].v2_val;756	default:757		if (!is_read)758			return info_table[index].v3_val;759 760		val = readl(qm->io_base + info_table[index].offset);761		return (val >> info_table[index].shift) & info_table[index].mask;762	}763}764EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info);765 766static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits,767			     u16 *high_bits, enum qm_basic_type type)768{769	u32 depth;770 771	depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver);772	*low_bits = depth & QM_XQ_DEPTH_MASK;773	*high_bits = (depth >> QM_XQ_DEPTH_SHIFT) & QM_XQ_DEPTH_MASK;774}775 776int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs,777		     u32 dev_algs_size)778{779	struct device *dev = &qm->pdev->dev;780	char *algs, *ptr;781	int i;782 783	if (!qm->uacce)784		return 0;785 786	if (dev_algs_size >= QM_DEV_ALG_MAX_LEN) {787		dev_err(dev, "algs size %u is equal or larger than %d.\n",788			dev_algs_size, QM_DEV_ALG_MAX_LEN);789		return -EINVAL;790	}791 792	algs = devm_kzalloc(dev, QM_DEV_ALG_MAX_LEN * sizeof(char), GFP_KERNEL);793	if (!algs)794		return -ENOMEM;795 796	for (i = 0; i < dev_algs_size; i++)797		if (alg_msk & dev_algs[i].alg_msk)798			strcat(algs, dev_algs[i].alg);799 800	ptr = strrchr(algs, '\n');801	if (ptr) {802		*ptr = '\0';803		qm->uacce->algs = algs;804	}805 806	return 0;807}808EXPORT_SYMBOL_GPL(hisi_qm_set_algs);809 810static u32 qm_get_irq_num(struct hisi_qm *qm)811{812	if (qm->fun_type == QM_HW_PF)813		return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver);814 815	return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver);816}817 818static int qm_pm_get_sync(struct hisi_qm *qm)819{820	struct device *dev = &qm->pdev->dev;821	int ret;822 823	if (!test_bit(QM_SUPPORT_RPM, &qm->caps))824		return 0;825 826	ret = pm_runtime_resume_and_get(dev);827	if (ret < 0) {828		dev_err(dev, "failed to get_sync(%d).\n", ret);829		return ret;830	}831 832	return 0;833}834 835static void qm_pm_put_sync(struct hisi_qm *qm)836{837	struct device *dev = &qm->pdev->dev;838 839	if (!test_bit(QM_SUPPORT_RPM, &qm->caps))840		return;841 842	pm_runtime_mark_last_busy(dev);843	pm_runtime_put_autosuspend(dev);844}845 846static void qm_cq_head_update(struct hisi_qp *qp)847{848	if (qp->qp_status.cq_head == qp->cq_depth - 1) {849		qp->qp_status.cqc_phase = !qp->qp_status.cqc_phase;850		qp->qp_status.cq_head = 0;851	} else {852		qp->qp_status.cq_head++;853	}854}855 856static void qm_poll_req_cb(struct hisi_qp *qp)857{858	struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;859	struct hisi_qm *qm = qp->qm;860 861	while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {862		dma_rmb();863		qp->req_cb(qp, qp->sqe + qm->sqe_size *864			   le16_to_cpu(cqe->sq_head));865		qm_cq_head_update(qp);866		cqe = qp->cqe + qp->qp_status.cq_head;867		qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ,868		      qp->qp_status.cq_head, 0);869		atomic_dec(&qp->qp_status.used);870 871		cond_resched();872	}873 874	/* set c_flag */875	qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1);876}877 878static void qm_work_process(struct work_struct *work)879{880	struct hisi_qm_poll_data *poll_data =881		container_of(work, struct hisi_qm_poll_data, work);882	struct hisi_qm *qm = poll_data->qm;883	u16 eqe_num = poll_data->eqe_num;884	struct hisi_qp *qp;885	int i;886 887	for (i = eqe_num - 1; i >= 0; i--) {888		qp = &qm->qp_array[poll_data->qp_finish_id[i]];889		if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP))890			continue;891 892		if (qp->event_cb) {893			qp->event_cb(qp);894			continue;895		}896 897		if (likely(qp->req_cb))898			qm_poll_req_cb(qp);899	}900}901 902static void qm_get_complete_eqe_num(struct hisi_qm *qm)903{904	struct qm_eqe *eqe = qm->eqe + qm->status.eq_head;905	struct hisi_qm_poll_data *poll_data = NULL;906	u16 eq_depth = qm->eq_depth;907	u16 cqn, eqe_num = 0;908 909	if (QM_EQE_PHASE(eqe) != qm->status.eqc_phase) {910		atomic64_inc(&qm->debug.dfx.err_irq_cnt);911		qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);912		return;913	}914 915	cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;916	if (unlikely(cqn >= qm->qp_num))917		return;918	poll_data = &qm->poll_data[cqn];919 920	while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) {921		cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK;922		poll_data->qp_finish_id[eqe_num] = cqn;923		eqe_num++;924 925		if (qm->status.eq_head == eq_depth - 1) {926			qm->status.eqc_phase = !qm->status.eqc_phase;927			eqe = qm->eqe;928			qm->status.eq_head = 0;929		} else {930			eqe++;931			qm->status.eq_head++;932		}933 934		if (eqe_num == (eq_depth >> 1) - 1)935			break;936	}937 938	poll_data->eqe_num = eqe_num;939	queue_work(qm->wq, &poll_data->work);940	qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);941}942 943static irqreturn_t qm_eq_irq(int irq, void *data)944{945	struct hisi_qm *qm = data;946 947	/* Get qp id of completed tasks and re-enable the interrupt */948	qm_get_complete_eqe_num(qm);949 950	return IRQ_HANDLED;951}952 953static irqreturn_t qm_mb_cmd_irq(int irq, void *data)954{955	struct hisi_qm *qm = data;956	u32 val;957 958	val = readl(qm->io_base + QM_IFC_INT_STATUS);959	val &= QM_IFC_INT_STATUS_MASK;960	if (!val)961		return IRQ_NONE;962 963	if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) {964		dev_warn(&qm->pdev->dev, "Driver is down, message cannot be processed!\n");965		return IRQ_HANDLED;966	}967 968	schedule_work(&qm->cmd_process);969 970	return IRQ_HANDLED;971}972 973static void qm_set_qp_disable(struct hisi_qp *qp, int offset)974{975	u32 *addr;976 977	if (qp->is_in_kernel)978		return;979 980	addr = (u32 *)(qp->qdma.va + qp->qdma.size) - offset;981	*addr = 1;982 983	/* make sure setup is completed */984	smp_wmb();985}986 987static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id)988{989	struct hisi_qp *qp = &qm->qp_array[qp_id];990 991	qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET);992	hisi_qm_stop_qp(qp);993	qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET);994}995 996static void qm_reset_function(struct hisi_qm *qm)997{998	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));999	struct device *dev = &qm->pdev->dev;1000	int ret;1001 1002	if (qm_check_dev_error(pf_qm))1003		return;1004 1005	ret = qm_reset_prepare_ready(qm);1006	if (ret) {1007		dev_err(dev, "reset function not ready\n");1008		return;1009	}1010 1011	ret = hisi_qm_stop(qm, QM_DOWN);1012	if (ret) {1013		dev_err(dev, "failed to stop qm when reset function\n");1014		goto clear_bit;1015	}1016 1017	ret = hisi_qm_start(qm);1018	if (ret)1019		dev_err(dev, "failed to start qm when reset function\n");1020 1021clear_bit:1022	qm_reset_bit_clear(qm);1023}1024 1025static irqreturn_t qm_aeq_thread(int irq, void *data)1026{1027	struct hisi_qm *qm = data;1028	struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;1029	u16 aeq_depth = qm->aeq_depth;1030	u32 type, qp_id;1031 1032	atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);1033 1034	while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {1035		type = (le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT) &1036			QM_AEQE_TYPE_MASK;1037		qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK;1038 1039		switch (type) {1040		case QM_EQ_OVERFLOW:1041			dev_err(&qm->pdev->dev, "eq overflow, reset function\n");1042			qm_reset_function(qm);1043			return IRQ_HANDLED;1044		case QM_CQ_OVERFLOW:1045			dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n",1046				qp_id);1047			fallthrough;1048		case QM_CQE_ERROR:1049			qm_disable_qp(qm, qp_id);1050			break;1051		default:1052			dev_err(&qm->pdev->dev, "unknown error type %u\n",1053				type);1054			break;1055		}1056 1057		if (qm->status.aeq_head == aeq_depth - 1) {1058			qm->status.aeqc_phase = !qm->status.aeqc_phase;1059			aeqe = qm->aeqe;1060			qm->status.aeq_head = 0;1061		} else {1062			aeqe++;1063			qm->status.aeq_head++;1064		}1065	}1066 1067	qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);1068 1069	return IRQ_HANDLED;1070}1071 1072static void qm_init_qp_status(struct hisi_qp *qp)1073{1074	struct hisi_qp_status *qp_status = &qp->qp_status;1075 1076	qp_status->sq_tail = 0;1077	qp_status->cq_head = 0;1078	qp_status->cqc_phase = true;1079	atomic_set(&qp_status->used, 0);1080}1081 1082static void qm_init_prefetch(struct hisi_qm *qm)1083{1084	struct device *dev = &qm->pdev->dev;1085	u32 page_type = 0x0;1086 1087	if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))1088		return;1089 1090	switch (PAGE_SIZE) {1091	case SZ_4K:1092		page_type = 0x0;1093		break;1094	case SZ_16K:1095		page_type = 0x1;1096		break;1097	case SZ_64K:1098		page_type = 0x2;1099		break;1100	default:1101		dev_err(dev, "system page size is not support: %lu, default set to 4KB",1102			PAGE_SIZE);1103	}1104 1105	writel(page_type, qm->io_base + QM_PAGE_SIZE);1106}1107 1108/*1109 * acc_shaper_para_calc() Get the IR value by the qos formula, the return value1110 * is the expected qos calculated.1111 * the formula:1112 * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps1113 *1114 *		IR_b * (2 ^ IR_u) * 80001115 * IR(Mbps) = -------------------------1116 *		  Tick * (2 ^ IR_s)1117 */1118static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)1119{1120	return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) /1121					(QM_QOS_TICK * (1 << cir_s));1122}1123 1124static u32 acc_shaper_calc_cbs_s(u32 ir)1125{1126	int table_size = ARRAY_SIZE(shaper_cbs_s);1127	int i;1128 1129	for (i = 0; i < table_size; i++) {1130		if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end)1131			return shaper_cbs_s[i].val;1132	}1133 1134	return QM_SHAPER_MIN_CBS_S;1135}1136 1137static u32 acc_shaper_calc_cir_s(u32 ir)1138{1139	int table_size = ARRAY_SIZE(shaper_cir_s);1140	int i;1141 1142	for (i = 0; i < table_size; i++) {1143		if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end)1144			return shaper_cir_s[i].val;1145	}1146 1147	return 0;1148}1149 1150static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)1151{1152	u32 cir_b, cir_u, cir_s, ir_calc;1153	u32 error_rate;1154 1155	factor->cbs_s = acc_shaper_calc_cbs_s(ir);1156	cir_s = acc_shaper_calc_cir_s(ir);1157 1158	for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) {1159		for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) {1160			ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);1161 1162			error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;1163			if (error_rate <= QM_QOS_MIN_ERROR_RATE) {1164				factor->cir_b = cir_b;1165				factor->cir_u = cir_u;1166				factor->cir_s = cir_s;1167				return 0;1168			}1169		}1170	}1171 1172	return -EINVAL;1173}1174 1175static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base,1176			    u32 number, struct qm_shaper_factor *factor)1177{1178	u64 tmp = 0;1179 1180	if (number > 0) {1181		switch (type) {1182		case SQC_VFT:1183			if (qm->ver == QM_HW_V1) {1184				tmp = QM_SQC_VFT_BUF_SIZE	|1185				      QM_SQC_VFT_SQC_SIZE	|1186				      QM_SQC_VFT_INDEX_NUMBER	|1187				      QM_SQC_VFT_VALID		|1188				      (u64)base << QM_SQC_VFT_START_SQN_SHIFT;1189			} else {1190				tmp = (u64)base << QM_SQC_VFT_START_SQN_SHIFT |1191				      QM_SQC_VFT_VALID |1192				      (u64)(number - 1) << QM_SQC_VFT_SQN_SHIFT;1193			}1194			break;1195		case CQC_VFT:1196			if (qm->ver == QM_HW_V1) {1197				tmp = QM_CQC_VFT_BUF_SIZE	|1198				      QM_CQC_VFT_SQC_SIZE	|1199				      QM_CQC_VFT_INDEX_NUMBER	|1200				      QM_CQC_VFT_VALID;1201			} else {1202				tmp = QM_CQC_VFT_VALID;1203			}1204			break;1205		case SHAPER_VFT:1206			if (factor) {1207				tmp = factor->cir_b |1208				(factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) |1209				(factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) |1210				(QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) |1211				(factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT);1212			}1213			break;1214		}1215	}1216 1217	writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L);1218	writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H);1219}1220 1221static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,1222			     u32 fun_num, u32 base, u32 number)1223{1224	struct qm_shaper_factor *factor = NULL;1225	unsigned int val;1226	int ret;1227 1228	if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))1229		factor = &qm->factor[fun_num];1230 1231	ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,1232					 val & BIT(0), POLL_PERIOD,1233					 POLL_TIMEOUT);1234	if (ret)1235		return ret;1236 1237	writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR);1238	writel(type, qm->io_base + QM_VFT_CFG_TYPE);1239	if (type == SHAPER_VFT)1240		fun_num |= base << QM_SHAPER_VFT_OFFSET;1241 1242	writel(fun_num, qm->io_base + QM_VFT_CFG);1243 1244	qm_vft_data_cfg(qm, type, base, number, factor);1245 1246	writel(0x0, qm->io_base + QM_VFT_CFG_RDY);1247	writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);1248 1249	return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,1250					  val & BIT(0), POLL_PERIOD,1251					  POLL_TIMEOUT);1252}1253 1254static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num)1255{1256	u32 qos = qm->factor[fun_num].func_qos;1257	int ret, i;1258 1259	ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]);1260	if (ret) {1261		dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n");1262		return ret;1263	}1264	writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG);1265	for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {1266		/* The base number of queue reuse for different alg type */1267		ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1);1268		if (ret)1269			return ret;1270	}1271 1272	return 0;1273}1274 1275/* The config should be conducted after qm_dev_mem_reset() */1276static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base,1277			      u32 number)1278{1279	int ret, i;1280 1281	for (i = SQC_VFT; i <= CQC_VFT; i++) {1282		ret = qm_set_vft_common(qm, i, fun_num, base, number);1283		if (ret)1284			return ret;1285	}1286 1287	/* init default shaper qos val */1288	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) {1289		ret = qm_shaper_init_vft(qm, fun_num);1290		if (ret)1291			goto back_sqc_cqc;1292	}1293 1294	return 0;1295back_sqc_cqc:1296	for (i = SQC_VFT; i <= CQC_VFT; i++)1297		qm_set_vft_common(qm, i, fun_num, 0, 0);1298 1299	return ret;1300}1301 1302static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number)1303{1304	u64 sqc_vft;1305	int ret;1306 1307	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);1308	if (ret)1309		return ret;1310 1311	sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |1312		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);1313	*base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);1314	*number = (QM_SQC_VFT_NUM_MASK_V2 &1315		   (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;1316 1317	return 0;1318}1319 1320static void qm_hw_error_init_v1(struct hisi_qm *qm)1321{1322	writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK);1323}1324 1325static void qm_hw_error_cfg(struct hisi_qm *qm)1326{1327	struct hisi_qm_err_info *err_info = &qm->err_info;1328 1329	qm->error_mask = err_info->nfe | err_info->ce | err_info->fe;1330	/* clear QM hw residual error source */1331	writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE);1332 1333	/* configure error type */1334	writel(err_info->ce, qm->io_base + QM_RAS_CE_ENABLE);1335	writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD);1336	writel(err_info->nfe, qm->io_base + QM_RAS_NFE_ENABLE);1337	writel(err_info->fe, qm->io_base + QM_RAS_FE_ENABLE);1338}1339 1340static void qm_hw_error_init_v2(struct hisi_qm *qm)1341{1342	u32 irq_unmask;1343 1344	qm_hw_error_cfg(qm);1345 1346	irq_unmask = ~qm->error_mask;1347	irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);1348	writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);1349}1350 1351static void qm_hw_error_uninit_v2(struct hisi_qm *qm)1352{1353	u32 irq_mask = qm->error_mask;1354 1355	irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK);1356	writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK);1357}1358 1359static void qm_hw_error_init_v3(struct hisi_qm *qm)1360{1361	u32 irq_unmask;1362 1363	qm_hw_error_cfg(qm);1364 1365	/* enable close master ooo when hardware error happened */1366	writel(qm->err_info.qm_shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL);1367 1368	irq_unmask = ~qm->error_mask;1369	irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK);1370	writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK);1371}1372 1373static void qm_hw_error_uninit_v3(struct hisi_qm *qm)1374{1375	u32 irq_mask = qm->error_mask;1376 1377	irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK);1378	writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK);1379 1380	/* disable close master ooo when hardware error happened */1381	writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL);1382}1383 1384static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status)1385{1386	const struct hisi_qm_hw_error *err;1387	struct device *dev = &qm->pdev->dev;1388	u32 reg_val, type, vf_num, qp_id;1389	int i;1390 1391	for (i = 0; i < ARRAY_SIZE(qm_hw_error); i++) {1392		err = &qm_hw_error[i];1393		if (!(err->int_msk & error_status))1394			continue;1395 1396		dev_err(dev, "%s [error status=0x%x] found\n",1397			err->msg, err->int_msk);1398 1399		if (err->int_msk & QM_DB_TIMEOUT) {1400			reg_val = readl(qm->io_base + QM_ABNORMAL_INF01);1401			type = (reg_val & QM_DB_TIMEOUT_TYPE) >>1402			       QM_DB_TIMEOUT_TYPE_SHIFT;1403			vf_num = reg_val & QM_DB_TIMEOUT_VF;1404			qp_id = reg_val >> QM_DB_TIMEOUT_QP_SHIFT;1405			dev_err(dev, "qm %s doorbell timeout in function %u qp %u\n",1406				qm_db_timeout[type], vf_num, qp_id);1407		} else if (err->int_msk & QM_OF_FIFO_OF) {1408			reg_val = readl(qm->io_base + QM_ABNORMAL_INF00);1409			type = (reg_val & QM_FIFO_OVERFLOW_TYPE) >>1410			       QM_FIFO_OVERFLOW_TYPE_SHIFT;1411			vf_num = reg_val & QM_FIFO_OVERFLOW_VF;1412			qp_id = reg_val >> QM_FIFO_OVERFLOW_QP_SHIFT;1413			if (type < ARRAY_SIZE(qm_fifo_overflow))1414				dev_err(dev, "qm %s fifo overflow in function %u qp %u\n",1415					qm_fifo_overflow[type], vf_num, qp_id);1416			else1417				dev_err(dev, "unknown error type\n");1418		} else if (err->int_msk & QM_AXI_RRESP_ERR) {1419			reg_val = readl(qm->io_base + QM_ABNORMAL_INF02);1420			if (reg_val & QM_AXI_POISON_ERR)1421				dev_err(dev, "qm axi poison error happened\n");1422		}1423	}1424}1425 1426static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)1427{1428	u32 error_status, tmp;1429 1430	/* read err sts */1431	tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS);1432	error_status = qm->error_mask & tmp;1433 1434	if (error_status) {1435		if (error_status & QM_ECC_MBIT)1436			qm->err_status.is_qm_ecc_mbit = true;1437 1438		qm_log_hw_error(qm, error_status);1439		if (error_status & qm->err_info.qm_reset_mask)1440			return ACC_ERR_NEED_RESET;1441 1442		writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE);1443		writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE);1444	}1445 1446	return ACC_ERR_RECOVERED;1447}1448 1449static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num)1450{1451	struct qm_mailbox mailbox;1452	int ret;1453 1454	qm_mb_pre_init(&mailbox, QM_MB_CMD_DST, 0, fun_num, 0);1455	mutex_lock(&qm->mailbox_lock);1456	ret = qm_mb_nolock(qm, &mailbox);1457	if (ret)1458		goto err_unlock;1459 1460	*msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |1461		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32);1462 1463err_unlock:1464	mutex_unlock(&qm->mailbox_lock);1465	return ret;1466}1467 1468static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask)1469{1470	u32 val;1471 1472	if (qm->fun_type == QM_HW_PF)1473		writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P);1474 1475	val = readl(qm->io_base + QM_IFC_INT_SOURCE_V);1476	val |= QM_IFC_INT_SOURCE_MASK;1477	writel(val, qm->io_base + QM_IFC_INT_SOURCE_V);1478}1479 1480static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id)1481{1482	struct device *dev = &qm->pdev->dev;1483	u32 cmd;1484	u64 msg;1485	int ret;1486 1487	ret = qm_get_mb_cmd(qm, &msg, vf_id);1488	if (ret) {1489		dev_err(dev, "failed to get msg from VF(%u)!\n", vf_id);1490		return;1491	}1492 1493	cmd = msg & QM_MB_CMD_DATA_MASK;1494	switch (cmd) {1495	case QM_VF_PREPARE_FAIL:1496		dev_err(dev, "failed to stop VF(%u)!\n", vf_id);1497		break;1498	case QM_VF_START_FAIL:1499		dev_err(dev, "failed to start VF(%u)!\n", vf_id);1500		break;1501	case QM_VF_PREPARE_DONE:1502	case QM_VF_START_DONE:1503		break;1504	default:1505		dev_err(dev, "unsupported cmd %u sent by VF(%u)!\n", cmd, vf_id);1506		break;1507	}1508}1509 1510static int qm_wait_vf_prepare_finish(struct hisi_qm *qm)1511{1512	struct device *dev = &qm->pdev->dev;1513	u32 vfs_num = qm->vfs_num;1514	int cnt = 0;1515	int ret = 0;1516	u64 val;1517	u32 i;1518 1519	if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))1520		return 0;1521 1522	while (true) {1523		val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);1524		/* All VFs send command to PF, break */1525		if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1))1526			break;1527 1528		if (++cnt > QM_MAX_PF_WAIT_COUNT) {1529			ret = -EBUSY;1530			break;1531		}1532 1533		msleep(QM_WAIT_DST_ACK);1534	}1535 1536	/* PF check VFs msg */1537	for (i = 1; i <= vfs_num; i++) {1538		if (val & BIT(i))1539			qm_handle_vf_msg(qm, i);1540		else1541			dev_err(dev, "VF(%u) not ping PF!\n", i);1542	}1543 1544	/* PF clear interrupt to ack VFs */1545	qm_clear_cmd_interrupt(qm, val);1546 1547	return ret;1548}1549 1550static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num)1551{1552	u32 val;1553 1554	val = readl(qm->io_base + QM_IFC_INT_CFG);1555	val &= ~QM_IFC_SEND_ALL_VFS;1556	val |= fun_num;1557	writel(val, qm->io_base + QM_IFC_INT_CFG);1558 1559	val = readl(qm->io_base + QM_IFC_INT_SET_P);1560	val |= QM_IFC_INT_SET_MASK;1561	writel(val, qm->io_base + QM_IFC_INT_SET_P);1562}1563 1564static void qm_trigger_pf_interrupt(struct hisi_qm *qm)1565{1566	u32 val;1567 1568	val = readl(qm->io_base + QM_IFC_INT_SET_V);1569	val |= QM_IFC_INT_SET_MASK;1570	writel(val, qm->io_base + QM_IFC_INT_SET_V);1571}1572 1573static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num)1574{1575	struct device *dev = &qm->pdev->dev;1576	struct qm_mailbox mailbox;1577	int cnt = 0;1578	u64 val;1579	int ret;1580 1581	qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, fun_num, 0);1582	mutex_lock(&qm->mailbox_lock);1583	ret = qm_mb_nolock(qm, &mailbox);1584	if (ret) {1585		dev_err(dev, "failed to send command to vf(%u)!\n", fun_num);1586		goto err_unlock;1587	}1588 1589	qm_trigger_vf_interrupt(qm, fun_num);1590	while (true) {1591		msleep(QM_WAIT_DST_ACK);1592		val = readq(qm->io_base + QM_IFC_READY_STATUS);1593		/* if VF respond, PF notifies VF successfully. */1594		if (!(val & BIT(fun_num)))1595			goto err_unlock;1596 1597		if (++cnt > QM_MAX_PF_WAIT_COUNT) {1598			dev_err(dev, "failed to get response from VF(%u)!\n", fun_num);1599			ret = -ETIMEDOUT;1600			break;1601		}1602	}1603 1604err_unlock:1605	mutex_unlock(&qm->mailbox_lock);1606	return ret;1607}1608 1609static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd)1610{1611	struct device *dev = &qm->pdev->dev;1612	u32 vfs_num = qm->vfs_num;1613	struct qm_mailbox mailbox;1614	u64 val = 0;1615	int cnt = 0;1616	int ret;1617	u32 i;1618 1619	qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, QM_MB_PING_ALL_VFS, 0);1620	mutex_lock(&qm->mailbox_lock);1621	/* PF sends command to all VFs by mailbox */1622	ret = qm_mb_nolock(qm, &mailbox);1623	if (ret) {1624		dev_err(dev, "failed to send command to VFs!\n");1625		mutex_unlock(&qm->mailbox_lock);1626		return ret;1627	}1628 1629	qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS);1630	while (true) {1631		msleep(QM_WAIT_DST_ACK);1632		val = readq(qm->io_base + QM_IFC_READY_STATUS);1633		/* If all VFs acked, PF notifies VFs successfully. */1634		if (!(val & GENMASK(vfs_num, 1))) {1635			mutex_unlock(&qm->mailbox_lock);1636			return 0;1637		}1638 1639		if (++cnt > QM_MAX_PF_WAIT_COUNT)1640			break;1641	}1642 1643	mutex_unlock(&qm->mailbox_lock);1644 1645	/* Check which vf respond timeout. */1646	for (i = 1; i <= vfs_num; i++) {1647		if (val & BIT(i))1648			dev_err(dev, "failed to get response from VF(%u)!\n", i);1649	}1650 1651	return -ETIMEDOUT;1652}1653 1654static int qm_ping_pf(struct hisi_qm *qm, u64 cmd)1655{1656	struct qm_mailbox mailbox;1657	int cnt = 0;1658	u32 val;1659	int ret;1660 1661	qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, 0, 0);1662	mutex_lock(&qm->mailbox_lock);1663	ret = qm_mb_nolock(qm, &mailbox);1664	if (ret) {1665		dev_err(&qm->pdev->dev, "failed to send command to PF!\n");1666		goto unlock;1667	}1668 1669	qm_trigger_pf_interrupt(qm);1670	/* Waiting for PF response */1671	while (true) {1672		msleep(QM_WAIT_DST_ACK);1673		val = readl(qm->io_base + QM_IFC_INT_SET_V);1674		if (!(val & QM_IFC_INT_STATUS_MASK))1675			break;1676 1677		if (++cnt > QM_MAX_VF_WAIT_COUNT) {1678			ret = -ETIMEDOUT;1679			break;1680		}1681	}1682 1683unlock:1684	mutex_unlock(&qm->mailbox_lock);1685	return ret;1686}1687 1688static int qm_drain_qm(struct hisi_qm *qm)1689{1690	return hisi_qm_mb(qm, QM_MB_CMD_FLUSH_QM, 0, 0, 0);1691}1692 1693static int qm_stop_qp(struct hisi_qp *qp)1694{1695	return hisi_qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0);1696}1697 1698static int qm_set_msi(struct hisi_qm *qm, bool set)1699{1700	struct pci_dev *pdev = qm->pdev;1701 1702	if (set) {1703		pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,1704				       0);1705	} else {1706		pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64,1707				       ACC_PEH_MSI_DISABLE);1708		if (qm->err_status.is_qm_ecc_mbit ||1709		    qm->err_status.is_dev_ecc_mbit)1710			return 0;1711 1712		mdelay(1);1713		if (readl(qm->io_base + QM_PEH_DFX_INFO0))1714			return -EFAULT;1715	}1716 1717	return 0;1718}1719 1720static void qm_wait_msi_finish(struct hisi_qm *qm)1721{1722	struct pci_dev *pdev = qm->pdev;1723	u32 cmd = ~0;1724	int cnt = 0;1725	u32 val;1726	int ret;1727 1728	while (true) {1729		pci_read_config_dword(pdev, pdev->msi_cap +1730				      PCI_MSI_PENDING_64, &cmd);1731		if (!cmd)1732			break;1733 1734		if (++cnt > MAX_WAIT_COUNTS) {1735			pci_warn(pdev, "failed to empty MSI PENDING!\n");1736			break;1737		}1738 1739		udelay(1);1740	}1741 1742	ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0,1743					 val, !(val & QM_PEH_DFX_MASK),1744					 POLL_PERIOD, POLL_TIMEOUT);1745	if (ret)1746		pci_warn(pdev, "failed to empty PEH MSI!\n");1747 1748	ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1,1749					 val, !(val & QM_PEH_MSI_FINISH_MASK),1750					 POLL_PERIOD, POLL_TIMEOUT);1751	if (ret)1752		pci_warn(pdev, "failed to finish MSI operation!\n");1753}1754 1755static int qm_set_msi_v3(struct hisi_qm *qm, bool set)1756{1757	struct pci_dev *pdev = qm->pdev;1758	int ret = -ETIMEDOUT;1759	u32 cmd, i;1760 1761	pci_read_config_dword(pdev, pdev->msi_cap, &cmd);1762	if (set)1763		cmd |= QM_MSI_CAP_ENABLE;1764	else1765		cmd &= ~QM_MSI_CAP_ENABLE;1766 1767	pci_write_config_dword(pdev, pdev->msi_cap, cmd);1768	if (set) {1769		for (i = 0; i < MAX_WAIT_COUNTS; i++) {1770			pci_read_config_dword(pdev, pdev->msi_cap, &cmd);1771			if (cmd & QM_MSI_CAP_ENABLE)1772				return 0;1773 1774			udelay(1);1775		}1776	} else {1777		udelay(WAIT_PERIOD_US_MIN);1778		qm_wait_msi_finish(qm);1779		ret = 0;1780	}1781 1782	return ret;1783}1784 1785static const struct hisi_qm_hw_ops qm_hw_ops_v1 = {1786	.qm_db = qm_db_v1,1787	.hw_error_init = qm_hw_error_init_v1,1788	.set_msi = qm_set_msi,1789};1790 1791static const struct hisi_qm_hw_ops qm_hw_ops_v2 = {1792	.get_vft = qm_get_vft_v2,1793	.qm_db = qm_db_v2,1794	.hw_error_init = qm_hw_error_init_v2,1795	.hw_error_uninit = qm_hw_error_uninit_v2,1796	.hw_error_handle = qm_hw_error_handle_v2,1797	.set_msi = qm_set_msi,1798};1799 1800static const struct hisi_qm_hw_ops qm_hw_ops_v3 = {1801	.get_vft = qm_get_vft_v2,1802	.qm_db = qm_db_v2,1803	.hw_error_init = qm_hw_error_init_v3,1804	.hw_error_uninit = qm_hw_error_uninit_v3,1805	.hw_error_handle = qm_hw_error_handle_v2,1806	.set_msi = qm_set_msi_v3,1807};1808 1809static void *qm_get_avail_sqe(struct hisi_qp *qp)1810{1811	struct hisi_qp_status *qp_status = &qp->qp_status;1812	u16 sq_tail = qp_status->sq_tail;1813 1814	if (unlikely(atomic_read(&qp->qp_status.used) == qp->sq_depth - 1))1815		return NULL;1816 1817	return qp->sqe + sq_tail * qp->qm->sqe_size;1818}1819 1820static void hisi_qm_unset_hw_reset(struct hisi_qp *qp)1821{1822	u64 *addr;1823 1824	/* Use last 64 bits of DUS to reset status. */1825	addr = (u64 *)(qp->qdma.va + qp->qdma.size) - QM_RESET_STOP_TX_OFFSET;1826	*addr = 0;1827}1828 1829static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type)1830{1831	struct device *dev = &qm->pdev->dev;1832	struct hisi_qp *qp;1833	int qp_id;1834 1835	if (atomic_read(&qm->status.flags) == QM_STOP) {1836		dev_info_ratelimited(dev, "failed to create qp as qm is stop!\n");1837		return ERR_PTR(-EPERM);1838	}1839 1840	if (qm->qp_in_used == qm->qp_num) {1841		dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",1842				     qm->qp_num);1843		atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);1844		return ERR_PTR(-EBUSY);1845	}1846 1847	qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC);1848	if (qp_id < 0) {1849		dev_info_ratelimited(dev, "All %u queues of QM are busy!\n",1850				    qm->qp_num);1851		atomic64_inc(&qm->debug.dfx.create_qp_err_cnt);1852		return ERR_PTR(-EBUSY);1853	}1854 1855	qp = &qm->qp_array[qp_id];1856	hisi_qm_unset_hw_reset(qp);1857	memset(qp->cqe, 0, sizeof(struct qm_cqe) * qp->cq_depth);1858 1859	qp->event_cb = NULL;1860	qp->req_cb = NULL;1861	qp->qp_id = qp_id;1862	qp->alg_type = alg_type;1863	qp->is_in_kernel = true;1864	qm->qp_in_used++;1865 1866	return qp;1867}1868 1869/**1870 * hisi_qm_create_qp() - Create a queue pair from qm.1871 * @qm: The qm we create a qp from.1872 * @alg_type: Accelerator specific algorithm type in sqc.1873 *1874 * Return created qp, negative error code if failed.1875 */1876static struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type)1877{1878	struct hisi_qp *qp;1879	int ret;1880 1881	ret = qm_pm_get_sync(qm);1882	if (ret)1883		return ERR_PTR(ret);1884 1885	down_write(&qm->qps_lock);1886	qp = qm_create_qp_nolock(qm, alg_type);1887	up_write(&qm->qps_lock);1888 1889	if (IS_ERR(qp))1890		qm_pm_put_sync(qm);1891 1892	return qp;1893}1894 1895/**1896 * hisi_qm_release_qp() - Release a qp back to its qm.1897 * @qp: The qp we want to release.1898 *1899 * This function releases the resource of a qp.1900 */1901static void hisi_qm_release_qp(struct hisi_qp *qp)1902{1903	struct hisi_qm *qm = qp->qm;1904 1905	down_write(&qm->qps_lock);1906 1907	qm->qp_in_used--;1908	idr_remove(&qm->qp_idr, qp->qp_id);1909 1910	up_write(&qm->qps_lock);1911 1912	qm_pm_put_sync(qm);1913}1914 1915static int qm_sq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)1916{1917	struct hisi_qm *qm = qp->qm;1918	enum qm_hw_ver ver = qm->ver;1919	struct qm_sqc sqc = {0};1920 1921	if (ver == QM_HW_V1) {1922		sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size));1923		sqc.w8 = cpu_to_le16(qp->sq_depth - 1);1924	} else {1925		sqc.dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth));1926		sqc.w8 = 0; /* rand_qc */1927	}1928	sqc.w13 = cpu_to_le16(QM_MK_SQC_W13(0, 1, qp->alg_type));1929	sqc.base_l = cpu_to_le32(lower_32_bits(qp->sqe_dma));1930	sqc.base_h = cpu_to_le32(upper_32_bits(qp->sqe_dma));1931	sqc.cq_num = cpu_to_le16(qp_id);1932	sqc.pasid = cpu_to_le16(pasid);1933 1934	if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)1935		sqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE <<1936				      QM_QC_PASID_ENABLE_SHIFT);1937 1938	return qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 0);1939}1940 1941static int qm_cq_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)1942{1943	struct hisi_qm *qm = qp->qm;1944	enum qm_hw_ver ver = qm->ver;1945	struct qm_cqc cqc = {0};1946 1947	if (ver == QM_HW_V1) {1948		cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V1(0, 0, 0, QM_QC_CQE_SIZE));1949		cqc.w8 = cpu_to_le16(qp->cq_depth - 1);1950	} else {1951		cqc.dw3 = cpu_to_le32(QM_MK_CQC_DW3_V2(QM_QC_CQE_SIZE, qp->cq_depth));1952		cqc.w8 = 0; /* rand_qc */1953	}1954	/*1955	 * Enable request finishing interrupts defaultly.1956	 * So, there will be some interrupts until disabling1957	 * this.1958	 */1959	cqc.dw6 = cpu_to_le32(1 << QM_CQ_PHASE_SHIFT | 1 << QM_CQ_FLAG_SHIFT);1960	cqc.base_l = cpu_to_le32(lower_32_bits(qp->cqe_dma));1961	cqc.base_h = cpu_to_le32(upper_32_bits(qp->cqe_dma));1962	cqc.pasid = cpu_to_le16(pasid);1963 1964	if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel)1965		cqc.w11 = cpu_to_le16(QM_QC_PASID_ENABLE);1966 1967	return qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 0);1968}1969 1970static int qm_qp_ctx_cfg(struct hisi_qp *qp, int qp_id, u32 pasid)1971{1972	int ret;1973 1974	qm_init_qp_status(qp);1975 1976	ret = qm_sq_ctx_cfg(qp, qp_id, pasid);1977	if (ret)1978		return ret;1979 1980	return qm_cq_ctx_cfg(qp, qp_id, pasid);1981}1982 1983static int qm_start_qp_nolock(struct hisi_qp *qp, unsigned long arg)1984{1985	struct hisi_qm *qm = qp->qm;1986	struct device *dev = &qm->pdev->dev;1987	int qp_id = qp->qp_id;1988	u32 pasid = arg;1989	int ret;1990 1991	if (atomic_read(&qm->status.flags) == QM_STOP) {1992		dev_info_ratelimited(dev, "failed to start qp as qm is stop!\n");1993		return -EPERM;1994	}1995 1996	ret = qm_qp_ctx_cfg(qp, qp_id, pasid);1997	if (ret)1998		return ret;1999 2000	atomic_set(&qp->qp_status.flags, QP_START);2001	dev_dbg(dev, "queue %d started\n", qp_id);2002 2003	return 0;2004}2005 2006/**2007 * hisi_qm_start_qp() - Start a qp into running.2008 * @qp: The qp we want to start to run.2009 * @arg: Accelerator specific argument.2010 *2011 * After this function, qp can receive request from user. Return 0 if2012 * successful, negative error code if failed.2013 */2014int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)2015{2016	struct hisi_qm *qm = qp->qm;2017	int ret;2018 2019	down_write(&qm->qps_lock);2020	ret = qm_start_qp_nolock(qp, arg);2021	up_write(&qm->qps_lock);2022 2023	return ret;2024}2025EXPORT_SYMBOL_GPL(hisi_qm_start_qp);2026 2027/**2028 * qp_stop_fail_cb() - call request cb.2029 * @qp: stopped failed qp.2030 *2031 * Callback function should be called whether task completed or not.2032 */2033static void qp_stop_fail_cb(struct hisi_qp *qp)2034{2035	int qp_used = atomic_read(&qp->qp_status.used);2036	u16 cur_tail = qp->qp_status.sq_tail;2037	u16 sq_depth = qp->sq_depth;2038	u16 cur_head = (cur_tail + sq_depth - qp_used) % sq_depth;2039	struct hisi_qm *qm = qp->qm;2040	u16 pos;2041	int i;2042 2043	for (i = 0; i < qp_used; i++) {2044		pos = (i + cur_head) % sq_depth;2045		qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos));2046		atomic_dec(&qp->qp_status.used);2047	}2048}2049 2050static int qm_wait_qp_empty(struct hisi_qm *qm, u32 *state, u32 qp_id)2051{2052	struct device *dev = &qm->pdev->dev;2053	struct qm_sqc sqc;2054	struct qm_cqc cqc;2055	int ret, i = 0;2056 2057	while (++i) {2058		ret = qm_set_and_get_xqc(qm, QM_MB_CMD_SQC, &sqc, qp_id, 1);2059		if (ret) {2060			dev_err_ratelimited(dev, "Failed to dump sqc!\n");2061			*state = QM_DUMP_SQC_FAIL;2062			return ret;2063		}2064 2065		ret = qm_set_and_get_xqc(qm, QM_MB_CMD_CQC, &cqc, qp_id, 1);2066		if (ret) {2067			dev_err_ratelimited(dev, "Failed to dump cqc!\n");2068			*state = QM_DUMP_CQC_FAIL;2069			return ret;2070		}2071 2072		if ((sqc.tail == cqc.tail) &&2073		    (QM_SQ_TAIL_IDX(sqc) == QM_CQ_TAIL_IDX(cqc)))2074			break;2075 2076		if (i == MAX_WAIT_COUNTS) {2077			dev_err(dev, "Fail to empty queue %u!\n", qp_id);2078			*state = QM_STOP_QUEUE_FAIL;2079			return -ETIMEDOUT;2080		}2081 2082		usleep_range(WAIT_PERIOD_US_MIN, WAIT_PERIOD_US_MAX);2083	}2084 2085	return 0;2086}2087 2088/**2089 * qm_drain_qp() - Drain a qp.2090 * @qp: The qp we want to drain.2091 *2092 * If the device does not support stopping queue by sending mailbox,2093 * determine whether the queue is cleared by judging the tail pointers of2094 * sq and cq.2095 */2096static int qm_drain_qp(struct hisi_qp *qp)2097{2098	struct hisi_qm *qm = qp->qm;2099	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));2100	u32 state = 0;2101	int ret;2102 2103	/* No need to judge if master OOO is blocked. */2104	if (qm_check_dev_error(pf_qm))2105		return 0;2106 2107	/* HW V3 supports drain qp by device */2108	if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) {2109		ret = qm_stop_qp(qp);2110		if (ret) {2111			dev_err(&qm->pdev->dev, "Failed to stop qp!\n");2112			state = QM_STOP_QUEUE_FAIL;2113			goto set_dev_state;2114		}2115		return ret;2116	}2117 2118	ret = qm_wait_qp_empty(qm, &state, qp->qp_id);2119	if (ret)2120		goto set_dev_state;2121 2122	return 0;2123 2124set_dev_state:2125	if (qm->debug.dev_dfx.dev_timeout)2126		qm->debug.dev_dfx.dev_state = state;2127 2128	return ret;2129}2130 2131static void qm_stop_qp_nolock(struct hisi_qp *qp)2132{2133	struct hisi_qm *qm = qp->qm;2134	struct device *dev = &qm->pdev->dev;2135	int ret;2136 2137	/*2138	 * It is allowed to stop and release qp when reset, If the qp is2139	 * stopped when reset but still want to be released then, the2140	 * is_resetting flag should be set negative so that this qp will not2141	 * be restarted after reset.2142	 */2143	if (atomic_read(&qp->qp_status.flags) != QP_START) {2144		qp->is_resetting = false;2145		return;2146	}2147 2148	atomic_set(&qp->qp_status.flags, QP_STOP);2149 2150	/* V3 supports direct stop function when FLR prepare */2151	if (qm->ver < QM_HW_V3 || qm->status.stop_reason == QM_NORMAL) {2152		ret = qm_drain_qp(qp);2153		if (ret)2154			dev_err(dev, "Failed to drain out data for stopping qp(%u)!\n", qp->qp_id);2155	}2156 2157	flush_workqueue(qm->wq);2158	if (unlikely(qp->is_resetting && atomic_read(&qp->qp_status.used)))2159		qp_stop_fail_cb(qp);2160 2161	dev_dbg(dev, "stop queue %u!", qp->qp_id);2162}2163 2164/**2165 * hisi_qm_stop_qp() - Stop a qp in qm.2166 * @qp: The qp we want to stop.2167 *2168 * This function is reverse of hisi_qm_start_qp.2169 */2170void hisi_qm_stop_qp(struct hisi_qp *qp)2171{2172	down_write(&qp->qm->qps_lock);2173	qm_stop_qp_nolock(qp);2174	up_write(&qp->qm->qps_lock);2175}2176EXPORT_SYMBOL_GPL(hisi_qm_stop_qp);2177 2178/**2179 * hisi_qp_send() - Queue up a task in the hardware queue.2180 * @qp: The qp in which to put the message.2181 * @msg: The message.2182 *2183 * This function will return -EBUSY if qp is currently full, and -EAGAIN2184 * if qp related qm is resetting.2185 *2186 * Note: This function may run with qm_irq_thread and ACC reset at same time.2187 *       It has no race with qm_irq_thread. However, during hisi_qp_send, ACC2188 *       reset may happen, we have no lock here considering performance. This2189 *       causes current qm_db sending fail or can not receive sended sqe. QM2190 *       sync/async receive function should handle the error sqe. ACC reset2191 *       done function should clear used sqe to 0.2192 */2193int hisi_qp_send(struct hisi_qp *qp, const void *msg)2194{2195	struct hisi_qp_status *qp_status = &qp->qp_status;2196	u16 sq_tail = qp_status->sq_tail;2197	u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth;2198	void *sqe = qm_get_avail_sqe(qp);2199 2200	if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP ||2201		     atomic_read(&qp->qm->status.flags) == QM_STOP ||2202		     qp->is_resetting)) {2203		dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n");2204		return -EAGAIN;2205	}2206 2207	if (!sqe)2208		return -EBUSY;2209 2210	memcpy(sqe, msg, qp->qm->sqe_size);2211 2212	qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0);2213	atomic_inc(&qp->qp_status.used);2214	qp_status->sq_tail = sq_tail_next;2215 2216	return 0;2217}2218EXPORT_SYMBOL_GPL(hisi_qp_send);2219 2220static void hisi_qm_cache_wb(struct hisi_qm *qm)2221{2222	unsigned int val;2223 2224	if (qm->ver == QM_HW_V1)2225		return;2226 2227	writel(0x1, qm->io_base + QM_CACHE_WB_START);2228	if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE,2229				       val, val & BIT(0), POLL_PERIOD,2230				       POLL_TIMEOUT))2231		dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n");2232}2233 2234static void qm_qp_event_notifier(struct hisi_qp *qp)2235{2236	wake_up_interruptible(&qp->uacce_q->wait);2237}2238 2239 /* This function returns free number of qp in qm. */2240static int hisi_qm_get_available_instances(struct uacce_device *uacce)2241{2242	struct hisi_qm *qm = uacce->priv;2243	int ret;2244 2245	down_read(&qm->qps_lock);2246	ret = qm->qp_num - qm->qp_in_used;2247	up_read(&qm->qps_lock);2248 2249	return ret;2250}2251 2252static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset)2253{2254	int i;2255 2256	for (i = 0; i < qm->qp_num; i++)2257		qm_set_qp_disable(&qm->qp_array[i], offset);2258}2259 2260static int hisi_qm_uacce_get_queue(struct uacce_device *uacce,2261				   unsigned long arg,2262				   struct uacce_queue *q)2263{2264	struct hisi_qm *qm = uacce->priv;2265	struct hisi_qp *qp;2266	u8 alg_type = 0;2267 2268	qp = hisi_qm_create_qp(qm, alg_type);2269	if (IS_ERR(qp))2270		return PTR_ERR(qp);2271 2272	q->priv = qp;2273	q->uacce = uacce;2274	qp->uacce_q = q;2275	qp->event_cb = qm_qp_event_notifier;2276	qp->pasid = arg;2277	qp->is_in_kernel = false;2278 2279	return 0;2280}2281 2282static void hisi_qm_uacce_put_queue(struct uacce_queue *q)2283{2284	struct hisi_qp *qp = q->priv;2285 2286	hisi_qm_release_qp(qp);2287}2288 2289/* map sq/cq/doorbell to user space */2290static int hisi_qm_uacce_mmap(struct uacce_queue *q,2291			      struct vm_area_struct *vma,2292			      struct uacce_qfile_region *qfr)2293{2294	struct hisi_qp *qp = q->priv;2295	struct hisi_qm *qm = qp->qm;2296	resource_size_t phys_base = qm->db_phys_base +2297				    qp->qp_id * qm->db_interval;2298	size_t sz = vma->vm_end - vma->vm_start;2299	struct pci_dev *pdev = qm->pdev;2300	struct device *dev = &pdev->dev;2301	unsigned long vm_pgoff;2302	int ret;2303 2304	switch (qfr->type) {2305	case UACCE_QFRT_MMIO:2306		if (qm->ver == QM_HW_V1) {2307			if (sz > PAGE_SIZE * QM_DOORBELL_PAGE_NR)2308				return -EINVAL;2309		} else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) {2310			if (sz > PAGE_SIZE * (QM_DOORBELL_PAGE_NR +2311			    QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE))2312				return -EINVAL;2313		} else {2314			if (sz > qm->db_interval)2315				return -EINVAL;2316		}2317 2318		vm_flags_set(vma, VM_IO);2319 2320		return remap_pfn_range(vma, vma->vm_start,2321				       phys_base >> PAGE_SHIFT,2322				       sz, pgprot_noncached(vma->vm_page_prot));2323	case UACCE_QFRT_DUS:2324		if (sz != qp->qdma.size)2325			return -EINVAL;2326 2327		/*2328		 * dma_mmap_coherent() requires vm_pgoff as 02329		 * restore vm_pfoff to initial value for mmap()2330		 */2331		vm_pgoff = vma->vm_pgoff;2332		vma->vm_pgoff = 0;2333		ret = dma_mmap_coherent(dev, vma, qp->qdma.va,2334					qp->qdma.dma, sz);2335		vma->vm_pgoff = vm_pgoff;2336		return ret;2337 2338	default:2339		return -EINVAL;2340	}2341}2342 2343static int hisi_qm_uacce_start_queue(struct uacce_queue *q)2344{2345	struct hisi_qp *qp = q->priv;2346 2347	return hisi_qm_start_qp(qp, qp->pasid);2348}2349 2350static void hisi_qm_uacce_stop_queue(struct uacce_queue *q)2351{2352	struct hisi_qp *qp = q->priv;2353	struct hisi_qm *qm = qp->qm;2354	struct qm_dev_dfx *dev_dfx = &qm->debug.dev_dfx;2355	u32 i = 0;2356 2357	hisi_qm_stop_qp(qp);2358 2359	if (!dev_dfx->dev_timeout || !dev_dfx->dev_state)2360		return;2361 2362	/*2363	 * After the queue fails to be stopped,2364	 * wait for a period of time before releasing the queue.2365	 */2366	while (++i) {2367		msleep(WAIT_PERIOD);2368 2369		/* Since dev_timeout maybe modified, check i >= dev_timeout */2370		if (i >= dev_dfx->dev_timeout) {2371			dev_err(&qm->pdev->dev, "Stop q %u timeout, state %u\n",2372			       qp->qp_id, dev_dfx->dev_state);2373			dev_dfx->dev_state = QM_FINISH_WAIT;2374			break;2375		}2376	}2377}2378 2379static int hisi_qm_is_q_updated(struct uacce_queue *q)2380{2381	struct hisi_qp *qp = q->priv;2382	struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head;2383	int updated = 0;2384 2385	while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) {2386		/* make sure to read data from memory */2387		dma_rmb();2388		qm_cq_head_update(qp);2389		cqe = qp->cqe + qp->qp_status.cq_head;2390		updated = 1;2391	}2392 2393	return updated;2394}2395 2396static void qm_set_sqctype(struct uacce_queue *q, u16 type)2397{2398	struct hisi_qm *qm = q->uacce->priv;2399	struct hisi_qp *qp = q->priv;2400 2401	down_write(&qm->qps_lock);2402	qp->alg_type = type;2403	up_write(&qm->qps_lock);2404}2405 2406static long hisi_qm_uacce_ioctl(struct uacce_queue *q, unsigned int cmd,2407				unsigned long arg)2408{2409	struct hisi_qp *qp = q->priv;2410	struct hisi_qp_info qp_info;2411	struct hisi_qp_ctx qp_ctx;2412 2413	if (cmd == UACCE_CMD_QM_SET_QP_CTX) {2414		if (copy_from_user(&qp_ctx, (void __user *)arg,2415				   sizeof(struct hisi_qp_ctx)))2416			return -EFAULT;2417 2418		if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1)2419			return -EINVAL;2420 2421		qm_set_sqctype(q, qp_ctx.qc_type);2422		qp_ctx.id = qp->qp_id;2423 2424		if (copy_to_user((void __user *)arg, &qp_ctx,2425				 sizeof(struct hisi_qp_ctx)))2426			return -EFAULT;2427 2428		return 0;2429	} else if (cmd == UACCE_CMD_QM_SET_QP_INFO) {2430		if (copy_from_user(&qp_info, (void __user *)arg,2431				   sizeof(struct hisi_qp_info)))2432			return -EFAULT;2433 2434		qp_info.sqe_size = qp->qm->sqe_size;2435		qp_info.sq_depth = qp->sq_depth;2436		qp_info.cq_depth = qp->cq_depth;2437 2438		if (copy_to_user((void __user *)arg, &qp_info,2439				  sizeof(struct hisi_qp_info)))2440			return -EFAULT;2441 2442		return 0;2443	}2444 2445	return -EINVAL;2446}2447 2448/**2449 * qm_hw_err_isolate() - Try to set the isolation status of the uacce device2450 * according to user's configuration of error threshold.2451 * @qm: the uacce device2452 */2453static int qm_hw_err_isolate(struct hisi_qm *qm)2454{2455	struct qm_hw_err *err, *tmp, *hw_err;2456	struct qm_err_isolate *isolate;2457	u32 count = 0;2458 2459	isolate = &qm->isolate_data;2460 2461#define SECONDS_PER_HOUR	36002462 2463	/* All the hw errs are processed by PF driver */2464	if (qm->uacce->is_vf || isolate->is_isolate || !isolate->err_threshold)2465		return 0;2466 2467	hw_err = kzalloc(sizeof(*hw_err), GFP_KERNEL);2468	if (!hw_err)2469		return -ENOMEM;2470 2471	/*2472	 * Time-stamp every slot AER error. Then check the AER error log when the2473	 * next device AER error occurred. if the device slot AER error count exceeds2474	 * the setting error threshold in one hour, the isolated state will be set2475	 * to true. And the AER error logs that exceed one hour will be cleared.2476	 */2477	mutex_lock(&isolate->isolate_lock);2478	hw_err->timestamp = jiffies;2479	list_for_each_entry_safe(err, tmp, &isolate->qm_hw_errs, list) {2480		if ((hw_err->timestamp - err->timestamp) / HZ >2481		    SECONDS_PER_HOUR) {2482			list_del(&err->list);2483			kfree(err);2484		} else {2485			count++;2486		}2487	}2488	list_add(&hw_err->list, &isolate->qm_hw_errs);2489	mutex_unlock(&isolate->isolate_lock);2490 2491	if (count >= isolate->err_threshold)2492		isolate->is_isolate = true;2493 2494	return 0;2495}2496 2497static void qm_hw_err_destroy(struct hisi_qm *qm)2498{2499	struct qm_hw_err *err, *tmp;2500 2501	mutex_lock(&qm->isolate_data.isolate_lock);2502	list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) {2503		list_del(&err->list);2504		kfree(err);2505	}2506	mutex_unlock(&qm->isolate_data.isolate_lock);2507}2508 2509static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce)2510{2511	struct hisi_qm *qm = uacce->priv;2512	struct hisi_qm *pf_qm;2513 2514	if (uacce->is_vf)2515		pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));2516	else2517		pf_qm = qm;2518 2519	return pf_qm->isolate_data.is_isolate ?2520			UACCE_DEV_ISOLATE : UACCE_DEV_NORMAL;2521}2522 2523static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num)2524{2525	struct hisi_qm *qm = uacce->priv;2526 2527	/* Must be set by PF */2528	if (uacce->is_vf)2529		return -EPERM;2530 2531	if (qm->isolate_data.is_isolate)2532		return -EPERM;2533 2534	qm->isolate_data.err_threshold = num;2535 2536	/* After the policy is updated, need to reset the hardware err list */2537	qm_hw_err_destroy(qm);2538 2539	return 0;2540}2541 2542static u32 hisi_qm_isolate_threshold_read(struct uacce_device *uacce)2543{2544	struct hisi_qm *qm = uacce->priv;2545	struct hisi_qm *pf_qm;2546 2547	if (uacce->is_vf) {2548		pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));2549		return pf_qm->isolate_data.err_threshold;2550	}2551 2552	return qm->isolate_data.err_threshold;2553}2554 2555static const struct uacce_ops uacce_qm_ops = {2556	.get_available_instances = hisi_qm_get_available_instances,2557	.get_queue = hisi_qm_uacce_get_queue,2558	.put_queue = hisi_qm_uacce_put_queue,2559	.start_queue = hisi_qm_uacce_start_queue,2560	.stop_queue = hisi_qm_uacce_stop_queue,2561	.mmap = hisi_qm_uacce_mmap,2562	.ioctl = hisi_qm_uacce_ioctl,2563	.is_q_updated = hisi_qm_is_q_updated,2564	.get_isolate_state = hisi_qm_get_isolate_state,2565	.isolate_err_threshold_write = hisi_qm_isolate_threshold_write,2566	.isolate_err_threshold_read = hisi_qm_isolate_threshold_read,2567};2568 2569static void qm_remove_uacce(struct hisi_qm *qm)2570{2571	struct uacce_device *uacce = qm->uacce;2572 2573	if (qm->use_sva) {2574		qm_hw_err_destroy(qm);2575		uacce_remove(uacce);2576		qm->uacce = NULL;2577	}2578}2579 2580static int qm_alloc_uacce(struct hisi_qm *qm)2581{2582	struct pci_dev *pdev = qm->pdev;2583	struct uacce_device *uacce;2584	unsigned long mmio_page_nr;2585	unsigned long dus_page_nr;2586	u16 sq_depth, cq_depth;2587	struct uacce_interface interface = {2588		.flags = UACCE_DEV_SVA,2589		.ops = &uacce_qm_ops,2590	};2591	int ret;2592 2593	ret = strscpy(interface.name, dev_driver_string(&pdev->dev),2594		      sizeof(interface.name));2595	if (ret < 0)2596		return -ENAMETOOLONG;2597 2598	uacce = uacce_alloc(&pdev->dev, &interface);2599	if (IS_ERR(uacce))2600		return PTR_ERR(uacce);2601 2602	if (uacce->flags & UACCE_DEV_SVA) {2603		qm->use_sva = true;2604	} else {2605		/* only consider sva case */2606		qm_remove_uacce(qm);2607		return -EINVAL;2608	}2609 2610	uacce->is_vf = pdev->is_virtfn;2611	uacce->priv = qm;2612 2613	if (qm->ver == QM_HW_V1)2614		uacce->api_ver = HISI_QM_API_VER_BASE;2615	else if (qm->ver == QM_HW_V2)2616		uacce->api_ver = HISI_QM_API_VER2_BASE;2617	else2618		uacce->api_ver = HISI_QM_API_VER3_BASE;2619 2620	if (qm->ver == QM_HW_V1)2621		mmio_page_nr = QM_DOORBELL_PAGE_NR;2622	else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))2623		mmio_page_nr = QM_DOORBELL_PAGE_NR +2624			QM_DOORBELL_SQ_CQ_BASE_V2 / PAGE_SIZE;2625	else2626		mmio_page_nr = qm->db_interval / PAGE_SIZE;2627 2628	qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP);2629 2630	/* Add one more page for device or qp status */2631	dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth +2632		       sizeof(struct qm_cqe) * cq_depth  + PAGE_SIZE) >>2633					 PAGE_SHIFT;2634 2635	uacce->qf_pg_num[UACCE_QFRT_MMIO] = mmio_page_nr;2636	uacce->qf_pg_num[UACCE_QFRT_DUS]  = dus_page_nr;2637 2638	qm->uacce = uacce;2639	INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs);2640	mutex_init(&qm->isolate_data.isolate_lock);2641 2642	return 0;2643}2644 2645/**2646 * qm_frozen() - Try to froze QM to cut continuous queue request. If2647 * there is user on the QM, return failure without doing anything.2648 * @qm: The qm needed to be fronzen.2649 *2650 * This function frozes QM, then we can do SRIOV disabling.2651 */2652static int qm_frozen(struct hisi_qm *qm)2653{2654	if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl))2655		return 0;2656 2657	down_write(&qm->qps_lock);2658 2659	if (!qm->qp_in_used) {2660		qm->qp_in_used = qm->qp_num;2661		up_write(&qm->qps_lock);2662		set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl);2663		return 0;2664	}2665 2666	up_write(&qm->qps_lock);2667 2668	return -EBUSY;2669}2670 2671static int qm_try_frozen_vfs(struct pci_dev *pdev,2672			     struct hisi_qm_list *qm_list)2673{2674	struct hisi_qm *qm, *vf_qm;2675	struct pci_dev *dev;2676	int ret = 0;2677 2678	if (!qm_list || !pdev)2679		return -EINVAL;2680 2681	/* Try to frozen all the VFs as disable SRIOV */2682	mutex_lock(&qm_list->lock);2683	list_for_each_entry(qm, &qm_list->list, list) {2684		dev = qm->pdev;2685		if (dev == pdev)2686			continue;2687		if (pci_physfn(dev) == pdev) {2688			vf_qm = pci_get_drvdata(dev);2689			ret = qm_frozen(vf_qm);2690			if (ret)2691				goto frozen_fail;2692		}2693	}2694 2695frozen_fail:2696	mutex_unlock(&qm_list->lock);2697 2698	return ret;2699}2700 2701/**2702 * hisi_qm_wait_task_finish() - Wait until the task is finished2703 * when removing the driver.2704 * @qm: The qm needed to wait for the task to finish.2705 * @qm_list: The list of all available devices.2706 */2707void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list)2708{2709	while (qm_frozen(qm) ||2710	       ((qm->fun_type == QM_HW_PF) &&2711	       qm_try_frozen_vfs(qm->pdev, qm_list))) {2712		msleep(WAIT_PERIOD);2713	}2714 2715	while (test_bit(QM_RST_SCHED, &qm->misc_ctl) ||2716	       test_bit(QM_RESETTING, &qm->misc_ctl))2717		msleep(WAIT_PERIOD);2718 2719	if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))2720		flush_work(&qm->cmd_process);2721 2722	udelay(REMOVE_WAIT_DELAY);2723}2724EXPORT_SYMBOL_GPL(hisi_qm_wait_task_finish);2725 2726static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num)2727{2728	struct device *dev = &qm->pdev->dev;2729	struct qm_dma *qdma;2730	int i;2731 2732	for (i = num - 1; i >= 0; i--) {2733		qdma = &qm->qp_array[i].qdma;2734		dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma);2735		kfree(qm->poll_data[i].qp_finish_id);2736	}2737 2738	kfree(qm->poll_data);2739	kfree(qm->qp_array);2740}2741 2742static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id,2743			       u16 sq_depth, u16 cq_depth)2744{2745	struct device *dev = &qm->pdev->dev;2746	size_t off = qm->sqe_size * sq_depth;2747	struct hisi_qp *qp;2748	int ret = -ENOMEM;2749 2750	qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16),2751						 GFP_KERNEL);2752	if (!qm->poll_data[id].qp_finish_id)2753		return -ENOMEM;2754 2755	qp = &qm->qp_array[id];2756	qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma,2757					 GFP_KERNEL);2758	if (!qp->qdma.va)2759		goto err_free_qp_finish_id;2760 2761	qp->sqe = qp->qdma.va;2762	qp->sqe_dma = qp->qdma.dma;2763	qp->cqe = qp->qdma.va + off;2764	qp->cqe_dma = qp->qdma.dma + off;2765	qp->qdma.size = dma_size;2766	qp->sq_depth = sq_depth;2767	qp->cq_depth = cq_depth;2768	qp->qm = qm;2769	qp->qp_id = id;2770 2771	return 0;2772 2773err_free_qp_finish_id:2774	kfree(qm->poll_data[id].qp_finish_id);2775	return ret;2776}2777 2778static void hisi_qm_pre_init(struct hisi_qm *qm)2779{2780	struct pci_dev *pdev = qm->pdev;2781 2782	if (qm->ver == QM_HW_V1)2783		qm->ops = &qm_hw_ops_v1;2784	else if (qm->ver == QM_HW_V2)2785		qm->ops = &qm_hw_ops_v2;2786	else2787		qm->ops = &qm_hw_ops_v3;2788 2789	pci_set_drvdata(pdev, qm);2790	mutex_init(&qm->mailbox_lock);2791	init_rwsem(&qm->qps_lock);2792	qm->qp_in_used = 0;2793	if (test_bit(QM_SUPPORT_RPM, &qm->caps)) {2794		if (!acpi_device_power_manageable(ACPI_COMPANION(&pdev->dev)))2795			dev_info(&pdev->dev, "_PS0 and _PR0 are not defined");2796	}2797}2798 2799static void qm_cmd_uninit(struct hisi_qm *qm)2800{2801	u32 val;2802 2803	if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))2804		return;2805 2806	val = readl(qm->io_base + QM_IFC_INT_MASK);2807	val |= QM_IFC_INT_DISABLE;2808	writel(val, qm->io_base + QM_IFC_INT_MASK);2809}2810 2811static void qm_cmd_init(struct hisi_qm *qm)2812{2813	u32 val;2814 2815	if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))2816		return;2817 2818	/* Clear communication interrupt source */2819	qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR);2820 2821	/* Enable pf to vf communication reg. */2822	val = readl(qm->io_base + QM_IFC_INT_MASK);2823	val &= ~QM_IFC_INT_DISABLE;2824	writel(val, qm->io_base + QM_IFC_INT_MASK);2825}2826 2827static void qm_put_pci_res(struct hisi_qm *qm)2828{2829	struct pci_dev *pdev = qm->pdev;2830 2831	if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))2832		iounmap(qm->db_io_base);2833 2834	iounmap(qm->io_base);2835	pci_release_mem_regions(pdev);2836}2837 2838static void hisi_qm_pci_uninit(struct hisi_qm *qm)2839{2840	struct pci_dev *pdev = qm->pdev;2841 2842	pci_free_irq_vectors(pdev);2843	qm_put_pci_res(qm);2844	pci_disable_device(pdev);2845}2846 2847static void hisi_qm_set_state(struct hisi_qm *qm, u8 state)2848{2849	if (qm->ver > QM_HW_V2 && qm->fun_type == QM_HW_VF)2850		writel(state, qm->io_base + QM_VF_STATE);2851}2852 2853static void hisi_qm_unint_work(struct hisi_qm *qm)2854{2855	destroy_workqueue(qm->wq);2856}2857 2858static void hisi_qm_free_rsv_buf(struct hisi_qm *qm)2859{2860	struct qm_dma *xqc_dma = &qm->xqc_buf.qcdma;2861	struct device *dev = &qm->pdev->dev;2862 2863	dma_free_coherent(dev, xqc_dma->size, xqc_dma->va, xqc_dma->dma);2864}2865 2866static void hisi_qm_memory_uninit(struct hisi_qm *qm)2867{2868	struct device *dev = &qm->pdev->dev;2869 2870	hisi_qp_memory_uninit(qm, qm->qp_num);2871	hisi_qm_free_rsv_buf(qm);2872	if (qm->qdma.va) {2873		hisi_qm_cache_wb(qm);2874		dma_free_coherent(dev, qm->qdma.size,2875				  qm->qdma.va, qm->qdma.dma);2876	}2877 2878	idr_destroy(&qm->qp_idr);2879 2880	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))2881		kfree(qm->factor);2882}2883 2884/**2885 * hisi_qm_uninit() - Uninitialize qm.2886 * @qm: The qm needed uninit.2887 *2888 * This function uninits qm related device resources.2889 */2890void hisi_qm_uninit(struct hisi_qm *qm)2891{2892	qm_cmd_uninit(qm);2893	hisi_qm_unint_work(qm);2894 2895	down_write(&qm->qps_lock);2896	hisi_qm_memory_uninit(qm);2897	hisi_qm_set_state(qm, QM_NOT_READY);2898	up_write(&qm->qps_lock);2899 2900	qm_remove_uacce(qm);2901	qm_irqs_unregister(qm);2902	hisi_qm_pci_uninit(qm);2903}2904EXPORT_SYMBOL_GPL(hisi_qm_uninit);2905 2906/**2907 * hisi_qm_get_vft() - Get vft from a qm.2908 * @qm: The qm we want to get its vft.2909 * @base: The base number of queue in vft.2910 * @number: The number of queues in vft.2911 *2912 * We can allocate multiple queues to a qm by configuring virtual function2913 * table. We get related configures by this function. Normally, we call this2914 * function in VF driver to get the queue information.2915 *2916 * qm hw v1 does not support this interface.2917 */2918static int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number)2919{2920	if (!base || !number)2921		return -EINVAL;2922 2923	if (!qm->ops->get_vft) {2924		dev_err(&qm->pdev->dev, "Don't support vft read!\n");2925		return -EINVAL;2926	}2927 2928	return qm->ops->get_vft(qm, base, number);2929}2930 2931/**2932 * hisi_qm_set_vft() - Set vft to a qm.2933 * @qm: The qm we want to set its vft.2934 * @fun_num: The function number.2935 * @base: The base number of queue in vft.2936 * @number: The number of queues in vft.2937 *2938 * This function is alway called in PF driver, it is used to assign queues2939 * among PF and VFs.2940 *2941 * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)2942 * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)2943 * (VF function number 0x2)2944 */2945static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base,2946		    u32 number)2947{2948	u32 max_q_num = qm->ctrl_qp_num;2949 2950	if (base >= max_q_num || number > max_q_num ||2951	    (base + number) > max_q_num)2952		return -EINVAL;2953 2954	return qm_set_sqc_cqc_vft(qm, fun_num, base, number);2955}2956 2957static void qm_init_eq_aeq_status(struct hisi_qm *qm)2958{2959	struct hisi_qm_status *status = &qm->status;2960 2961	status->eq_head = 0;2962	status->aeq_head = 0;2963	status->eqc_phase = true;2964	status->aeqc_phase = true;2965}2966 2967static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm)2968{2969	/* Clear eq/aeq interrupt source */2970	qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);2971	qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);2972 2973	writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);2974	writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);2975}2976 2977static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm)2978{2979	writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);2980	writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);2981}2982 2983static int qm_eq_ctx_cfg(struct hisi_qm *qm)2984{2985	struct qm_eqc eqc = {0};2986 2987	eqc.base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma));2988	eqc.base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma));2989	if (qm->ver == QM_HW_V1)2990		eqc.dw3 = cpu_to_le32(QM_EQE_AEQE_SIZE);2991	eqc.dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT));2992 2993	return qm_set_and_get_xqc(qm, QM_MB_CMD_EQC, &eqc, 0, 0);2994}2995 2996static int qm_aeq_ctx_cfg(struct hisi_qm *qm)2997{2998	struct qm_aeqc aeqc = {0};2999 3000	aeqc.base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma));3001	aeqc.base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma));3002	aeqc.dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT));3003 3004	return qm_set_and_get_xqc(qm, QM_MB_CMD_AEQC, &aeqc, 0, 0);3005}3006 3007static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm)3008{3009	struct device *dev = &qm->pdev->dev;3010	int ret;3011 3012	qm_init_eq_aeq_status(qm);3013 3014	ret = qm_eq_ctx_cfg(qm);3015	if (ret) {3016		dev_err(dev, "Set eqc failed!\n");3017		return ret;3018	}3019 3020	return qm_aeq_ctx_cfg(qm);3021}3022 3023static int __hisi_qm_start(struct hisi_qm *qm)3024{3025	int ret;3026 3027	WARN_ON(!qm->qdma.va);3028 3029	if (qm->fun_type == QM_HW_PF) {3030		ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);3031		if (ret)3032			return ret;3033	}3034 3035	ret = qm_eq_aeq_ctx_cfg(qm);3036	if (ret)3037		return ret;3038 3039	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0);3040	if (ret)3041		return ret;3042 3043	ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0);3044	if (ret)3045		return ret;3046 3047	qm_init_prefetch(qm);3048	qm_enable_eq_aeq_interrupts(qm);3049 3050	return 0;3051}3052 3053/**3054 * hisi_qm_start() - start qm3055 * @qm: The qm to be started.3056 *3057 * This function starts a qm, then we can allocate qp from this qm.3058 */3059int hisi_qm_start(struct hisi_qm *qm)3060{3061	struct device *dev = &qm->pdev->dev;3062	int ret = 0;3063 3064	down_write(&qm->qps_lock);3065 3066	dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num);3067 3068	if (!qm->qp_num) {3069		dev_err(dev, "qp_num should not be 0\n");3070		ret = -EINVAL;3071		goto err_unlock;3072	}3073 3074	ret = __hisi_qm_start(qm);3075	if (ret)3076		goto err_unlock;3077 3078	atomic_set(&qm->status.flags, QM_WORK);3079	hisi_qm_set_state(qm, QM_READY);3080 3081err_unlock:3082	up_write(&qm->qps_lock);3083	return ret;3084}3085EXPORT_SYMBOL_GPL(hisi_qm_start);3086 3087static int qm_restart(struct hisi_qm *qm)3088{3089	struct device *dev = &qm->pdev->dev;3090	struct hisi_qp *qp;3091	int ret, i;3092 3093	ret = hisi_qm_start(qm);3094	if (ret < 0)3095		return ret;3096 3097	down_write(&qm->qps_lock);3098	for (i = 0; i < qm->qp_num; i++) {3099		qp = &qm->qp_array[i];3100		if (atomic_read(&qp->qp_status.flags) == QP_STOP &&3101		    qp->is_resetting == true) {3102			ret = qm_start_qp_nolock(qp, 0);3103			if (ret < 0) {3104				dev_err(dev, "Failed to start qp%d!\n", i);3105 3106				up_write(&qm->qps_lock);3107				return ret;3108			}3109			qp->is_resetting = false;3110		}3111	}3112	up_write(&qm->qps_lock);3113 3114	return 0;3115}3116 3117/* Stop started qps in reset flow */3118static void qm_stop_started_qp(struct hisi_qm *qm)3119{3120	struct hisi_qp *qp;3121	int i;3122 3123	for (i = 0; i < qm->qp_num; i++) {3124		qp = &qm->qp_array[i];3125		if (atomic_read(&qp->qp_status.flags) == QP_START) {3126			qp->is_resetting = true;3127			qm_stop_qp_nolock(qp);3128		}3129	}3130}3131 3132/**3133 * qm_clear_queues() - Clear all queues memory in a qm.3134 * @qm: The qm in which the queues will be cleared.3135 *3136 * This function clears all queues memory in a qm. Reset of accelerator can3137 * use this to clear queues.3138 */3139static void qm_clear_queues(struct hisi_qm *qm)3140{3141	struct hisi_qp *qp;3142	int i;3143 3144	for (i = 0; i < qm->qp_num; i++) {3145		qp = &qm->qp_array[i];3146		if (qp->is_in_kernel && qp->is_resetting)3147			memset(qp->qdma.va, 0, qp->qdma.size);3148	}3149 3150	memset(qm->qdma.va, 0, qm->qdma.size);3151}3152 3153/**3154 * hisi_qm_stop() - Stop a qm.3155 * @qm: The qm which will be stopped.3156 * @r: The reason to stop qm.3157 *3158 * This function stops qm and its qps, then qm can not accept request.3159 * Related resources are not released at this state, we can use hisi_qm_start3160 * to let qm start again.3161 */3162int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)3163{3164	struct device *dev = &qm->pdev->dev;3165	int ret = 0;3166 3167	down_write(&qm->qps_lock);3168 3169	if (atomic_read(&qm->status.flags) == QM_STOP)3170		goto err_unlock;3171 3172	/* Stop all the request sending at first. */3173	atomic_set(&qm->status.flags, QM_STOP);3174	qm->status.stop_reason = r;3175 3176	if (qm->status.stop_reason != QM_NORMAL) {3177		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);3178		/*3179		 * When performing soft reset, the hardware will no longer3180		 * do tasks, and the tasks in the device will be flushed3181		 * out directly since the master ooo is closed.3182		 */3183		if (test_bit(QM_SUPPORT_STOP_FUNC, &qm->caps) &&3184		    r != QM_SOFT_RESET) {3185			ret = qm_drain_qm(qm);3186			if (ret) {3187				dev_err(dev, "failed to drain qm!\n");3188				goto err_unlock;3189			}3190		}3191 3192		qm_stop_started_qp(qm);3193 3194		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);3195	}3196 3197	qm_disable_eq_aeq_interrupts(qm);3198	if (qm->fun_type == QM_HW_PF) {3199		ret = hisi_qm_set_vft(qm, 0, 0, 0);3200		if (ret < 0) {3201			dev_err(dev, "Failed to set vft!\n");3202			ret = -EBUSY;3203			goto err_unlock;3204		}3205	}3206 3207	qm_clear_queues(qm);3208	qm->status.stop_reason = QM_NORMAL;3209 3210err_unlock:3211	up_write(&qm->qps_lock);3212	return ret;3213}3214EXPORT_SYMBOL_GPL(hisi_qm_stop);3215 3216static void qm_hw_error_init(struct hisi_qm *qm)3217{3218	if (!qm->ops->hw_error_init) {3219		dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n");3220		return;3221	}3222 3223	qm->ops->hw_error_init(qm);3224}3225 3226static void qm_hw_error_uninit(struct hisi_qm *qm)3227{3228	if (!qm->ops->hw_error_uninit) {3229		dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n");3230		return;3231	}3232 3233	qm->ops->hw_error_uninit(qm);3234}3235 3236static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm)3237{3238	if (!qm->ops->hw_error_handle) {3239		dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n");3240		return ACC_ERR_NONE;3241	}3242 3243	return qm->ops->hw_error_handle(qm);3244}3245 3246/**3247 * hisi_qm_dev_err_init() - Initialize device error configuration.3248 * @qm: The qm for which we want to do error initialization.3249 *3250 * Initialize QM and device error related configuration.3251 */3252void hisi_qm_dev_err_init(struct hisi_qm *qm)3253{3254	if (qm->fun_type == QM_HW_VF)3255		return;3256 3257	qm_hw_error_init(qm);3258 3259	if (!qm->err_ini->hw_err_enable) {3260		dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n");3261		return;3262	}3263	qm->err_ini->hw_err_enable(qm);3264}3265EXPORT_SYMBOL_GPL(hisi_qm_dev_err_init);3266 3267/**3268 * hisi_qm_dev_err_uninit() - Uninitialize device error configuration.3269 * @qm: The qm for which we want to do error uninitialization.3270 *3271 * Uninitialize QM and device error related configuration.3272 */3273void hisi_qm_dev_err_uninit(struct hisi_qm *qm)3274{3275	if (qm->fun_type == QM_HW_VF)3276		return;3277 3278	qm_hw_error_uninit(qm);3279 3280	if (!qm->err_ini->hw_err_disable) {3281		dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n");3282		return;3283	}3284	qm->err_ini->hw_err_disable(qm);3285}3286EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit);3287 3288/**3289 * hisi_qm_free_qps() - free multiple queue pairs.3290 * @qps: The queue pairs need to be freed.3291 * @qp_num: The num of queue pairs.3292 */3293void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num)3294{3295	int i;3296 3297	if (!qps || qp_num <= 0)3298		return;3299 3300	for (i = qp_num - 1; i >= 0; i--)3301		hisi_qm_release_qp(qps[i]);3302}3303EXPORT_SYMBOL_GPL(hisi_qm_free_qps);3304 3305static void free_list(struct list_head *head)3306{3307	struct hisi_qm_resource *res, *tmp;3308 3309	list_for_each_entry_safe(res, tmp, head, list) {3310		list_del(&res->list);3311		kfree(res);3312	}3313}3314 3315static int hisi_qm_sort_devices(int node, struct list_head *head,3316				struct hisi_qm_list *qm_list)3317{3318	struct hisi_qm_resource *res, *tmp;3319	struct hisi_qm *qm;3320	struct list_head *n;3321	struct device *dev;3322	int dev_node;3323 3324	list_for_each_entry(qm, &qm_list->list, list) {3325		dev = &qm->pdev->dev;3326 3327		dev_node = dev_to_node(dev);3328		if (dev_node < 0)3329			dev_node = 0;3330 3331		res = kzalloc(sizeof(*res), GFP_KERNEL);3332		if (!res)3333			return -ENOMEM;3334 3335		res->qm = qm;3336		res->distance = node_distance(dev_node, node);3337		n = head;3338		list_for_each_entry(tmp, head, list) {3339			if (res->distance < tmp->distance) {3340				n = &tmp->list;3341				break;3342			}3343		}3344		list_add_tail(&res->list, n);3345	}3346 3347	return 0;3348}3349 3350/**3351 * hisi_qm_alloc_qps_node() - Create multiple queue pairs.3352 * @qm_list: The list of all available devices.3353 * @qp_num: The number of queue pairs need created.3354 * @alg_type: The algorithm type.3355 * @node: The numa node.3356 * @qps: The queue pairs need created.3357 *3358 * This function will sort all available device according to numa distance.3359 * Then try to create all queue pairs from one device, if all devices do3360 * not meet the requirements will return error.3361 */3362int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num,3363			   u8 alg_type, int node, struct hisi_qp **qps)3364{3365	struct hisi_qm_resource *tmp;3366	int ret = -ENODEV;3367	LIST_HEAD(head);3368	int i;3369 3370	if (!qps || !qm_list || qp_num <= 0)3371		return -EINVAL;3372 3373	mutex_lock(&qm_list->lock);3374	if (hisi_qm_sort_devices(node, &head, qm_list)) {3375		mutex_unlock(&qm_list->lock);3376		goto err;3377	}3378 3379	list_for_each_entry(tmp, &head, list) {3380		for (i = 0; i < qp_num; i++) {3381			qps[i] = hisi_qm_create_qp(tmp->qm, alg_type);3382			if (IS_ERR(qps[i])) {3383				hisi_qm_free_qps(qps, i);3384				break;3385			}3386		}3387 3388		if (i == qp_num) {3389			ret = 0;3390			break;3391		}3392	}3393 3394	mutex_unlock(&qm_list->lock);3395	if (ret)3396		pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n",3397			node, alg_type, qp_num);3398 3399err:3400	free_list(&head);3401	return ret;3402}3403EXPORT_SYMBOL_GPL(hisi_qm_alloc_qps_node);3404 3405static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs)3406{3407	u32 remain_q_num, vfs_q_num, act_q_num, q_num, i, j;3408	u32 max_qp_num = qm->max_qp_num;3409	u32 q_base = qm->qp_num;3410	int ret;3411 3412	if (!num_vfs)3413		return -EINVAL;3414 3415	vfs_q_num = qm->ctrl_qp_num - qm->qp_num;3416 3417	/* If vfs_q_num is less than num_vfs, return error. */3418	if (vfs_q_num < num_vfs)3419		return -EINVAL;3420 3421	q_num = vfs_q_num / num_vfs;3422	remain_q_num = vfs_q_num % num_vfs;3423 3424	for (i = num_vfs; i > 0; i--) {3425		/*3426		 * if q_num + remain_q_num > max_qp_num in last vf, divide the3427		 * remaining queues equally.3428		 */3429		if (i == num_vfs && q_num + remain_q_num <= max_qp_num) {3430			act_q_num = q_num + remain_q_num;3431			remain_q_num = 0;3432		} else if (remain_q_num > 0) {3433			act_q_num = q_num + 1;3434			remain_q_num--;3435		} else {3436			act_q_num = q_num;3437		}3438 3439		act_q_num = min(act_q_num, max_qp_num);3440		ret = hisi_qm_set_vft(qm, i, q_base, act_q_num);3441		if (ret) {3442			for (j = num_vfs; j > i; j--)3443				hisi_qm_set_vft(qm, j, 0, 0);3444			return ret;3445		}3446		q_base += act_q_num;3447	}3448 3449	return 0;3450}3451 3452static int qm_clear_vft_config(struct hisi_qm *qm)3453{3454	int ret;3455	u32 i;3456 3457	for (i = 1; i <= qm->vfs_num; i++) {3458		ret = hisi_qm_set_vft(qm, i, 0, 0);3459		if (ret)3460			return ret;3461	}3462	qm->vfs_num = 0;3463 3464	return 0;3465}3466 3467static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos)3468{3469	struct device *dev = &qm->pdev->dev;3470	u32 ir = qos * QM_QOS_RATE;3471	int ret, total_vfs, i;3472 3473	total_vfs = pci_sriov_get_totalvfs(qm->pdev);3474	if (fun_index > total_vfs)3475		return -EINVAL;3476 3477	qm->factor[fun_index].func_qos = qos;3478 3479	ret = qm_get_shaper_para(ir, &qm->factor[fun_index]);3480	if (ret) {3481		dev_err(dev, "failed to calculate shaper parameter!\n");3482		return -EINVAL;3483	}3484 3485	for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) {3486		/* The base number of queue reuse for different alg type */3487		ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1);3488		if (ret) {3489			dev_err(dev, "type: %d, failed to set shaper vft!\n", i);3490			return -EINVAL;3491		}3492	}3493 3494	return 0;3495}3496 3497static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index)3498{3499	u64 cir_u = 0, cir_b = 0, cir_s = 0;3500	u64 shaper_vft, ir_calc, ir;3501	unsigned int val;3502	u32 error_rate;3503	int ret;3504 3505	ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,3506					 val & BIT(0), POLL_PERIOD,3507					 POLL_TIMEOUT);3508	if (ret)3509		return 0;3510 3511	writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR);3512	writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE);3513	writel(fun_index, qm->io_base + QM_VFT_CFG);3514 3515	writel(0x0, qm->io_base + QM_VFT_CFG_RDY);3516	writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE);3517 3518	ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val,3519					 val & BIT(0), POLL_PERIOD,3520					 POLL_TIMEOUT);3521	if (ret)3522		return 0;3523 3524	shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) |3525		  ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32);3526 3527	cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK;3528	cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK;3529	cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT;3530 3531	cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK;3532	cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT;3533 3534	ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);3535 3536	ir = qm->factor[fun_index].func_qos * QM_QOS_RATE;3537 3538	error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;3539	if (error_rate > QM_QOS_MIN_ERROR_RATE) {3540		pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate);3541		return 0;3542	}3543 3544	return ir;3545}3546 3547static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num)3548{3549	struct device *dev = &qm->pdev->dev;3550	u64 mb_cmd;3551	u32 qos;3552	int ret;3553 3554	qos = qm_get_shaper_vft_qos(qm, fun_num);3555	if (!qos) {3556		dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num);3557		return;3558	}3559 3560	mb_cmd = QM_PF_SET_QOS | (u64)qos << QM_MB_CMD_DATA_SHIFT;3561	ret = qm_ping_single_vf(qm, mb_cmd, fun_num);3562	if (ret)3563		dev_err(dev, "failed to send cmd to VF(%u)!\n", fun_num);3564}3565 3566static int qm_vf_read_qos(struct hisi_qm *qm)3567{3568	int cnt = 0;3569	int ret = -EINVAL;3570 3571	/* reset mailbox qos val */3572	qm->mb_qos = 0;3573 3574	/* vf ping pf to get function qos */3575	ret = qm_ping_pf(qm, QM_VF_GET_QOS);3576	if (ret) {3577		pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n");3578		return ret;3579	}3580 3581	while (true) {3582		msleep(QM_WAIT_DST_ACK);3583		if (qm->mb_qos)3584			break;3585 3586		if (++cnt > QM_MAX_VF_WAIT_COUNT) {3587			pci_err(qm->pdev, "PF ping VF timeout!\n");3588			return  -ETIMEDOUT;3589		}3590	}3591 3592	return ret;3593}3594 3595static ssize_t qm_algqos_read(struct file *filp, char __user *buf,3596			       size_t count, loff_t *pos)3597{3598	struct hisi_qm *qm = filp->private_data;3599	char tbuf[QM_DBG_READ_LEN];3600	u32 qos_val, ir;3601	int ret;3602 3603	ret = hisi_qm_get_dfx_access(qm);3604	if (ret)3605		return ret;3606 3607	/* Mailbox and reset cannot be operated at the same time */3608	if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {3609		pci_err(qm->pdev, "dev resetting, read alg qos failed!\n");3610		ret = -EAGAIN;3611		goto err_put_dfx_access;3612	}3613 3614	if (qm->fun_type == QM_HW_PF) {3615		ir = qm_get_shaper_vft_qos(qm, 0);3616	} else {3617		ret = qm_vf_read_qos(qm);3618		if (ret)3619			goto err_get_status;3620		ir = qm->mb_qos;3621	}3622 3623	qos_val = ir / QM_QOS_RATE;3624	ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val);3625 3626	ret = simple_read_from_buffer(buf, count, pos, tbuf, ret);3627 3628err_get_status:3629	clear_bit(QM_RESETTING, &qm->misc_ctl);3630err_put_dfx_access:3631	hisi_qm_put_dfx_access(qm);3632	return ret;3633}3634 3635static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf,3636			       unsigned long *val,3637			       unsigned int *fun_index)3638{3639	const struct bus_type *bus_type = qm->pdev->dev.bus;3640	char tbuf_bdf[QM_DBG_READ_LEN] = {0};3641	char val_buf[QM_DBG_READ_LEN] = {0};3642	struct pci_dev *pdev;3643	struct device *dev;3644	int ret;3645 3646	ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf);3647	if (ret != QM_QOS_PARAM_NUM)3648		return -EINVAL;3649 3650	ret = kstrtoul(val_buf, 10, val);3651	if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) {3652		pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");3653		return -EINVAL;3654	}3655 3656	dev = bus_find_device_by_name(bus_type, NULL, tbuf_bdf);3657	if (!dev) {3658		pci_err(qm->pdev, "input pci bdf number is error!\n");3659		return -ENODEV;3660	}3661 3662	pdev = container_of(dev, struct pci_dev, dev);3663 3664	*fun_index = pdev->devfn;3665 3666	return 0;3667}3668 3669static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,3670			       size_t count, loff_t *pos)3671{3672	struct hisi_qm *qm = filp->private_data;3673	char tbuf[QM_DBG_READ_LEN];3674	unsigned int fun_index;3675	unsigned long val;3676	int len, ret;3677 3678	if (*pos != 0)3679		return 0;3680 3681	if (count >= QM_DBG_READ_LEN)3682		return -ENOSPC;3683 3684	len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);3685	if (len < 0)3686		return len;3687 3688	tbuf[len] = '\0';3689	ret = qm_get_qos_value(qm, tbuf, &val, &fun_index);3690	if (ret)3691		return ret;3692 3693	/* Mailbox and reset cannot be operated at the same time */3694	if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {3695		pci_err(qm->pdev, "dev resetting, write alg qos failed!\n");3696		return -EAGAIN;3697	}3698 3699	ret = qm_pm_get_sync(qm);3700	if (ret) {3701		ret = -EINVAL;3702		goto err_get_status;3703	}3704 3705	ret = qm_func_shaper_enable(qm, fun_index, val);3706	if (ret) {3707		pci_err(qm->pdev, "failed to enable function shaper!\n");3708		ret = -EINVAL;3709		goto err_put_sync;3710	}3711 3712	pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n",3713		 fun_index, val);3714	ret = count;3715 3716err_put_sync:3717	qm_pm_put_sync(qm);3718err_get_status:3719	clear_bit(QM_RESETTING, &qm->misc_ctl);3720	return ret;3721}3722 3723static const struct file_operations qm_algqos_fops = {3724	.owner = THIS_MODULE,3725	.open = simple_open,3726	.read = qm_algqos_read,3727	.write = qm_algqos_write,3728};3729 3730/**3731 * hisi_qm_set_algqos_init() - Initialize function qos debugfs files.3732 * @qm: The qm for which we want to add debugfs files.3733 *3734 * Create function qos debugfs files, VF ping PF to get function qos.3735 */3736void hisi_qm_set_algqos_init(struct hisi_qm *qm)3737{3738	if (qm->fun_type == QM_HW_PF)3739		debugfs_create_file("alg_qos", 0644, qm->debug.debug_root,3740				    qm, &qm_algqos_fops);3741	else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps))3742		debugfs_create_file("alg_qos", 0444, qm->debug.debug_root,3743				    qm, &qm_algqos_fops);3744}3745 3746static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func)3747{3748	int i;3749 3750	for (i = 1; i <= total_func; i++)3751		qm->factor[i].func_qos = QM_QOS_MAX_VAL;3752}3753 3754/**3755 * hisi_qm_sriov_enable() - enable virtual functions3756 * @pdev: the PCIe device3757 * @max_vfs: the number of virtual functions to enable3758 *3759 * Returns the number of enabled VFs. If there are VFs enabled already or3760 * max_vfs is more than the total number of device can be enabled, returns3761 * failure.3762 */3763int hisi_qm_sriov_enable(struct pci_dev *pdev, int max_vfs)3764{3765	struct hisi_qm *qm = pci_get_drvdata(pdev);3766	int pre_existing_vfs, num_vfs, total_vfs, ret;3767 3768	ret = qm_pm_get_sync(qm);3769	if (ret)3770		return ret;3771 3772	total_vfs = pci_sriov_get_totalvfs(pdev);3773	pre_existing_vfs = pci_num_vf(pdev);3774	if (pre_existing_vfs) {3775		pci_err(pdev, "%d VFs already enabled. Please disable pre-enabled VFs!\n",3776			pre_existing_vfs);3777		goto err_put_sync;3778	}3779 3780	if (max_vfs > total_vfs) {3781		pci_err(pdev, "%d VFs is more than total VFs %d!\n", max_vfs, total_vfs);3782		ret = -ERANGE;3783		goto err_put_sync;3784	}3785 3786	num_vfs = max_vfs;3787 3788	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))3789		hisi_qm_init_vf_qos(qm, num_vfs);3790 3791	ret = qm_vf_q_assign(qm, num_vfs);3792	if (ret) {3793		pci_err(pdev, "Can't assign queues for VF!\n");3794		goto err_put_sync;3795	}3796 3797	ret = pci_enable_sriov(pdev, num_vfs);3798	if (ret) {3799		pci_err(pdev, "Can't enable VF!\n");3800		qm_clear_vft_config(qm);3801		goto err_put_sync;3802	}3803	qm->vfs_num = num_vfs;3804 3805	pci_info(pdev, "VF enabled, vfs_num(=%d)!\n", num_vfs);3806 3807	return num_vfs;3808 3809err_put_sync:3810	qm_pm_put_sync(qm);3811	return ret;3812}3813EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable);3814 3815/**3816 * hisi_qm_sriov_disable - disable virtual functions3817 * @pdev: the PCI device.3818 * @is_frozen: true when all the VFs are frozen.3819 *3820 * Return failure if there are VFs assigned already or VF is in used.3821 */3822int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen)3823{3824	struct hisi_qm *qm = pci_get_drvdata(pdev);3825 3826	if (pci_vfs_assigned(pdev)) {3827		pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n");3828		return -EPERM;3829	}3830 3831	/* While VF is in used, SRIOV cannot be disabled. */3832	if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) {3833		pci_err(pdev, "Task is using its VF!\n");3834		return -EBUSY;3835	}3836 3837	pci_disable_sriov(pdev);3838 3839	qm->vfs_num = 0;3840	qm_pm_put_sync(qm);3841 3842	return qm_clear_vft_config(qm);3843}3844EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable);3845 3846/**3847 * hisi_qm_sriov_configure - configure the number of VFs3848 * @pdev: The PCI device3849 * @num_vfs: The number of VFs need enabled3850 *3851 * Enable SR-IOV according to num_vfs, 0 means disable.3852 */3853int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs)3854{3855	if (num_vfs == 0)3856		return hisi_qm_sriov_disable(pdev, false);3857	else3858		return hisi_qm_sriov_enable(pdev, num_vfs);3859}3860EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure);3861 3862static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm)3863{3864	u32 err_sts;3865 3866	if (!qm->err_ini->get_dev_hw_err_status) {3867		dev_err(&qm->pdev->dev, "Device doesn't support get hw error status!\n");3868		return ACC_ERR_NONE;3869	}3870 3871	/* get device hardware error status */3872	err_sts = qm->err_ini->get_dev_hw_err_status(qm);3873	if (err_sts) {3874		if (err_sts & qm->err_info.ecc_2bits_mask)3875			qm->err_status.is_dev_ecc_mbit = true;3876 3877		if (qm->err_ini->log_dev_hw_err)3878			qm->err_ini->log_dev_hw_err(qm, err_sts);3879 3880		if (err_sts & qm->err_info.dev_reset_mask)3881			return ACC_ERR_NEED_RESET;3882 3883		if (qm->err_ini->clear_dev_hw_err_status)3884			qm->err_ini->clear_dev_hw_err_status(qm, err_sts);3885	}3886 3887	return ACC_ERR_RECOVERED;3888}3889 3890static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm)3891{3892	enum acc_err_result qm_ret, dev_ret;3893 3894	/* log qm error */3895	qm_ret = qm_hw_error_handle(qm);3896 3897	/* log device error */3898	dev_ret = qm_dev_err_handle(qm);3899 3900	return (qm_ret == ACC_ERR_NEED_RESET ||3901		dev_ret == ACC_ERR_NEED_RESET) ?3902		ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED;3903}3904 3905/**3906 * hisi_qm_dev_err_detected() - Get device and qm error status then log it.3907 * @pdev: The PCI device which need report error.3908 * @state: The connectivity between CPU and device.3909 *3910 * We register this function into PCIe AER handlers, It will report device or3911 * qm hardware error status when error occur.3912 */3913pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev,3914					  pci_channel_state_t state)3915{3916	struct hisi_qm *qm = pci_get_drvdata(pdev);3917	enum acc_err_result ret;3918 3919	if (pdev->is_virtfn)3920		return PCI_ERS_RESULT_NONE;3921 3922	pci_info(pdev, "PCI error detected, state(=%u)!!\n", state);3923	if (state == pci_channel_io_perm_failure)3924		return PCI_ERS_RESULT_DISCONNECT;3925 3926	ret = qm_process_dev_error(qm);3927	if (ret == ACC_ERR_NEED_RESET)3928		return PCI_ERS_RESULT_NEED_RESET;3929 3930	return PCI_ERS_RESULT_RECOVERED;3931}3932EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected);3933 3934static int qm_check_req_recv(struct hisi_qm *qm)3935{3936	struct pci_dev *pdev = qm->pdev;3937	int ret;3938	u32 val;3939 3940	if (qm->ver >= QM_HW_V3)3941		return 0;3942 3943	writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID);3944	ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,3945					 (val == ACC_VENDOR_ID_VALUE),3946					 POLL_PERIOD, POLL_TIMEOUT);3947	if (ret) {3948		dev_err(&pdev->dev, "Fails to read QM reg!\n");3949		return ret;3950	}3951 3952	writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID);3953	ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val,3954					 (val == PCI_VENDOR_ID_HUAWEI),3955					 POLL_PERIOD, POLL_TIMEOUT);3956	if (ret)3957		dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n");3958 3959	return ret;3960}3961 3962static int qm_set_pf_mse(struct hisi_qm *qm, bool set)3963{3964	struct pci_dev *pdev = qm->pdev;3965	u16 cmd;3966	int i;3967 3968	pci_read_config_word(pdev, PCI_COMMAND, &cmd);3969	if (set)3970		cmd |= PCI_COMMAND_MEMORY;3971	else3972		cmd &= ~PCI_COMMAND_MEMORY;3973 3974	pci_write_config_word(pdev, PCI_COMMAND, cmd);3975	for (i = 0; i < MAX_WAIT_COUNTS; i++) {3976		pci_read_config_word(pdev, PCI_COMMAND, &cmd);3977		if (set == ((cmd & PCI_COMMAND_MEMORY) >> 1))3978			return 0;3979 3980		udelay(1);3981	}3982 3983	return -ETIMEDOUT;3984}3985 3986static int qm_set_vf_mse(struct hisi_qm *qm, bool set)3987{3988	struct pci_dev *pdev = qm->pdev;3989	u16 sriov_ctrl;3990	int pos;3991	int i;3992 3993	/*3994	 * Since function qm_set_vf_mse is called only after SRIOV is enabled,3995	 * pci_find_ext_capability cannot return 0, pos does not need to be3996	 * checked.3997	 */3998	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);3999	pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);4000	if (set)4001		sriov_ctrl |= PCI_SRIOV_CTRL_MSE;4002	else4003		sriov_ctrl &= ~PCI_SRIOV_CTRL_MSE;4004	pci_write_config_word(pdev, pos + PCI_SRIOV_CTRL, sriov_ctrl);4005 4006	for (i = 0; i < MAX_WAIT_COUNTS; i++) {4007		pci_read_config_word(pdev, pos + PCI_SRIOV_CTRL, &sriov_ctrl);4008		if (set == (sriov_ctrl & PCI_SRIOV_CTRL_MSE) >>4009		    ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT)4010			return 0;4011 4012		udelay(1);4013	}4014 4015	return -ETIMEDOUT;4016}4017 4018static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm)4019{4020	u32 nfe_enb = 0;4021 4022	/* Kunpeng930 hardware automatically close master ooo when NFE occurs */4023	if (qm->ver >= QM_HW_V3)4024		return;4025 4026	if (!qm->err_status.is_dev_ecc_mbit &&4027	    qm->err_status.is_qm_ecc_mbit &&4028	    qm->err_ini->close_axi_master_ooo) {4029		qm->err_ini->close_axi_master_ooo(qm);4030	} else if (qm->err_status.is_dev_ecc_mbit &&4031		   !qm->err_status.is_qm_ecc_mbit &&4032		   !qm->err_ini->close_axi_master_ooo) {4033		nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE);4034		writel(nfe_enb & QM_RAS_NFE_MBIT_DISABLE,4035		       qm->io_base + QM_RAS_NFE_ENABLE);4036		writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET);4037	}4038}4039 4040static int qm_vf_reset_prepare(struct hisi_qm *qm,4041			       enum qm_stop_reason stop_reason)4042{4043	struct hisi_qm_list *qm_list = qm->qm_list;4044	struct pci_dev *pdev = qm->pdev;4045	struct pci_dev *virtfn;4046	struct hisi_qm *vf_qm;4047	int ret = 0;4048 4049	mutex_lock(&qm_list->lock);4050	list_for_each_entry(vf_qm, &qm_list->list, list) {4051		virtfn = vf_qm->pdev;4052		if (virtfn == pdev)4053			continue;4054 4055		if (pci_physfn(virtfn) == pdev) {4056			/* save VFs PCIE BAR configuration */4057			pci_save_state(virtfn);4058 4059			ret = hisi_qm_stop(vf_qm, stop_reason);4060			if (ret)4061				goto stop_fail;4062		}4063	}4064 4065stop_fail:4066	mutex_unlock(&qm_list->lock);4067	return ret;4068}4069 4070static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd,4071			   enum qm_stop_reason stop_reason)4072{4073	struct pci_dev *pdev = qm->pdev;4074	int ret;4075 4076	if (!qm->vfs_num)4077		return 0;4078 4079	/* Kunpeng930 supports to notify VFs to stop before PF reset */4080	if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) {4081		ret = qm_ping_all_vfs(qm, cmd);4082		if (ret)4083			pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n");4084	} else {4085		ret = qm_vf_reset_prepare(qm, stop_reason);4086		if (ret)4087			pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret);4088	}4089 4090	return ret;4091}4092 4093static int qm_controller_reset_prepare(struct hisi_qm *qm)4094{4095	struct pci_dev *pdev = qm->pdev;4096	int ret;4097 4098	ret = qm_reset_prepare_ready(qm);4099	if (ret) {4100		pci_err(pdev, "Controller reset not ready!\n");4101		return ret;4102	}4103 4104	qm_dev_ecc_mbit_handle(qm);4105 4106	/* PF obtains the information of VF by querying the register. */4107	qm_cmd_uninit(qm);4108 4109	/* Whether VFs stop successfully, soft reset will continue. */4110	ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET);4111	if (ret)4112		pci_err(pdev, "failed to stop vfs by pf in soft reset.\n");4113 4114	ret = hisi_qm_stop(qm, QM_SOFT_RESET);4115	if (ret) {4116		pci_err(pdev, "Fails to stop QM!\n");4117		qm_reset_bit_clear(qm);4118		return ret;4119	}4120 4121	if (qm->use_sva) {4122		ret = qm_hw_err_isolate(qm);4123		if (ret)4124			pci_err(pdev, "failed to isolate hw err!\n");4125	}4126 4127	ret = qm_wait_vf_prepare_finish(qm);4128	if (ret)4129		pci_err(pdev, "failed to stop by vfs in soft reset!\n");4130 4131	clear_bit(QM_RST_SCHED, &qm->misc_ctl);4132 4133	return 0;4134}4135 4136static int qm_master_ooo_check(struct hisi_qm *qm)4137{4138	u32 val;4139	int ret;4140 4141	/* Check the ooo register of the device before resetting the device. */4142	writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN, qm->io_base + ACC_MASTER_GLOBAL_CTRL);4143	ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN,4144					 val, (val == ACC_MASTER_TRANS_RETURN_RW),4145					 POLL_PERIOD, POLL_TIMEOUT);4146	if (ret)4147		pci_warn(qm->pdev, "Bus lock! Please reset system.\n");4148 4149	return ret;4150}4151 4152static int qm_soft_reset_prepare(struct hisi_qm *qm)4153{4154	struct pci_dev *pdev = qm->pdev;4155	int ret;4156 4157	/* Ensure all doorbells and mailboxes received by QM */4158	ret = qm_check_req_recv(qm);4159	if (ret)4160		return ret;4161 4162	if (qm->vfs_num) {4163		ret = qm_set_vf_mse(qm, false);4164		if (ret) {4165			pci_err(pdev, "Fails to disable vf MSE bit.\n");4166			return ret;4167		}4168	}4169 4170	ret = qm->ops->set_msi(qm, false);4171	if (ret) {4172		pci_err(pdev, "Fails to disable PEH MSI bit.\n");4173		return ret;4174	}4175 4176	ret = qm_master_ooo_check(qm);4177	if (ret)4178		return ret;4179 4180	if (qm->err_ini->close_sva_prefetch)4181		qm->err_ini->close_sva_prefetch(qm);4182 4183	ret = qm_set_pf_mse(qm, false);4184	if (ret)4185		pci_err(pdev, "Fails to disable pf MSE bit.\n");4186 4187	return ret;4188}4189 4190static int qm_reset_device(struct hisi_qm *qm)4191{4192	struct pci_dev *pdev = qm->pdev;4193 4194	/* The reset related sub-control registers are not in PCI BAR */4195	if (ACPI_HANDLE(&pdev->dev)) {4196		unsigned long long value = 0;4197		acpi_status s;4198 4199		s = acpi_evaluate_integer(ACPI_HANDLE(&pdev->dev),4200					  qm->err_info.acpi_rst,4201					  NULL, &value);4202		if (ACPI_FAILURE(s)) {4203			pci_err(pdev, "NO controller reset method!\n");4204			return -EIO;4205		}4206 4207		if (value) {4208			pci_err(pdev, "Reset step %llu failed!\n", value);4209			return -EIO;4210		}4211 4212		return 0;4213	}4214 4215	pci_err(pdev, "No reset method!\n");4216	return -EINVAL;4217}4218 4219static int qm_soft_reset(struct hisi_qm *qm)4220{4221	int ret;4222 4223	ret = qm_soft_reset_prepare(qm);4224	if (ret)4225		return ret;4226 4227	return qm_reset_device(qm);4228}4229 4230static int qm_vf_reset_done(struct hisi_qm *qm)4231{4232	struct hisi_qm_list *qm_list = qm->qm_list;4233	struct pci_dev *pdev = qm->pdev;4234	struct pci_dev *virtfn;4235	struct hisi_qm *vf_qm;4236	int ret = 0;4237 4238	mutex_lock(&qm_list->lock);4239	list_for_each_entry(vf_qm, &qm_list->list, list) {4240		virtfn = vf_qm->pdev;4241		if (virtfn == pdev)4242			continue;4243 4244		if (pci_physfn(virtfn) == pdev) {4245			/* enable VFs PCIE BAR configuration */4246			pci_restore_state(virtfn);4247 4248			ret = qm_restart(vf_qm);4249			if (ret)4250				goto restart_fail;4251		}4252	}4253 4254restart_fail:4255	mutex_unlock(&qm_list->lock);4256	return ret;4257}4258 4259static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd)4260{4261	struct pci_dev *pdev = qm->pdev;4262	int ret;4263 4264	if (!qm->vfs_num)4265		return 0;4266 4267	ret = qm_vf_q_assign(qm, qm->vfs_num);4268	if (ret) {4269		pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret);4270		return ret;4271	}4272 4273	/* Kunpeng930 supports to notify VFs to start after PF reset. */4274	if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) {4275		ret = qm_ping_all_vfs(qm, cmd);4276		if (ret)4277			pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n");4278	} else {4279		ret = qm_vf_reset_done(qm);4280		if (ret)4281			pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret);4282	}4283 4284	return ret;4285}4286 4287static int qm_dev_hw_init(struct hisi_qm *qm)4288{4289	return qm->err_ini->hw_init(qm);4290}4291 4292static void qm_restart_prepare(struct hisi_qm *qm)4293{4294	u32 value;4295 4296	if (qm->err_ini->open_sva_prefetch)4297		qm->err_ini->open_sva_prefetch(qm);4298 4299	if (qm->ver >= QM_HW_V3)4300		return;4301 4302	if (!qm->err_status.is_qm_ecc_mbit &&4303	    !qm->err_status.is_dev_ecc_mbit)4304		return;4305 4306	/* temporarily close the OOO port used for PEH to write out MSI */4307	value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);4308	writel(value & ~qm->err_info.msi_wr_port,4309	       qm->io_base + ACC_AM_CFG_PORT_WR_EN);4310 4311	/* clear dev ecc 2bit error source if having */4312	value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask;4313	if (value && qm->err_ini->clear_dev_hw_err_status)4314		qm->err_ini->clear_dev_hw_err_status(qm, value);4315 4316	/* clear QM ecc mbit error source */4317	writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE);4318 4319	/* clear AM Reorder Buffer ecc mbit source */4320	writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS);4321}4322 4323static void qm_restart_done(struct hisi_qm *qm)4324{4325	u32 value;4326 4327	if (qm->ver >= QM_HW_V3)4328		goto clear_flags;4329 4330	if (!qm->err_status.is_qm_ecc_mbit &&4331	    !qm->err_status.is_dev_ecc_mbit)4332		return;4333 4334	/* open the OOO port for PEH to write out MSI */4335	value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN);4336	value |= qm->err_info.msi_wr_port;4337	writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN);4338 4339clear_flags:4340	qm->err_status.is_qm_ecc_mbit = false;4341	qm->err_status.is_dev_ecc_mbit = false;4342}4343 4344static int qm_controller_reset_done(struct hisi_qm *qm)4345{4346	struct pci_dev *pdev = qm->pdev;4347	int ret;4348 4349	ret = qm->ops->set_msi(qm, true);4350	if (ret) {4351		pci_err(pdev, "Fails to enable PEH MSI bit!\n");4352		return ret;4353	}4354 4355	ret = qm_set_pf_mse(qm, true);4356	if (ret) {4357		pci_err(pdev, "Fails to enable pf MSE bit!\n");4358		return ret;4359	}4360 4361	if (qm->vfs_num) {4362		ret = qm_set_vf_mse(qm, true);4363		if (ret) {4364			pci_err(pdev, "Fails to enable vf MSE bit!\n");4365			return ret;4366		}4367	}4368 4369	ret = qm_dev_hw_init(qm);4370	if (ret) {4371		pci_err(pdev, "Failed to init device\n");4372		return ret;4373	}4374 4375	qm_restart_prepare(qm);4376	hisi_qm_dev_err_init(qm);4377	if (qm->err_ini->open_axi_master_ooo)4378		qm->err_ini->open_axi_master_ooo(qm);4379 4380	ret = qm_dev_mem_reset(qm);4381	if (ret) {4382		pci_err(pdev, "failed to reset device memory\n");4383		return ret;4384	}4385 4386	ret = qm_restart(qm);4387	if (ret) {4388		pci_err(pdev, "Failed to start QM!\n");4389		return ret;4390	}4391 4392	ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);4393	if (ret)4394		pci_err(pdev, "failed to start vfs by pf in soft reset.\n");4395 4396	ret = qm_wait_vf_prepare_finish(qm);4397	if (ret)4398		pci_err(pdev, "failed to start by vfs in soft reset!\n");4399 4400	qm_cmd_init(qm);4401	qm_restart_done(qm);4402 4403	qm_reset_bit_clear(qm);4404 4405	return 0;4406}4407 4408static int qm_controller_reset(struct hisi_qm *qm)4409{4410	struct pci_dev *pdev = qm->pdev;4411	int ret;4412 4413	pci_info(pdev, "Controller resetting...\n");4414 4415	ret = qm_controller_reset_prepare(qm);4416	if (ret) {4417		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);4418		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);4419		clear_bit(QM_RST_SCHED, &qm->misc_ctl);4420		return ret;4421	}4422 4423	hisi_qm_show_last_dfx_regs(qm);4424	if (qm->err_ini->show_last_dfx_regs)4425		qm->err_ini->show_last_dfx_regs(qm);4426 4427	ret = qm_soft_reset(qm);4428	if (ret)4429		goto err_reset;4430 4431	ret = qm_controller_reset_done(qm);4432	if (ret)4433		goto err_reset;4434 4435	pci_info(pdev, "Controller reset complete\n");4436 4437	return 0;4438 4439err_reset:4440	pci_err(pdev, "Controller reset failed (%d)\n", ret);4441	qm_reset_bit_clear(qm);4442 4443	/* if resetting fails, isolate the device */4444	if (qm->use_sva)4445		qm->isolate_data.is_isolate = true;4446	return ret;4447}4448 4449/**4450 * hisi_qm_dev_slot_reset() - slot reset4451 * @pdev: the PCIe device4452 *4453 * This function offers QM relate PCIe device reset interface. Drivers which4454 * use QM can use this function as slot_reset in its struct pci_error_handlers.4455 */4456pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev)4457{4458	struct hisi_qm *qm = pci_get_drvdata(pdev);4459	int ret;4460 4461	if (pdev->is_virtfn)4462		return PCI_ERS_RESULT_RECOVERED;4463 4464	/* reset pcie device controller */4465	ret = qm_controller_reset(qm);4466	if (ret) {4467		pci_err(pdev, "Controller reset failed (%d)\n", ret);4468		return PCI_ERS_RESULT_DISCONNECT;4469	}4470 4471	return PCI_ERS_RESULT_RECOVERED;4472}4473EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset);4474 4475void hisi_qm_reset_prepare(struct pci_dev *pdev)4476{4477	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));4478	struct hisi_qm *qm = pci_get_drvdata(pdev);4479	u32 delay = 0;4480	int ret;4481 4482	hisi_qm_dev_err_uninit(pf_qm);4483 4484	/*4485	 * Check whether there is an ECC mbit error, If it occurs, need to4486	 * wait for soft reset to fix it.4487	 */4488	while (qm_check_dev_error(pf_qm)) {4489		msleep(++delay);4490		if (delay > QM_RESET_WAIT_TIMEOUT)4491			return;4492	}4493 4494	ret = qm_reset_prepare_ready(qm);4495	if (ret) {4496		pci_err(pdev, "FLR not ready!\n");4497		return;4498	}4499 4500	/* PF obtains the information of VF by querying the register. */4501	if (qm->fun_type == QM_HW_PF)4502		qm_cmd_uninit(qm);4503 4504	ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_DOWN);4505	if (ret)4506		pci_err(pdev, "failed to stop vfs by pf in FLR.\n");4507 4508	ret = hisi_qm_stop(qm, QM_DOWN);4509	if (ret) {4510		pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret);4511		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);4512		hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);4513		return;4514	}4515 4516	ret = qm_wait_vf_prepare_finish(qm);4517	if (ret)4518		pci_err(pdev, "failed to stop by vfs in FLR!\n");4519 4520	pci_info(pdev, "FLR resetting...\n");4521}4522EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare);4523 4524static bool qm_flr_reset_complete(struct pci_dev *pdev)4525{4526	struct pci_dev *pf_pdev = pci_physfn(pdev);4527	struct hisi_qm *qm = pci_get_drvdata(pf_pdev);4528	u32 id;4529 4530	pci_read_config_dword(qm->pdev, PCI_COMMAND, &id);4531	if (id == QM_PCI_COMMAND_INVALID) {4532		pci_err(pdev, "Device can not be used!\n");4533		return false;4534	}4535 4536	return true;4537}4538 4539void hisi_qm_reset_done(struct pci_dev *pdev)4540{4541	struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));4542	struct hisi_qm *qm = pci_get_drvdata(pdev);4543	int ret;4544 4545	if (qm->fun_type == QM_HW_PF) {4546		ret = qm_dev_hw_init(qm);4547		if (ret) {4548			pci_err(pdev, "Failed to init PF, ret = %d.\n", ret);4549			goto flr_done;4550		}4551	}4552 4553	hisi_qm_dev_err_init(pf_qm);4554 4555	ret = qm_restart(qm);4556	if (ret) {4557		pci_err(pdev, "Failed to start QM, ret = %d.\n", ret);4558		goto flr_done;4559	}4560 4561	ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE);4562	if (ret)4563		pci_err(pdev, "failed to start vfs by pf in FLR.\n");4564 4565	ret = qm_wait_vf_prepare_finish(qm);4566	if (ret)4567		pci_err(pdev, "failed to start by vfs in FLR!\n");4568 4569flr_done:4570	if (qm->fun_type == QM_HW_PF)4571		qm_cmd_init(qm);4572 4573	if (qm_flr_reset_complete(pdev))4574		pci_info(pdev, "FLR reset complete\n");4575 4576	qm_reset_bit_clear(qm);4577}4578EXPORT_SYMBOL_GPL(hisi_qm_reset_done);4579 4580static irqreturn_t qm_abnormal_irq(int irq, void *data)4581{4582	struct hisi_qm *qm = data;4583	enum acc_err_result ret;4584 4585	atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt);4586	ret = qm_process_dev_error(qm);4587	if (ret == ACC_ERR_NEED_RESET &&4588	    !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) &&4589	    !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl))4590		schedule_work(&qm->rst_work);4591 4592	return IRQ_HANDLED;4593}4594 4595/**4596 * hisi_qm_dev_shutdown() - Shutdown device.4597 * @pdev: The device will be shutdown.4598 *4599 * This function will stop qm when OS shutdown or rebooting.4600 */4601void hisi_qm_dev_shutdown(struct pci_dev *pdev)4602{4603	struct hisi_qm *qm = pci_get_drvdata(pdev);4604	int ret;4605 4606	ret = hisi_qm_stop(qm, QM_DOWN);4607	if (ret)4608		dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n");4609 4610	hisi_qm_cache_wb(qm);4611}4612EXPORT_SYMBOL_GPL(hisi_qm_dev_shutdown);4613 4614static void hisi_qm_controller_reset(struct work_struct *rst_work)4615{4616	struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work);4617	int ret;4618 4619	ret = qm_pm_get_sync(qm);4620	if (ret) {4621		clear_bit(QM_RST_SCHED, &qm->misc_ctl);4622		return;4623	}4624 4625	/* reset pcie device controller */4626	ret = qm_controller_reset(qm);4627	if (ret)4628		dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret);4629 4630	qm_pm_put_sync(qm);4631}4632 4633static void qm_pf_reset_vf_prepare(struct hisi_qm *qm,4634				   enum qm_stop_reason stop_reason)4635{4636	enum qm_mb_cmd cmd = QM_VF_PREPARE_DONE;4637	struct pci_dev *pdev = qm->pdev;4638	int ret;4639 4640	ret = qm_reset_prepare_ready(qm);4641	if (ret) {4642		dev_err(&pdev->dev, "reset prepare not ready!\n");4643		atomic_set(&qm->status.flags, QM_STOP);4644		cmd = QM_VF_PREPARE_FAIL;4645		goto err_prepare;4646	}4647 4648	ret = hisi_qm_stop(qm, stop_reason);4649	if (ret) {4650		dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret);4651		atomic_set(&qm->status.flags, QM_STOP);4652		cmd = QM_VF_PREPARE_FAIL;4653		goto err_prepare;4654	} else {4655		goto out;4656	}4657 4658err_prepare:4659	hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET);4660	hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);4661out:4662	pci_save_state(pdev);4663	ret = qm_ping_pf(qm, cmd);4664	if (ret)4665		dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n");4666}4667 4668static void qm_pf_reset_vf_done(struct hisi_qm *qm)4669{4670	enum qm_mb_cmd cmd = QM_VF_START_DONE;4671	struct pci_dev *pdev = qm->pdev;4672	int ret;4673 4674	pci_restore_state(pdev);4675	ret = hisi_qm_start(qm);4676	if (ret) {4677		dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret);4678		cmd = QM_VF_START_FAIL;4679	}4680 4681	qm_cmd_init(qm);4682	ret = qm_ping_pf(qm, cmd);4683	if (ret)4684		dev_warn(&pdev->dev, "PF responds timeout in reset done!\n");4685 4686	qm_reset_bit_clear(qm);4687}4688 4689static int qm_wait_pf_reset_finish(struct hisi_qm *qm)4690{4691	struct device *dev = &qm->pdev->dev;4692	u32 val, cmd;4693	u64 msg;4694	int ret;4695 4696	/* Wait for reset to finish */4697	ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val,4698					 val == BIT(0), QM_VF_RESET_WAIT_US,4699					 QM_VF_RESET_WAIT_TIMEOUT_US);4700	/* hardware completion status should be available by this time */4701	if (ret) {4702		dev_err(dev, "couldn't get reset done status from PF, timeout!\n");4703		return -ETIMEDOUT;4704	}4705 4706	/*4707	 * Whether message is got successfully,4708	 * VF needs to ack PF by clearing the interrupt.4709	 */4710	ret = qm_get_mb_cmd(qm, &msg, 0);4711	qm_clear_cmd_interrupt(qm, 0);4712	if (ret) {4713		dev_err(dev, "failed to get msg from PF in reset done!\n");4714		return ret;4715	}4716 4717	cmd = msg & QM_MB_CMD_DATA_MASK;4718	if (cmd != QM_PF_RESET_DONE) {4719		dev_err(dev, "the cmd(%u) is not reset done!\n", cmd);4720		ret = -EINVAL;4721	}4722 4723	return ret;4724}4725 4726static void qm_pf_reset_vf_process(struct hisi_qm *qm,4727				   enum qm_stop_reason stop_reason)4728{4729	struct device *dev = &qm->pdev->dev;4730	int ret;4731 4732	dev_info(dev, "device reset start...\n");4733 4734	/* The message is obtained by querying the register during resetting */4735	qm_cmd_uninit(qm);4736	qm_pf_reset_vf_prepare(qm, stop_reason);4737 4738	ret = qm_wait_pf_reset_finish(qm);4739	if (ret)4740		goto err_get_status;4741 4742	qm_pf_reset_vf_done(qm);4743 4744	dev_info(dev, "device reset done.\n");4745 4746	return;4747 4748err_get_status:4749	qm_cmd_init(qm);4750	qm_reset_bit_clear(qm);4751}4752 4753static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num)4754{4755	struct device *dev = &qm->pdev->dev;4756	u64 msg;4757	u32 cmd;4758	int ret;4759 4760	/*4761	 * Get the msg from source by sending mailbox. Whether message is got4762	 * successfully, destination needs to ack source by clearing the interrupt.4763	 */4764	ret = qm_get_mb_cmd(qm, &msg, fun_num);4765	qm_clear_cmd_interrupt(qm, BIT(fun_num));4766	if (ret) {4767		dev_err(dev, "failed to get msg from source!\n");4768		return;4769	}4770 4771	cmd = msg & QM_MB_CMD_DATA_MASK;4772	switch (cmd) {4773	case QM_PF_FLR_PREPARE:4774		qm_pf_reset_vf_process(qm, QM_DOWN);4775		break;4776	case QM_PF_SRST_PREPARE:4777		qm_pf_reset_vf_process(qm, QM_SOFT_RESET);4778		break;4779	case QM_VF_GET_QOS:4780		qm_vf_get_qos(qm, fun_num);4781		break;4782	case QM_PF_SET_QOS:4783		qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT;4784		break;4785	default:4786		dev_err(dev, "unsupported cmd %u sent by function(%u)!\n", cmd, fun_num);4787		break;4788	}4789}4790 4791static void qm_cmd_process(struct work_struct *cmd_process)4792{4793	struct hisi_qm *qm = container_of(cmd_process,4794					struct hisi_qm, cmd_process);4795	u32 vfs_num = qm->vfs_num;4796	u64 val;4797	u32 i;4798 4799	if (qm->fun_type == QM_HW_PF) {4800		val = readq(qm->io_base + QM_IFC_INT_SOURCE_P);4801		if (!val)4802			return;4803 4804		for (i = 1; i <= vfs_num; i++) {4805			if (val & BIT(i))4806				qm_handle_cmd_msg(qm, i);4807		}4808 4809		return;4810	}4811 4812	qm_handle_cmd_msg(qm, 0);4813}4814 4815/**4816 * hisi_qm_alg_register() - Register alg to crypto.4817 * @qm: The qm needs add.4818 * @qm_list: The qm list.4819 * @guard: Guard of qp_num.4820 *4821 * Register algorithm to crypto when the function is satisfy guard.4822 */4823int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard)4824{4825	struct device *dev = &qm->pdev->dev;4826 4827	if (qm->ver <= QM_HW_V2 && qm->use_sva) {4828		dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n");4829		return 0;4830	}4831 4832	if (qm->qp_num < guard) {4833		dev_info(dev, "qp_num is less than task need.\n");4834		return 0;4835	}4836 4837	return qm_list->register_to_crypto(qm);4838}4839EXPORT_SYMBOL_GPL(hisi_qm_alg_register);4840 4841/**4842 * hisi_qm_alg_unregister() - Unregister alg from crypto.4843 * @qm: The qm needs delete.4844 * @qm_list: The qm list.4845 * @guard: Guard of qp_num.4846 *4847 * Unregister algorithm from crypto when the last function is satisfy guard.4848 */4849void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list, int guard)4850{4851	if (qm->ver <= QM_HW_V2 && qm->use_sva)4852		return;4853 4854	if (qm->qp_num < guard)4855		return;4856 4857	qm_list->unregister_from_crypto(qm);4858}4859EXPORT_SYMBOL_GPL(hisi_qm_alg_unregister);4860 4861static void qm_unregister_abnormal_irq(struct hisi_qm *qm)4862{4863	struct pci_dev *pdev = qm->pdev;4864	u32 irq_vector, val;4865 4866	if (qm->fun_type == QM_HW_VF)4867		return;4868 4869	val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val;4870	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))4871		return;4872 4873	irq_vector = val & QM_IRQ_VECTOR_MASK;4874	free_irq(pci_irq_vector(pdev, irq_vector), qm);4875}4876 4877static int qm_register_abnormal_irq(struct hisi_qm *qm)4878{4879	struct pci_dev *pdev = qm->pdev;4880	u32 irq_vector, val;4881	int ret;4882 4883	if (qm->fun_type == QM_HW_VF)4884		return 0;4885 4886	val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val;4887	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK))4888		return 0;4889 4890	irq_vector = val & QM_IRQ_VECTOR_MASK;4891	ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm);4892	if (ret)4893		dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret);4894 4895	return ret;4896}4897 4898static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm)4899{4900	struct pci_dev *pdev = qm->pdev;4901	u32 irq_vector, val;4902 4903	val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val;4904	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))4905		return;4906 4907	irq_vector = val & QM_IRQ_VECTOR_MASK;4908	free_irq(pci_irq_vector(pdev, irq_vector), qm);4909}4910 4911static int qm_register_mb_cmd_irq(struct hisi_qm *qm)4912{4913	struct pci_dev *pdev = qm->pdev;4914	u32 irq_vector, val;4915	int ret;4916 4917	val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val;4918	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))4919		return 0;4920 4921	irq_vector = val & QM_IRQ_VECTOR_MASK;4922	ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm);4923	if (ret)4924		dev_err(&pdev->dev, "failed to request function communication irq, ret = %d", ret);4925 4926	return ret;4927}4928 4929static void qm_unregister_aeq_irq(struct hisi_qm *qm)4930{4931	struct pci_dev *pdev = qm->pdev;4932	u32 irq_vector, val;4933 4934	val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val;4935	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))4936		return;4937 4938	irq_vector = val & QM_IRQ_VECTOR_MASK;4939	free_irq(pci_irq_vector(pdev, irq_vector), qm);4940}4941 4942static int qm_register_aeq_irq(struct hisi_qm *qm)4943{4944	struct pci_dev *pdev = qm->pdev;4945	u32 irq_vector, val;4946	int ret;4947 4948	val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val;4949	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))4950		return 0;4951 4952	irq_vector = val & QM_IRQ_VECTOR_MASK;4953	ret = request_threaded_irq(pci_irq_vector(pdev, irq_vector), NULL,4954						   qm_aeq_thread, IRQF_ONESHOT, qm->dev_name, qm);4955	if (ret)4956		dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret);4957 4958	return ret;4959}4960 4961static void qm_unregister_eq_irq(struct hisi_qm *qm)4962{4963	struct pci_dev *pdev = qm->pdev;4964	u32 irq_vector, val;4965 4966	val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val;4967	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))4968		return;4969 4970	irq_vector = val & QM_IRQ_VECTOR_MASK;4971	free_irq(pci_irq_vector(pdev, irq_vector), qm);4972}4973 4974static int qm_register_eq_irq(struct hisi_qm *qm)4975{4976	struct pci_dev *pdev = qm->pdev;4977	u32 irq_vector, val;4978	int ret;4979 4980	val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val;4981	if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK))4982		return 0;4983 4984	irq_vector = val & QM_IRQ_VECTOR_MASK;4985	ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_eq_irq, 0, qm->dev_name, qm);4986	if (ret)4987		dev_err(&pdev->dev, "failed to request eq irq, ret = %d", ret);4988 4989	return ret;4990}4991 4992static void qm_irqs_unregister(struct hisi_qm *qm)4993{4994	qm_unregister_mb_cmd_irq(qm);4995	qm_unregister_abnormal_irq(qm);4996	qm_unregister_aeq_irq(qm);4997	qm_unregister_eq_irq(qm);4998}4999 5000static int qm_irqs_register(struct hisi_qm *qm)5001{5002	int ret;5003 5004	ret = qm_register_eq_irq(qm);5005	if (ret)5006		return ret;5007 5008	ret = qm_register_aeq_irq(qm);5009	if (ret)5010		goto free_eq_irq;5011 5012	ret = qm_register_abnormal_irq(qm);5013	if (ret)5014		goto free_aeq_irq;5015 5016	ret = qm_register_mb_cmd_irq(qm);5017	if (ret)5018		goto free_abnormal_irq;5019 5020	return 0;5021 5022free_abnormal_irq:5023	qm_unregister_abnormal_irq(qm);5024free_aeq_irq:5025	qm_unregister_aeq_irq(qm);5026free_eq_irq:5027	qm_unregister_eq_irq(qm);5028	return ret;5029}5030 5031static int qm_get_qp_num(struct hisi_qm *qm)5032{5033	struct device *dev = &qm->pdev->dev;5034	bool is_db_isolation;5035 5036	/* VF's qp_num assigned by PF in v2, and VF can get qp_num by vft. */5037	if (qm->fun_type == QM_HW_VF) {5038		if (qm->ver != QM_HW_V1)5039			/* v2 starts to support get vft by mailbox */5040			return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num);5041 5042		return 0;5043	}5044 5045	is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps);5046	qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true);5047	qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info,5048					     QM_FUNC_MAX_QP_CAP, is_db_isolation);5049 5050	if (qm->qp_num <= qm->max_qp_num)5051		return 0;5052 5053	if (test_bit(QM_MODULE_PARAM, &qm->misc_ctl)) {5054		/* Check whether the set qp number is valid */5055		dev_err(dev, "qp num(%u) is more than max qp num(%u)!\n",5056			qm->qp_num, qm->max_qp_num);5057		return -EINVAL;5058	}5059 5060	dev_info(dev, "Default qp num(%u) is too big, reset it to Function's max qp num(%u)!\n",5061		 qm->qp_num, qm->max_qp_num);5062	qm->qp_num = qm->max_qp_num;5063	qm->debug.curr_qm_qp_num = qm->qp_num;5064 5065	return 0;5066}5067 5068static int qm_pre_store_irq_type_caps(struct hisi_qm *qm)5069{5070	struct hisi_qm_cap_record *qm_cap;5071	struct pci_dev *pdev = qm->pdev;5072	size_t i, size;5073 5074	size = ARRAY_SIZE(qm_pre_store_caps);5075	qm_cap = devm_kzalloc(&pdev->dev, sizeof(*qm_cap) * size, GFP_KERNEL);5076	if (!qm_cap)5077		return -ENOMEM;5078 5079	for (i = 0; i < size; i++) {5080		qm_cap[i].type = qm_pre_store_caps[i];5081		qm_cap[i].cap_val = hisi_qm_get_hw_info(qm, qm_basic_info,5082							qm_pre_store_caps[i], qm->cap_ver);5083	}5084 5085	qm->cap_tables.qm_cap_table = qm_cap;5086 5087	return 0;5088}5089 5090static int qm_get_hw_caps(struct hisi_qm *qm)5091{5092	const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ?5093						  qm_cap_info_pf : qm_cap_info_vf;5094	u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) :5095				   ARRAY_SIZE(qm_cap_info_vf);5096	u32 val, i;5097 5098	/* Doorbell isolate register is a independent register. */5099	val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true);5100	if (val)5101		set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps);5102 5103	if (qm->ver >= QM_HW_V3) {5104		val = readl(qm->io_base + QM_FUNC_CAPS_REG);5105		qm->cap_ver = val & QM_CAPBILITY_VERSION;5106	}5107 5108	/* Get PF/VF common capbility */5109	for (i = 1; i < ARRAY_SIZE(qm_cap_info_comm); i++) {5110		val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver);5111		if (val)5112			set_bit(qm_cap_info_comm[i].type, &qm->caps);5113	}5114 5115	/* Get PF/VF different capbility */5116	for (i = 0; i < size; i++) {5117		val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver);5118		if (val)5119			set_bit(cap_info[i].type, &qm->caps);5120	}5121 5122	/* Fetch and save the value of irq type related capability registers */5123	return qm_pre_store_irq_type_caps(qm);5124}5125 5126static int qm_get_pci_res(struct hisi_qm *qm)5127{5128	struct pci_dev *pdev = qm->pdev;5129	struct device *dev = &pdev->dev;5130	int ret;5131 5132	ret = pci_request_mem_regions(pdev, qm->dev_name);5133	if (ret < 0) {5134		dev_err(dev, "Failed to request mem regions!\n");5135		return ret;5136	}5137 5138	qm->phys_base = pci_resource_start(pdev, PCI_BAR_2);5139	qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2));5140	if (!qm->io_base) {5141		ret = -EIO;5142		goto err_request_mem_regions;5143	}5144 5145	ret = qm_get_hw_caps(qm);5146	if (ret)5147		goto err_ioremap;5148 5149	if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) {5150		qm->db_interval = QM_QP_DB_INTERVAL;5151		qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4);5152		qm->db_io_base = ioremap(qm->db_phys_base,5153					 pci_resource_len(pdev, PCI_BAR_4));5154		if (!qm->db_io_base) {5155			ret = -EIO;5156			goto err_ioremap;5157		}5158	} else {5159		qm->db_phys_base = qm->phys_base;5160		qm->db_io_base = qm->io_base;5161		qm->db_interval = 0;5162	}5163 5164	ret = qm_get_qp_num(qm);5165	if (ret)5166		goto err_db_ioremap;5167 5168	return 0;5169 5170err_db_ioremap:5171	if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps))5172		iounmap(qm->db_io_base);5173err_ioremap:5174	iounmap(qm->io_base);5175err_request_mem_regions:5176	pci_release_mem_regions(pdev);5177	return ret;5178}5179 5180static int qm_clear_device(struct hisi_qm *qm)5181{5182	acpi_handle handle = ACPI_HANDLE(&qm->pdev->dev);5183	int ret;5184 5185	if (qm->fun_type == QM_HW_VF)5186		return 0;5187 5188	/* Device does not support reset, return */5189	if (!qm->err_ini->err_info_init)5190		return 0;5191	qm->err_ini->err_info_init(qm);5192 5193	if (!handle)5194		return 0;5195 5196	/* No reset method, return */5197	if (!acpi_has_method(handle, qm->err_info.acpi_rst))5198		return 0;5199 5200	ret = qm_master_ooo_check(qm);5201	if (ret) {5202		writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL);5203		return ret;5204	}5205 5206	return qm_reset_device(qm);5207}5208 5209static int hisi_qm_pci_init(struct hisi_qm *qm)5210{5211	struct pci_dev *pdev = qm->pdev;5212	struct device *dev = &pdev->dev;5213	unsigned int num_vec;5214	int ret;5215 5216	ret = pci_enable_device_mem(pdev);5217	if (ret < 0) {5218		dev_err(dev, "Failed to enable device mem!\n");5219		return ret;5220	}5221 5222	ret = qm_get_pci_res(qm);5223	if (ret)5224		goto err_disable_pcidev;5225 5226	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));5227	if (ret < 0)5228		goto err_get_pci_res;5229	pci_set_master(pdev);5230 5231	num_vec = qm_get_irq_num(qm);5232	ret = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSI);5233	if (ret < 0) {5234		dev_err(dev, "Failed to enable MSI vectors!\n");5235		goto err_get_pci_res;5236	}5237 5238	ret = qm_clear_device(qm);5239	if (ret)5240		goto err_free_vectors;5241 5242	return 0;5243 5244err_free_vectors:5245	pci_free_irq_vectors(pdev);5246err_get_pci_res:5247	qm_put_pci_res(qm);5248err_disable_pcidev:5249	pci_disable_device(pdev);5250	return ret;5251}5252 5253static int hisi_qm_init_work(struct hisi_qm *qm)5254{5255	int i;5256 5257	for (i = 0; i < qm->qp_num; i++)5258		INIT_WORK(&qm->poll_data[i].work, qm_work_process);5259 5260	if (qm->fun_type == QM_HW_PF)5261		INIT_WORK(&qm->rst_work, hisi_qm_controller_reset);5262 5263	if (qm->ver > QM_HW_V2)5264		INIT_WORK(&qm->cmd_process, qm_cmd_process);5265 5266	qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM |5267				 WQ_UNBOUND, num_online_cpus(),5268				 pci_name(qm->pdev));5269	if (!qm->wq) {5270		pci_err(qm->pdev, "failed to alloc workqueue!\n");5271		return -ENOMEM;5272	}5273 5274	return 0;5275}5276 5277static int hisi_qp_alloc_memory(struct hisi_qm *qm)5278{5279	struct device *dev = &qm->pdev->dev;5280	u16 sq_depth, cq_depth;5281	size_t qp_dma_size;5282	int i, ret;5283 5284	qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL);5285	if (!qm->qp_array)5286		return -ENOMEM;5287 5288	qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL);5289	if (!qm->poll_data) {5290		kfree(qm->qp_array);5291		return -ENOMEM;5292	}5293 5294	qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP);5295 5296	/* one more page for device or qp statuses */5297	qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth;5298	qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE;5299	for (i = 0; i < qm->qp_num; i++) {5300		qm->poll_data[i].qm = qm;5301		ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth);5302		if (ret)5303			goto err_init_qp_mem;5304 5305		dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size);5306	}5307 5308	return 0;5309err_init_qp_mem:5310	hisi_qp_memory_uninit(qm, i);5311 5312	return ret;5313}5314 5315static int hisi_qm_alloc_rsv_buf(struct hisi_qm *qm)5316{5317	struct qm_rsv_buf *xqc_buf = &qm->xqc_buf;5318	struct qm_dma *xqc_dma = &xqc_buf->qcdma;5319	struct device *dev = &qm->pdev->dev;5320	size_t off = 0;5321 5322#define QM_XQC_BUF_INIT(xqc_buf, type) do { \5323	(xqc_buf)->type = ((xqc_buf)->qcdma.va + (off)); \5324	(xqc_buf)->type##_dma = (xqc_buf)->qcdma.dma + (off); \5325	off += QMC_ALIGN(sizeof(struct qm_##type)); \5326} while (0)5327 5328	xqc_dma->size = QMC_ALIGN(sizeof(struct qm_eqc)) +5329			QMC_ALIGN(sizeof(struct qm_aeqc)) +5330			QMC_ALIGN(sizeof(struct qm_sqc)) +5331			QMC_ALIGN(sizeof(struct qm_cqc));5332	xqc_dma->va = dma_alloc_coherent(dev, xqc_dma->size,5333					 &xqc_dma->dma, GFP_KERNEL);5334	if (!xqc_dma->va)5335		return -ENOMEM;5336 5337	QM_XQC_BUF_INIT(xqc_buf, eqc);5338	QM_XQC_BUF_INIT(xqc_buf, aeqc);5339	QM_XQC_BUF_INIT(xqc_buf, sqc);5340	QM_XQC_BUF_INIT(xqc_buf, cqc);5341 5342	return 0;5343}5344 5345static int hisi_qm_memory_init(struct hisi_qm *qm)5346{5347	struct device *dev = &qm->pdev->dev;5348	int ret, total_func;5349	size_t off = 0;5350 5351	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) {5352		total_func = pci_sriov_get_totalvfs(qm->pdev) + 1;5353		qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL);5354		if (!qm->factor)5355			return -ENOMEM;5356 5357		/* Only the PF value needs to be initialized */5358		qm->factor[0].func_qos = QM_QOS_MAX_VAL;5359	}5360 5361#define QM_INIT_BUF(qm, type, num) do { \5362	(qm)->type = ((qm)->qdma.va + (off)); \5363	(qm)->type##_dma = (qm)->qdma.dma + (off); \5364	off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \5365} while (0)5366 5367	idr_init(&qm->qp_idr);5368	qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP);5369	qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) +5370			QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) +5371			QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) +5372			QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num);5373	qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma,5374					 GFP_ATOMIC);5375	dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size);5376	if (!qm->qdma.va) {5377		ret = -ENOMEM;5378		goto err_destroy_idr;5379	}5380 5381	QM_INIT_BUF(qm, eqe, qm->eq_depth);5382	QM_INIT_BUF(qm, aeqe, qm->aeq_depth);5383	QM_INIT_BUF(qm, sqc, qm->qp_num);5384	QM_INIT_BUF(qm, cqc, qm->qp_num);5385 5386	ret = hisi_qm_alloc_rsv_buf(qm);5387	if (ret)5388		goto err_free_qdma;5389 5390	ret = hisi_qp_alloc_memory(qm);5391	if (ret)5392		goto err_free_reserve_buf;5393 5394	return 0;5395 5396err_free_reserve_buf:5397	hisi_qm_free_rsv_buf(qm);5398err_free_qdma:5399	dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma);5400err_destroy_idr:5401	idr_destroy(&qm->qp_idr);5402	if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps))5403		kfree(qm->factor);5404 5405	return ret;5406}5407 5408/**5409 * hisi_qm_init() - Initialize configures about qm.5410 * @qm: The qm needing init.5411 *5412 * This function init qm, then we can call hisi_qm_start to put qm into work.5413 */5414int hisi_qm_init(struct hisi_qm *qm)5415{5416	struct pci_dev *pdev = qm->pdev;5417	struct device *dev = &pdev->dev;5418	int ret;5419 5420	hisi_qm_pre_init(qm);5421 5422	ret = hisi_qm_pci_init(qm);5423	if (ret)5424		return ret;5425 5426	ret = qm_irqs_register(qm);5427	if (ret)5428		goto err_pci_init;5429 5430	if (qm->fun_type == QM_HW_PF) {5431		/* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */5432		writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG);5433		qm_disable_clock_gate(qm);5434		ret = qm_dev_mem_reset(qm);5435		if (ret) {5436			dev_err(dev, "failed to reset device memory\n");5437			goto err_irq_register;5438		}5439	}5440 5441	if (qm->mode == UACCE_MODE_SVA) {5442		ret = qm_alloc_uacce(qm);5443		if (ret < 0)5444			dev_warn(dev, "fail to alloc uacce (%d)\n", ret);5445	}5446 5447	ret = hisi_qm_memory_init(qm);5448	if (ret)5449		goto err_alloc_uacce;5450 5451	ret = hisi_qm_init_work(qm);5452	if (ret)5453		goto err_free_qm_memory;5454 5455	qm_cmd_init(qm);5456 5457	return 0;5458 5459err_free_qm_memory:5460	hisi_qm_memory_uninit(qm);5461err_alloc_uacce:5462	qm_remove_uacce(qm);5463err_irq_register:5464	qm_irqs_unregister(qm);5465err_pci_init:5466	hisi_qm_pci_uninit(qm);5467	return ret;5468}5469EXPORT_SYMBOL_GPL(hisi_qm_init);5470 5471/**5472 * hisi_qm_get_dfx_access() - Try to get dfx access.5473 * @qm: pointer to accelerator device.5474 *5475 * Try to get dfx access, then user can get message.5476 *5477 * If device is in suspended, return failure, otherwise5478 * bump up the runtime PM usage counter.5479 */5480int hisi_qm_get_dfx_access(struct hisi_qm *qm)5481{5482	struct device *dev = &qm->pdev->dev;5483 5484	if (pm_runtime_suspended(dev)) {5485		dev_info(dev, "can not read/write - device in suspended.\n");5486		return -EAGAIN;5487	}5488 5489	return qm_pm_get_sync(qm);5490}5491EXPORT_SYMBOL_GPL(hisi_qm_get_dfx_access);5492 5493/**5494 * hisi_qm_put_dfx_access() - Put dfx access.5495 * @qm: pointer to accelerator device.5496 *5497 * Put dfx access, drop runtime PM usage counter.5498 */5499void hisi_qm_put_dfx_access(struct hisi_qm *qm)5500{5501	qm_pm_put_sync(qm);5502}5503EXPORT_SYMBOL_GPL(hisi_qm_put_dfx_access);5504 5505/**5506 * hisi_qm_pm_init() - Initialize qm runtime PM.5507 * @qm: pointer to accelerator device.5508 *5509 * Function that initialize qm runtime PM.5510 */5511void hisi_qm_pm_init(struct hisi_qm *qm)5512{5513	struct device *dev = &qm->pdev->dev;5514 5515	if (!test_bit(QM_SUPPORT_RPM, &qm->caps))5516		return;5517 5518	pm_runtime_set_autosuspend_delay(dev, QM_AUTOSUSPEND_DELAY);5519	pm_runtime_use_autosuspend(dev);5520	pm_runtime_put_noidle(dev);5521}5522EXPORT_SYMBOL_GPL(hisi_qm_pm_init);5523 5524/**5525 * hisi_qm_pm_uninit() - Uninitialize qm runtime PM.5526 * @qm: pointer to accelerator device.5527 *5528 * Function that uninitialize qm runtime PM.5529 */5530void hisi_qm_pm_uninit(struct hisi_qm *qm)5531{5532	struct device *dev = &qm->pdev->dev;5533 5534	if (!test_bit(QM_SUPPORT_RPM, &qm->caps))5535		return;5536 5537	pm_runtime_get_noresume(dev);5538	pm_runtime_dont_use_autosuspend(dev);5539}5540EXPORT_SYMBOL_GPL(hisi_qm_pm_uninit);5541 5542static int qm_prepare_for_suspend(struct hisi_qm *qm)5543{5544	struct pci_dev *pdev = qm->pdev;5545	int ret;5546 5547	ret = qm->ops->set_msi(qm, false);5548	if (ret) {5549		pci_err(pdev, "failed to disable MSI before suspending!\n");5550		return ret;5551	}5552 5553	ret = qm_master_ooo_check(qm);5554	if (ret)5555		return ret;5556 5557	ret = qm_set_pf_mse(qm, false);5558	if (ret)5559		pci_err(pdev, "failed to disable MSE before suspending!\n");5560 5561	return ret;5562}5563 5564static int qm_rebuild_for_resume(struct hisi_qm *qm)5565{5566	struct pci_dev *pdev = qm->pdev;5567	int ret;5568 5569	ret = qm_set_pf_mse(qm, true);5570	if (ret) {5571		pci_err(pdev, "failed to enable MSE after resuming!\n");5572		return ret;5573	}5574 5575	ret = qm->ops->set_msi(qm, true);5576	if (ret) {5577		pci_err(pdev, "failed to enable MSI after resuming!\n");5578		return ret;5579	}5580 5581	ret = qm_dev_hw_init(qm);5582	if (ret) {5583		pci_err(pdev, "failed to init device after resuming\n");5584		return ret;5585	}5586 5587	qm_cmd_init(qm);5588	hisi_qm_dev_err_init(qm);5589	/* Set the doorbell timeout to QM_DB_TIMEOUT_CFG ns. */5590	writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG);5591	qm_disable_clock_gate(qm);5592	ret = qm_dev_mem_reset(qm);5593	if (ret)5594		pci_err(pdev, "failed to reset device memory\n");5595 5596	return ret;5597}5598 5599/**5600 * hisi_qm_suspend() - Runtime suspend of given device.5601 * @dev: device to suspend.5602 *5603 * Function that suspend the device.5604 */5605int hisi_qm_suspend(struct device *dev)5606{5607	struct pci_dev *pdev = to_pci_dev(dev);5608	struct hisi_qm *qm = pci_get_drvdata(pdev);5609	int ret;5610 5611	pci_info(pdev, "entering suspended state\n");5612 5613	ret = hisi_qm_stop(qm, QM_NORMAL);5614	if (ret) {5615		pci_err(pdev, "failed to stop qm(%d)\n", ret);5616		return ret;5617	}5618 5619	ret = qm_prepare_for_suspend(qm);5620	if (ret)5621		pci_err(pdev, "failed to prepare suspended(%d)\n", ret);5622 5623	return ret;5624}5625EXPORT_SYMBOL_GPL(hisi_qm_suspend);5626 5627/**5628 * hisi_qm_resume() - Runtime resume of given device.5629 * @dev: device to resume.5630 *5631 * Function that resume the device.5632 */5633int hisi_qm_resume(struct device *dev)5634{5635	struct pci_dev *pdev = to_pci_dev(dev);5636	struct hisi_qm *qm = pci_get_drvdata(pdev);5637	int ret;5638 5639	pci_info(pdev, "resuming from suspend state\n");5640 5641	ret = qm_rebuild_for_resume(qm);5642	if (ret) {5643		pci_err(pdev, "failed to rebuild resume(%d)\n", ret);5644		return ret;5645	}5646 5647	ret = hisi_qm_start(qm);5648	if (ret) {5649		if (qm_check_dev_error(qm)) {5650			pci_info(pdev, "failed to start qm due to device error, device will be reset!\n");5651			return 0;5652		}5653 5654		pci_err(pdev, "failed to start qm(%d)!\n", ret);5655	}5656 5657	return ret;5658}5659EXPORT_SYMBOL_GPL(hisi_qm_resume);5660 5661MODULE_LICENSE("GPL v2");5662MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");5663MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");5664