brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0+2// Copyright (c) 2016-2017 Hisilicon Limited.3 4#include <linux/dma-mapping.h>5#include <linux/etherdevice.h>6#include <linux/interrupt.h>7#ifdef CONFIG_RFS_ACCEL8#include <linux/cpu_rmap.h>9#endif10#include <linux/if_vlan.h>11#include <linux/irq.h>12#include <linux/ip.h>13#include <linux/ipv6.h>14#include <linux/module.h>15#include <linux/pci.h>16#include <linux/skbuff.h>17#include <linux/sctp.h>18#include <net/gre.h>19#include <net/gro.h>20#include <net/ip6_checksum.h>21#include <net/page_pool/helpers.h>22#include <net/pkt_cls.h>23#include <net/pkt_sched.h>24#include <net/tcp.h>25#include <net/vxlan.h>26#include <net/geneve.h>27 28#include "hnae3.h"29#include "hns3_enet.h"30/* All hns3 tracepoints are defined by the include below, which31 * must be included exactly once across the whole kernel with32 * CREATE_TRACE_POINTS defined33 */34#define CREATE_TRACE_POINTS35#include "hns3_trace.h"36 37#define hns3_set_field(origin, shift, val)	((origin) |= (val) << (shift))38#define hns3_tx_bd_count(S)	DIV_ROUND_UP(S, HNS3_MAX_BD_SIZE)39 40#define hns3_rl_err(fmt, ...)						\41	do {								\42		if (net_ratelimit())					\43			netdev_err(fmt, ##__VA_ARGS__);			\44	} while (0)45 46static void hns3_clear_all_ring(struct hnae3_handle *h, bool force);47 48static const char hns3_driver_name[] = "hns3";49static const char hns3_driver_string[] =50			"Hisilicon Ethernet Network Driver for Hip08 Family";51static const char hns3_copyright[] = "Copyright (c) 2017 Huawei Corporation.";52static struct hnae3_client client;53 54static int debug = -1;55module_param(debug, int, 0);56MODULE_PARM_DESC(debug, " Network interface message level setting");57 58static unsigned int tx_sgl = 1;59module_param(tx_sgl, uint, 0600);60MODULE_PARM_DESC(tx_sgl, "Minimum number of frags when using dma_map_sg() to optimize the IOMMU mapping");61 62static bool page_pool_enabled = true;63module_param(page_pool_enabled, bool, 0400);64 65#define HNS3_SGL_SIZE(nfrag)	(sizeof(struct scatterlist) * (nfrag) +	\66				 sizeof(struct sg_table))67#define HNS3_MAX_SGL_SIZE	ALIGN(HNS3_SGL_SIZE(HNS3_MAX_TSO_BD_NUM), \68				      dma_get_cache_alignment())69 70#define DEFAULT_MSG_LEVEL (NETIF_MSG_PROBE | NETIF_MSG_LINK | \71			   NETIF_MSG_IFDOWN | NETIF_MSG_IFUP)72 73#define HNS3_INNER_VLAN_TAG	174#define HNS3_OUTER_VLAN_TAG	275 76#define HNS3_MIN_TX_LEN		33U77#define HNS3_MIN_TUN_PKT_LEN	65U78 79/* hns3_pci_tbl - PCI Device ID Table80 *81 * Last entry must be all 0s82 *83 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,84 *   Class, Class Mask, private data (not used) }85 */86static const struct pci_device_id hns3_pci_tbl[] = {87	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},88	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},89	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),90	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},91	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),92	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},93	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),94	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},95	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),96	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},97	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),98	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},99	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA),100	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},101	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF), 0},102	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),103	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},104	/* required last entry */105	{0,}106};107MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);108 109#define HNS3_RX_PTYPE_ENTRY(ptype, l, s, t, h) \110	{	ptype, \111		l, \112		CHECKSUM_##s, \113		HNS3_L3_TYPE_##t, \114		1, \115		h}116 117#define HNS3_RX_PTYPE_UNUSED_ENTRY(ptype) \118		{ ptype, 0, CHECKSUM_NONE, HNS3_L3_TYPE_PARSE_FAIL, 0, \119		  PKT_HASH_TYPE_NONE }120 121static const struct hns3_rx_ptype hns3_rx_ptype_tbl[] = {122	HNS3_RX_PTYPE_UNUSED_ENTRY(0),123	HNS3_RX_PTYPE_ENTRY(1, 0, COMPLETE, ARP, PKT_HASH_TYPE_NONE),124	HNS3_RX_PTYPE_ENTRY(2, 0, COMPLETE, RARP, PKT_HASH_TYPE_NONE),125	HNS3_RX_PTYPE_ENTRY(3, 0, COMPLETE, LLDP, PKT_HASH_TYPE_NONE),126	HNS3_RX_PTYPE_ENTRY(4, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE),127	HNS3_RX_PTYPE_ENTRY(5, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE),128	HNS3_RX_PTYPE_ENTRY(6, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE),129	HNS3_RX_PTYPE_ENTRY(7, 0, COMPLETE, CNM, PKT_HASH_TYPE_NONE),130	HNS3_RX_PTYPE_ENTRY(8, 0, NONE, PARSE_FAIL, PKT_HASH_TYPE_NONE),131	HNS3_RX_PTYPE_UNUSED_ENTRY(9),132	HNS3_RX_PTYPE_UNUSED_ENTRY(10),133	HNS3_RX_PTYPE_UNUSED_ENTRY(11),134	HNS3_RX_PTYPE_UNUSED_ENTRY(12),135	HNS3_RX_PTYPE_UNUSED_ENTRY(13),136	HNS3_RX_PTYPE_UNUSED_ENTRY(14),137	HNS3_RX_PTYPE_UNUSED_ENTRY(15),138	HNS3_RX_PTYPE_ENTRY(16, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE),139	HNS3_RX_PTYPE_ENTRY(17, 0, COMPLETE, IPV4, PKT_HASH_TYPE_NONE),140	HNS3_RX_PTYPE_ENTRY(18, 0, COMPLETE, IPV4, PKT_HASH_TYPE_NONE),141	HNS3_RX_PTYPE_ENTRY(19, 0, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),142	HNS3_RX_PTYPE_ENTRY(20, 0, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),143	HNS3_RX_PTYPE_ENTRY(21, 0, NONE, IPV4, PKT_HASH_TYPE_NONE),144	HNS3_RX_PTYPE_ENTRY(22, 0, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),145	HNS3_RX_PTYPE_ENTRY(23, 0, NONE, IPV4, PKT_HASH_TYPE_L3),146	HNS3_RX_PTYPE_ENTRY(24, 0, NONE, IPV4, PKT_HASH_TYPE_L3),147	HNS3_RX_PTYPE_ENTRY(25, 0, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),148	HNS3_RX_PTYPE_UNUSED_ENTRY(26),149	HNS3_RX_PTYPE_UNUSED_ENTRY(27),150	HNS3_RX_PTYPE_UNUSED_ENTRY(28),151	HNS3_RX_PTYPE_ENTRY(29, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE),152	HNS3_RX_PTYPE_ENTRY(30, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE),153	HNS3_RX_PTYPE_ENTRY(31, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3),154	HNS3_RX_PTYPE_ENTRY(32, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3),155	HNS3_RX_PTYPE_ENTRY(33, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),156	HNS3_RX_PTYPE_ENTRY(34, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),157	HNS3_RX_PTYPE_ENTRY(35, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),158	HNS3_RX_PTYPE_ENTRY(36, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3),159	HNS3_RX_PTYPE_ENTRY(37, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3),160	HNS3_RX_PTYPE_UNUSED_ENTRY(38),161	HNS3_RX_PTYPE_ENTRY(39, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),162	HNS3_RX_PTYPE_ENTRY(40, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),163	HNS3_RX_PTYPE_ENTRY(41, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),164	HNS3_RX_PTYPE_ENTRY(42, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),165	HNS3_RX_PTYPE_ENTRY(43, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),166	HNS3_RX_PTYPE_ENTRY(44, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),167	HNS3_RX_PTYPE_ENTRY(45, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),168	HNS3_RX_PTYPE_UNUSED_ENTRY(46),169	HNS3_RX_PTYPE_UNUSED_ENTRY(47),170	HNS3_RX_PTYPE_UNUSED_ENTRY(48),171	HNS3_RX_PTYPE_UNUSED_ENTRY(49),172	HNS3_RX_PTYPE_UNUSED_ENTRY(50),173	HNS3_RX_PTYPE_UNUSED_ENTRY(51),174	HNS3_RX_PTYPE_UNUSED_ENTRY(52),175	HNS3_RX_PTYPE_UNUSED_ENTRY(53),176	HNS3_RX_PTYPE_UNUSED_ENTRY(54),177	HNS3_RX_PTYPE_UNUSED_ENTRY(55),178	HNS3_RX_PTYPE_UNUSED_ENTRY(56),179	HNS3_RX_PTYPE_UNUSED_ENTRY(57),180	HNS3_RX_PTYPE_UNUSED_ENTRY(58),181	HNS3_RX_PTYPE_UNUSED_ENTRY(59),182	HNS3_RX_PTYPE_UNUSED_ENTRY(60),183	HNS3_RX_PTYPE_UNUSED_ENTRY(61),184	HNS3_RX_PTYPE_UNUSED_ENTRY(62),185	HNS3_RX_PTYPE_UNUSED_ENTRY(63),186	HNS3_RX_PTYPE_UNUSED_ENTRY(64),187	HNS3_RX_PTYPE_UNUSED_ENTRY(65),188	HNS3_RX_PTYPE_UNUSED_ENTRY(66),189	HNS3_RX_PTYPE_UNUSED_ENTRY(67),190	HNS3_RX_PTYPE_UNUSED_ENTRY(68),191	HNS3_RX_PTYPE_UNUSED_ENTRY(69),192	HNS3_RX_PTYPE_UNUSED_ENTRY(70),193	HNS3_RX_PTYPE_UNUSED_ENTRY(71),194	HNS3_RX_PTYPE_UNUSED_ENTRY(72),195	HNS3_RX_PTYPE_UNUSED_ENTRY(73),196	HNS3_RX_PTYPE_UNUSED_ENTRY(74),197	HNS3_RX_PTYPE_UNUSED_ENTRY(75),198	HNS3_RX_PTYPE_UNUSED_ENTRY(76),199	HNS3_RX_PTYPE_UNUSED_ENTRY(77),200	HNS3_RX_PTYPE_UNUSED_ENTRY(78),201	HNS3_RX_PTYPE_UNUSED_ENTRY(79),202	HNS3_RX_PTYPE_UNUSED_ENTRY(80),203	HNS3_RX_PTYPE_UNUSED_ENTRY(81),204	HNS3_RX_PTYPE_UNUSED_ENTRY(82),205	HNS3_RX_PTYPE_UNUSED_ENTRY(83),206	HNS3_RX_PTYPE_UNUSED_ENTRY(84),207	HNS3_RX_PTYPE_UNUSED_ENTRY(85),208	HNS3_RX_PTYPE_UNUSED_ENTRY(86),209	HNS3_RX_PTYPE_UNUSED_ENTRY(87),210	HNS3_RX_PTYPE_UNUSED_ENTRY(88),211	HNS3_RX_PTYPE_UNUSED_ENTRY(89),212	HNS3_RX_PTYPE_UNUSED_ENTRY(90),213	HNS3_RX_PTYPE_UNUSED_ENTRY(91),214	HNS3_RX_PTYPE_UNUSED_ENTRY(92),215	HNS3_RX_PTYPE_UNUSED_ENTRY(93),216	HNS3_RX_PTYPE_UNUSED_ENTRY(94),217	HNS3_RX_PTYPE_UNUSED_ENTRY(95),218	HNS3_RX_PTYPE_UNUSED_ENTRY(96),219	HNS3_RX_PTYPE_UNUSED_ENTRY(97),220	HNS3_RX_PTYPE_UNUSED_ENTRY(98),221	HNS3_RX_PTYPE_UNUSED_ENTRY(99),222	HNS3_RX_PTYPE_UNUSED_ENTRY(100),223	HNS3_RX_PTYPE_UNUSED_ENTRY(101),224	HNS3_RX_PTYPE_UNUSED_ENTRY(102),225	HNS3_RX_PTYPE_UNUSED_ENTRY(103),226	HNS3_RX_PTYPE_UNUSED_ENTRY(104),227	HNS3_RX_PTYPE_UNUSED_ENTRY(105),228	HNS3_RX_PTYPE_UNUSED_ENTRY(106),229	HNS3_RX_PTYPE_UNUSED_ENTRY(107),230	HNS3_RX_PTYPE_UNUSED_ENTRY(108),231	HNS3_RX_PTYPE_UNUSED_ENTRY(109),232	HNS3_RX_PTYPE_UNUSED_ENTRY(110),233	HNS3_RX_PTYPE_ENTRY(111, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),234	HNS3_RX_PTYPE_ENTRY(112, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),235	HNS3_RX_PTYPE_ENTRY(113, 0, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),236	HNS3_RX_PTYPE_ENTRY(114, 0, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),237	HNS3_RX_PTYPE_ENTRY(115, 0, NONE, IPV6, PKT_HASH_TYPE_L3),238	HNS3_RX_PTYPE_ENTRY(116, 0, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),239	HNS3_RX_PTYPE_ENTRY(117, 0, NONE, IPV6, PKT_HASH_TYPE_L3),240	HNS3_RX_PTYPE_ENTRY(118, 0, NONE, IPV6, PKT_HASH_TYPE_L3),241	HNS3_RX_PTYPE_ENTRY(119, 0, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),242	HNS3_RX_PTYPE_UNUSED_ENTRY(120),243	HNS3_RX_PTYPE_UNUSED_ENTRY(121),244	HNS3_RX_PTYPE_UNUSED_ENTRY(122),245	HNS3_RX_PTYPE_ENTRY(123, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE),246	HNS3_RX_PTYPE_ENTRY(124, 0, COMPLETE, PARSE_FAIL, PKT_HASH_TYPE_NONE),247	HNS3_RX_PTYPE_ENTRY(125, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3),248	HNS3_RX_PTYPE_ENTRY(126, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3),249	HNS3_RX_PTYPE_ENTRY(127, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),250	HNS3_RX_PTYPE_ENTRY(128, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),251	HNS3_RX_PTYPE_ENTRY(129, 1, UNNECESSARY, IPV4, PKT_HASH_TYPE_L4),252	HNS3_RX_PTYPE_ENTRY(130, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3),253	HNS3_RX_PTYPE_ENTRY(131, 0, COMPLETE, IPV4, PKT_HASH_TYPE_L3),254	HNS3_RX_PTYPE_UNUSED_ENTRY(132),255	HNS3_RX_PTYPE_ENTRY(133, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),256	HNS3_RX_PTYPE_ENTRY(134, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),257	HNS3_RX_PTYPE_ENTRY(135, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),258	HNS3_RX_PTYPE_ENTRY(136, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),259	HNS3_RX_PTYPE_ENTRY(137, 1, UNNECESSARY, IPV6, PKT_HASH_TYPE_L4),260	HNS3_RX_PTYPE_ENTRY(138, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),261	HNS3_RX_PTYPE_ENTRY(139, 0, COMPLETE, IPV6, PKT_HASH_TYPE_L3),262	HNS3_RX_PTYPE_UNUSED_ENTRY(140),263	HNS3_RX_PTYPE_UNUSED_ENTRY(141),264	HNS3_RX_PTYPE_UNUSED_ENTRY(142),265	HNS3_RX_PTYPE_UNUSED_ENTRY(143),266	HNS3_RX_PTYPE_UNUSED_ENTRY(144),267	HNS3_RX_PTYPE_UNUSED_ENTRY(145),268	HNS3_RX_PTYPE_UNUSED_ENTRY(146),269	HNS3_RX_PTYPE_UNUSED_ENTRY(147),270	HNS3_RX_PTYPE_UNUSED_ENTRY(148),271	HNS3_RX_PTYPE_UNUSED_ENTRY(149),272	HNS3_RX_PTYPE_UNUSED_ENTRY(150),273	HNS3_RX_PTYPE_UNUSED_ENTRY(151),274	HNS3_RX_PTYPE_UNUSED_ENTRY(152),275	HNS3_RX_PTYPE_UNUSED_ENTRY(153),276	HNS3_RX_PTYPE_UNUSED_ENTRY(154),277	HNS3_RX_PTYPE_UNUSED_ENTRY(155),278	HNS3_RX_PTYPE_UNUSED_ENTRY(156),279	HNS3_RX_PTYPE_UNUSED_ENTRY(157),280	HNS3_RX_PTYPE_UNUSED_ENTRY(158),281	HNS3_RX_PTYPE_UNUSED_ENTRY(159),282	HNS3_RX_PTYPE_UNUSED_ENTRY(160),283	HNS3_RX_PTYPE_UNUSED_ENTRY(161),284	HNS3_RX_PTYPE_UNUSED_ENTRY(162),285	HNS3_RX_PTYPE_UNUSED_ENTRY(163),286	HNS3_RX_PTYPE_UNUSED_ENTRY(164),287	HNS3_RX_PTYPE_UNUSED_ENTRY(165),288	HNS3_RX_PTYPE_UNUSED_ENTRY(166),289	HNS3_RX_PTYPE_UNUSED_ENTRY(167),290	HNS3_RX_PTYPE_UNUSED_ENTRY(168),291	HNS3_RX_PTYPE_UNUSED_ENTRY(169),292	HNS3_RX_PTYPE_UNUSED_ENTRY(170),293	HNS3_RX_PTYPE_UNUSED_ENTRY(171),294	HNS3_RX_PTYPE_UNUSED_ENTRY(172),295	HNS3_RX_PTYPE_UNUSED_ENTRY(173),296	HNS3_RX_PTYPE_UNUSED_ENTRY(174),297	HNS3_RX_PTYPE_UNUSED_ENTRY(175),298	HNS3_RX_PTYPE_UNUSED_ENTRY(176),299	HNS3_RX_PTYPE_UNUSED_ENTRY(177),300	HNS3_RX_PTYPE_UNUSED_ENTRY(178),301	HNS3_RX_PTYPE_UNUSED_ENTRY(179),302	HNS3_RX_PTYPE_UNUSED_ENTRY(180),303	HNS3_RX_PTYPE_UNUSED_ENTRY(181),304	HNS3_RX_PTYPE_UNUSED_ENTRY(182),305	HNS3_RX_PTYPE_UNUSED_ENTRY(183),306	HNS3_RX_PTYPE_UNUSED_ENTRY(184),307	HNS3_RX_PTYPE_UNUSED_ENTRY(185),308	HNS3_RX_PTYPE_UNUSED_ENTRY(186),309	HNS3_RX_PTYPE_UNUSED_ENTRY(187),310	HNS3_RX_PTYPE_UNUSED_ENTRY(188),311	HNS3_RX_PTYPE_UNUSED_ENTRY(189),312	HNS3_RX_PTYPE_UNUSED_ENTRY(190),313	HNS3_RX_PTYPE_UNUSED_ENTRY(191),314	HNS3_RX_PTYPE_UNUSED_ENTRY(192),315	HNS3_RX_PTYPE_UNUSED_ENTRY(193),316	HNS3_RX_PTYPE_UNUSED_ENTRY(194),317	HNS3_RX_PTYPE_UNUSED_ENTRY(195),318	HNS3_RX_PTYPE_UNUSED_ENTRY(196),319	HNS3_RX_PTYPE_UNUSED_ENTRY(197),320	HNS3_RX_PTYPE_UNUSED_ENTRY(198),321	HNS3_RX_PTYPE_UNUSED_ENTRY(199),322	HNS3_RX_PTYPE_UNUSED_ENTRY(200),323	HNS3_RX_PTYPE_UNUSED_ENTRY(201),324	HNS3_RX_PTYPE_UNUSED_ENTRY(202),325	HNS3_RX_PTYPE_UNUSED_ENTRY(203),326	HNS3_RX_PTYPE_UNUSED_ENTRY(204),327	HNS3_RX_PTYPE_UNUSED_ENTRY(205),328	HNS3_RX_PTYPE_UNUSED_ENTRY(206),329	HNS3_RX_PTYPE_UNUSED_ENTRY(207),330	HNS3_RX_PTYPE_UNUSED_ENTRY(208),331	HNS3_RX_PTYPE_UNUSED_ENTRY(209),332	HNS3_RX_PTYPE_UNUSED_ENTRY(210),333	HNS3_RX_PTYPE_UNUSED_ENTRY(211),334	HNS3_RX_PTYPE_UNUSED_ENTRY(212),335	HNS3_RX_PTYPE_UNUSED_ENTRY(213),336	HNS3_RX_PTYPE_UNUSED_ENTRY(214),337	HNS3_RX_PTYPE_UNUSED_ENTRY(215),338	HNS3_RX_PTYPE_UNUSED_ENTRY(216),339	HNS3_RX_PTYPE_UNUSED_ENTRY(217),340	HNS3_RX_PTYPE_UNUSED_ENTRY(218),341	HNS3_RX_PTYPE_UNUSED_ENTRY(219),342	HNS3_RX_PTYPE_UNUSED_ENTRY(220),343	HNS3_RX_PTYPE_UNUSED_ENTRY(221),344	HNS3_RX_PTYPE_UNUSED_ENTRY(222),345	HNS3_RX_PTYPE_UNUSED_ENTRY(223),346	HNS3_RX_PTYPE_UNUSED_ENTRY(224),347	HNS3_RX_PTYPE_UNUSED_ENTRY(225),348	HNS3_RX_PTYPE_UNUSED_ENTRY(226),349	HNS3_RX_PTYPE_UNUSED_ENTRY(227),350	HNS3_RX_PTYPE_UNUSED_ENTRY(228),351	HNS3_RX_PTYPE_UNUSED_ENTRY(229),352	HNS3_RX_PTYPE_UNUSED_ENTRY(230),353	HNS3_RX_PTYPE_UNUSED_ENTRY(231),354	HNS3_RX_PTYPE_UNUSED_ENTRY(232),355	HNS3_RX_PTYPE_UNUSED_ENTRY(233),356	HNS3_RX_PTYPE_UNUSED_ENTRY(234),357	HNS3_RX_PTYPE_UNUSED_ENTRY(235),358	HNS3_RX_PTYPE_UNUSED_ENTRY(236),359	HNS3_RX_PTYPE_UNUSED_ENTRY(237),360	HNS3_RX_PTYPE_UNUSED_ENTRY(238),361	HNS3_RX_PTYPE_UNUSED_ENTRY(239),362	HNS3_RX_PTYPE_UNUSED_ENTRY(240),363	HNS3_RX_PTYPE_UNUSED_ENTRY(241),364	HNS3_RX_PTYPE_UNUSED_ENTRY(242),365	HNS3_RX_PTYPE_UNUSED_ENTRY(243),366	HNS3_RX_PTYPE_UNUSED_ENTRY(244),367	HNS3_RX_PTYPE_UNUSED_ENTRY(245),368	HNS3_RX_PTYPE_UNUSED_ENTRY(246),369	HNS3_RX_PTYPE_UNUSED_ENTRY(247),370	HNS3_RX_PTYPE_UNUSED_ENTRY(248),371	HNS3_RX_PTYPE_UNUSED_ENTRY(249),372	HNS3_RX_PTYPE_UNUSED_ENTRY(250),373	HNS3_RX_PTYPE_UNUSED_ENTRY(251),374	HNS3_RX_PTYPE_UNUSED_ENTRY(252),375	HNS3_RX_PTYPE_UNUSED_ENTRY(253),376	HNS3_RX_PTYPE_UNUSED_ENTRY(254),377	HNS3_RX_PTYPE_UNUSED_ENTRY(255),378};379 380#define HNS3_INVALID_PTYPE \381		ARRAY_SIZE(hns3_rx_ptype_tbl)382 383static irqreturn_t hns3_irq_handle(int irq, void *vector)384{385	struct hns3_enet_tqp_vector *tqp_vector = vector;386 387	napi_schedule_irqoff(&tqp_vector->napi);388	tqp_vector->event_cnt++;389 390	return IRQ_HANDLED;391}392 393static void hns3_nic_uninit_irq(struct hns3_nic_priv *priv)394{395	struct hns3_enet_tqp_vector *tqp_vectors;396	unsigned int i;397 398	for (i = 0; i < priv->vector_num; i++) {399		tqp_vectors = &priv->tqp_vector[i];400 401		if (tqp_vectors->irq_init_flag != HNS3_VECTOR_INITED)402			continue;403 404		/* clear the affinity mask */405		irq_set_affinity_hint(tqp_vectors->vector_irq, NULL);406 407		/* release the irq resource */408		free_irq(tqp_vectors->vector_irq, tqp_vectors);409		tqp_vectors->irq_init_flag = HNS3_VECTOR_NOT_INITED;410	}411}412 413static int hns3_nic_init_irq(struct hns3_nic_priv *priv)414{415	struct hns3_enet_tqp_vector *tqp_vectors;416	int txrx_int_idx = 0;417	int rx_int_idx = 0;418	int tx_int_idx = 0;419	unsigned int i;420	int ret;421 422	for (i = 0; i < priv->vector_num; i++) {423		tqp_vectors = &priv->tqp_vector[i];424 425		if (tqp_vectors->irq_init_flag == HNS3_VECTOR_INITED)426			continue;427 428		if (tqp_vectors->tx_group.ring && tqp_vectors->rx_group.ring) {429			snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,430				 "%s-%s-%s-%d", hns3_driver_name,431				 pci_name(priv->ae_handle->pdev),432				 "TxRx", txrx_int_idx++);433			txrx_int_idx++;434		} else if (tqp_vectors->rx_group.ring) {435			snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,436				 "%s-%s-%s-%d", hns3_driver_name,437				 pci_name(priv->ae_handle->pdev),438				 "Rx", rx_int_idx++);439		} else if (tqp_vectors->tx_group.ring) {440			snprintf(tqp_vectors->name, HNAE3_INT_NAME_LEN,441				 "%s-%s-%s-%d", hns3_driver_name,442				 pci_name(priv->ae_handle->pdev),443				 "Tx", tx_int_idx++);444		} else {445			/* Skip this unused q_vector */446			continue;447		}448 449		tqp_vectors->name[HNAE3_INT_NAME_LEN - 1] = '\0';450 451		irq_set_status_flags(tqp_vectors->vector_irq, IRQ_NOAUTOEN);452		ret = request_irq(tqp_vectors->vector_irq, hns3_irq_handle, 0,453				  tqp_vectors->name, tqp_vectors);454		if (ret) {455			netdev_err(priv->netdev, "request irq(%d) fail\n",456				   tqp_vectors->vector_irq);457			hns3_nic_uninit_irq(priv);458			return ret;459		}460 461		irq_set_affinity_hint(tqp_vectors->vector_irq,462				      &tqp_vectors->affinity_mask);463 464		tqp_vectors->irq_init_flag = HNS3_VECTOR_INITED;465	}466 467	return 0;468}469 470static void hns3_mask_vector_irq(struct hns3_enet_tqp_vector *tqp_vector,471				 u32 mask_en)472{473	writel(mask_en, tqp_vector->mask_addr);474}475 476static void hns3_vector_enable(struct hns3_enet_tqp_vector *tqp_vector)477{478	napi_enable(&tqp_vector->napi);479	enable_irq(tqp_vector->vector_irq);480 481	/* enable vector */482	hns3_mask_vector_irq(tqp_vector, 1);483}484 485static void hns3_vector_disable(struct hns3_enet_tqp_vector *tqp_vector)486{487	/* disable vector */488	hns3_mask_vector_irq(tqp_vector, 0);489 490	disable_irq(tqp_vector->vector_irq);491	napi_disable(&tqp_vector->napi);492	cancel_work_sync(&tqp_vector->rx_group.dim.work);493	cancel_work_sync(&tqp_vector->tx_group.dim.work);494}495 496void hns3_set_vector_coalesce_rl(struct hns3_enet_tqp_vector *tqp_vector,497				 u32 rl_value)498{499	u32 rl_reg = hns3_rl_usec_to_reg(rl_value);500 501	/* this defines the configuration for RL (Interrupt Rate Limiter).502	 * Rl defines rate of interrupts i.e. number of interrupts-per-second503	 * GL and RL(Rate Limiter) are 2 ways to acheive interrupt coalescing504	 */505	if (rl_reg > 0 && !tqp_vector->tx_group.coal.adapt_enable &&506	    !tqp_vector->rx_group.coal.adapt_enable)507		/* According to the hardware, the range of rl_reg is508		 * 0-59 and the unit is 4.509		 */510		rl_reg |=  HNS3_INT_RL_ENABLE_MASK;511 512	writel(rl_reg, tqp_vector->mask_addr + HNS3_VECTOR_RL_OFFSET);513}514 515void hns3_set_vector_coalesce_rx_gl(struct hns3_enet_tqp_vector *tqp_vector,516				    u32 gl_value)517{518	u32 new_val;519 520	if (tqp_vector->rx_group.coal.unit_1us)521		new_val = gl_value | HNS3_INT_GL_1US;522	else523		new_val = hns3_gl_usec_to_reg(gl_value);524 525	writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL0_OFFSET);526}527 528void hns3_set_vector_coalesce_tx_gl(struct hns3_enet_tqp_vector *tqp_vector,529				    u32 gl_value)530{531	u32 new_val;532 533	if (tqp_vector->tx_group.coal.unit_1us)534		new_val = gl_value | HNS3_INT_GL_1US;535	else536		new_val = hns3_gl_usec_to_reg(gl_value);537 538	writel(new_val, tqp_vector->mask_addr + HNS3_VECTOR_GL1_OFFSET);539}540 541void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector,542				    u32 ql_value)543{544	writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_TX_QL_OFFSET);545}546 547void hns3_set_vector_coalesce_rx_ql(struct hns3_enet_tqp_vector *tqp_vector,548				    u32 ql_value)549{550	writel(ql_value, tqp_vector->mask_addr + HNS3_VECTOR_RX_QL_OFFSET);551}552 553static void hns3_vector_coalesce_init(struct hns3_enet_tqp_vector *tqp_vector,554				      struct hns3_nic_priv *priv)555{556	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);557	struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;558	struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;559	struct hns3_enet_coalesce *ptx_coal = &priv->tx_coal;560	struct hns3_enet_coalesce *prx_coal = &priv->rx_coal;561 562	tx_coal->adapt_enable = ptx_coal->adapt_enable;563	rx_coal->adapt_enable = prx_coal->adapt_enable;564 565	tx_coal->int_gl = ptx_coal->int_gl;566	rx_coal->int_gl = prx_coal->int_gl;567 568	rx_coal->flow_level = prx_coal->flow_level;569	tx_coal->flow_level = ptx_coal->flow_level;570 571	/* device version above V3(include V3), GL can configure 1us572	 * unit, so uses 1us unit.573	 */574	if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3) {575		tx_coal->unit_1us = 1;576		rx_coal->unit_1us = 1;577	}578 579	if (ae_dev->dev_specs.int_ql_max) {580		tx_coal->ql_enable = 1;581		rx_coal->ql_enable = 1;582		tx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;583		rx_coal->int_ql_max = ae_dev->dev_specs.int_ql_max;584		tx_coal->int_ql = ptx_coal->int_ql;585		rx_coal->int_ql = prx_coal->int_ql;586	}587}588 589static void590hns3_vector_coalesce_init_hw(struct hns3_enet_tqp_vector *tqp_vector,591			     struct hns3_nic_priv *priv)592{593	struct hns3_enet_coalesce *tx_coal = &tqp_vector->tx_group.coal;594	struct hns3_enet_coalesce *rx_coal = &tqp_vector->rx_group.coal;595	struct hnae3_handle *h = priv->ae_handle;596 597	hns3_set_vector_coalesce_tx_gl(tqp_vector, tx_coal->int_gl);598	hns3_set_vector_coalesce_rx_gl(tqp_vector, rx_coal->int_gl);599	hns3_set_vector_coalesce_rl(tqp_vector, h->kinfo.int_rl_setting);600 601	if (tx_coal->ql_enable)602		hns3_set_vector_coalesce_tx_ql(tqp_vector, tx_coal->int_ql);603 604	if (rx_coal->ql_enable)605		hns3_set_vector_coalesce_rx_ql(tqp_vector, rx_coal->int_ql);606}607 608static int hns3_nic_set_real_num_queue(struct net_device *netdev)609{610	struct hnae3_handle *h = hns3_get_handle(netdev);611	struct hnae3_knic_private_info *kinfo = &h->kinfo;612	struct hnae3_tc_info *tc_info = &kinfo->tc_info;613	unsigned int queue_size = kinfo->num_tqps;614	int i, ret;615 616	if (tc_info->num_tc <= 1 && !tc_info->mqprio_active) {617		netdev_reset_tc(netdev);618	} else {619		ret = netdev_set_num_tc(netdev, tc_info->num_tc);620		if (ret) {621			netdev_err(netdev,622				   "netdev_set_num_tc fail, ret=%d!\n", ret);623			return ret;624		}625 626		for (i = 0; i < tc_info->num_tc; i++)627			netdev_set_tc_queue(netdev, i, tc_info->tqp_count[i],628					    tc_info->tqp_offset[i]);629	}630 631	ret = netif_set_real_num_tx_queues(netdev, queue_size);632	if (ret) {633		netdev_err(netdev,634			   "netif_set_real_num_tx_queues fail, ret=%d!\n", ret);635		return ret;636	}637 638	ret = netif_set_real_num_rx_queues(netdev, queue_size);639	if (ret) {640		netdev_err(netdev,641			   "netif_set_real_num_rx_queues fail, ret=%d!\n", ret);642		return ret;643	}644 645	return 0;646}647 648u16 hns3_get_max_available_channels(struct hnae3_handle *h)649{650	u16 alloc_tqps, max_rss_size, rss_size;651 652	h->ae_algo->ops->get_tqps_and_rss_info(h, &alloc_tqps, &max_rss_size);653	rss_size = alloc_tqps / h->kinfo.tc_info.num_tc;654 655	return min_t(u16, rss_size, max_rss_size);656}657 658static void hns3_tqp_enable(struct hnae3_queue *tqp)659{660	u32 rcb_reg;661 662	rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);663	rcb_reg |= BIT(HNS3_RING_EN_B);664	hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);665}666 667static void hns3_tqp_disable(struct hnae3_queue *tqp)668{669	u32 rcb_reg;670 671	rcb_reg = hns3_read_dev(tqp, HNS3_RING_EN_REG);672	rcb_reg &= ~BIT(HNS3_RING_EN_B);673	hns3_write_dev(tqp, HNS3_RING_EN_REG, rcb_reg);674}675 676static void hns3_free_rx_cpu_rmap(struct net_device *netdev)677{678#ifdef CONFIG_RFS_ACCEL679	free_irq_cpu_rmap(netdev->rx_cpu_rmap);680	netdev->rx_cpu_rmap = NULL;681#endif682}683 684static int hns3_set_rx_cpu_rmap(struct net_device *netdev)685{686#ifdef CONFIG_RFS_ACCEL687	struct hns3_nic_priv *priv = netdev_priv(netdev);688	struct hns3_enet_tqp_vector *tqp_vector;689	int i, ret;690 691	if (!netdev->rx_cpu_rmap) {692		netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(priv->vector_num);693		if (!netdev->rx_cpu_rmap)694			return -ENOMEM;695	}696 697	for (i = 0; i < priv->vector_num; i++) {698		tqp_vector = &priv->tqp_vector[i];699		ret = irq_cpu_rmap_add(netdev->rx_cpu_rmap,700				       tqp_vector->vector_irq);701		if (ret) {702			hns3_free_rx_cpu_rmap(netdev);703			return ret;704		}705	}706#endif707	return 0;708}709 710static int hns3_nic_net_up(struct net_device *netdev)711{712	struct hns3_nic_priv *priv = netdev_priv(netdev);713	struct hnae3_handle *h = priv->ae_handle;714	int i, j;715	int ret;716 717	ret = hns3_nic_reset_all_ring(h);718	if (ret)719		return ret;720 721	clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);722 723	/* enable the vectors */724	for (i = 0; i < priv->vector_num; i++)725		hns3_vector_enable(&priv->tqp_vector[i]);726 727	/* enable rcb */728	for (j = 0; j < h->kinfo.num_tqps; j++)729		hns3_tqp_enable(h->kinfo.tqp[j]);730 731	/* start the ae_dev */732	ret = h->ae_algo->ops->start ? h->ae_algo->ops->start(h) : 0;733	if (ret) {734		set_bit(HNS3_NIC_STATE_DOWN, &priv->state);735		while (j--)736			hns3_tqp_disable(h->kinfo.tqp[j]);737 738		for (j = i - 1; j >= 0; j--)739			hns3_vector_disable(&priv->tqp_vector[j]);740	}741 742	return ret;743}744 745static void hns3_config_xps(struct hns3_nic_priv *priv)746{747	int i;748 749	for (i = 0; i < priv->vector_num; i++) {750		struct hns3_enet_tqp_vector *tqp_vector = &priv->tqp_vector[i];751		struct hns3_enet_ring *ring = tqp_vector->tx_group.ring;752 753		while (ring) {754			int ret;755 756			ret = netif_set_xps_queue(priv->netdev,757						  &tqp_vector->affinity_mask,758						  ring->tqp->tqp_index);759			if (ret)760				netdev_warn(priv->netdev,761					    "set xps queue failed: %d", ret);762 763			ring = ring->next;764		}765	}766}767 768static int hns3_nic_net_open(struct net_device *netdev)769{770	struct hns3_nic_priv *priv = netdev_priv(netdev);771	struct hnae3_handle *h = hns3_get_handle(netdev);772	struct hnae3_knic_private_info *kinfo;773	int i, ret;774 775	if (hns3_nic_resetting(netdev))776		return -EBUSY;777 778	if (!test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {779		netdev_warn(netdev, "net open repeatedly!\n");780		return 0;781	}782 783	netif_carrier_off(netdev);784 785	ret = hns3_nic_set_real_num_queue(netdev);786	if (ret)787		return ret;788 789	ret = hns3_nic_net_up(netdev);790	if (ret) {791		netdev_err(netdev, "net up fail, ret=%d!\n", ret);792		return ret;793	}794 795	kinfo = &h->kinfo;796	for (i = 0; i < HNAE3_MAX_USER_PRIO; i++)797		netdev_set_prio_tc_map(netdev, i, kinfo->tc_info.prio_tc[i]);798 799	if (h->ae_algo->ops->set_timer_task)800		h->ae_algo->ops->set_timer_task(priv->ae_handle, true);801 802	hns3_config_xps(priv);803 804	netif_dbg(h, drv, netdev, "net open\n");805 806	return 0;807}808 809static void hns3_reset_tx_queue(struct hnae3_handle *h)810{811	struct net_device *ndev = h->kinfo.netdev;812	struct hns3_nic_priv *priv = netdev_priv(ndev);813	struct netdev_queue *dev_queue;814	u32 i;815 816	for (i = 0; i < h->kinfo.num_tqps; i++) {817		dev_queue = netdev_get_tx_queue(ndev,818						priv->ring[i].queue_index);819		netdev_tx_reset_queue(dev_queue);820	}821}822 823static void hns3_nic_net_down(struct net_device *netdev)824{825	struct hns3_nic_priv *priv = netdev_priv(netdev);826	struct hnae3_handle *h = hns3_get_handle(netdev);827	const struct hnae3_ae_ops *ops;828	int i;829 830	/* disable vectors */831	for (i = 0; i < priv->vector_num; i++)832		hns3_vector_disable(&priv->tqp_vector[i]);833 834	/* disable rcb */835	for (i = 0; i < h->kinfo.num_tqps; i++)836		hns3_tqp_disable(h->kinfo.tqp[i]);837 838	/* stop ae_dev */839	ops = priv->ae_handle->ae_algo->ops;840	if (ops->stop)841		ops->stop(priv->ae_handle);842 843	/* delay ring buffer clearing to hns3_reset_notify_uninit_enet844	 * during reset process, because driver may not be able845	 * to disable the ring through firmware when downing the netdev.846	 */847	if (!hns3_nic_resetting(netdev))848		hns3_clear_all_ring(priv->ae_handle, false);849 850	hns3_reset_tx_queue(priv->ae_handle);851}852 853static int hns3_nic_net_stop(struct net_device *netdev)854{855	struct hns3_nic_priv *priv = netdev_priv(netdev);856	struct hnae3_handle *h = hns3_get_handle(netdev);857 858	if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))859		return 0;860 861	netif_dbg(h, drv, netdev, "net stop\n");862 863	if (h->ae_algo->ops->set_timer_task)864		h->ae_algo->ops->set_timer_task(priv->ae_handle, false);865 866	netif_carrier_off(netdev);867	netif_tx_disable(netdev);868 869	hns3_nic_net_down(netdev);870 871	return 0;872}873 874static int hns3_nic_uc_sync(struct net_device *netdev,875			    const unsigned char *addr)876{877	struct hnae3_handle *h = hns3_get_handle(netdev);878 879	if (h->ae_algo->ops->add_uc_addr)880		return h->ae_algo->ops->add_uc_addr(h, addr);881 882	return 0;883}884 885static int hns3_nic_uc_unsync(struct net_device *netdev,886			      const unsigned char *addr)887{888	struct hnae3_handle *h = hns3_get_handle(netdev);889 890	/* need ignore the request of removing device address, because891	 * we store the device address and other addresses of uc list892	 * in the function's mac filter list.893	 */894	if (ether_addr_equal(addr, netdev->dev_addr))895		return 0;896 897	if (h->ae_algo->ops->rm_uc_addr)898		return h->ae_algo->ops->rm_uc_addr(h, addr);899 900	return 0;901}902 903static int hns3_nic_mc_sync(struct net_device *netdev,904			    const unsigned char *addr)905{906	struct hnae3_handle *h = hns3_get_handle(netdev);907 908	if (h->ae_algo->ops->add_mc_addr)909		return h->ae_algo->ops->add_mc_addr(h, addr);910 911	return 0;912}913 914static int hns3_nic_mc_unsync(struct net_device *netdev,915			      const unsigned char *addr)916{917	struct hnae3_handle *h = hns3_get_handle(netdev);918 919	if (h->ae_algo->ops->rm_mc_addr)920		return h->ae_algo->ops->rm_mc_addr(h, addr);921 922	return 0;923}924 925static u8 hns3_get_netdev_flags(struct net_device *netdev)926{927	u8 flags = 0;928 929	if (netdev->flags & IFF_PROMISC)930		flags = HNAE3_USER_UPE | HNAE3_USER_MPE | HNAE3_BPE;931	else if (netdev->flags & IFF_ALLMULTI)932		flags = HNAE3_USER_MPE;933 934	return flags;935}936 937static void hns3_nic_set_rx_mode(struct net_device *netdev)938{939	struct hnae3_handle *h = hns3_get_handle(netdev);940	u8 new_flags;941 942	new_flags = hns3_get_netdev_flags(netdev);943 944	__dev_uc_sync(netdev, hns3_nic_uc_sync, hns3_nic_uc_unsync);945	__dev_mc_sync(netdev, hns3_nic_mc_sync, hns3_nic_mc_unsync);946 947	/* User mode Promisc mode enable and vlan filtering is disabled to948	 * let all packets in.949	 */950	h->netdev_flags = new_flags;951	hns3_request_update_promisc_mode(h);952}953 954void hns3_request_update_promisc_mode(struct hnae3_handle *handle)955{956	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;957 958	if (ops->request_update_promisc_mode)959		ops->request_update_promisc_mode(handle);960}961 962static u32 hns3_tx_spare_space(struct hns3_enet_ring *ring)963{964	struct hns3_tx_spare *tx_spare = ring->tx_spare;965	u32 ntc, ntu;966 967	/* This smp_load_acquire() pairs with smp_store_release() in968	 * hns3_tx_spare_update() called in tx desc cleaning process.969	 */970	ntc = smp_load_acquire(&tx_spare->last_to_clean);971	ntu = tx_spare->next_to_use;972 973	if (ntc > ntu)974		return ntc - ntu - 1;975 976	/* The free tx buffer is divided into two part, so pick the977	 * larger one.978	 */979	return max(ntc, tx_spare->len - ntu) - 1;980}981 982static void hns3_tx_spare_update(struct hns3_enet_ring *ring)983{984	struct hns3_tx_spare *tx_spare = ring->tx_spare;985 986	if (!tx_spare ||987	    tx_spare->last_to_clean == tx_spare->next_to_clean)988		return;989 990	/* This smp_store_release() pairs with smp_load_acquire() in991	 * hns3_tx_spare_space() called in xmit process.992	 */993	smp_store_release(&tx_spare->last_to_clean,994			  tx_spare->next_to_clean);995}996 997static bool hns3_can_use_tx_bounce(struct hns3_enet_ring *ring,998				   struct sk_buff *skb,999				   u32 space)1000{1001	u32 len = skb->len <= ring->tx_copybreak ? skb->len :1002				skb_headlen(skb);1003 1004	if (len > ring->tx_copybreak)1005		return false;1006 1007	if (ALIGN(len, dma_get_cache_alignment()) > space) {1008		hns3_ring_stats_update(ring, tx_spare_full);1009		return false;1010	}1011 1012	return true;1013}1014 1015static bool hns3_can_use_tx_sgl(struct hns3_enet_ring *ring,1016				struct sk_buff *skb,1017				u32 space)1018{1019	if (skb->len <= ring->tx_copybreak || !tx_sgl ||1020	    (!skb_has_frag_list(skb) &&1021	     skb_shinfo(skb)->nr_frags < tx_sgl))1022		return false;1023 1024	if (space < HNS3_MAX_SGL_SIZE) {1025		hns3_ring_stats_update(ring, tx_spare_full);1026		return false;1027	}1028 1029	return true;1030}1031 1032static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring)1033{1034	u32 alloc_size = ring->tqp->handle->kinfo.tx_spare_buf_size;1035	struct hns3_tx_spare *tx_spare;1036	struct page *page;1037	dma_addr_t dma;1038	int order;1039 1040	if (!alloc_size)1041		return;1042 1043	order = get_order(alloc_size);1044	if (order > MAX_PAGE_ORDER) {1045		if (net_ratelimit())1046			dev_warn(ring_to_dev(ring), "failed to allocate tx spare buffer, exceed to max order\n");1047		return;1048	}1049 1050	tx_spare = devm_kzalloc(ring_to_dev(ring), sizeof(*tx_spare),1051				GFP_KERNEL);1052	if (!tx_spare) {1053		/* The driver still work without the tx spare buffer */1054		dev_warn(ring_to_dev(ring), "failed to allocate hns3_tx_spare\n");1055		goto devm_kzalloc_error;1056	}1057 1058	page = alloc_pages_node(dev_to_node(ring_to_dev(ring)),1059				GFP_KERNEL, order);1060	if (!page) {1061		dev_warn(ring_to_dev(ring), "failed to allocate tx spare pages\n");1062		goto alloc_pages_error;1063	}1064 1065	dma = dma_map_page(ring_to_dev(ring), page, 0,1066			   PAGE_SIZE << order, DMA_TO_DEVICE);1067	if (dma_mapping_error(ring_to_dev(ring), dma)) {1068		dev_warn(ring_to_dev(ring), "failed to map pages for tx spare\n");1069		goto dma_mapping_error;1070	}1071 1072	tx_spare->dma = dma;1073	tx_spare->buf = page_address(page);1074	tx_spare->len = PAGE_SIZE << order;1075	ring->tx_spare = tx_spare;1076	return;1077 1078dma_mapping_error:1079	put_page(page);1080alloc_pages_error:1081	devm_kfree(ring_to_dev(ring), tx_spare);1082devm_kzalloc_error:1083	ring->tqp->handle->kinfo.tx_spare_buf_size = 0;1084}1085 1086/* Use hns3_tx_spare_space() to make sure there is enough buffer1087 * before calling below function to allocate tx buffer.1088 */1089static void *hns3_tx_spare_alloc(struct hns3_enet_ring *ring,1090				 unsigned int size, dma_addr_t *dma,1091				 u32 *cb_len)1092{1093	struct hns3_tx_spare *tx_spare = ring->tx_spare;1094	u32 ntu = tx_spare->next_to_use;1095 1096	size = ALIGN(size, dma_get_cache_alignment());1097	*cb_len = size;1098 1099	/* Tx spare buffer wraps back here because the end of1100	 * freed tx buffer is not enough.1101	 */1102	if (ntu + size > tx_spare->len) {1103		*cb_len += (tx_spare->len - ntu);1104		ntu = 0;1105	}1106 1107	tx_spare->next_to_use = ntu + size;1108	if (tx_spare->next_to_use == tx_spare->len)1109		tx_spare->next_to_use = 0;1110 1111	*dma = tx_spare->dma + ntu;1112 1113	return tx_spare->buf + ntu;1114}1115 1116static void hns3_tx_spare_rollback(struct hns3_enet_ring *ring, u32 len)1117{1118	struct hns3_tx_spare *tx_spare = ring->tx_spare;1119 1120	if (len > tx_spare->next_to_use) {1121		len -= tx_spare->next_to_use;1122		tx_spare->next_to_use = tx_spare->len - len;1123	} else {1124		tx_spare->next_to_use -= len;1125	}1126}1127 1128static void hns3_tx_spare_reclaim_cb(struct hns3_enet_ring *ring,1129				     struct hns3_desc_cb *cb)1130{1131	struct hns3_tx_spare *tx_spare = ring->tx_spare;1132	u32 ntc = tx_spare->next_to_clean;1133	u32 len = cb->length;1134 1135	tx_spare->next_to_clean += len;1136 1137	if (tx_spare->next_to_clean >= tx_spare->len) {1138		tx_spare->next_to_clean -= tx_spare->len;1139 1140		if (tx_spare->next_to_clean) {1141			ntc = 0;1142			len = tx_spare->next_to_clean;1143		}1144	}1145 1146	/* This tx spare buffer is only really reclaimed after calling1147	 * hns3_tx_spare_update(), so it is still safe to use the info in1148	 * the tx buffer to do the dma sync or sg unmapping after1149	 * tx_spare->next_to_clean is moved forword.1150	 */1151	if (cb->type & (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL)) {1152		dma_addr_t dma = tx_spare->dma + ntc;1153 1154		dma_sync_single_for_cpu(ring_to_dev(ring), dma, len,1155					DMA_TO_DEVICE);1156	} else {1157		struct sg_table *sgt = tx_spare->buf + ntc;1158 1159		dma_unmap_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents,1160			     DMA_TO_DEVICE);1161	}1162}1163 1164static int hns3_set_tso(struct sk_buff *skb, u32 *paylen_fdop_ol4cs,1165			u16 *mss, u32 *type_cs_vlan_tso, u32 *send_bytes)1166{1167	u32 l4_offset, hdr_len;1168	union l3_hdr_info l3;1169	union l4_hdr_info l4;1170	u32 l4_paylen;1171	int ret;1172 1173	if (!skb_is_gso(skb))1174		return 0;1175 1176	ret = skb_cow_head(skb, 0);1177	if (unlikely(ret < 0))1178		return ret;1179 1180	l3.hdr = skb_network_header(skb);1181	l4.hdr = skb_transport_header(skb);1182 1183	/* Software should clear the IPv4's checksum field when tso is1184	 * needed.1185	 */1186	if (l3.v4->version == 4)1187		l3.v4->check = 0;1188 1189	/* tunnel packet */1190	if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |1191					 SKB_GSO_GRE_CSUM |1192					 SKB_GSO_UDP_TUNNEL |1193					 SKB_GSO_UDP_TUNNEL_CSUM)) {1194		/* reset l3&l4 pointers from outer to inner headers */1195		l3.hdr = skb_inner_network_header(skb);1196		l4.hdr = skb_inner_transport_header(skb);1197 1198		/* Software should clear the IPv4's checksum field when1199		 * tso is needed.1200		 */1201		if (l3.v4->version == 4)1202			l3.v4->check = 0;1203	}1204 1205	/* normal or tunnel packet */1206	l4_offset = l4.hdr - skb->data;1207 1208	/* remove payload length from inner pseudo checksum when tso */1209	l4_paylen = skb->len - l4_offset;1210 1211	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {1212		hdr_len = sizeof(*l4.udp) + l4_offset;1213		csum_replace_by_diff(&l4.udp->check,1214				     (__force __wsum)htonl(l4_paylen));1215	} else {1216		hdr_len = (l4.tcp->doff << 2) + l4_offset;1217		csum_replace_by_diff(&l4.tcp->check,1218				     (__force __wsum)htonl(l4_paylen));1219	}1220 1221	*send_bytes = (skb_shinfo(skb)->gso_segs - 1) * hdr_len + skb->len;1222 1223	/* find the txbd field values */1224	*paylen_fdop_ol4cs = skb->len - hdr_len;1225	hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_TSO_B, 1);1226 1227	/* offload outer UDP header checksum */1228	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)1229		hns3_set_field(*paylen_fdop_ol4cs, HNS3_TXD_OL4CS_B, 1);1230 1231	/* get MSS for TSO */1232	*mss = skb_shinfo(skb)->gso_size;1233 1234	trace_hns3_tso(skb);1235 1236	return 0;1237}1238 1239static int hns3_get_l4_protocol(struct sk_buff *skb, u8 *ol4_proto,1240				u8 *il4_proto)1241{1242	union l3_hdr_info l3;1243	unsigned char *l4_hdr;1244	unsigned char *exthdr;1245	u8 l4_proto_tmp;1246	__be16 frag_off;1247 1248	/* find outer header point */1249	l3.hdr = skb_network_header(skb);1250	l4_hdr = skb_transport_header(skb);1251 1252	if (skb->protocol == htons(ETH_P_IPV6)) {1253		exthdr = l3.hdr + sizeof(*l3.v6);1254		l4_proto_tmp = l3.v6->nexthdr;1255		if (l4_hdr != exthdr)1256			ipv6_skip_exthdr(skb, exthdr - skb->data,1257					 &l4_proto_tmp, &frag_off);1258	} else if (skb->protocol == htons(ETH_P_IP)) {1259		l4_proto_tmp = l3.v4->protocol;1260	} else {1261		return -EINVAL;1262	}1263 1264	*ol4_proto = l4_proto_tmp;1265 1266	/* tunnel packet */1267	if (!skb->encapsulation) {1268		*il4_proto = 0;1269		return 0;1270	}1271 1272	/* find inner header point */1273	l3.hdr = skb_inner_network_header(skb);1274	l4_hdr = skb_inner_transport_header(skb);1275 1276	if (l3.v6->version == 6) {1277		exthdr = l3.hdr + sizeof(*l3.v6);1278		l4_proto_tmp = l3.v6->nexthdr;1279		if (l4_hdr != exthdr)1280			ipv6_skip_exthdr(skb, exthdr - skb->data,1281					 &l4_proto_tmp, &frag_off);1282	} else if (l3.v4->version == 4) {1283		l4_proto_tmp = l3.v4->protocol;1284	}1285 1286	*il4_proto = l4_proto_tmp;1287 1288	return 0;1289}1290 1291/* when skb->encapsulation is 0, skb->ip_summed is CHECKSUM_PARTIAL1292 * and it is udp packet, which has a dest port as the IANA assigned.1293 * the hardware is expected to do the checksum offload, but the1294 * hardware will not do the checksum offload when udp dest port is1295 * 4789, 4790 or 6081.1296 */1297static bool hns3_tunnel_csum_bug(struct sk_buff *skb)1298{1299	struct hns3_nic_priv *priv = netdev_priv(skb->dev);1300	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);1301	union l4_hdr_info l4;1302 1303	/* device version above V3(include V3), the hardware can1304	 * do this checksum offload.1305	 */1306	if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)1307		return false;1308 1309	l4.hdr = skb_transport_header(skb);1310 1311	if (!(!skb->encapsulation &&1312	      (l4.udp->dest == htons(IANA_VXLAN_UDP_PORT) ||1313	      l4.udp->dest == htons(GENEVE_UDP_PORT) ||1314	      l4.udp->dest == htons(IANA_VXLAN_GPE_UDP_PORT))))1315		return false;1316 1317	return true;1318}1319 1320static void hns3_set_outer_l2l3l4(struct sk_buff *skb, u8 ol4_proto,1321				  u32 *ol_type_vlan_len_msec)1322{1323	u32 l2_len, l3_len, l4_len;1324	unsigned char *il2_hdr;1325	union l3_hdr_info l3;1326	union l4_hdr_info l4;1327 1328	l3.hdr = skb_network_header(skb);1329	l4.hdr = skb_transport_header(skb);1330 1331	/* compute OL2 header size, defined in 2 Bytes */1332	l2_len = l3.hdr - skb->data;1333	hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L2LEN_S, l2_len >> 1);1334 1335	/* compute OL3 header size, defined in 4 Bytes */1336	l3_len = l4.hdr - l3.hdr;1337	hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L3LEN_S, l3_len >> 2);1338 1339	il2_hdr = skb_inner_mac_header(skb);1340	/* compute OL4 header size, defined in 4 Bytes */1341	l4_len = il2_hdr - l4.hdr;1342	hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_L4LEN_S, l4_len >> 2);1343 1344	/* define outer network header type */1345	if (skb->protocol == htons(ETH_P_IP)) {1346		if (skb_is_gso(skb))1347			hns3_set_field(*ol_type_vlan_len_msec,1348				       HNS3_TXD_OL3T_S,1349				       HNS3_OL3T_IPV4_CSUM);1350		else1351			hns3_set_field(*ol_type_vlan_len_msec,1352				       HNS3_TXD_OL3T_S,1353				       HNS3_OL3T_IPV4_NO_CSUM);1354	} else if (skb->protocol == htons(ETH_P_IPV6)) {1355		hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_OL3T_S,1356			       HNS3_OL3T_IPV6);1357	}1358 1359	if (ol4_proto == IPPROTO_UDP)1360		hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,1361			       HNS3_TUN_MAC_IN_UDP);1362	else if (ol4_proto == IPPROTO_GRE)1363		hns3_set_field(*ol_type_vlan_len_msec, HNS3_TXD_TUNTYPE_S,1364			       HNS3_TUN_NVGRE);1365}1366 1367static void hns3_set_l3_type(struct sk_buff *skb, union l3_hdr_info l3,1368			     u32 *type_cs_vlan_tso)1369{1370	if (l3.v4->version == 4) {1371		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,1372			       HNS3_L3T_IPV4);1373 1374		/* the stack computes the IP header already, the only time we1375		 * need the hardware to recompute it is in the case of TSO.1376		 */1377		if (skb_is_gso(skb))1378			hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3CS_B, 1);1379	} else if (l3.v6->version == 6) {1380		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S,1381			       HNS3_L3T_IPV6);1382	}1383}1384 1385static int hns3_set_l4_csum_length(struct sk_buff *skb, union l4_hdr_info l4,1386				   u32 l4_proto, u32 *type_cs_vlan_tso)1387{1388	/* compute inner(/normal) L4 header size, defined in 4 Bytes */1389	switch (l4_proto) {1390	case IPPROTO_TCP:1391		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);1392		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,1393			       HNS3_L4T_TCP);1394		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,1395			       l4.tcp->doff);1396		break;1397	case IPPROTO_UDP:1398		if (hns3_tunnel_csum_bug(skb)) {1399			int ret = skb_put_padto(skb, HNS3_MIN_TUN_PKT_LEN);1400 1401			return ret ? ret : skb_checksum_help(skb);1402		}1403 1404		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);1405		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,1406			       HNS3_L4T_UDP);1407		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,1408			       (sizeof(struct udphdr) >> 2));1409		break;1410	case IPPROTO_SCTP:1411		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4CS_B, 1);1412		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4T_S,1413			       HNS3_L4T_SCTP);1414		hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L4LEN_S,1415			       (sizeof(struct sctphdr) >> 2));1416		break;1417	default:1418		/* drop the skb tunnel packet if hardware don't support,1419		 * because hardware can't calculate csum when TSO.1420		 */1421		if (skb_is_gso(skb))1422			return -EDOM;1423 1424		/* the stack computes the IP header already,1425		 * driver calculate l4 checksum when not TSO.1426		 */1427		return skb_checksum_help(skb);1428	}1429 1430	return 0;1431}1432 1433static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto,1434			   u8 il4_proto, u32 *type_cs_vlan_tso,1435			   u32 *ol_type_vlan_len_msec)1436{1437	unsigned char *l2_hdr = skb->data;1438	u32 l4_proto = ol4_proto;1439	union l4_hdr_info l4;1440	union l3_hdr_info l3;1441	u32 l2_len, l3_len;1442 1443	l4.hdr = skb_transport_header(skb);1444	l3.hdr = skb_network_header(skb);1445 1446	/* handle encapsulation skb */1447	if (skb->encapsulation) {1448		/* If this is a not UDP/GRE encapsulation skb */1449		if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) {1450			/* drop the skb tunnel packet if hardware don't support,1451			 * because hardware can't calculate csum when TSO.1452			 */1453			if (skb_is_gso(skb))1454				return -EDOM;1455 1456			/* the stack computes the IP header already,1457			 * driver calculate l4 checksum when not TSO.1458			 */1459			return skb_checksum_help(skb);1460		}1461 1462		hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec);1463 1464		/* switch to inner header */1465		l2_hdr = skb_inner_mac_header(skb);1466		l3.hdr = skb_inner_network_header(skb);1467		l4.hdr = skb_inner_transport_header(skb);1468		l4_proto = il4_proto;1469	}1470 1471	hns3_set_l3_type(skb, l3, type_cs_vlan_tso);1472 1473	/* compute inner(/normal) L2 header size, defined in 2 Bytes */1474	l2_len = l3.hdr - l2_hdr;1475	hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_S, l2_len >> 1);1476 1477	/* compute inner(/normal) L3 header size, defined in 4 Bytes */1478	l3_len = l4.hdr - l3.hdr;1479	hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_S, l3_len >> 2);1480 1481	return hns3_set_l4_csum_length(skb, l4, l4_proto, type_cs_vlan_tso);1482}1483 1484static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring,1485			     struct sk_buff *skb)1486{1487	struct hnae3_handle *handle = tx_ring->tqp->handle;1488	struct hnae3_ae_dev *ae_dev;1489	struct vlan_ethhdr *vhdr;1490	int rc;1491 1492	if (!(skb->protocol == htons(ETH_P_8021Q) ||1493	      skb_vlan_tag_present(skb)))1494		return 0;1495 1496	/* For HW limitation on HNAE3_DEVICE_VERSION_V2, if port based insert1497	 * VLAN enabled, only one VLAN header is allowed in skb, otherwise it1498	 * will cause RAS error.1499	 */1500	ae_dev = pci_get_drvdata(handle->pdev);1501	if (unlikely(skb_vlan_tagged_multi(skb) &&1502		     ae_dev->dev_version <= HNAE3_DEVICE_VERSION_V2 &&1503		     handle->port_base_vlan_state ==1504		     HNAE3_PORT_BASE_VLAN_ENABLE))1505		return -EINVAL;1506 1507	if (skb->protocol == htons(ETH_P_8021Q) &&1508	    !(handle->kinfo.netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {1509		/* When HW VLAN acceleration is turned off, and the stack1510		 * sets the protocol to 802.1q, the driver just need to1511		 * set the protocol to the encapsulated ethertype.1512		 */1513		skb->protocol = vlan_get_protocol(skb);1514		return 0;1515	}1516 1517	if (skb_vlan_tag_present(skb)) {1518		/* Based on hw strategy, use out_vtag in two layer tag case,1519		 * and use inner_vtag in one tag case.1520		 */1521		if (skb->protocol == htons(ETH_P_8021Q) &&1522		    handle->port_base_vlan_state ==1523		    HNAE3_PORT_BASE_VLAN_DISABLE)1524			rc = HNS3_OUTER_VLAN_TAG;1525		else1526			rc = HNS3_INNER_VLAN_TAG;1527 1528		skb->protocol = vlan_get_protocol(skb);1529		return rc;1530	}1531 1532	rc = skb_cow_head(skb, 0);1533	if (unlikely(rc < 0))1534		return rc;1535 1536	vhdr = skb_vlan_eth_hdr(skb);1537	vhdr->h_vlan_TCI |= cpu_to_be16((skb->priority << VLAN_PRIO_SHIFT)1538					 & VLAN_PRIO_MASK);1539 1540	skb->protocol = vlan_get_protocol(skb);1541	return 0;1542}1543 1544/* check if the hardware is capable of checksum offloading */1545static bool hns3_check_hw_tx_csum(struct sk_buff *skb)1546{1547	struct hns3_nic_priv *priv = netdev_priv(skb->dev);1548 1549	/* Kindly note, due to backward compatibility of the TX descriptor,1550	 * HW checksum of the non-IP packets and GSO packets is handled at1551	 * different place in the following code1552	 */1553	if (skb_csum_is_sctp(skb) || skb_is_gso(skb) ||1554	    !test_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state))1555		return false;1556 1557	return true;1558}1559 1560struct hns3_desc_param {1561	u32 paylen_ol4cs;1562	u32 ol_type_vlan_len_msec;1563	u32 type_cs_vlan_tso;1564	u16 mss_hw_csum;1565	u16 inner_vtag;1566	u16 out_vtag;1567};1568 1569static void hns3_init_desc_data(struct sk_buff *skb, struct hns3_desc_param *pa)1570{1571	pa->paylen_ol4cs = skb->len;1572	pa->ol_type_vlan_len_msec = 0;1573	pa->type_cs_vlan_tso = 0;1574	pa->mss_hw_csum = 0;1575	pa->inner_vtag = 0;1576	pa->out_vtag = 0;1577}1578 1579static int hns3_handle_vlan_info(struct hns3_enet_ring *ring,1580				 struct sk_buff *skb,1581				 struct hns3_desc_param *param)1582{1583	int ret;1584 1585	ret = hns3_handle_vtags(ring, skb);1586	if (unlikely(ret < 0)) {1587		hns3_ring_stats_update(ring, tx_vlan_err);1588		return ret;1589	} else if (ret == HNS3_INNER_VLAN_TAG) {1590		param->inner_vtag = skb_vlan_tag_get(skb);1591		param->inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &1592				VLAN_PRIO_MASK;1593		hns3_set_field(param->type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1);1594	} else if (ret == HNS3_OUTER_VLAN_TAG) {1595		param->out_vtag = skb_vlan_tag_get(skb);1596		param->out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) &1597				VLAN_PRIO_MASK;1598		hns3_set_field(param->ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B,1599			       1);1600	}1601	return 0;1602}1603 1604static int hns3_handle_csum_partial(struct hns3_enet_ring *ring,1605				    struct sk_buff *skb,1606				    struct hns3_desc_cb *desc_cb,1607				    struct hns3_desc_param *param)1608{1609	u8 ol4_proto, il4_proto;1610	int ret;1611 1612	if (hns3_check_hw_tx_csum(skb)) {1613		/* set checksum start and offset, defined in 2 Bytes */1614		hns3_set_field(param->type_cs_vlan_tso, HNS3_TXD_CSUM_START_S,1615			       skb_checksum_start_offset(skb) >> 1);1616		hns3_set_field(param->ol_type_vlan_len_msec,1617			       HNS3_TXD_CSUM_OFFSET_S,1618			       skb->csum_offset >> 1);1619		param->mss_hw_csum |= BIT(HNS3_TXD_HW_CS_B);1620		return 0;1621	}1622 1623	skb_reset_mac_len(skb);1624 1625	ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto);1626	if (unlikely(ret < 0)) {1627		hns3_ring_stats_update(ring, tx_l4_proto_err);1628		return ret;1629	}1630 1631	ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto,1632			      &param->type_cs_vlan_tso,1633			      &param->ol_type_vlan_len_msec);1634	if (unlikely(ret < 0)) {1635		hns3_ring_stats_update(ring, tx_l2l3l4_err);1636		return ret;1637	}1638 1639	ret = hns3_set_tso(skb, &param->paylen_ol4cs, &param->mss_hw_csum,1640			   &param->type_cs_vlan_tso, &desc_cb->send_bytes);1641	if (unlikely(ret < 0)) {1642		hns3_ring_stats_update(ring, tx_tso_err);1643		return ret;1644	}1645	return 0;1646}1647 1648static int hns3_fill_skb_desc(struct hns3_enet_ring *ring,1649			      struct sk_buff *skb, struct hns3_desc *desc,1650			      struct hns3_desc_cb *desc_cb)1651{1652	struct hns3_desc_param param;1653	int ret;1654 1655	hns3_init_desc_data(skb, &param);1656	ret = hns3_handle_vlan_info(ring, skb, &param);1657	if (unlikely(ret < 0))1658		return ret;1659 1660	desc_cb->send_bytes = skb->len;1661 1662	if (skb->ip_summed == CHECKSUM_PARTIAL) {1663		ret = hns3_handle_csum_partial(ring, skb, desc_cb, &param);1664		if (ret)1665			return ret;1666	}1667 1668	/* Set txbd */1669	desc->tx.ol_type_vlan_len_msec =1670		cpu_to_le32(param.ol_type_vlan_len_msec);1671	desc->tx.type_cs_vlan_tso_len = cpu_to_le32(param.type_cs_vlan_tso);1672	desc->tx.paylen_ol4cs = cpu_to_le32(param.paylen_ol4cs);1673	desc->tx.mss_hw_csum = cpu_to_le16(param.mss_hw_csum);1674	desc->tx.vlan_tag = cpu_to_le16(param.inner_vtag);1675	desc->tx.outer_vlan_tag = cpu_to_le16(param.out_vtag);1676 1677	return 0;1678}1679 1680static int hns3_fill_desc(struct hns3_enet_ring *ring, dma_addr_t dma,1681			  unsigned int size)1682{1683#define HNS3_LIKELY_BD_NUM	11684 1685	struct hns3_desc *desc = &ring->desc[ring->next_to_use];1686	unsigned int frag_buf_num;1687	int k, sizeoflast;1688 1689	if (likely(size <= HNS3_MAX_BD_SIZE)) {1690		desc->addr = cpu_to_le64(dma);1691		desc->tx.send_size = cpu_to_le16(size);1692		desc->tx.bdtp_fe_sc_vld_ra_ri =1693			cpu_to_le16(BIT(HNS3_TXD_VLD_B));1694 1695		trace_hns3_tx_desc(ring, ring->next_to_use);1696		ring_ptr_move_fw(ring, next_to_use);1697		return HNS3_LIKELY_BD_NUM;1698	}1699 1700	frag_buf_num = hns3_tx_bd_count(size);1701	sizeoflast = size % HNS3_MAX_BD_SIZE;1702	sizeoflast = sizeoflast ? sizeoflast : HNS3_MAX_BD_SIZE;1703 1704	/* When frag size is bigger than hardware limit, split this frag */1705	for (k = 0; k < frag_buf_num; k++) {1706		/* now, fill the descriptor */1707		desc->addr = cpu_to_le64(dma + HNS3_MAX_BD_SIZE * k);1708		desc->tx.send_size = cpu_to_le16((k == frag_buf_num - 1) ?1709				     (u16)sizeoflast : (u16)HNS3_MAX_BD_SIZE);1710		desc->tx.bdtp_fe_sc_vld_ra_ri =1711				cpu_to_le16(BIT(HNS3_TXD_VLD_B));1712 1713		trace_hns3_tx_desc(ring, ring->next_to_use);1714		/* move ring pointer to next */1715		ring_ptr_move_fw(ring, next_to_use);1716 1717		desc = &ring->desc[ring->next_to_use];1718	}1719 1720	return frag_buf_num;1721}1722 1723static int hns3_map_and_fill_desc(struct hns3_enet_ring *ring, void *priv,1724				  unsigned int type)1725{1726	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];1727	struct device *dev = ring_to_dev(ring);1728	unsigned int size;1729	dma_addr_t dma;1730 1731	if (type & (DESC_TYPE_FRAGLIST_SKB | DESC_TYPE_SKB)) {1732		struct sk_buff *skb = (struct sk_buff *)priv;1733 1734		size = skb_headlen(skb);1735		if (!size)1736			return 0;1737 1738		dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE);1739	} else if (type & DESC_TYPE_BOUNCE_HEAD) {1740		/* Head data has been filled in hns3_handle_tx_bounce(),1741		 * just return 0 here.1742		 */1743		return 0;1744	} else {1745		skb_frag_t *frag = (skb_frag_t *)priv;1746 1747		size = skb_frag_size(frag);1748		if (!size)1749			return 0;1750 1751		dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE);1752	}1753 1754	if (unlikely(dma_mapping_error(dev, dma))) {1755		hns3_ring_stats_update(ring, sw_err_cnt);1756		return -ENOMEM;1757	}1758 1759	desc_cb->priv = priv;1760	desc_cb->length = size;1761	desc_cb->dma = dma;1762	desc_cb->type = type;1763 1764	return hns3_fill_desc(ring, dma, size);1765}1766 1767static unsigned int hns3_skb_bd_num(struct sk_buff *skb, unsigned int *bd_size,1768				    unsigned int bd_num)1769{1770	unsigned int size;1771	int i;1772 1773	size = skb_headlen(skb);1774	while (size > HNS3_MAX_BD_SIZE) {1775		bd_size[bd_num++] = HNS3_MAX_BD_SIZE;1776		size -= HNS3_MAX_BD_SIZE;1777 1778		if (bd_num > HNS3_MAX_TSO_BD_NUM)1779			return bd_num;1780	}1781 1782	if (size) {1783		bd_size[bd_num++] = size;1784		if (bd_num > HNS3_MAX_TSO_BD_NUM)1785			return bd_num;1786	}1787 1788	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {1789		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];1790		size = skb_frag_size(frag);1791		if (!size)1792			continue;1793 1794		while (size > HNS3_MAX_BD_SIZE) {1795			bd_size[bd_num++] = HNS3_MAX_BD_SIZE;1796			size -= HNS3_MAX_BD_SIZE;1797 1798			if (bd_num > HNS3_MAX_TSO_BD_NUM)1799				return bd_num;1800		}1801 1802		bd_size[bd_num++] = size;1803		if (bd_num > HNS3_MAX_TSO_BD_NUM)1804			return bd_num;1805	}1806 1807	return bd_num;1808}1809 1810static unsigned int hns3_tx_bd_num(struct sk_buff *skb, unsigned int *bd_size,1811				   u8 max_non_tso_bd_num, unsigned int bd_num,1812				   unsigned int recursion_level)1813{1814#define HNS3_MAX_RECURSION_LEVEL	241815 1816	struct sk_buff *frag_skb;1817 1818	/* If the total len is within the max bd limit */1819	if (likely(skb->len <= HNS3_MAX_BD_SIZE && !recursion_level &&1820		   !skb_has_frag_list(skb) &&1821		   skb_shinfo(skb)->nr_frags < max_non_tso_bd_num))1822		return skb_shinfo(skb)->nr_frags + 1U;1823 1824	if (unlikely(recursion_level >= HNS3_MAX_RECURSION_LEVEL))1825		return UINT_MAX;1826 1827	bd_num = hns3_skb_bd_num(skb, bd_size, bd_num);1828	if (!skb_has_frag_list(skb) || bd_num > HNS3_MAX_TSO_BD_NUM)1829		return bd_num;1830 1831	skb_walk_frags(skb, frag_skb) {1832		bd_num = hns3_tx_bd_num(frag_skb, bd_size, max_non_tso_bd_num,1833					bd_num, recursion_level + 1);1834		if (bd_num > HNS3_MAX_TSO_BD_NUM)1835			return bd_num;1836	}1837 1838	return bd_num;1839}1840 1841static unsigned int hns3_gso_hdr_len(struct sk_buff *skb)1842{1843	if (!skb->encapsulation)1844		return skb_tcp_all_headers(skb);1845 1846	return skb_inner_tcp_all_headers(skb);1847}1848 1849/* HW need every continuous max_non_tso_bd_num buffer data to be larger1850 * than MSS, we simplify it by ensuring skb_headlen + the first continuous1851 * max_non_tso_bd_num - 1 frags to be larger than gso header len + mss,1852 * and the remaining continuous max_non_tso_bd_num - 1 frags to be larger1853 * than MSS except the last max_non_tso_bd_num - 1 frags.1854 */1855static bool hns3_skb_need_linearized(struct sk_buff *skb, unsigned int *bd_size,1856				     unsigned int bd_num, u8 max_non_tso_bd_num)1857{1858	unsigned int tot_len = 0;1859	int i;1860 1861	for (i = 0; i < max_non_tso_bd_num - 1U; i++)1862		tot_len += bd_size[i];1863 1864	/* ensure the first max_non_tso_bd_num frags is greater than1865	 * mss + header1866	 */1867	if (tot_len + bd_size[max_non_tso_bd_num - 1U] <1868	    skb_shinfo(skb)->gso_size + hns3_gso_hdr_len(skb))1869		return true;1870 1871	/* ensure every continuous max_non_tso_bd_num - 1 buffer is greater1872	 * than mss except the last one.1873	 */1874	for (i = 0; i < bd_num - max_non_tso_bd_num; i++) {1875		tot_len -= bd_size[i];1876		tot_len += bd_size[i + max_non_tso_bd_num - 1U];1877 1878		if (tot_len < skb_shinfo(skb)->gso_size)1879			return true;1880	}1881 1882	return false;1883}1884 1885void hns3_shinfo_pack(struct skb_shared_info *shinfo, __u32 *size)1886{1887	int i;1888 1889	for (i = 0; i < MAX_SKB_FRAGS; i++)1890		size[i] = skb_frag_size(&shinfo->frags[i]);1891}1892 1893static int hns3_skb_linearize(struct hns3_enet_ring *ring,1894			      struct sk_buff *skb,1895			      unsigned int bd_num)1896{1897	/* 'bd_num == UINT_MAX' means the skb' fraglist has a1898	 * recursion level of over HNS3_MAX_RECURSION_LEVEL.1899	 */1900	if (bd_num == UINT_MAX) {1901		hns3_ring_stats_update(ring, over_max_recursion);1902		return -ENOMEM;1903	}1904 1905	/* The skb->len has exceeded the hw limitation, linearization1906	 * will not help.1907	 */1908	if (skb->len > HNS3_MAX_TSO_SIZE ||1909	    (!skb_is_gso(skb) && skb->len > HNS3_MAX_NON_TSO_SIZE)) {1910		hns3_ring_stats_update(ring, hw_limitation);1911		return -ENOMEM;1912	}1913 1914	if (__skb_linearize(skb)) {1915		hns3_ring_stats_update(ring, sw_err_cnt);1916		return -ENOMEM;1917	}1918 1919	return 0;1920}1921 1922static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring,1923				  struct net_device *netdev,1924				  struct sk_buff *skb)1925{1926	struct hns3_nic_priv *priv = netdev_priv(netdev);1927	u8 max_non_tso_bd_num = priv->max_non_tso_bd_num;1928	unsigned int bd_size[HNS3_MAX_TSO_BD_NUM + 1U];1929	unsigned int bd_num;1930 1931	bd_num = hns3_tx_bd_num(skb, bd_size, max_non_tso_bd_num, 0, 0);1932	if (unlikely(bd_num > max_non_tso_bd_num)) {1933		if (bd_num <= HNS3_MAX_TSO_BD_NUM && skb_is_gso(skb) &&1934		    !hns3_skb_need_linearized(skb, bd_size, bd_num,1935					      max_non_tso_bd_num)) {1936			trace_hns3_over_max_bd(skb);1937			goto out;1938		}1939 1940		if (hns3_skb_linearize(ring, skb, bd_num))1941			return -ENOMEM;1942 1943		bd_num = hns3_tx_bd_count(skb->len);1944 1945		hns3_ring_stats_update(ring, tx_copy);1946	}1947 1948out:1949	if (likely(ring_space(ring) >= bd_num))1950		return bd_num;1951 1952	netif_stop_subqueue(netdev, ring->queue_index);1953	smp_mb(); /* Memory barrier before checking ring_space */1954 1955	/* Start queue in case hns3_clean_tx_ring has just made room1956	 * available and has not seen the queue stopped state performed1957	 * by netif_stop_subqueue above.1958	 */1959	if (ring_space(ring) >= bd_num && netif_carrier_ok(netdev) &&1960	    !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {1961		netif_start_subqueue(netdev, ring->queue_index);1962		return bd_num;1963	}1964 1965	hns3_ring_stats_update(ring, tx_busy);1966 1967	return -EBUSY;1968}1969 1970static void hns3_clear_desc(struct hns3_enet_ring *ring, int next_to_use_orig)1971{1972	struct device *dev = ring_to_dev(ring);1973	unsigned int i;1974 1975	for (i = 0; i < ring->desc_num; i++) {1976		struct hns3_desc *desc = &ring->desc[ring->next_to_use];1977		struct hns3_desc_cb *desc_cb;1978 1979		memset(desc, 0, sizeof(*desc));1980 1981		/* check if this is where we started */1982		if (ring->next_to_use == next_to_use_orig)1983			break;1984 1985		/* rollback one */1986		ring_ptr_move_bw(ring, next_to_use);1987 1988		desc_cb = &ring->desc_cb[ring->next_to_use];1989 1990		if (!desc_cb->dma)1991			continue;1992 1993		/* unmap the descriptor dma address */1994		if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB))1995			dma_unmap_single(dev, desc_cb->dma, desc_cb->length,1996					 DMA_TO_DEVICE);1997		else if (desc_cb->type &1998			 (DESC_TYPE_BOUNCE_HEAD | DESC_TYPE_BOUNCE_ALL))1999			hns3_tx_spare_rollback(ring, desc_cb->length);2000		else if (desc_cb->length)2001			dma_unmap_page(dev, desc_cb->dma, desc_cb->length,2002				       DMA_TO_DEVICE);2003 2004		desc_cb->length = 0;2005		desc_cb->dma = 0;2006		desc_cb->type = DESC_TYPE_UNKNOWN;2007	}2008}2009 2010static int hns3_fill_skb_to_desc(struct hns3_enet_ring *ring,2011				 struct sk_buff *skb, unsigned int type)2012{2013	struct sk_buff *frag_skb;2014	int i, ret, bd_num = 0;2015 2016	ret = hns3_map_and_fill_desc(ring, skb, type);2017	if (unlikely(ret < 0))2018		return ret;2019 2020	bd_num += ret;2021 2022	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {2023		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];2024 2025		ret = hns3_map_and_fill_desc(ring, frag, DESC_TYPE_PAGE);2026		if (unlikely(ret < 0))2027			return ret;2028 2029		bd_num += ret;2030	}2031 2032	skb_walk_frags(skb, frag_skb) {2033		ret = hns3_fill_skb_to_desc(ring, frag_skb,2034					    DESC_TYPE_FRAGLIST_SKB);2035		if (unlikely(ret < 0))2036			return ret;2037 2038		bd_num += ret;2039	}2040 2041	return bd_num;2042}2043 2044static void hns3_tx_push_bd(struct hns3_enet_ring *ring, int num)2045{2046#define HNS3_BYTES_PER_64BIT		82047 2048	struct hns3_desc desc[HNS3_MAX_PUSH_BD_NUM] = {};2049	int offset = 0;2050 2051	/* make sure everything is visible to device before2052	 * excuting tx push or updating doorbell2053	 */2054	dma_wmb();2055 2056	do {2057		int idx = (ring->next_to_use - num + ring->desc_num) %2058			  ring->desc_num;2059 2060		u64_stats_update_begin(&ring->syncp);2061		ring->stats.tx_push++;2062		u64_stats_update_end(&ring->syncp);2063		memcpy(&desc[offset], &ring->desc[idx],2064		       sizeof(struct hns3_desc));2065		offset++;2066	} while (--num);2067 2068	__iowrite64_copy(ring->tqp->mem_base, desc,2069			 (sizeof(struct hns3_desc) * HNS3_MAX_PUSH_BD_NUM) /2070			 HNS3_BYTES_PER_64BIT);2071}2072 2073static void hns3_tx_mem_doorbell(struct hns3_enet_ring *ring)2074{2075#define HNS3_MEM_DOORBELL_OFFSET	642076 2077	__le64 bd_num = cpu_to_le64((u64)ring->pending_buf);2078 2079	/* make sure everything is visible to device before2080	 * excuting tx push or updating doorbell2081	 */2082	dma_wmb();2083 2084	__iowrite64_copy(ring->tqp->mem_base + HNS3_MEM_DOORBELL_OFFSET,2085			 &bd_num, 1);2086	u64_stats_update_begin(&ring->syncp);2087	ring->stats.tx_mem_doorbell += ring->pending_buf;2088	u64_stats_update_end(&ring->syncp);2089}2090 2091static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num,2092			     bool doorbell)2093{2094	struct net_device *netdev = ring_to_netdev(ring);2095	struct hns3_nic_priv *priv = netdev_priv(netdev);2096 2097	/* when tx push is enabled, the packet whose number of BD below2098	 * HNS3_MAX_PUSH_BD_NUM can be pushed directly.2099	 */2100	if (test_bit(HNS3_NIC_STATE_TX_PUSH_ENABLE, &priv->state) && num &&2101	    !ring->pending_buf && num <= HNS3_MAX_PUSH_BD_NUM && doorbell) {2102		/* This smp_store_release() pairs with smp_load_aquire() in2103		 * hns3_nic_reclaim_desc(). Ensure that the BD valid bit2104		 * is updated.2105		 */2106		smp_store_release(&ring->last_to_use, ring->next_to_use);2107		hns3_tx_push_bd(ring, num);2108		return;2109	}2110 2111	ring->pending_buf += num;2112 2113	if (!doorbell) {2114		hns3_ring_stats_update(ring, tx_more);2115		return;2116	}2117 2118	/* This smp_store_release() pairs with smp_load_aquire() in2119	 * hns3_nic_reclaim_desc(). Ensure that the BD valid bit is updated.2120	 */2121	smp_store_release(&ring->last_to_use, ring->next_to_use);2122 2123	if (ring->tqp->mem_base)2124		hns3_tx_mem_doorbell(ring);2125	else2126		writel(ring->pending_buf,2127		       ring->tqp->io_base + HNS3_RING_TX_RING_TAIL_REG);2128 2129	ring->pending_buf = 0;2130}2131 2132static void hns3_tsyn(struct net_device *netdev, struct sk_buff *skb,2133		      struct hns3_desc *desc)2134{2135	struct hnae3_handle *h = hns3_get_handle(netdev);2136 2137	if (!(h->ae_algo->ops->set_tx_hwts_info &&2138	      h->ae_algo->ops->set_tx_hwts_info(h, skb)))2139		return;2140 2141	desc->tx.bdtp_fe_sc_vld_ra_ri |= cpu_to_le16(BIT(HNS3_TXD_TSYN_B));2142}2143 2144static int hns3_handle_tx_bounce(struct hns3_enet_ring *ring,2145				 struct sk_buff *skb)2146{2147	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];2148	unsigned int type = DESC_TYPE_BOUNCE_HEAD;2149	unsigned int size = skb_headlen(skb);2150	dma_addr_t dma;2151	int bd_num = 0;2152	u32 cb_len;2153	void *buf;2154	int ret;2155 2156	if (skb->len <= ring->tx_copybreak) {2157		size = skb->len;2158		type = DESC_TYPE_BOUNCE_ALL;2159	}2160 2161	/* hns3_can_use_tx_bounce() is called to ensure the below2162	 * function can always return the tx buffer.2163	 */2164	buf = hns3_tx_spare_alloc(ring, size, &dma, &cb_len);2165 2166	ret = skb_copy_bits(skb, 0, buf, size);2167	if (unlikely(ret < 0)) {2168		hns3_tx_spare_rollback(ring, cb_len);2169		hns3_ring_stats_update(ring, copy_bits_err);2170		return ret;2171	}2172 2173	desc_cb->priv = skb;2174	desc_cb->length = cb_len;2175	desc_cb->dma = dma;2176	desc_cb->type = type;2177 2178	bd_num += hns3_fill_desc(ring, dma, size);2179 2180	if (type == DESC_TYPE_BOUNCE_HEAD) {2181		ret = hns3_fill_skb_to_desc(ring, skb,2182					    DESC_TYPE_BOUNCE_HEAD);2183		if (unlikely(ret < 0))2184			return ret;2185 2186		bd_num += ret;2187	}2188 2189	dma_sync_single_for_device(ring_to_dev(ring), dma, size,2190				   DMA_TO_DEVICE);2191 2192	hns3_ring_stats_update(ring, tx_bounce);2193 2194	return bd_num;2195}2196 2197static int hns3_handle_tx_sgl(struct hns3_enet_ring *ring,2198			      struct sk_buff *skb)2199{2200	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];2201	u32 nfrag = skb_shinfo(skb)->nr_frags + 1;2202	struct sg_table *sgt;2203	int i, bd_num = 0;2204	dma_addr_t dma;2205	u32 cb_len;2206	int nents;2207 2208	if (skb_has_frag_list(skb))2209		nfrag = HNS3_MAX_TSO_BD_NUM;2210 2211	/* hns3_can_use_tx_sgl() is called to ensure the below2212	 * function can always return the tx buffer.2213	 */2214	sgt = hns3_tx_spare_alloc(ring, HNS3_SGL_SIZE(nfrag),2215				  &dma, &cb_len);2216 2217	/* scatterlist follows by the sg table */2218	sgt->sgl = (struct scatterlist *)(sgt + 1);2219	sg_init_table(sgt->sgl, nfrag);2220	nents = skb_to_sgvec(skb, sgt->sgl, 0, skb->len);2221	if (unlikely(nents < 0)) {2222		hns3_tx_spare_rollback(ring, cb_len);2223		hns3_ring_stats_update(ring, skb2sgl_err);2224		return -ENOMEM;2225	}2226 2227	sgt->orig_nents = nents;2228	sgt->nents = dma_map_sg(ring_to_dev(ring), sgt->sgl, sgt->orig_nents,2229				DMA_TO_DEVICE);2230	if (unlikely(!sgt->nents)) {2231		hns3_tx_spare_rollback(ring, cb_len);2232		hns3_ring_stats_update(ring, map_sg_err);2233		return -ENOMEM;2234	}2235 2236	desc_cb->priv = skb;2237	desc_cb->length = cb_len;2238	desc_cb->dma = dma;2239	desc_cb->type = DESC_TYPE_SGL_SKB;2240 2241	for (i = 0; i < sgt->nents; i++)2242		bd_num += hns3_fill_desc(ring, sg_dma_address(sgt->sgl + i),2243					 sg_dma_len(sgt->sgl + i));2244	hns3_ring_stats_update(ring, tx_sgl);2245 2246	return bd_num;2247}2248 2249static int hns3_handle_desc_filling(struct hns3_enet_ring *ring,2250				    struct sk_buff *skb)2251{2252	u32 space;2253 2254	if (!ring->tx_spare)2255		goto out;2256 2257	space = hns3_tx_spare_space(ring);2258 2259	if (hns3_can_use_tx_sgl(ring, skb, space))2260		return hns3_handle_tx_sgl(ring, skb);2261 2262	if (hns3_can_use_tx_bounce(ring, skb, space))2263		return hns3_handle_tx_bounce(ring, skb);2264 2265out:2266	return hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB);2267}2268 2269static int hns3_handle_skb_desc(struct hns3_enet_ring *ring,2270				struct sk_buff *skb,2271				struct hns3_desc_cb *desc_cb,2272				int next_to_use_head)2273{2274	int ret;2275 2276	ret = hns3_fill_skb_desc(ring, skb, &ring->desc[ring->next_to_use],2277				 desc_cb);2278	if (unlikely(ret < 0))2279		goto fill_err;2280 2281	/* 'ret < 0' means filling error, 'ret == 0' means skb->len is2282	 * zero, which is unlikely, and 'ret > 0' means how many tx desc2283	 * need to be notified to the hw.2284	 */2285	ret = hns3_handle_desc_filling(ring, skb);2286	if (likely(ret > 0))2287		return ret;2288 2289fill_err:2290	hns3_clear_desc(ring, next_to_use_head);2291	return ret;2292}2293 2294netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev)2295{2296	struct hns3_nic_priv *priv = netdev_priv(netdev);2297	struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping];2298	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use];2299	struct netdev_queue *dev_queue;2300	int pre_ntu, ret;2301	bool doorbell;2302 2303	/* Hardware can only handle short frames above 32 bytes */2304	if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) {2305		hns3_tx_doorbell(ring, 0, !netdev_xmit_more());2306 2307		hns3_ring_stats_update(ring, sw_err_cnt);2308 2309		return NETDEV_TX_OK;2310	}2311 2312	/* Prefetch the data used later */2313	prefetch(skb->data);2314 2315	ret = hns3_nic_maybe_stop_tx(ring, netdev, skb);2316	if (unlikely(ret <= 0)) {2317		if (ret == -EBUSY) {2318			hns3_tx_doorbell(ring, 0, true);2319			return NETDEV_TX_BUSY;2320		}2321 2322		hns3_rl_err(netdev, "xmit error: %d!\n", ret);2323		goto out_err_tx_ok;2324	}2325 2326	ret = hns3_handle_skb_desc(ring, skb, desc_cb, ring->next_to_use);2327	if (unlikely(ret <= 0))2328		goto out_err_tx_ok;2329 2330	pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) :2331					(ring->desc_num - 1);2332 2333	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))2334		hns3_tsyn(netdev, skb, &ring->desc[pre_ntu]);2335 2336	ring->desc[pre_ntu].tx.bdtp_fe_sc_vld_ra_ri |=2337				cpu_to_le16(BIT(HNS3_TXD_FE_B));2338	trace_hns3_tx_desc(ring, pre_ntu);2339 2340	skb_tx_timestamp(skb);2341 2342	/* Complete translate all packets */2343	dev_queue = netdev_get_tx_queue(netdev, ring->queue_index);2344	doorbell = __netdev_tx_sent_queue(dev_queue, desc_cb->send_bytes,2345					  netdev_xmit_more());2346	hns3_tx_doorbell(ring, ret, doorbell);2347 2348	return NETDEV_TX_OK;2349 2350out_err_tx_ok:2351	dev_kfree_skb_any(skb);2352	hns3_tx_doorbell(ring, 0, !netdev_xmit_more());2353	return NETDEV_TX_OK;2354}2355 2356static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p)2357{2358	char format_mac_addr_perm[HNAE3_FORMAT_MAC_ADDR_LEN];2359	char format_mac_addr_sa[HNAE3_FORMAT_MAC_ADDR_LEN];2360	struct hnae3_handle *h = hns3_get_handle(netdev);2361	struct sockaddr *mac_addr = p;2362	int ret;2363 2364	if (!mac_addr || !is_valid_ether_addr((const u8 *)mac_addr->sa_data))2365		return -EADDRNOTAVAIL;2366 2367	if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) {2368		hnae3_format_mac_addr(format_mac_addr_sa, mac_addr->sa_data);2369		netdev_info(netdev, "already using mac address %s\n",2370			    format_mac_addr_sa);2371		return 0;2372	}2373 2374	/* For VF device, if there is a perm_addr, then the user will not2375	 * be allowed to change the address.2376	 */2377	if (!hns3_is_phys_func(h->pdev) &&2378	    !is_zero_ether_addr(netdev->perm_addr)) {2379		hnae3_format_mac_addr(format_mac_addr_perm, netdev->perm_addr);2380		hnae3_format_mac_addr(format_mac_addr_sa, mac_addr->sa_data);2381		netdev_err(netdev, "has permanent MAC %s, user MAC %s not allow\n",2382			   format_mac_addr_perm, format_mac_addr_sa);2383		return -EPERM;2384	}2385 2386	ret = h->ae_algo->ops->set_mac_addr(h, mac_addr->sa_data, false);2387	if (ret) {2388		netdev_err(netdev, "set_mac_address fail, ret=%d!\n", ret);2389		return ret;2390	}2391 2392	eth_hw_addr_set(netdev, mac_addr->sa_data);2393 2394	return 0;2395}2396 2397static int hns3_nic_do_ioctl(struct net_device *netdev,2398			     struct ifreq *ifr, int cmd)2399{2400	struct hnae3_handle *h = hns3_get_handle(netdev);2401 2402	if (!netif_running(netdev))2403		return -EINVAL;2404 2405	if (!h->ae_algo->ops->do_ioctl)2406		return -EOPNOTSUPP;2407 2408	return h->ae_algo->ops->do_ioctl(h, ifr, cmd);2409}2410 2411static int hns3_nic_set_features(struct net_device *netdev,2412				 netdev_features_t features)2413{2414	netdev_features_t changed = netdev->features ^ features;2415	struct hns3_nic_priv *priv = netdev_priv(netdev);2416	struct hnae3_handle *h = priv->ae_handle;2417	bool enable;2418	int ret;2419 2420	if (changed & (NETIF_F_GRO_HW) && h->ae_algo->ops->set_gro_en) {2421		enable = !!(features & NETIF_F_GRO_HW);2422		ret = h->ae_algo->ops->set_gro_en(h, enable);2423		if (ret)2424			return ret;2425	}2426 2427	if ((changed & NETIF_F_HW_VLAN_CTAG_RX) &&2428	    h->ae_algo->ops->enable_hw_strip_rxvtag) {2429		enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);2430		ret = h->ae_algo->ops->enable_hw_strip_rxvtag(h, enable);2431		if (ret)2432			return ret;2433	}2434 2435	if ((changed & NETIF_F_NTUPLE) && h->ae_algo->ops->enable_fd) {2436		enable = !!(features & NETIF_F_NTUPLE);2437		h->ae_algo->ops->enable_fd(h, enable);2438	}2439 2440	if ((netdev->features & NETIF_F_HW_TC) > (features & NETIF_F_HW_TC) &&2441	    h->ae_algo->ops->cls_flower_active(h)) {2442		netdev_err(netdev,2443			   "there are offloaded TC filters active, cannot disable HW TC offload");2444		return -EINVAL;2445	}2446 2447	if ((changed & NETIF_F_HW_VLAN_CTAG_FILTER) &&2448	    h->ae_algo->ops->enable_vlan_filter) {2449		enable = !!(features & NETIF_F_HW_VLAN_CTAG_FILTER);2450		ret = h->ae_algo->ops->enable_vlan_filter(h, enable);2451		if (ret)2452			return ret;2453	}2454 2455	netdev->features = features;2456	return 0;2457}2458 2459static netdev_features_t hns3_features_check(struct sk_buff *skb,2460					     struct net_device *dev,2461					     netdev_features_t features)2462{2463#define HNS3_MAX_HDR_LEN	480U2464#define HNS3_MAX_L4_HDR_LEN	60U2465 2466	size_t len;2467 2468	if (skb->ip_summed != CHECKSUM_PARTIAL)2469		return features;2470 2471	if (skb->encapsulation)2472		len = skb_inner_transport_offset(skb);2473	else2474		len = skb_transport_offset(skb);2475 2476	/* Assume L4 is 60 byte as TCP is the only protocol with a2477	 * a flexible value, and it's max len is 60 bytes.2478	 */2479	len += HNS3_MAX_L4_HDR_LEN;2480 2481	/* Hardware only supports checksum on the skb with a max header2482	 * len of 480 bytes.2483	 */2484	if (len > HNS3_MAX_HDR_LEN)2485		features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);2486 2487	return features;2488}2489 2490static void hns3_fetch_stats(struct rtnl_link_stats64 *stats,2491			     struct hns3_enet_ring *ring, bool is_tx)2492{2493	unsigned int start;2494 2495	do {2496		start = u64_stats_fetch_begin(&ring->syncp);2497		if (is_tx) {2498			stats->tx_bytes += ring->stats.tx_bytes;2499			stats->tx_packets += ring->stats.tx_pkts;2500			stats->tx_dropped += ring->stats.sw_err_cnt;2501			stats->tx_dropped += ring->stats.tx_vlan_err;2502			stats->tx_dropped += ring->stats.tx_l4_proto_err;2503			stats->tx_dropped += ring->stats.tx_l2l3l4_err;2504			stats->tx_dropped += ring->stats.tx_tso_err;2505			stats->tx_dropped += ring->stats.over_max_recursion;2506			stats->tx_dropped += ring->stats.hw_limitation;2507			stats->tx_dropped += ring->stats.copy_bits_err;2508			stats->tx_dropped += ring->stats.skb2sgl_err;2509			stats->tx_dropped += ring->stats.map_sg_err;2510			stats->tx_errors += ring->stats.sw_err_cnt;2511			stats->tx_errors += ring->stats.tx_vlan_err;2512			stats->tx_errors += ring->stats.tx_l4_proto_err;2513			stats->tx_errors += ring->stats.tx_l2l3l4_err;2514			stats->tx_errors += ring->stats.tx_tso_err;2515			stats->tx_errors += ring->stats.over_max_recursion;2516			stats->tx_errors += ring->stats.hw_limitation;2517			stats->tx_errors += ring->stats.copy_bits_err;2518			stats->tx_errors += ring->stats.skb2sgl_err;2519			stats->tx_errors += ring->stats.map_sg_err;2520		} else {2521			stats->rx_bytes += ring->stats.rx_bytes;2522			stats->rx_packets += ring->stats.rx_pkts;2523			stats->rx_dropped += ring->stats.l2_err;2524			stats->rx_errors += ring->stats.l2_err;2525			stats->rx_errors += ring->stats.l3l4_csum_err;2526			stats->rx_crc_errors += ring->stats.l2_err;2527			stats->multicast += ring->stats.rx_multicast;2528			stats->rx_length_errors += ring->stats.err_pkt_len;2529		}2530	} while (u64_stats_fetch_retry(&ring->syncp, start));2531}2532 2533static void hns3_nic_get_stats64(struct net_device *netdev,2534				 struct rtnl_link_stats64 *stats)2535{2536	struct hns3_nic_priv *priv = netdev_priv(netdev);2537	int queue_num = priv->ae_handle->kinfo.num_tqps;2538	struct hnae3_handle *handle = priv->ae_handle;2539	struct rtnl_link_stats64 ring_total_stats;2540	struct hns3_enet_ring *ring;2541	unsigned int idx;2542 2543	if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state))2544		return;2545 2546	handle->ae_algo->ops->update_stats(handle);2547 2548	memset(&ring_total_stats, 0, sizeof(ring_total_stats));2549	for (idx = 0; idx < queue_num; idx++) {2550		/* fetch the tx stats */2551		ring = &priv->ring[idx];2552		hns3_fetch_stats(&ring_total_stats, ring, true);2553 2554		/* fetch the rx stats */2555		ring = &priv->ring[idx + queue_num];2556		hns3_fetch_stats(&ring_total_stats, ring, false);2557	}2558 2559	stats->tx_bytes = ring_total_stats.tx_bytes;2560	stats->tx_packets = ring_total_stats.tx_packets;2561	stats->rx_bytes = ring_total_stats.rx_bytes;2562	stats->rx_packets = ring_total_stats.rx_packets;2563 2564	stats->rx_errors = ring_total_stats.rx_errors;2565	stats->multicast = ring_total_stats.multicast;2566	stats->rx_length_errors = ring_total_stats.rx_length_errors;2567	stats->rx_crc_errors = ring_total_stats.rx_crc_errors;2568	stats->rx_missed_errors = netdev->stats.rx_missed_errors;2569 2570	stats->tx_errors = ring_total_stats.tx_errors;2571	stats->rx_dropped = ring_total_stats.rx_dropped;2572	stats->tx_dropped = ring_total_stats.tx_dropped;2573	stats->collisions = netdev->stats.collisions;2574	stats->rx_over_errors = netdev->stats.rx_over_errors;2575	stats->rx_frame_errors = netdev->stats.rx_frame_errors;2576	stats->rx_fifo_errors = netdev->stats.rx_fifo_errors;2577	stats->tx_aborted_errors = netdev->stats.tx_aborted_errors;2578	stats->tx_carrier_errors = netdev->stats.tx_carrier_errors;2579	stats->tx_fifo_errors = netdev->stats.tx_fifo_errors;2580	stats->tx_heartbeat_errors = netdev->stats.tx_heartbeat_errors;2581	stats->tx_window_errors = netdev->stats.tx_window_errors;2582	stats->rx_compressed = netdev->stats.rx_compressed;2583	stats->tx_compressed = netdev->stats.tx_compressed;2584}2585 2586static int hns3_setup_tc(struct net_device *netdev, void *type_data)2587{2588	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;2589	struct hnae3_knic_private_info *kinfo;2590	u8 tc = mqprio_qopt->qopt.num_tc;2591	u16 mode = mqprio_qopt->mode;2592	u8 hw = mqprio_qopt->qopt.hw;2593	struct hnae3_handle *h;2594 2595	if (!((hw == TC_MQPRIO_HW_OFFLOAD_TCS &&2596	       mode == TC_MQPRIO_MODE_CHANNEL) || (!hw && tc == 0)))2597		return -EOPNOTSUPP;2598 2599	if (tc > HNAE3_MAX_TC)2600		return -EINVAL;2601 2602	if (!netdev)2603		return -EINVAL;2604 2605	h = hns3_get_handle(netdev);2606	kinfo = &h->kinfo;2607 2608	netif_dbg(h, drv, netdev, "setup tc: num_tc=%u\n", tc);2609 2610	return (kinfo->dcb_ops && kinfo->dcb_ops->setup_tc) ?2611		kinfo->dcb_ops->setup_tc(h, mqprio_qopt) : -EOPNOTSUPP;2612}2613 2614static int hns3_setup_tc_cls_flower(struct hns3_nic_priv *priv,2615				    struct flow_cls_offload *flow)2616{2617	int tc = tc_classid_to_hwtc(priv->netdev, flow->classid);2618	struct hnae3_handle *h = hns3_get_handle(priv->netdev);2619 2620	switch (flow->command) {2621	case FLOW_CLS_REPLACE:2622		if (h->ae_algo->ops->add_cls_flower)2623			return h->ae_algo->ops->add_cls_flower(h, flow, tc);2624		break;2625	case FLOW_CLS_DESTROY:2626		if (h->ae_algo->ops->del_cls_flower)2627			return h->ae_algo->ops->del_cls_flower(h, flow);2628		break;2629	default:2630		break;2631	}2632 2633	return -EOPNOTSUPP;2634}2635 2636static int hns3_setup_tc_block_cb(enum tc_setup_type type, void *type_data,2637				  void *cb_priv)2638{2639	struct hns3_nic_priv *priv = cb_priv;2640 2641	if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))2642		return -EOPNOTSUPP;2643 2644	switch (type) {2645	case TC_SETUP_CLSFLOWER:2646		return hns3_setup_tc_cls_flower(priv, type_data);2647	default:2648		return -EOPNOTSUPP;2649	}2650}2651 2652static LIST_HEAD(hns3_block_cb_list);2653 2654static int hns3_nic_setup_tc(struct net_device *dev, enum tc_setup_type type,2655			     void *type_data)2656{2657	struct hns3_nic_priv *priv = netdev_priv(dev);2658	int ret;2659 2660	switch (type) {2661	case TC_SETUP_QDISC_MQPRIO:2662		ret = hns3_setup_tc(dev, type_data);2663		break;2664	case TC_SETUP_BLOCK:2665		ret = flow_block_cb_setup_simple(type_data,2666						 &hns3_block_cb_list,2667						 hns3_setup_tc_block_cb,2668						 priv, priv, true);2669		break;2670	default:2671		return -EOPNOTSUPP;2672	}2673 2674	return ret;2675}2676 2677static int hns3_vlan_rx_add_vid(struct net_device *netdev,2678				__be16 proto, u16 vid)2679{2680	struct hnae3_handle *h = hns3_get_handle(netdev);2681	int ret = -EIO;2682 2683	if (h->ae_algo->ops->set_vlan_filter)2684		ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, false);2685 2686	return ret;2687}2688 2689static int hns3_vlan_rx_kill_vid(struct net_device *netdev,2690				 __be16 proto, u16 vid)2691{2692	struct hnae3_handle *h = hns3_get_handle(netdev);2693	int ret = -EIO;2694 2695	if (h->ae_algo->ops->set_vlan_filter)2696		ret = h->ae_algo->ops->set_vlan_filter(h, proto, vid, true);2697 2698	return ret;2699}2700 2701static int hns3_ndo_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,2702				u8 qos, __be16 vlan_proto)2703{2704	struct hnae3_handle *h = hns3_get_handle(netdev);2705	int ret = -EIO;2706 2707	netif_dbg(h, drv, netdev,2708		  "set vf vlan: vf=%d, vlan=%u, qos=%u, vlan_proto=0x%x\n",2709		  vf, vlan, qos, ntohs(vlan_proto));2710 2711	if (h->ae_algo->ops->set_vf_vlan_filter)2712		ret = h->ae_algo->ops->set_vf_vlan_filter(h, vf, vlan,2713							  qos, vlan_proto);2714 2715	return ret;2716}2717 2718static int hns3_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable)2719{2720	struct hnae3_handle *handle = hns3_get_handle(netdev);2721 2722	if (hns3_nic_resetting(netdev))2723		return -EBUSY;2724 2725	if (!handle->ae_algo->ops->set_vf_spoofchk)2726		return -EOPNOTSUPP;2727 2728	return handle->ae_algo->ops->set_vf_spoofchk(handle, vf, enable);2729}2730 2731static int hns3_set_vf_trust(struct net_device *netdev, int vf, bool enable)2732{2733	struct hnae3_handle *handle = hns3_get_handle(netdev);2734 2735	if (!handle->ae_algo->ops->set_vf_trust)2736		return -EOPNOTSUPP;2737 2738	return handle->ae_algo->ops->set_vf_trust(handle, vf, enable);2739}2740 2741static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu)2742{2743	struct hnae3_handle *h = hns3_get_handle(netdev);2744	int ret;2745 2746	if (hns3_nic_resetting(netdev))2747		return -EBUSY;2748 2749	if (!h->ae_algo->ops->set_mtu)2750		return -EOPNOTSUPP;2751 2752	netif_dbg(h, drv, netdev,2753		  "change mtu from %u to %d\n", netdev->mtu, new_mtu);2754 2755	ret = h->ae_algo->ops->set_mtu(h, new_mtu);2756	if (ret)2757		netdev_err(netdev, "failed to change MTU in hardware %d\n",2758			   ret);2759	else2760		WRITE_ONCE(netdev->mtu, new_mtu);2761 2762	return ret;2763}2764 2765static int hns3_get_timeout_queue(struct net_device *ndev)2766{2767	int i;2768 2769	/* Find the stopped queue the same way the stack does */2770	for (i = 0; i < ndev->num_tx_queues; i++) {2771		struct netdev_queue *q;2772		unsigned long trans_start;2773 2774		q = netdev_get_tx_queue(ndev, i);2775		trans_start = READ_ONCE(q->trans_start);2776		if (netif_xmit_stopped(q) &&2777		    time_after(jiffies,2778			       (trans_start + ndev->watchdog_timeo))) {2779#ifdef CONFIG_BQL2780			struct dql *dql = &q->dql;2781 2782			netdev_info(ndev, "DQL info last_cnt: %u, queued: %u, adj_limit: %u, completed: %u\n",2783				    dql->last_obj_cnt, dql->num_queued,2784				    dql->adj_limit, dql->num_completed);2785#endif2786			netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n",2787				    q->state,2788				    jiffies_to_msecs(jiffies - trans_start));2789			break;2790		}2791	}2792 2793	return i;2794}2795 2796static void hns3_dump_queue_stats(struct net_device *ndev,2797				  struct hns3_enet_ring *tx_ring,2798				  int timeout_queue)2799{2800	struct napi_struct *napi = &tx_ring->tqp_vector->napi;2801	struct hns3_nic_priv *priv = netdev_priv(ndev);2802 2803	netdev_info(ndev,2804		    "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, napi state: %lu\n",2805		    priv->tx_timeout_count, timeout_queue, tx_ring->next_to_use,2806		    tx_ring->next_to_clean, napi->state);2807 2808	netdev_info(ndev,2809		    "tx_pkts: %llu, tx_bytes: %llu, sw_err_cnt: %llu, tx_pending: %d\n",2810		    tx_ring->stats.tx_pkts, tx_ring->stats.tx_bytes,2811		    tx_ring->stats.sw_err_cnt, tx_ring->pending_buf);2812 2813	netdev_info(ndev,2814		    "seg_pkt_cnt: %llu, tx_more: %llu, restart_queue: %llu, tx_busy: %llu\n",2815		    tx_ring->stats.seg_pkt_cnt, tx_ring->stats.tx_more,2816		    tx_ring->stats.restart_queue, tx_ring->stats.tx_busy);2817 2818	netdev_info(ndev, "tx_push: %llu, tx_mem_doorbell: %llu\n",2819		    tx_ring->stats.tx_push, tx_ring->stats.tx_mem_doorbell);2820}2821 2822static void hns3_dump_queue_reg(struct net_device *ndev,2823				struct hns3_enet_ring *tx_ring)2824{2825	netdev_info(ndev,2826		    "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n",2827		    hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_BD_NUM_REG),2828		    hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_HEAD_REG),2829		    hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_TAIL_REG),2830		    hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_BD_ERR_REG),2831		    readl(tx_ring->tqp_vector->mask_addr));2832	netdev_info(ndev,2833		    "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n",2834		    hns3_tqp_read_reg(tx_ring, HNS3_RING_EN_REG),2835		    hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_TC_REG),2836		    hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_FBDNUM_REG),2837		    hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_OFFSET_REG),2838		    hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_EBDNUM_REG),2839		    hns3_tqp_read_reg(tx_ring,2840				      HNS3_RING_TX_RING_EBD_OFFSET_REG));2841}2842 2843static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev)2844{2845	struct hns3_nic_priv *priv = netdev_priv(ndev);2846	struct hnae3_handle *h = hns3_get_handle(ndev);2847	struct hns3_enet_ring *tx_ring;2848	int timeout_queue;2849 2850	timeout_queue = hns3_get_timeout_queue(ndev);2851	if (timeout_queue >= ndev->num_tx_queues) {2852		netdev_info(ndev,2853			    "no netdev TX timeout queue found, timeout count: %llu\n",2854			    priv->tx_timeout_count);2855		return false;2856	}2857 2858	priv->tx_timeout_count++;2859 2860	tx_ring = &priv->ring[timeout_queue];2861	hns3_dump_queue_stats(ndev, tx_ring, timeout_queue);2862 2863	/* When mac received many pause frames continuous, it's unable to send2864	 * packets, which may cause tx timeout2865	 */2866	if (h->ae_algo->ops->get_mac_stats) {2867		struct hns3_mac_stats mac_stats;2868 2869		h->ae_algo->ops->get_mac_stats(h, &mac_stats);2870		netdev_info(ndev, "tx_pause_cnt: %llu, rx_pause_cnt: %llu\n",2871			    mac_stats.tx_pause_cnt, mac_stats.rx_pause_cnt);2872	}2873 2874	hns3_dump_queue_reg(ndev, tx_ring);2875 2876	return true;2877}2878 2879static void hns3_nic_net_timeout(struct net_device *ndev, unsigned int txqueue)2880{2881	struct hns3_nic_priv *priv = netdev_priv(ndev);2882	struct hnae3_handle *h = priv->ae_handle;2883 2884	if (!hns3_get_tx_timeo_queue_info(ndev))2885		return;2886 2887	/* request the reset, and let the hclge to determine2888	 * which reset level should be done2889	 */2890	if (h->ae_algo->ops->reset_event)2891		h->ae_algo->ops->reset_event(h->pdev, h);2892}2893 2894#ifdef CONFIG_RFS_ACCEL2895static int hns3_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,2896			      u16 rxq_index, u32 flow_id)2897{2898	struct hnae3_handle *h = hns3_get_handle(dev);2899	struct flow_keys fkeys;2900 2901	if (!h->ae_algo->ops->add_arfs_entry)2902		return -EOPNOTSUPP;2903 2904	if (skb->encapsulation)2905		return -EPROTONOSUPPORT;2906 2907	if (!skb_flow_dissect_flow_keys(skb, &fkeys, 0))2908		return -EPROTONOSUPPORT;2909 2910	if ((fkeys.basic.n_proto != htons(ETH_P_IP) &&2911	     fkeys.basic.n_proto != htons(ETH_P_IPV6)) ||2912	    (fkeys.basic.ip_proto != IPPROTO_TCP &&2913	     fkeys.basic.ip_proto != IPPROTO_UDP))2914		return -EPROTONOSUPPORT;2915 2916	return h->ae_algo->ops->add_arfs_entry(h, rxq_index, flow_id, &fkeys);2917}2918#endif2919 2920static int hns3_nic_get_vf_config(struct net_device *ndev, int vf,2921				  struct ifla_vf_info *ivf)2922{2923	struct hnae3_handle *h = hns3_get_handle(ndev);2924 2925	if (!h->ae_algo->ops->get_vf_config)2926		return -EOPNOTSUPP;2927 2928	return h->ae_algo->ops->get_vf_config(h, vf, ivf);2929}2930 2931static int hns3_nic_set_vf_link_state(struct net_device *ndev, int vf,2932				      int link_state)2933{2934	struct hnae3_handle *h = hns3_get_handle(ndev);2935 2936	if (!h->ae_algo->ops->set_vf_link_state)2937		return -EOPNOTSUPP;2938 2939	return h->ae_algo->ops->set_vf_link_state(h, vf, link_state);2940}2941 2942static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf,2943				int min_tx_rate, int max_tx_rate)2944{2945	struct hnae3_handle *h = hns3_get_handle(ndev);2946 2947	if (!h->ae_algo->ops->set_vf_rate)2948		return -EOPNOTSUPP;2949 2950	return h->ae_algo->ops->set_vf_rate(h, vf, min_tx_rate, max_tx_rate,2951					    false);2952}2953 2954static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac)2955{2956	struct hnae3_handle *h = hns3_get_handle(netdev);2957	char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];2958 2959	if (!h->ae_algo->ops->set_vf_mac)2960		return -EOPNOTSUPP;2961 2962	if (is_multicast_ether_addr(mac)) {2963		hnae3_format_mac_addr(format_mac_addr, mac);2964		netdev_err(netdev,2965			   "Invalid MAC:%s specified. Could not set MAC\n",2966			   format_mac_addr);2967		return -EINVAL;2968	}2969 2970	return h->ae_algo->ops->set_vf_mac(h, vf_id, mac);2971}2972 2973#define HNS3_INVALID_DSCP		0xff2974#define HNS3_DSCP_SHIFT			22975 2976static u8 hns3_get_skb_dscp(struct sk_buff *skb)2977{2978	__be16 protocol = skb->protocol;2979	u8 dscp = HNS3_INVALID_DSCP;2980 2981	if (protocol == htons(ETH_P_8021Q))2982		protocol = vlan_get_protocol(skb);2983 2984	if (protocol == htons(ETH_P_IP))2985		dscp = ipv4_get_dsfield(ip_hdr(skb)) >> HNS3_DSCP_SHIFT;2986	else if (protocol == htons(ETH_P_IPV6))2987		dscp = ipv6_get_dsfield(ipv6_hdr(skb)) >> HNS3_DSCP_SHIFT;2988 2989	return dscp;2990}2991 2992static u16 hns3_nic_select_queue(struct net_device *netdev,2993				 struct sk_buff *skb,2994				 struct net_device *sb_dev)2995{2996	struct hnae3_handle *h = hns3_get_handle(netdev);2997	u8 dscp;2998 2999	if (h->kinfo.tc_map_mode != HNAE3_TC_MAP_MODE_DSCP ||3000	    !h->ae_algo->ops->get_dscp_prio)3001		goto out;3002 3003	dscp = hns3_get_skb_dscp(skb);3004	if (unlikely(dscp >= HNAE3_MAX_DSCP))3005		goto out;3006 3007	skb->priority = h->kinfo.dscp_prio[dscp];3008	if (skb->priority == HNAE3_PRIO_ID_INVALID)3009		skb->priority = 0;3010 3011out:3012	return netdev_pick_tx(netdev, skb, sb_dev);3013}3014 3015static const struct net_device_ops hns3_nic_netdev_ops = {3016	.ndo_open		= hns3_nic_net_open,3017	.ndo_stop		= hns3_nic_net_stop,3018	.ndo_start_xmit		= hns3_nic_net_xmit,3019	.ndo_tx_timeout		= hns3_nic_net_timeout,3020	.ndo_set_mac_address	= hns3_nic_net_set_mac_address,3021	.ndo_eth_ioctl		= hns3_nic_do_ioctl,3022	.ndo_change_mtu		= hns3_nic_change_mtu,3023	.ndo_set_features	= hns3_nic_set_features,3024	.ndo_features_check	= hns3_features_check,3025	.ndo_get_stats64	= hns3_nic_get_stats64,3026	.ndo_setup_tc		= hns3_nic_setup_tc,3027	.ndo_set_rx_mode	= hns3_nic_set_rx_mode,3028	.ndo_vlan_rx_add_vid	= hns3_vlan_rx_add_vid,3029	.ndo_vlan_rx_kill_vid	= hns3_vlan_rx_kill_vid,3030	.ndo_set_vf_vlan	= hns3_ndo_set_vf_vlan,3031	.ndo_set_vf_spoofchk	= hns3_set_vf_spoofchk,3032	.ndo_set_vf_trust	= hns3_set_vf_trust,3033#ifdef CONFIG_RFS_ACCEL3034	.ndo_rx_flow_steer	= hns3_rx_flow_steer,3035#endif3036	.ndo_get_vf_config	= hns3_nic_get_vf_config,3037	.ndo_set_vf_link_state	= hns3_nic_set_vf_link_state,3038	.ndo_set_vf_rate	= hns3_nic_set_vf_rate,3039	.ndo_set_vf_mac		= hns3_nic_set_vf_mac,3040	.ndo_select_queue	= hns3_nic_select_queue,3041};3042 3043bool hns3_is_phys_func(struct pci_dev *pdev)3044{3045	u32 dev_id = pdev->device;3046 3047	switch (dev_id) {3048	case HNAE3_DEV_ID_GE:3049	case HNAE3_DEV_ID_25GE:3050	case HNAE3_DEV_ID_25GE_RDMA:3051	case HNAE3_DEV_ID_25GE_RDMA_MACSEC:3052	case HNAE3_DEV_ID_50GE_RDMA:3053	case HNAE3_DEV_ID_50GE_RDMA_MACSEC:3054	case HNAE3_DEV_ID_100G_RDMA_MACSEC:3055	case HNAE3_DEV_ID_200G_RDMA:3056		return true;3057	case HNAE3_DEV_ID_VF:3058	case HNAE3_DEV_ID_RDMA_DCB_PFC_VF:3059		return false;3060	default:3061		dev_warn(&pdev->dev, "un-recognized pci device-id %u",3062			 dev_id);3063	}3064 3065	return false;3066}3067 3068static void hns3_disable_sriov(struct pci_dev *pdev)3069{3070	/* If our VFs are assigned we cannot shut down SR-IOV3071	 * without causing issues, so just leave the hardware3072	 * available but disabled3073	 */3074	if (pci_vfs_assigned(pdev)) {3075		dev_warn(&pdev->dev,3076			 "disabling driver while VFs are assigned\n");3077		return;3078	}3079 3080	pci_disable_sriov(pdev);3081}3082 3083/* hns3_probe - Device initialization routine3084 * @pdev: PCI device information struct3085 * @ent: entry in hns3_pci_tbl3086 *3087 * hns3_probe initializes a PF identified by a pci_dev structure.3088 * The OS initialization, configuring of the PF private structure,3089 * and a hardware reset occur.3090 *3091 * Returns 0 on success, negative on failure3092 */3093static int hns3_probe(struct pci_dev *pdev, const struct pci_device_id *ent)3094{3095	struct hnae3_ae_dev *ae_dev;3096	int ret;3097 3098	ae_dev = devm_kzalloc(&pdev->dev, sizeof(*ae_dev), GFP_KERNEL);3099	if (!ae_dev)3100		return -ENOMEM;3101 3102	ae_dev->pdev = pdev;3103	ae_dev->flag = ent->driver_data;3104	pci_set_drvdata(pdev, ae_dev);3105 3106	ret = hnae3_register_ae_dev(ae_dev);3107	if (ret)3108		pci_set_drvdata(pdev, NULL);3109 3110	return ret;3111}3112 3113/**3114 * hns3_clean_vf_config3115 * @pdev: pointer to a pci_dev structure3116 * @num_vfs: number of VFs allocated3117 *3118 * Clean residual vf config after disable sriov3119 **/3120static void hns3_clean_vf_config(struct pci_dev *pdev, int num_vfs)3121{3122	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);3123 3124	if (ae_dev->ops->clean_vf_config)3125		ae_dev->ops->clean_vf_config(ae_dev, num_vfs);3126}3127 3128/* hns3_remove - Device removal routine3129 * @pdev: PCI device information struct3130 */3131static void hns3_remove(struct pci_dev *pdev)3132{3133	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);3134 3135	if (hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))3136		hns3_disable_sriov(pdev);3137 3138	hnae3_unregister_ae_dev(ae_dev);3139	pci_set_drvdata(pdev, NULL);3140}3141 3142/**3143 * hns3_pci_sriov_configure3144 * @pdev: pointer to a pci_dev structure3145 * @num_vfs: number of VFs to allocate3146 *3147 * Enable or change the number of VFs. Called when the user updates the number3148 * of VFs in sysfs.3149 **/3150static int hns3_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)3151{3152	int ret;3153 3154	if (!(hns3_is_phys_func(pdev) && IS_ENABLED(CONFIG_PCI_IOV))) {3155		dev_warn(&pdev->dev, "Can not config SRIOV\n");3156		return -EINVAL;3157	}3158 3159	if (num_vfs) {3160		ret = pci_enable_sriov(pdev, num_vfs);3161		if (ret)3162			dev_err(&pdev->dev, "SRIOV enable failed %d\n", ret);3163		else3164			return num_vfs;3165	} else if (!pci_vfs_assigned(pdev)) {3166		int num_vfs_pre = pci_num_vf(pdev);3167 3168		pci_disable_sriov(pdev);3169		hns3_clean_vf_config(pdev, num_vfs_pre);3170	} else {3171		dev_warn(&pdev->dev,3172			 "Unable to free VFs because some are assigned to VMs.\n");3173	}3174 3175	return 0;3176}3177 3178static void hns3_shutdown(struct pci_dev *pdev)3179{3180	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);3181 3182	hnae3_unregister_ae_dev(ae_dev);3183	pci_set_drvdata(pdev, NULL);3184 3185	if (system_state == SYSTEM_POWER_OFF)3186		pci_set_power_state(pdev, PCI_D3hot);3187}3188 3189static int __maybe_unused hns3_suspend(struct device *dev)3190{3191	struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);3192 3193	if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {3194		dev_info(dev, "Begin to suspend.\n");3195		if (ae_dev->ops && ae_dev->ops->reset_prepare)3196			ae_dev->ops->reset_prepare(ae_dev, HNAE3_FUNC_RESET);3197	}3198 3199	return 0;3200}3201 3202static int __maybe_unused hns3_resume(struct device *dev)3203{3204	struct hnae3_ae_dev *ae_dev = dev_get_drvdata(dev);3205 3206	if (ae_dev && hns3_is_phys_func(ae_dev->pdev)) {3207		dev_info(dev, "Begin to resume.\n");3208		if (ae_dev->ops && ae_dev->ops->reset_done)3209			ae_dev->ops->reset_done(ae_dev);3210	}3211 3212	return 0;3213}3214 3215static pci_ers_result_t hns3_error_detected(struct pci_dev *pdev,3216					    pci_channel_state_t state)3217{3218	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);3219	pci_ers_result_t ret;3220 3221	dev_info(&pdev->dev, "PCI error detected, state(=%u)!!\n", state);3222 3223	if (state == pci_channel_io_perm_failure)3224		return PCI_ERS_RESULT_DISCONNECT;3225 3226	if (!ae_dev || !ae_dev->ops) {3227		dev_err(&pdev->dev,3228			"Can't recover - error happened before device initialized\n");3229		return PCI_ERS_RESULT_NONE;3230	}3231 3232	if (ae_dev->ops->handle_hw_ras_error)3233		ret = ae_dev->ops->handle_hw_ras_error(ae_dev);3234	else3235		return PCI_ERS_RESULT_NONE;3236 3237	return ret;3238}3239 3240static pci_ers_result_t hns3_slot_reset(struct pci_dev *pdev)3241{3242	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);3243	const struct hnae3_ae_ops *ops;3244	enum hnae3_reset_type reset_type;3245	struct device *dev = &pdev->dev;3246 3247	if (!ae_dev || !ae_dev->ops)3248		return PCI_ERS_RESULT_NONE;3249 3250	ops = ae_dev->ops;3251	/* request the reset */3252	if (ops->reset_event && ops->get_reset_level &&3253	    ops->set_default_reset_request) {3254		if (ae_dev->hw_err_reset_req) {3255			reset_type = ops->get_reset_level(ae_dev,3256						&ae_dev->hw_err_reset_req);3257			ops->set_default_reset_request(ae_dev, reset_type);3258			dev_info(dev, "requesting reset due to PCI error\n");3259			ops->reset_event(pdev, NULL);3260		}3261 3262		return PCI_ERS_RESULT_RECOVERED;3263	}3264 3265	return PCI_ERS_RESULT_DISCONNECT;3266}3267 3268static void hns3_reset_prepare(struct pci_dev *pdev)3269{3270	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);3271 3272	dev_info(&pdev->dev, "FLR prepare\n");3273	if (ae_dev && ae_dev->ops && ae_dev->ops->reset_prepare)3274		ae_dev->ops->reset_prepare(ae_dev, HNAE3_FLR_RESET);3275}3276 3277static void hns3_reset_done(struct pci_dev *pdev)3278{3279	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);3280 3281	dev_info(&pdev->dev, "FLR done\n");3282	if (ae_dev && ae_dev->ops && ae_dev->ops->reset_done)3283		ae_dev->ops->reset_done(ae_dev);3284}3285 3286static const struct pci_error_handlers hns3_err_handler = {3287	.error_detected = hns3_error_detected,3288	.slot_reset     = hns3_slot_reset,3289	.reset_prepare	= hns3_reset_prepare,3290	.reset_done	= hns3_reset_done,3291};3292 3293static SIMPLE_DEV_PM_OPS(hns3_pm_ops, hns3_suspend, hns3_resume);3294 3295static struct pci_driver hns3_driver = {3296	.name     = hns3_driver_name,3297	.id_table = hns3_pci_tbl,3298	.probe    = hns3_probe,3299	.remove   = hns3_remove,3300	.shutdown = hns3_shutdown,3301	.driver.pm  = &hns3_pm_ops,3302	.sriov_configure = hns3_pci_sriov_configure,3303	.err_handler    = &hns3_err_handler,3304};3305 3306/* set default feature to hns3 */3307static void hns3_set_default_feature(struct net_device *netdev)3308{3309	struct hnae3_handle *h = hns3_get_handle(netdev);3310	struct pci_dev *pdev = h->pdev;3311	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);3312 3313	netdev->priv_flags |= IFF_UNICAST_FLT;3314 3315	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |3316		NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |3317		NETIF_F_RXCSUM | NETIF_F_SG | NETIF_F_GSO |3318		NETIF_F_GRO | NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_GSO_GRE |3319		NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_UDP_TUNNEL |3320		NETIF_F_SCTP_CRC | NETIF_F_FRAGLIST;3321 3322	if (hnae3_ae_dev_gro_supported(ae_dev))3323		netdev->features |= NETIF_F_GRO_HW;3324 3325	if (hnae3_ae_dev_fd_supported(ae_dev))3326		netdev->features |= NETIF_F_NTUPLE;3327 3328	if (test_bit(HNAE3_DEV_SUPPORT_UDP_GSO_B, ae_dev->caps))3329		netdev->features |= NETIF_F_GSO_UDP_L4;3330 3331	if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps))3332		netdev->features |= NETIF_F_HW_CSUM;3333	else3334		netdev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;3335 3336	if (test_bit(HNAE3_DEV_SUPPORT_UDP_TUNNEL_CSUM_B, ae_dev->caps))3337		netdev->features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;3338 3339	if (test_bit(HNAE3_DEV_SUPPORT_FD_FORWARD_TC_B, ae_dev->caps))3340		netdev->features |= NETIF_F_HW_TC;3341 3342	netdev->hw_features |= netdev->features;3343	if (!test_bit(HNAE3_DEV_SUPPORT_VLAN_FLTR_MDF_B, ae_dev->caps))3344		netdev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;3345 3346	netdev->vlan_features |= netdev->features &3347		~(NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_CTAG_TX |3348		  NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_GRO_HW | NETIF_F_NTUPLE |3349		  NETIF_F_HW_TC);3350 3351	netdev->hw_enc_features |= netdev->vlan_features | NETIF_F_TSO_MANGLEID;3352 3353	/* The device_version V3 hardware can't offload the checksum for IP in3354	 * GRE packets, but can do it for NvGRE. So default to disable the3355	 * checksum and GSO offload for GRE.3356	 */3357	if (ae_dev->dev_version > HNAE3_DEVICE_VERSION_V2) {3358		netdev->features &= ~NETIF_F_GSO_GRE;3359		netdev->features &= ~NETIF_F_GSO_GRE_CSUM;3360	}3361}3362 3363static int hns3_alloc_buffer(struct hns3_enet_ring *ring,3364			     struct hns3_desc_cb *cb)3365{3366	unsigned int order = hns3_page_order(ring);3367	struct page *p;3368 3369	if (ring->page_pool) {3370		p = page_pool_dev_alloc_frag(ring->page_pool,3371					     &cb->page_offset,3372					     hns3_buf_size(ring));3373		if (unlikely(!p))3374			return -ENOMEM;3375 3376		cb->priv = p;3377		cb->buf = page_address(p);3378		cb->dma = page_pool_get_dma_addr(p);3379		cb->type = DESC_TYPE_PP_FRAG;3380		cb->reuse_flag = 0;3381		return 0;3382	}3383 3384	p = dev_alloc_pages(order);3385	if (!p)3386		return -ENOMEM;3387 3388	cb->priv = p;3389	cb->page_offset = 0;3390	cb->reuse_flag = 0;3391	cb->buf  = page_address(p);3392	cb->length = hns3_page_size(ring);3393	cb->type = DESC_TYPE_PAGE;3394	page_ref_add(p, USHRT_MAX - 1);3395	cb->pagecnt_bias = USHRT_MAX;3396 3397	return 0;3398}3399 3400static void hns3_free_buffer(struct hns3_enet_ring *ring,3401			     struct hns3_desc_cb *cb, int budget)3402{3403	if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_HEAD |3404			DESC_TYPE_BOUNCE_ALL | DESC_TYPE_SGL_SKB))3405		napi_consume_skb(cb->priv, budget);3406	else if (!HNAE3_IS_TX_RING(ring)) {3407		if (cb->type & DESC_TYPE_PAGE && cb->pagecnt_bias)3408			__page_frag_cache_drain(cb->priv, cb->pagecnt_bias);3409		else if (cb->type & DESC_TYPE_PP_FRAG)3410			page_pool_put_full_page(ring->page_pool, cb->priv,3411						false);3412	}3413	memset(cb, 0, sizeof(*cb));3414}3415 3416static int hns3_map_buffer(struct hns3_enet_ring *ring, struct hns3_desc_cb *cb)3417{3418	cb->dma = dma_map_page(ring_to_dev(ring), cb->priv, 0,3419			       cb->length, ring_to_dma_dir(ring));3420 3421	if (unlikely(dma_mapping_error(ring_to_dev(ring), cb->dma)))3422		return -EIO;3423 3424	return 0;3425}3426 3427static void hns3_unmap_buffer(struct hns3_enet_ring *ring,3428			      struct hns3_desc_cb *cb)3429{3430	if (cb->type & (DESC_TYPE_SKB | DESC_TYPE_FRAGLIST_SKB))3431		dma_unmap_single(ring_to_dev(ring), cb->dma, cb->length,3432				 ring_to_dma_dir(ring));3433	else if ((cb->type & DESC_TYPE_PAGE) && cb->length)3434		dma_unmap_page(ring_to_dev(ring), cb->dma, cb->length,3435			       ring_to_dma_dir(ring));3436	else if (cb->type & (DESC_TYPE_BOUNCE_ALL | DESC_TYPE_BOUNCE_HEAD |3437			     DESC_TYPE_SGL_SKB))3438		hns3_tx_spare_reclaim_cb(ring, cb);3439}3440 3441static void hns3_buffer_detach(struct hns3_enet_ring *ring, int i)3442{3443	hns3_unmap_buffer(ring, &ring->desc_cb[i]);3444	ring->desc[i].addr = 0;3445	ring->desc_cb[i].refill = 0;3446}3447 3448static void hns3_free_buffer_detach(struct hns3_enet_ring *ring, int i,3449				    int budget)3450{3451	struct hns3_desc_cb *cb = &ring->desc_cb[i];3452 3453	if (!ring->desc_cb[i].dma)3454		return;3455 3456	hns3_buffer_detach(ring, i);3457	hns3_free_buffer(ring, cb, budget);3458}3459 3460static void hns3_free_buffers(struct hns3_enet_ring *ring)3461{3462	int i;3463 3464	for (i = 0; i < ring->desc_num; i++)3465		hns3_free_buffer_detach(ring, i, 0);3466}3467 3468/* free desc along with its attached buffer */3469static void hns3_free_desc(struct hns3_enet_ring *ring)3470{3471	int size = ring->desc_num * sizeof(ring->desc[0]);3472 3473	hns3_free_buffers(ring);3474 3475	if (ring->desc) {3476		dma_free_coherent(ring_to_dev(ring), size,3477				  ring->desc, ring->desc_dma_addr);3478		ring->desc = NULL;3479	}3480}3481 3482static int hns3_alloc_desc(struct hns3_enet_ring *ring)3483{3484	int size = ring->desc_num * sizeof(ring->desc[0]);3485 3486	ring->desc = dma_alloc_coherent(ring_to_dev(ring), size,3487					&ring->desc_dma_addr, GFP_KERNEL);3488	if (!ring->desc)3489		return -ENOMEM;3490 3491	return 0;3492}3493 3494static int hns3_alloc_and_map_buffer(struct hns3_enet_ring *ring,3495				   struct hns3_desc_cb *cb)3496{3497	int ret;3498 3499	ret = hns3_alloc_buffer(ring, cb);3500	if (ret || ring->page_pool)3501		goto out;3502 3503	ret = hns3_map_buffer(ring, cb);3504	if (ret)3505		goto out_with_buf;3506 3507	return 0;3508 3509out_with_buf:3510	hns3_free_buffer(ring, cb, 0);3511out:3512	return ret;3513}3514 3515static int hns3_alloc_and_attach_buffer(struct hns3_enet_ring *ring, int i)3516{3517	int ret = hns3_alloc_and_map_buffer(ring, &ring->desc_cb[i]);3518 3519	if (ret)3520		return ret;3521 3522	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +3523					 ring->desc_cb[i].page_offset);3524	ring->desc_cb[i].refill = 1;3525 3526	return 0;3527}3528 3529/* Allocate memory for raw pkg, and map with dma */3530static int hns3_alloc_ring_buffers(struct hns3_enet_ring *ring)3531{3532	int i, j, ret;3533 3534	for (i = 0; i < ring->desc_num; i++) {3535		ret = hns3_alloc_and_attach_buffer(ring, i);3536		if (ret)3537			goto out_buffer_fail;3538 3539		if (!(i % HNS3_RESCHED_BD_NUM))3540			cond_resched();3541	}3542 3543	return 0;3544 3545out_buffer_fail:3546	for (j = i - 1; j >= 0; j--)3547		hns3_free_buffer_detach(ring, j, 0);3548	return ret;3549}3550 3551/* detach a in-used buffer and replace with a reserved one */3552static void hns3_replace_buffer(struct hns3_enet_ring *ring, int i,3553				struct hns3_desc_cb *res_cb)3554{3555	hns3_unmap_buffer(ring, &ring->desc_cb[i]);3556	ring->desc_cb[i] = *res_cb;3557	ring->desc_cb[i].refill = 1;3558	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +3559					 ring->desc_cb[i].page_offset);3560	ring->desc[i].rx.bd_base_info = 0;3561}3562 3563static void hns3_reuse_buffer(struct hns3_enet_ring *ring, int i)3564{3565	ring->desc_cb[i].reuse_flag = 0;3566	ring->desc_cb[i].refill = 1;3567	ring->desc[i].addr = cpu_to_le64(ring->desc_cb[i].dma +3568					 ring->desc_cb[i].page_offset);3569	ring->desc[i].rx.bd_base_info = 0;3570 3571	dma_sync_single_for_device(ring_to_dev(ring),3572			ring->desc_cb[i].dma + ring->desc_cb[i].page_offset,3573			hns3_buf_size(ring),3574			DMA_FROM_DEVICE);3575}3576 3577static bool hns3_nic_reclaim_desc(struct hns3_enet_ring *ring,3578				  int *bytes, int *pkts, int budget)3579{3580	/* This smp_load_acquire() pairs with smp_store_release() in3581	 * hns3_tx_doorbell().3582	 */3583	int ltu = smp_load_acquire(&ring->last_to_use);3584	int ntc = ring->next_to_clean;3585	struct hns3_desc_cb *desc_cb;3586	bool reclaimed = false;3587	struct hns3_desc *desc;3588 3589	while (ltu != ntc) {3590		desc = &ring->desc[ntc];3591 3592		if (le16_to_cpu(desc->tx.bdtp_fe_sc_vld_ra_ri) &3593				BIT(HNS3_TXD_VLD_B))3594			break;3595 3596		desc_cb = &ring->desc_cb[ntc];3597 3598		if (desc_cb->type & (DESC_TYPE_SKB | DESC_TYPE_BOUNCE_ALL |3599				     DESC_TYPE_BOUNCE_HEAD |3600				     DESC_TYPE_SGL_SKB)) {3601			(*pkts)++;3602			(*bytes) += desc_cb->send_bytes;3603		}3604 3605		/* desc_cb will be cleaned, after hnae3_free_buffer_detach */3606		hns3_free_buffer_detach(ring, ntc, budget);3607 3608		if (++ntc == ring->desc_num)3609			ntc = 0;3610 3611		/* Issue prefetch for next Tx descriptor */3612		prefetch(&ring->desc_cb[ntc]);3613		reclaimed = true;3614	}3615 3616	if (unlikely(!reclaimed))3617		return false;3618 3619	/* This smp_store_release() pairs with smp_load_acquire() in3620	 * ring_space called by hns3_nic_net_xmit.3621	 */3622	smp_store_release(&ring->next_to_clean, ntc);3623 3624	hns3_tx_spare_update(ring);3625 3626	return true;3627}3628 3629void hns3_clean_tx_ring(struct hns3_enet_ring *ring, int budget)3630{3631	struct net_device *netdev = ring_to_netdev(ring);3632	struct hns3_nic_priv *priv = netdev_priv(netdev);3633	struct netdev_queue *dev_queue;3634	int bytes, pkts;3635 3636	bytes = 0;3637	pkts = 0;3638 3639	if (unlikely(!hns3_nic_reclaim_desc(ring, &bytes, &pkts, budget)))3640		return;3641 3642	ring->tqp_vector->tx_group.total_bytes += bytes;3643	ring->tqp_vector->tx_group.total_packets += pkts;3644 3645	u64_stats_update_begin(&ring->syncp);3646	ring->stats.tx_bytes += bytes;3647	ring->stats.tx_pkts += pkts;3648	u64_stats_update_end(&ring->syncp);3649 3650	dev_queue = netdev_get_tx_queue(netdev, ring->tqp->tqp_index);3651	netdev_tx_completed_queue(dev_queue, pkts, bytes);3652 3653	if (unlikely(netif_carrier_ok(netdev) &&3654		     ring_space(ring) > HNS3_MAX_TSO_BD_NUM)) {3655		/* Make sure that anybody stopping the queue after this3656		 * sees the new next_to_clean.3657		 */3658		smp_mb();3659		if (netif_tx_queue_stopped(dev_queue) &&3660		    !test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) {3661			netif_tx_wake_queue(dev_queue);3662			ring->stats.restart_queue++;3663		}3664	}3665}3666 3667static int hns3_desc_unused(struct hns3_enet_ring *ring)3668{3669	int ntc = ring->next_to_clean;3670	int ntu = ring->next_to_use;3671 3672	if (unlikely(ntc == ntu && !ring->desc_cb[ntc].refill))3673		return ring->desc_num;3674 3675	return ((ntc >= ntu) ? 0 : ring->desc_num) + ntc - ntu;3676}3677 3678/* Return true if there is any allocation failure */3679static bool hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring,3680				      int cleand_count)3681{3682	struct hns3_desc_cb *desc_cb;3683	struct hns3_desc_cb res_cbs;3684	int i, ret;3685 3686	for (i = 0; i < cleand_count; i++) {3687		desc_cb = &ring->desc_cb[ring->next_to_use];3688		if (desc_cb->reuse_flag) {3689			hns3_ring_stats_update(ring, reuse_pg_cnt);3690 3691			hns3_reuse_buffer(ring, ring->next_to_use);3692		} else {3693			ret = hns3_alloc_and_map_buffer(ring, &res_cbs);3694			if (ret) {3695				hns3_ring_stats_update(ring, sw_err_cnt);3696 3697				hns3_rl_err(ring_to_netdev(ring),3698					    "alloc rx buffer failed: %d\n",3699					    ret);3700 3701				writel(i, ring->tqp->io_base +3702				       HNS3_RING_RX_RING_HEAD_REG);3703				return true;3704			}3705			hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);3706 3707			hns3_ring_stats_update(ring, non_reuse_pg);3708		}3709 3710		ring_ptr_move_fw(ring, next_to_use);3711	}3712 3713	writel(i, ring->tqp->io_base + HNS3_RING_RX_RING_HEAD_REG);3714	return false;3715}3716 3717static bool hns3_can_reuse_page(struct hns3_desc_cb *cb)3718{3719	return page_count(cb->priv) == cb->pagecnt_bias;3720}3721 3722static int hns3_handle_rx_copybreak(struct sk_buff *skb, int i,3723				    struct hns3_enet_ring *ring,3724				    int pull_len,3725				    struct hns3_desc_cb *desc_cb)3726{3727	struct hns3_desc *desc = &ring->desc[ring->next_to_clean];3728	u32 frag_offset = desc_cb->page_offset + pull_len;3729	int size = le16_to_cpu(desc->rx.size);3730	u32 frag_size = size - pull_len;3731	void *frag = napi_alloc_frag(frag_size);3732 3733	if (unlikely(!frag)) {3734		hns3_ring_stats_update(ring, frag_alloc_err);3735 3736		hns3_rl_err(ring_to_netdev(ring),3737			    "failed to allocate rx frag\n");3738		return -ENOMEM;3739	}3740 3741	desc_cb->reuse_flag = 1;3742	memcpy(frag, desc_cb->buf + frag_offset, frag_size);3743	skb_add_rx_frag(skb, i, virt_to_page(frag),3744			offset_in_page(frag), frag_size, frag_size);3745 3746	hns3_ring_stats_update(ring, frag_alloc);3747	return 0;3748}3749 3750static void hns3_nic_reuse_page(struct sk_buff *skb, int i,3751				struct hns3_enet_ring *ring, int pull_len,3752				struct hns3_desc_cb *desc_cb)3753{3754	struct hns3_desc *desc = &ring->desc[ring->next_to_clean];3755	u32 frag_offset = desc_cb->page_offset + pull_len;3756	int size = le16_to_cpu(desc->rx.size);3757	u32 truesize = hns3_buf_size(ring);3758	u32 frag_size = size - pull_len;3759	int ret = 0;3760	bool reused;3761 3762	if (ring->page_pool) {3763		skb_add_rx_frag(skb, i, desc_cb->priv, frag_offset,3764				frag_size, truesize);3765		return;3766	}3767 3768	/* Avoid re-using remote or pfmem page */3769	if (unlikely(!dev_page_is_reusable(desc_cb->priv)))3770		goto out;3771 3772	reused = hns3_can_reuse_page(desc_cb);3773 3774	/* Rx page can be reused when:3775	 * 1. Rx page is only owned by the driver when page_offset3776	 *    is zero, which means 0 @ truesize will be used by3777	 *    stack after skb_add_rx_frag() is called, and the rest3778	 *    of rx page can be reused by driver.3779	 * Or3780	 * 2. Rx page is only owned by the driver when page_offset3781	 *    is non-zero, which means page_offset @ truesize will3782	 *    be used by stack after skb_add_rx_frag() is called,3783	 *    and 0 @ truesize can be reused by driver.3784	 */3785	if ((!desc_cb->page_offset && reused) ||3786	    ((desc_cb->page_offset + truesize + truesize) <=3787	     hns3_page_size(ring) && desc_cb->page_offset)) {3788		desc_cb->page_offset += truesize;3789		desc_cb->reuse_flag = 1;3790	} else if (desc_cb->page_offset && reused) {3791		desc_cb->page_offset = 0;3792		desc_cb->reuse_flag = 1;3793	} else if (frag_size <= ring->rx_copybreak) {3794		ret = hns3_handle_rx_copybreak(skb, i, ring, pull_len, desc_cb);3795		if (!ret)3796			return;3797	}3798 3799out:3800	desc_cb->pagecnt_bias--;3801 3802	if (unlikely(!desc_cb->pagecnt_bias)) {3803		page_ref_add(desc_cb->priv, USHRT_MAX);3804		desc_cb->pagecnt_bias = USHRT_MAX;3805	}3806 3807	skb_add_rx_frag(skb, i, desc_cb->priv, frag_offset,3808			frag_size, truesize);3809 3810	if (unlikely(!desc_cb->reuse_flag))3811		__page_frag_cache_drain(desc_cb->priv, desc_cb->pagecnt_bias);3812}3813 3814static int hns3_gro_complete(struct sk_buff *skb, u32 l234info)3815{3816	__be16 type = skb->protocol;3817	struct tcphdr *th;3818	int depth = 0;3819 3820	while (eth_type_vlan(type)) {3821		struct vlan_hdr *vh;3822 3823		if ((depth + VLAN_HLEN) > skb_headlen(skb))3824			return -EFAULT;3825 3826		vh = (struct vlan_hdr *)(skb->data + depth);3827		type = vh->h_vlan_encapsulated_proto;3828		depth += VLAN_HLEN;3829	}3830 3831	skb_set_network_header(skb, depth);3832 3833	if (type == htons(ETH_P_IP)) {3834		const struct iphdr *iph = ip_hdr(skb);3835 3836		depth += sizeof(struct iphdr);3837		skb_set_transport_header(skb, depth);3838		th = tcp_hdr(skb);3839		th->check = ~tcp_v4_check(skb->len - depth, iph->saddr,3840					  iph->daddr, 0);3841	} else if (type == htons(ETH_P_IPV6)) {3842		const struct ipv6hdr *iph = ipv6_hdr(skb);3843 3844		depth += sizeof(struct ipv6hdr);3845		skb_set_transport_header(skb, depth);3846		th = tcp_hdr(skb);3847		th->check = ~tcp_v6_check(skb->len - depth, &iph->saddr,3848					  &iph->daddr, 0);3849	} else {3850		hns3_rl_err(skb->dev,3851			    "Error: FW GRO supports only IPv4/IPv6, not 0x%04x, depth: %d\n",3852			    be16_to_cpu(type), depth);3853		return -EFAULT;3854	}3855 3856	skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;3857	if (th->cwr)3858		skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;3859 3860	if (l234info & BIT(HNS3_RXD_GRO_FIXID_B))3861		skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID;3862 3863	skb->csum_start = (unsigned char *)th - skb->head;3864	skb->csum_offset = offsetof(struct tcphdr, check);3865	skb->ip_summed = CHECKSUM_PARTIAL;3866 3867	trace_hns3_gro(skb);3868 3869	return 0;3870}3871 3872static void hns3_checksum_complete(struct hns3_enet_ring *ring,3873				   struct sk_buff *skb, u32 ptype, u16 csum)3874{3875	if (ptype == HNS3_INVALID_PTYPE ||3876	    hns3_rx_ptype_tbl[ptype].ip_summed != CHECKSUM_COMPLETE)3877		return;3878 3879	hns3_ring_stats_update(ring, csum_complete);3880	skb->ip_summed = CHECKSUM_COMPLETE;3881	skb->csum = csum_unfold((__force __sum16)csum);3882}3883 3884static void hns3_rx_handle_csum(struct sk_buff *skb, u32 l234info,3885				u32 ol_info, u32 ptype)3886{3887	int l3_type, l4_type;3888	int ol4_type;3889 3890	if (ptype != HNS3_INVALID_PTYPE) {3891		skb->csum_level = hns3_rx_ptype_tbl[ptype].csum_level;3892		skb->ip_summed = hns3_rx_ptype_tbl[ptype].ip_summed;3893 3894		return;3895	}3896 3897	ol4_type = hnae3_get_field(ol_info, HNS3_RXD_OL4ID_M,3898				   HNS3_RXD_OL4ID_S);3899	switch (ol4_type) {3900	case HNS3_OL4_TYPE_MAC_IN_UDP:3901	case HNS3_OL4_TYPE_NVGRE:3902		skb->csum_level = 1;3903		fallthrough;3904	case HNS3_OL4_TYPE_NO_TUN:3905		l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,3906					  HNS3_RXD_L3ID_S);3907		l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,3908					  HNS3_RXD_L4ID_S);3909		/* Can checksum ipv4 or ipv6 + UDP/TCP/SCTP packets */3910		if ((l3_type == HNS3_L3_TYPE_IPV4 ||3911		     l3_type == HNS3_L3_TYPE_IPV6) &&3912		    (l4_type == HNS3_L4_TYPE_UDP ||3913		     l4_type == HNS3_L4_TYPE_TCP ||3914		     l4_type == HNS3_L4_TYPE_SCTP))3915			skb->ip_summed = CHECKSUM_UNNECESSARY;3916		break;3917	default:3918		break;3919	}3920}3921 3922static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb,3923			     u32 l234info, u32 bd_base_info, u32 ol_info,3924			     u16 csum)3925{3926	struct net_device *netdev = ring_to_netdev(ring);3927	struct hns3_nic_priv *priv = netdev_priv(netdev);3928	u32 ptype = HNS3_INVALID_PTYPE;3929 3930	skb->ip_summed = CHECKSUM_NONE;3931 3932	skb_checksum_none_assert(skb);3933 3934	if (!(netdev->features & NETIF_F_RXCSUM))3935		return;3936 3937	if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state))3938		ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M,3939					HNS3_RXD_PTYPE_S);3940 3941	hns3_checksum_complete(ring, skb, ptype, csum);3942 3943	/* check if hardware has done checksum */3944	if (!(bd_base_info & BIT(HNS3_RXD_L3L4P_B)))3945		return;3946 3947	if (unlikely(l234info & (BIT(HNS3_RXD_L3E_B) | BIT(HNS3_RXD_L4E_B) |3948				 BIT(HNS3_RXD_OL3E_B) |3949				 BIT(HNS3_RXD_OL4E_B)))) {3950		skb->ip_summed = CHECKSUM_NONE;3951		hns3_ring_stats_update(ring, l3l4_csum_err);3952 3953		return;3954	}3955 3956	hns3_rx_handle_csum(skb, l234info, ol_info, ptype);3957}3958 3959static void hns3_rx_skb(struct hns3_enet_ring *ring, struct sk_buff *skb)3960{3961	if (skb_has_frag_list(skb))3962		napi_gro_flush(&ring->tqp_vector->napi, false);3963 3964	napi_gro_receive(&ring->tqp_vector->napi, skb);3965}3966 3967static bool hns3_parse_vlan_tag(struct hns3_enet_ring *ring,3968				struct hns3_desc *desc, u32 l234info,3969				u16 *vlan_tag)3970{3971	struct hnae3_handle *handle = ring->tqp->handle;3972	struct pci_dev *pdev = ring->tqp->handle->pdev;3973	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);3974 3975	if (unlikely(ae_dev->dev_version < HNAE3_DEVICE_VERSION_V2)) {3976		*vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);3977		if (!(*vlan_tag & VLAN_VID_MASK))3978			*vlan_tag = le16_to_cpu(desc->rx.vlan_tag);3979 3980		return (*vlan_tag != 0);3981	}3982 3983#define HNS3_STRP_OUTER_VLAN	0x13984#define HNS3_STRP_INNER_VLAN	0x23985#define HNS3_STRP_BOTH		0x33986 3987	/* Hardware always insert VLAN tag into RX descriptor when3988	 * remove the tag from packet, driver needs to determine3989	 * reporting which tag to stack.3990	 */3991	switch (hnae3_get_field(l234info, HNS3_RXD_STRP_TAGP_M,3992				HNS3_RXD_STRP_TAGP_S)) {3993	case HNS3_STRP_OUTER_VLAN:3994		if (handle->port_base_vlan_state !=3995				HNAE3_PORT_BASE_VLAN_DISABLE)3996			return false;3997 3998		*vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);3999		return true;4000	case HNS3_STRP_INNER_VLAN:4001		if (handle->port_base_vlan_state !=4002				HNAE3_PORT_BASE_VLAN_DISABLE)4003			return false;4004 4005		*vlan_tag = le16_to_cpu(desc->rx.vlan_tag);4006		return true;4007	case HNS3_STRP_BOTH:4008		if (handle->port_base_vlan_state ==4009				HNAE3_PORT_BASE_VLAN_DISABLE)4010			*vlan_tag = le16_to_cpu(desc->rx.ot_vlan_tag);4011		else4012			*vlan_tag = le16_to_cpu(desc->rx.vlan_tag);4013 4014		return true;4015	default:4016		return false;4017	}4018}4019 4020static void hns3_rx_ring_move_fw(struct hns3_enet_ring *ring)4021{4022	ring->desc[ring->next_to_clean].rx.bd_base_info &=4023		cpu_to_le32(~BIT(HNS3_RXD_VLD_B));4024	ring->desc_cb[ring->next_to_clean].refill = 0;4025	ring->next_to_clean += 1;4026 4027	if (unlikely(ring->next_to_clean == ring->desc_num))4028		ring->next_to_clean = 0;4029}4030 4031static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length,4032			  unsigned char *va)4033{4034	struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_clean];4035	struct net_device *netdev = ring_to_netdev(ring);4036	struct sk_buff *skb;4037 4038	ring->skb = napi_alloc_skb(&ring->tqp_vector->napi, HNS3_RX_HEAD_SIZE);4039	skb = ring->skb;4040	if (unlikely(!skb)) {4041		hns3_rl_err(netdev, "alloc rx skb fail\n");4042		hns3_ring_stats_update(ring, sw_err_cnt);4043 4044		return -ENOMEM;4045	}4046 4047	trace_hns3_rx_desc(ring);4048	prefetchw(skb->data);4049 4050	ring->pending_buf = 1;4051	ring->frag_num = 0;4052	ring->tail_skb = NULL;4053	if (length <= HNS3_RX_HEAD_SIZE) {4054		memcpy(__skb_put(skb, length), va, ALIGN(length, sizeof(long)));4055 4056		/* We can reuse buffer as-is, just make sure it is reusable */4057		if (dev_page_is_reusable(desc_cb->priv))4058			desc_cb->reuse_flag = 1;4059		else if (desc_cb->type & DESC_TYPE_PP_FRAG)4060			page_pool_put_full_page(ring->page_pool, desc_cb->priv,4061						false);4062		else /* This page cannot be reused so discard it */4063			__page_frag_cache_drain(desc_cb->priv,4064						desc_cb->pagecnt_bias);4065 4066		hns3_rx_ring_move_fw(ring);4067		return 0;4068	}4069 4070	if (ring->page_pool)4071		skb_mark_for_recycle(skb);4072 4073	hns3_ring_stats_update(ring, seg_pkt_cnt);4074 4075	ring->pull_len = eth_get_headlen(netdev, va, HNS3_RX_HEAD_SIZE);4076	__skb_put(skb, ring->pull_len);4077	hns3_nic_reuse_page(skb, ring->frag_num++, ring, ring->pull_len,4078			    desc_cb);4079	hns3_rx_ring_move_fw(ring);4080 4081	return 0;4082}4083 4084static int hns3_add_frag(struct hns3_enet_ring *ring)4085{4086	struct sk_buff *skb = ring->skb;4087	struct sk_buff *head_skb = skb;4088	struct sk_buff *new_skb;4089	struct hns3_desc_cb *desc_cb;4090	struct hns3_desc *desc;4091	u32 bd_base_info;4092 4093	do {4094		desc = &ring->desc[ring->next_to_clean];4095		desc_cb = &ring->desc_cb[ring->next_to_clean];4096		bd_base_info = le32_to_cpu(desc->rx.bd_base_info);4097		/* make sure HW write desc complete */4098		dma_rmb();4099		if (!(bd_base_info & BIT(HNS3_RXD_VLD_B)))4100			return -ENXIO;4101 4102		if (unlikely(ring->frag_num >= MAX_SKB_FRAGS)) {4103			new_skb = napi_alloc_skb(&ring->tqp_vector->napi, 0);4104			if (unlikely(!new_skb)) {4105				hns3_rl_err(ring_to_netdev(ring),4106					    "alloc rx fraglist skb fail\n");4107				return -ENXIO;4108			}4109 4110			if (ring->page_pool)4111				skb_mark_for_recycle(new_skb);4112 4113			ring->frag_num = 0;4114 4115			if (ring->tail_skb) {4116				ring->tail_skb->next = new_skb;4117				ring->tail_skb = new_skb;4118			} else {4119				skb_shinfo(skb)->frag_list = new_skb;4120				ring->tail_skb = new_skb;4121			}4122		}4123 4124		if (ring->tail_skb) {4125			head_skb->truesize += hns3_buf_size(ring);4126			head_skb->data_len += le16_to_cpu(desc->rx.size);4127			head_skb->len += le16_to_cpu(desc->rx.size);4128			skb = ring->tail_skb;4129		}4130 4131		dma_sync_single_for_cpu(ring_to_dev(ring),4132				desc_cb->dma + desc_cb->page_offset,4133				hns3_buf_size(ring),4134				DMA_FROM_DEVICE);4135 4136		hns3_nic_reuse_page(skb, ring->frag_num++, ring, 0, desc_cb);4137		trace_hns3_rx_desc(ring);4138		hns3_rx_ring_move_fw(ring);4139		ring->pending_buf++;4140	} while (!(bd_base_info & BIT(HNS3_RXD_FE_B)));4141 4142	return 0;4143}4144 4145static int hns3_set_gro_and_checksum(struct hns3_enet_ring *ring,4146				     struct sk_buff *skb, u32 l234info,4147				     u32 bd_base_info, u32 ol_info, u16 csum)4148{4149	struct net_device *netdev = ring_to_netdev(ring);4150	struct hns3_nic_priv *priv = netdev_priv(netdev);4151	u32 l3_type;4152 4153	skb_shinfo(skb)->gso_size = hnae3_get_field(bd_base_info,4154						    HNS3_RXD_GRO_SIZE_M,4155						    HNS3_RXD_GRO_SIZE_S);4156	/* if there is no HW GRO, do not set gro params */4157	if (!skb_shinfo(skb)->gso_size) {4158		hns3_rx_checksum(ring, skb, l234info, bd_base_info, ol_info,4159				 csum);4160		return 0;4161	}4162 4163	NAPI_GRO_CB(skb)->count = hnae3_get_field(l234info,4164						  HNS3_RXD_GRO_COUNT_M,4165						  HNS3_RXD_GRO_COUNT_S);4166 4167	if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state)) {4168		u32 ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M,4169					    HNS3_RXD_PTYPE_S);4170 4171		l3_type = hns3_rx_ptype_tbl[ptype].l3_type;4172	} else {4173		l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,4174					  HNS3_RXD_L3ID_S);4175	}4176 4177	if (l3_type == HNS3_L3_TYPE_IPV4)4178		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;4179	else if (l3_type == HNS3_L3_TYPE_IPV6)4180		skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;4181	else4182		return -EFAULT;4183 4184	return  hns3_gro_complete(skb, l234info);4185}4186 4187static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring,4188				     struct sk_buff *skb, u32 rss_hash,4189				     u32 l234info, u32 ol_info)4190{4191	enum pkt_hash_types rss_type = PKT_HASH_TYPE_NONE;4192	struct net_device *netdev = ring_to_netdev(ring);4193	struct hns3_nic_priv *priv = netdev_priv(netdev);4194 4195	if (test_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state)) {4196		u32 ptype = hnae3_get_field(ol_info, HNS3_RXD_PTYPE_M,4197					    HNS3_RXD_PTYPE_S);4198 4199		rss_type = hns3_rx_ptype_tbl[ptype].hash_type;4200	} else {4201		int l3_type = hnae3_get_field(l234info, HNS3_RXD_L3ID_M,4202					      HNS3_RXD_L3ID_S);4203		int l4_type = hnae3_get_field(l234info, HNS3_RXD_L4ID_M,4204					      HNS3_RXD_L4ID_S);4205 4206		if (l3_type == HNS3_L3_TYPE_IPV4 ||4207		    l3_type == HNS3_L3_TYPE_IPV6) {4208			if (l4_type == HNS3_L4_TYPE_UDP ||4209			    l4_type == HNS3_L4_TYPE_TCP ||4210			    l4_type == HNS3_L4_TYPE_SCTP)4211				rss_type = PKT_HASH_TYPE_L4;4212			else if (l4_type == HNS3_L4_TYPE_IGMP ||4213				 l4_type == HNS3_L4_TYPE_ICMP)4214				rss_type = PKT_HASH_TYPE_L3;4215		}4216	}4217 4218	skb_set_hash(skb, rss_hash, rss_type);4219}4220 4221static void hns3_handle_rx_ts_info(struct net_device *netdev,4222				   struct hns3_desc *desc, struct sk_buff *skb,4223				   u32 bd_base_info)4224{4225	if (unlikely(bd_base_info & BIT(HNS3_RXD_TS_VLD_B))) {4226		struct hnae3_handle *h = hns3_get_handle(netdev);4227		u32 nsec = le32_to_cpu(desc->ts_nsec);4228		u32 sec = le32_to_cpu(desc->ts_sec);4229 4230		if (h->ae_algo->ops->get_rx_hwts)4231			h->ae_algo->ops->get_rx_hwts(h, skb, nsec, sec);4232	}4233}4234 4235static void hns3_handle_rx_vlan_tag(struct hns3_enet_ring *ring,4236				    struct hns3_desc *desc, struct sk_buff *skb,4237				    u32 l234info)4238{4239	struct net_device *netdev = ring_to_netdev(ring);4240 4241	/* Based on hw strategy, the tag offloaded will be stored at4242	 * ot_vlan_tag in two layer tag case, and stored at vlan_tag4243	 * in one layer tag case.4244	 */4245	if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {4246		u16 vlan_tag;4247 4248		if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag))4249			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),4250					       vlan_tag);4251	}4252}4253 4254static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb)4255{4256	struct net_device *netdev = ring_to_netdev(ring);4257	enum hns3_pkt_l2t_type l2_frame_type;4258	u32 bd_base_info, l234info, ol_info;4259	struct hns3_desc *desc;4260	unsigned int len;4261	int pre_ntc, ret;4262	u16 csum;4263 4264	/* bdinfo handled below is only valid on the last BD of the4265	 * current packet, and ring->next_to_clean indicates the first4266	 * descriptor of next packet, so need - 1 below.4267	 */4268	pre_ntc = ring->next_to_clean ? (ring->next_to_clean - 1) :4269					(ring->desc_num - 1);4270	desc = &ring->desc[pre_ntc];4271	bd_base_info = le32_to_cpu(desc->rx.bd_base_info);4272	l234info = le32_to_cpu(desc->rx.l234_info);4273	ol_info = le32_to_cpu(desc->rx.ol_info);4274	csum = le16_to_cpu(desc->csum);4275 4276	hns3_handle_rx_ts_info(netdev, desc, skb, bd_base_info);4277 4278	hns3_handle_rx_vlan_tag(ring, desc, skb, l234info);4279 4280	if (unlikely(!desc->rx.pkt_len || (l234info & (BIT(HNS3_RXD_TRUNCAT_B) |4281				  BIT(HNS3_RXD_L2E_B))))) {4282		u64_stats_update_begin(&ring->syncp);4283		if (l234info & BIT(HNS3_RXD_L2E_B))4284			ring->stats.l2_err++;4285		else4286			ring->stats.err_pkt_len++;4287		u64_stats_update_end(&ring->syncp);4288 4289		return -EFAULT;4290	}4291 4292	len = skb->len;4293 4294	/* Do update ip stack process */4295	skb->protocol = eth_type_trans(skb, netdev);4296 4297	/* This is needed in order to enable forwarding support */4298	ret = hns3_set_gro_and_checksum(ring, skb, l234info,4299					bd_base_info, ol_info, csum);4300	if (unlikely(ret)) {4301		hns3_ring_stats_update(ring, rx_err_cnt);4302		return ret;4303	}4304 4305	l2_frame_type = hnae3_get_field(l234info, HNS3_RXD_DMAC_M,4306					HNS3_RXD_DMAC_S);4307 4308	u64_stats_update_begin(&ring->syncp);4309	ring->stats.rx_pkts++;4310	ring->stats.rx_bytes += len;4311 4312	if (l2_frame_type == HNS3_L2_TYPE_MULTICAST)4313		ring->stats.rx_multicast++;4314 4315	u64_stats_update_end(&ring->syncp);4316 4317	ring->tqp_vector->rx_group.total_bytes += len;4318 4319	hns3_set_rx_skb_rss_type(ring, skb, le32_to_cpu(desc->rx.rss_hash),4320				 l234info, ol_info);4321	return 0;4322}4323 4324static int hns3_handle_rx_bd(struct hns3_enet_ring *ring)4325{4326	struct sk_buff *skb = ring->skb;4327	struct hns3_desc_cb *desc_cb;4328	struct hns3_desc *desc;4329	unsigned int length;4330	u32 bd_base_info;4331	int ret;4332 4333	desc = &ring->desc[ring->next_to_clean];4334	desc_cb = &ring->desc_cb[ring->next_to_clean];4335 4336	prefetch(desc);4337 4338	if (!skb) {4339		bd_base_info = le32_to_cpu(desc->rx.bd_base_info);4340		/* Check valid BD */4341		if (unlikely(!(bd_base_info & BIT(HNS3_RXD_VLD_B))))4342			return -ENXIO;4343 4344		dma_rmb();4345		length = le16_to_cpu(desc->rx.size);4346 4347		ring->va = desc_cb->buf + desc_cb->page_offset;4348 4349		dma_sync_single_for_cpu(ring_to_dev(ring),4350				desc_cb->dma + desc_cb->page_offset,4351				hns3_buf_size(ring),4352				DMA_FROM_DEVICE);4353 4354		/* Prefetch first cache line of first page.4355		 * Idea is to cache few bytes of the header of the packet.4356		 * Our L1 Cache line size is 64B so need to prefetch twice to make4357		 * it 128B. But in actual we can have greater size of caches with4358		 * 128B Level 1 cache lines. In such a case, single fetch would4359		 * suffice to cache in the relevant part of the header.4360		 */4361		net_prefetch(ring->va);4362 4363		ret = hns3_alloc_skb(ring, length, ring->va);4364		skb = ring->skb;4365 4366		if (ret < 0) /* alloc buffer fail */4367			return ret;4368		if (!(bd_base_info & BIT(HNS3_RXD_FE_B))) { /* need add frag */4369			ret = hns3_add_frag(ring);4370			if (ret)4371				return ret;4372		}4373	} else {4374		ret = hns3_add_frag(ring);4375		if (ret)4376			return ret;4377	}4378 4379	/* As the head data may be changed when GRO enable, copy4380	 * the head data in after other data rx completed4381	 */4382	if (skb->len > HNS3_RX_HEAD_SIZE)4383		memcpy(skb->data, ring->va,4384		       ALIGN(ring->pull_len, sizeof(long)));4385 4386	ret = hns3_handle_bdinfo(ring, skb);4387	if (unlikely(ret)) {4388		dev_kfree_skb_any(skb);4389		return ret;4390	}4391 4392	skb_record_rx_queue(skb, ring->tqp->tqp_index);4393	return 0;4394}4395 4396int hns3_clean_rx_ring(struct hns3_enet_ring *ring, int budget,4397		       void (*rx_fn)(struct hns3_enet_ring *, struct sk_buff *))4398{4399#define RCB_NOF_ALLOC_RX_BUFF_ONCE 164400	int unused_count = hns3_desc_unused(ring);4401	bool failure = false;4402	int recv_pkts = 0;4403	int err;4404 4405	unused_count -= ring->pending_buf;4406 4407	while (recv_pkts < budget) {4408		/* Reuse or realloc buffers */4409		if (unused_count >= RCB_NOF_ALLOC_RX_BUFF_ONCE) {4410			failure = failure ||4411				hns3_nic_alloc_rx_buffers(ring, unused_count);4412			unused_count = 0;4413		}4414 4415		/* Poll one pkt */4416		err = hns3_handle_rx_bd(ring);4417		/* Do not get FE for the packet or failed to alloc skb */4418		if (unlikely(!ring->skb || err == -ENXIO)) {4419			goto out;4420		} else if (likely(!err)) {4421			rx_fn(ring, ring->skb);4422			recv_pkts++;4423		}4424 4425		unused_count += ring->pending_buf;4426		ring->skb = NULL;4427		ring->pending_buf = 0;4428	}4429 4430out:4431	/* sync head pointer before exiting, since hardware will calculate4432	 * FBD number with head pointer4433	 */4434	if (unused_count > 0)4435		failure = failure ||4436			  hns3_nic_alloc_rx_buffers(ring, unused_count);4437 4438	return failure ? budget : recv_pkts;4439}4440 4441static void hns3_update_rx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)4442{4443	struct hns3_enet_ring_group *rx_group = &tqp_vector->rx_group;4444	struct dim_sample sample = {};4445 4446	if (!rx_group->coal.adapt_enable)4447		return;4448 4449	dim_update_sample(tqp_vector->event_cnt, rx_group->total_packets,4450			  rx_group->total_bytes, &sample);4451	net_dim(&rx_group->dim, sample);4452}4453 4454static void hns3_update_tx_int_coalesce(struct hns3_enet_tqp_vector *tqp_vector)4455{4456	struct hns3_enet_ring_group *tx_group = &tqp_vector->tx_group;4457	struct dim_sample sample = {};4458 4459	if (!tx_group->coal.adapt_enable)4460		return;4461 4462	dim_update_sample(tqp_vector->event_cnt, tx_group->total_packets,4463			  tx_group->total_bytes, &sample);4464	net_dim(&tx_group->dim, sample);4465}4466 4467static int hns3_nic_common_poll(struct napi_struct *napi, int budget)4468{4469	struct hns3_nic_priv *priv = netdev_priv(napi->dev);4470	struct hns3_enet_ring *ring;4471	int rx_pkt_total = 0;4472 4473	struct hns3_enet_tqp_vector *tqp_vector =4474		container_of(napi, struct hns3_enet_tqp_vector, napi);4475	bool clean_complete = true;4476	int rx_budget = budget;4477 4478	if (unlikely(test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {4479		napi_complete(napi);4480		return 0;4481	}4482 4483	/* Since the actual Tx work is minimal, we can give the Tx a larger4484	 * budget and be more aggressive about cleaning up the Tx descriptors.4485	 */4486	hns3_for_each_ring(ring, tqp_vector->tx_group)4487		hns3_clean_tx_ring(ring, budget);4488 4489	/* make sure rx ring budget not smaller than 1 */4490	if (tqp_vector->num_tqps > 1)4491		rx_budget = max(budget / tqp_vector->num_tqps, 1);4492 4493	hns3_for_each_ring(ring, tqp_vector->rx_group) {4494		int rx_cleaned = hns3_clean_rx_ring(ring, rx_budget,4495						    hns3_rx_skb);4496		if (rx_cleaned >= rx_budget)4497			clean_complete = false;4498 4499		rx_pkt_total += rx_cleaned;4500	}4501 4502	tqp_vector->rx_group.total_packets += rx_pkt_total;4503 4504	if (!clean_complete)4505		return budget;4506 4507	if (napi_complete(napi) &&4508	    likely(!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))) {4509		hns3_update_rx_int_coalesce(tqp_vector);4510		hns3_update_tx_int_coalesce(tqp_vector);4511 4512		hns3_mask_vector_irq(tqp_vector, 1);4513	}4514 4515	return rx_pkt_total;4516}4517 4518static int hns3_create_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,4519				  struct hnae3_ring_chain_node **head,4520				  bool is_tx)4521{4522	u32 bit_value = is_tx ? HNAE3_RING_TYPE_TX : HNAE3_RING_TYPE_RX;4523	u32 field_value = is_tx ? HNAE3_RING_GL_TX : HNAE3_RING_GL_RX;4524	struct hnae3_ring_chain_node *cur_chain = *head;4525	struct pci_dev *pdev = tqp_vector->handle->pdev;4526	struct hnae3_ring_chain_node *chain;4527	struct hns3_enet_ring *ring;4528 4529	ring = is_tx ? tqp_vector->tx_group.ring : tqp_vector->rx_group.ring;4530 4531	if (cur_chain) {4532		while (cur_chain->next)4533			cur_chain = cur_chain->next;4534	}4535 4536	while (ring) {4537		chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL);4538		if (!chain)4539			return -ENOMEM;4540		if (cur_chain)4541			cur_chain->next = chain;4542		else4543			*head = chain;4544		chain->tqp_index = ring->tqp->tqp_index;4545		hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B,4546				bit_value);4547		hnae3_set_field(chain->int_gl_idx,4548				HNAE3_RING_GL_IDX_M,4549				HNAE3_RING_GL_IDX_S, field_value);4550 4551		cur_chain = chain;4552 4553		ring = ring->next;4554	}4555 4556	return 0;4557}4558 4559static struct hnae3_ring_chain_node *4560hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector)4561{4562	struct pci_dev *pdev = tqp_vector->handle->pdev;4563	struct hnae3_ring_chain_node *cur_chain = NULL;4564	struct hnae3_ring_chain_node *chain;4565 4566	if (hns3_create_ring_chain(tqp_vector, &cur_chain, true))4567		goto err_free_chain;4568 4569	if (hns3_create_ring_chain(tqp_vector, &cur_chain, false))4570		goto err_free_chain;4571 4572	return cur_chain;4573 4574err_free_chain:4575	while (cur_chain) {4576		chain = cur_chain->next;4577		devm_kfree(&pdev->dev, cur_chain);4578		cur_chain = chain;4579	}4580 4581	return NULL;4582}4583 4584static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector,4585					struct hnae3_ring_chain_node *head)4586{4587	struct pci_dev *pdev = tqp_vector->handle->pdev;4588	struct hnae3_ring_chain_node *chain_tmp, *chain;4589 4590	chain = head;4591 4592	while (chain) {4593		chain_tmp = chain->next;4594		devm_kfree(&pdev->dev, chain);4595		chain = chain_tmp;4596	}4597}4598 4599static void hns3_add_ring_to_group(struct hns3_enet_ring_group *group,4600				   struct hns3_enet_ring *ring)4601{4602	ring->next = group->ring;4603	group->ring = ring;4604 4605	group->count++;4606}4607 4608static void hns3_nic_set_cpumask(struct hns3_nic_priv *priv)4609{4610	struct pci_dev *pdev = priv->ae_handle->pdev;4611	struct hns3_enet_tqp_vector *tqp_vector;4612	int num_vectors = priv->vector_num;4613	int numa_node;4614	int vector_i;4615 4616	numa_node = dev_to_node(&pdev->dev);4617 4618	for (vector_i = 0; vector_i < num_vectors; vector_i++) {4619		tqp_vector = &priv->tqp_vector[vector_i];4620		cpumask_set_cpu(cpumask_local_spread(vector_i, numa_node),4621				&tqp_vector->affinity_mask);4622	}4623}4624 4625static void hns3_rx_dim_work(struct work_struct *work)4626{4627	struct dim *dim = container_of(work, struct dim, work);4628	struct hns3_enet_ring_group *group = container_of(dim,4629		struct hns3_enet_ring_group, dim);4630	struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;4631	struct dim_cq_moder cur_moder =4632		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);4633 4634	hns3_set_vector_coalesce_rx_gl(group->ring->tqp_vector, cur_moder.usec);4635	tqp_vector->rx_group.coal.int_gl = cur_moder.usec;4636 4637	if (cur_moder.pkts < tqp_vector->rx_group.coal.int_ql_max) {4638		hns3_set_vector_coalesce_rx_ql(tqp_vector, cur_moder.pkts);4639		tqp_vector->rx_group.coal.int_ql = cur_moder.pkts;4640	}4641 4642	dim->state = DIM_START_MEASURE;4643}4644 4645static void hns3_tx_dim_work(struct work_struct *work)4646{4647	struct dim *dim = container_of(work, struct dim, work);4648	struct hns3_enet_ring_group *group = container_of(dim,4649		struct hns3_enet_ring_group, dim);4650	struct hns3_enet_tqp_vector *tqp_vector = group->ring->tqp_vector;4651	struct dim_cq_moder cur_moder =4652		net_dim_get_tx_moderation(dim->mode, dim->profile_ix);4653 4654	hns3_set_vector_coalesce_tx_gl(tqp_vector, cur_moder.usec);4655	tqp_vector->tx_group.coal.int_gl = cur_moder.usec;4656 4657	if (cur_moder.pkts < tqp_vector->tx_group.coal.int_ql_max) {4658		hns3_set_vector_coalesce_tx_ql(tqp_vector, cur_moder.pkts);4659		tqp_vector->tx_group.coal.int_ql = cur_moder.pkts;4660	}4661 4662	dim->state = DIM_START_MEASURE;4663}4664 4665static void hns3_nic_init_dim(struct hns3_enet_tqp_vector *tqp_vector)4666{4667	INIT_WORK(&tqp_vector->rx_group.dim.work, hns3_rx_dim_work);4668	INIT_WORK(&tqp_vector->tx_group.dim.work, hns3_tx_dim_work);4669}4670 4671static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv)4672{4673	struct hnae3_handle *h = priv->ae_handle;4674	struct hns3_enet_tqp_vector *tqp_vector;4675	int ret;4676	int i;4677 4678	hns3_nic_set_cpumask(priv);4679 4680	for (i = 0; i < priv->vector_num; i++) {4681		tqp_vector = &priv->tqp_vector[i];4682		hns3_vector_coalesce_init_hw(tqp_vector, priv);4683		tqp_vector->num_tqps = 0;4684		hns3_nic_init_dim(tqp_vector);4685	}4686 4687	for (i = 0; i < h->kinfo.num_tqps; i++) {4688		u16 vector_i = i % priv->vector_num;4689		u16 tqp_num = h->kinfo.num_tqps;4690 4691		tqp_vector = &priv->tqp_vector[vector_i];4692 4693		hns3_add_ring_to_group(&tqp_vector->tx_group,4694				       &priv->ring[i]);4695 4696		hns3_add_ring_to_group(&tqp_vector->rx_group,4697				       &priv->ring[i + tqp_num]);4698 4699		priv->ring[i].tqp_vector = tqp_vector;4700		priv->ring[i + tqp_num].tqp_vector = tqp_vector;4701		tqp_vector->num_tqps++;4702	}4703 4704	for (i = 0; i < priv->vector_num; i++) {4705		struct hnae3_ring_chain_node *vector_ring_chain;4706 4707		tqp_vector = &priv->tqp_vector[i];4708 4709		tqp_vector->rx_group.total_bytes = 0;4710		tqp_vector->rx_group.total_packets = 0;4711		tqp_vector->tx_group.total_bytes = 0;4712		tqp_vector->tx_group.total_packets = 0;4713		tqp_vector->handle = h;4714 4715		vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector);4716		if (!vector_ring_chain) {4717			ret = -ENOMEM;4718			goto map_ring_fail;4719		}4720 4721		ret = h->ae_algo->ops->map_ring_to_vector(h,4722			tqp_vector->vector_irq, vector_ring_chain);4723 4724		hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain);4725 4726		if (ret)4727			goto map_ring_fail;4728 4729		netif_napi_add(priv->netdev, &tqp_vector->napi,4730			       hns3_nic_common_poll);4731	}4732 4733	return 0;4734 4735map_ring_fail:4736	while (i--)4737		netif_napi_del(&priv->tqp_vector[i].napi);4738 4739	return ret;4740}4741 4742static void hns3_nic_init_coal_cfg(struct hns3_nic_priv *priv)4743{4744	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);4745	struct hns3_enet_coalesce *tx_coal = &priv->tx_coal;4746	struct hns3_enet_coalesce *rx_coal = &priv->rx_coal;4747 4748	/* initialize the configuration for interrupt coalescing.4749	 * 1. GL (Interrupt Gap Limiter)4750	 * 2. RL (Interrupt Rate Limiter)4751	 * 3. QL (Interrupt Quantity Limiter)4752	 *4753	 * Default: enable interrupt coalescing self-adaptive and GL4754	 */4755	tx_coal->adapt_enable = 1;4756	rx_coal->adapt_enable = 1;4757 4758	tx_coal->int_gl = HNS3_INT_GL_50K;4759	rx_coal->int_gl = HNS3_INT_GL_50K;4760 4761	rx_coal->flow_level = HNS3_FLOW_LOW;4762	tx_coal->flow_level = HNS3_FLOW_LOW;4763 4764	if (ae_dev->dev_specs.int_ql_max) {4765		tx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG;4766		rx_coal->int_ql = HNS3_INT_QL_DEFAULT_CFG;4767	}4768}4769 4770static int hns3_nic_alloc_vector_data(struct hns3_nic_priv *priv)4771{4772	struct hnae3_handle *h = priv->ae_handle;4773	struct hns3_enet_tqp_vector *tqp_vector;4774	struct hnae3_vector_info *vector;4775	struct pci_dev *pdev = h->pdev;4776	u16 tqp_num = h->kinfo.num_tqps;4777	u16 vector_num;4778	int ret = 0;4779	u16 i;4780 4781	/* RSS size, cpu online and vector_num should be the same */4782	/* Should consider 2p/4p later */4783	vector_num = min_t(u16, num_online_cpus(), tqp_num);4784 4785	vector = devm_kcalloc(&pdev->dev, vector_num, sizeof(*vector),4786			      GFP_KERNEL);4787	if (!vector)4788		return -ENOMEM;4789 4790	/* save the actual available vector number */4791	vector_num = h->ae_algo->ops->get_vector(h, vector_num, vector);4792 4793	priv->vector_num = vector_num;4794	priv->tqp_vector = (struct hns3_enet_tqp_vector *)4795		devm_kcalloc(&pdev->dev, vector_num, sizeof(*priv->tqp_vector),4796			     GFP_KERNEL);4797	if (!priv->tqp_vector) {4798		ret = -ENOMEM;4799		goto out;4800	}4801 4802	for (i = 0; i < priv->vector_num; i++) {4803		tqp_vector = &priv->tqp_vector[i];4804		tqp_vector->idx = i;4805		tqp_vector->mask_addr = vector[i].io_addr;4806		tqp_vector->vector_irq = vector[i].vector;4807		hns3_vector_coalesce_init(tqp_vector, priv);4808	}4809 4810out:4811	devm_kfree(&pdev->dev, vector);4812	return ret;4813}4814 4815static void hns3_clear_ring_group(struct hns3_enet_ring_group *group)4816{4817	group->ring = NULL;4818	group->count = 0;4819}4820 4821static void hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv)4822{4823	struct hnae3_ring_chain_node *vector_ring_chain;4824	struct hnae3_handle *h = priv->ae_handle;4825	struct hns3_enet_tqp_vector *tqp_vector;4826	int i;4827 4828	for (i = 0; i < priv->vector_num; i++) {4829		tqp_vector = &priv->tqp_vector[i];4830 4831		if (!tqp_vector->rx_group.ring && !tqp_vector->tx_group.ring)4832			continue;4833 4834		/* Since the mapping can be overwritten, when fail to get the4835		 * chain between vector and ring, we should go on to deal with4836		 * the remaining options.4837		 */4838		vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector);4839		if (!vector_ring_chain)4840			dev_warn(priv->dev, "failed to get ring chain\n");4841 4842		h->ae_algo->ops->unmap_ring_from_vector(h,4843			tqp_vector->vector_irq, vector_ring_chain);4844 4845		hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain);4846 4847		hns3_clear_ring_group(&tqp_vector->rx_group);4848		hns3_clear_ring_group(&tqp_vector->tx_group);4849		netif_napi_del(&priv->tqp_vector[i].napi);4850	}4851}4852 4853static void hns3_nic_dealloc_vector_data(struct hns3_nic_priv *priv)4854{4855	struct hnae3_handle *h = priv->ae_handle;4856	struct pci_dev *pdev = h->pdev;4857	int i, ret;4858 4859	for (i = 0; i < priv->vector_num; i++) {4860		struct hns3_enet_tqp_vector *tqp_vector;4861 4862		tqp_vector = &priv->tqp_vector[i];4863		ret = h->ae_algo->ops->put_vector(h, tqp_vector->vector_irq);4864		if (ret)4865			return;4866	}4867 4868	devm_kfree(&pdev->dev, priv->tqp_vector);4869}4870 4871static void hns3_ring_get_cfg(struct hnae3_queue *q, struct hns3_nic_priv *priv,4872			      unsigned int ring_type)4873{4874	int queue_num = priv->ae_handle->kinfo.num_tqps;4875	struct hns3_enet_ring *ring;4876	int desc_num;4877 4878	if (ring_type == HNAE3_RING_TYPE_TX) {4879		ring = &priv->ring[q->tqp_index];4880		desc_num = priv->ae_handle->kinfo.num_tx_desc;4881		ring->queue_index = q->tqp_index;4882		ring->tx_copybreak = priv->tx_copybreak;4883		ring->last_to_use = 0;4884	} else {4885		ring = &priv->ring[q->tqp_index + queue_num];4886		desc_num = priv->ae_handle->kinfo.num_rx_desc;4887		ring->queue_index = q->tqp_index;4888		ring->rx_copybreak = priv->rx_copybreak;4889	}4890 4891	hnae3_set_bit(ring->flag, HNAE3_RING_TYPE_B, ring_type);4892 4893	ring->tqp = q;4894	ring->desc = NULL;4895	ring->desc_cb = NULL;4896	ring->dev = priv->dev;4897	ring->desc_dma_addr = 0;4898	ring->buf_size = q->buf_size;4899	ring->desc_num = desc_num;4900	ring->next_to_use = 0;4901	ring->next_to_clean = 0;4902}4903 4904static void hns3_queue_to_ring(struct hnae3_queue *tqp,4905			       struct hns3_nic_priv *priv)4906{4907	hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_TX);4908	hns3_ring_get_cfg(tqp, priv, HNAE3_RING_TYPE_RX);4909}4910 4911static int hns3_get_ring_config(struct hns3_nic_priv *priv)4912{4913	struct hnae3_handle *h = priv->ae_handle;4914	struct pci_dev *pdev = h->pdev;4915	int i;4916 4917	priv->ring = devm_kzalloc(&pdev->dev,4918				  array3_size(h->kinfo.num_tqps,4919					      sizeof(*priv->ring), 2),4920				  GFP_KERNEL);4921	if (!priv->ring)4922		return -ENOMEM;4923 4924	for (i = 0; i < h->kinfo.num_tqps; i++)4925		hns3_queue_to_ring(h->kinfo.tqp[i], priv);4926 4927	return 0;4928}4929 4930static void hns3_put_ring_config(struct hns3_nic_priv *priv)4931{4932	if (!priv->ring)4933		return;4934 4935	devm_kfree(priv->dev, priv->ring);4936	priv->ring = NULL;4937}4938 4939static void hns3_alloc_page_pool(struct hns3_enet_ring *ring)4940{4941	struct page_pool_params pp_params = {4942		.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,4943		.order = hns3_page_order(ring),4944		.pool_size = ring->desc_num * hns3_buf_size(ring) /4945				(PAGE_SIZE << hns3_page_order(ring)),4946		.nid = dev_to_node(ring_to_dev(ring)),4947		.dev = ring_to_dev(ring),4948		.dma_dir = DMA_FROM_DEVICE,4949		.offset = 0,4950		.max_len = PAGE_SIZE << hns3_page_order(ring),4951	};4952 4953	ring->page_pool = page_pool_create(&pp_params);4954	if (IS_ERR(ring->page_pool)) {4955		dev_warn(ring_to_dev(ring), "page pool creation failed: %ld\n",4956			 PTR_ERR(ring->page_pool));4957		ring->page_pool = NULL;4958	}4959}4960 4961static int hns3_alloc_ring_memory(struct hns3_enet_ring *ring)4962{4963	int ret;4964 4965	if (ring->desc_num <= 0 || ring->buf_size <= 0)4966		return -EINVAL;4967 4968	ring->desc_cb = devm_kcalloc(ring_to_dev(ring), ring->desc_num,4969				     sizeof(ring->desc_cb[0]), GFP_KERNEL);4970	if (!ring->desc_cb) {4971		ret = -ENOMEM;4972		goto out;4973	}4974 4975	ret = hns3_alloc_desc(ring);4976	if (ret)4977		goto out_with_desc_cb;4978 4979	if (!HNAE3_IS_TX_RING(ring)) {4980		if (page_pool_enabled)4981			hns3_alloc_page_pool(ring);4982 4983		ret = hns3_alloc_ring_buffers(ring);4984		if (ret)4985			goto out_with_desc;4986	} else {4987		hns3_init_tx_spare_buffer(ring);4988	}4989 4990	return 0;4991 4992out_with_desc:4993	hns3_free_desc(ring);4994out_with_desc_cb:4995	devm_kfree(ring_to_dev(ring), ring->desc_cb);4996	ring->desc_cb = NULL;4997out:4998	return ret;4999}5000 5001void hns3_fini_ring(struct hns3_enet_ring *ring)5002{5003	hns3_free_desc(ring);5004	devm_kfree(ring_to_dev(ring), ring->desc_cb);5005	ring->desc_cb = NULL;5006	ring->next_to_clean = 0;5007	ring->next_to_use = 0;5008	ring->last_to_use = 0;5009	ring->pending_buf = 0;5010	if (!HNAE3_IS_TX_RING(ring) && ring->skb) {5011		dev_kfree_skb_any(ring->skb);5012		ring->skb = NULL;5013	} else if (HNAE3_IS_TX_RING(ring) && ring->tx_spare) {5014		struct hns3_tx_spare *tx_spare = ring->tx_spare;5015 5016		dma_unmap_page(ring_to_dev(ring), tx_spare->dma, tx_spare->len,5017			       DMA_TO_DEVICE);5018		free_pages((unsigned long)tx_spare->buf,5019			   get_order(tx_spare->len));5020		devm_kfree(ring_to_dev(ring), tx_spare);5021		ring->tx_spare = NULL;5022	}5023 5024	if (!HNAE3_IS_TX_RING(ring) && ring->page_pool) {5025		page_pool_destroy(ring->page_pool);5026		ring->page_pool = NULL;5027	}5028}5029 5030static int hns3_buf_size2type(u32 buf_size)5031{5032	int bd_size_type;5033 5034	switch (buf_size) {5035	case 512:5036		bd_size_type = HNS3_BD_SIZE_512_TYPE;5037		break;5038	case 1024:5039		bd_size_type = HNS3_BD_SIZE_1024_TYPE;5040		break;5041	case 2048:5042		bd_size_type = HNS3_BD_SIZE_2048_TYPE;5043		break;5044	case 4096:5045		bd_size_type = HNS3_BD_SIZE_4096_TYPE;5046		break;5047	default:5048		bd_size_type = HNS3_BD_SIZE_2048_TYPE;5049	}5050 5051	return bd_size_type;5052}5053 5054static void hns3_init_ring_hw(struct hns3_enet_ring *ring)5055{5056	dma_addr_t dma = ring->desc_dma_addr;5057	struct hnae3_queue *q = ring->tqp;5058 5059	if (!HNAE3_IS_TX_RING(ring)) {5060		hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_L_REG, (u32)dma);5061		hns3_write_dev(q, HNS3_RING_RX_RING_BASEADDR_H_REG,5062			       (u32)((dma >> 31) >> 1));5063 5064		hns3_write_dev(q, HNS3_RING_RX_RING_BD_LEN_REG,5065			       hns3_buf_size2type(ring->buf_size));5066		hns3_write_dev(q, HNS3_RING_RX_RING_BD_NUM_REG,5067			       ring->desc_num / 8 - 1);5068	} else {5069		hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_L_REG,5070			       (u32)dma);5071		hns3_write_dev(q, HNS3_RING_TX_RING_BASEADDR_H_REG,5072			       (u32)((dma >> 31) >> 1));5073 5074		hns3_write_dev(q, HNS3_RING_TX_RING_BD_NUM_REG,5075			       ring->desc_num / 8 - 1);5076	}5077}5078 5079static void hns3_init_tx_ring_tc(struct hns3_nic_priv *priv)5080{5081	struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;5082	struct hnae3_tc_info *tc_info = &kinfo->tc_info;5083	int i;5084 5085	for (i = 0; i < tc_info->num_tc; i++) {5086		int j;5087 5088		for (j = 0; j < tc_info->tqp_count[i]; j++) {5089			struct hnae3_queue *q;5090 5091			q = priv->ring[tc_info->tqp_offset[i] + j].tqp;5092			hns3_write_dev(q, HNS3_RING_TX_RING_TC_REG, i);5093		}5094	}5095}5096 5097int hns3_init_all_ring(struct hns3_nic_priv *priv)5098{5099	struct hnae3_handle *h = priv->ae_handle;5100	int ring_num = h->kinfo.num_tqps * 2;5101	int i, j;5102	int ret;5103 5104	for (i = 0; i < ring_num; i++) {5105		ret = hns3_alloc_ring_memory(&priv->ring[i]);5106		if (ret) {5107			dev_err(priv->dev,5108				"Alloc ring memory fail! ret=%d\n", ret);5109			goto out_when_alloc_ring_memory;5110		}5111 5112		u64_stats_init(&priv->ring[i].syncp);5113		cond_resched();5114	}5115 5116	return 0;5117 5118out_when_alloc_ring_memory:5119	for (j = i - 1; j >= 0; j--)5120		hns3_fini_ring(&priv->ring[j]);5121 5122	return -ENOMEM;5123}5124 5125static void hns3_uninit_all_ring(struct hns3_nic_priv *priv)5126{5127	struct hnae3_handle *h = priv->ae_handle;5128	int i;5129 5130	for (i = 0; i < h->kinfo.num_tqps; i++) {5131		hns3_fini_ring(&priv->ring[i]);5132		hns3_fini_ring(&priv->ring[i + h->kinfo.num_tqps]);5133	}5134}5135 5136/* Set mac addr if it is configured. or leave it to the AE driver */5137static int hns3_init_mac_addr(struct net_device *netdev)5138{5139	struct hns3_nic_priv *priv = netdev_priv(netdev);5140	char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];5141	struct hnae3_handle *h = priv->ae_handle;5142	u8 mac_addr_temp[ETH_ALEN] = {0};5143	int ret = 0;5144 5145	if (h->ae_algo->ops->get_mac_addr)5146		h->ae_algo->ops->get_mac_addr(h, mac_addr_temp);5147 5148	/* Check if the MAC address is valid, if not get a random one */5149	if (!is_valid_ether_addr(mac_addr_temp)) {5150		eth_hw_addr_random(netdev);5151		hnae3_format_mac_addr(format_mac_addr, netdev->dev_addr);5152		dev_warn(priv->dev, "using random MAC address %s\n",5153			 format_mac_addr);5154	} else if (!ether_addr_equal(netdev->dev_addr, mac_addr_temp)) {5155		eth_hw_addr_set(netdev, mac_addr_temp);5156		ether_addr_copy(netdev->perm_addr, mac_addr_temp);5157	} else {5158		return 0;5159	}5160 5161	if (h->ae_algo->ops->set_mac_addr)5162		ret = h->ae_algo->ops->set_mac_addr(h, netdev->dev_addr, true);5163 5164	return ret;5165}5166 5167static int hns3_init_phy(struct net_device *netdev)5168{5169	struct hnae3_handle *h = hns3_get_handle(netdev);5170	int ret = 0;5171 5172	if (h->ae_algo->ops->mac_connect_phy)5173		ret = h->ae_algo->ops->mac_connect_phy(h);5174 5175	return ret;5176}5177 5178static void hns3_uninit_phy(struct net_device *netdev)5179{5180	struct hnae3_handle *h = hns3_get_handle(netdev);5181 5182	if (h->ae_algo->ops->mac_disconnect_phy)5183		h->ae_algo->ops->mac_disconnect_phy(h);5184}5185 5186static int hns3_client_start(struct hnae3_handle *handle)5187{5188	if (!handle->ae_algo->ops->client_start)5189		return 0;5190 5191	return handle->ae_algo->ops->client_start(handle);5192}5193 5194static void hns3_client_stop(struct hnae3_handle *handle)5195{5196	if (!handle->ae_algo->ops->client_stop)5197		return;5198 5199	handle->ae_algo->ops->client_stop(handle);5200}5201 5202static void hns3_info_show(struct hns3_nic_priv *priv)5203{5204	struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo;5205	char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN];5206 5207	hnae3_format_mac_addr(format_mac_addr, priv->netdev->dev_addr);5208	dev_info(priv->dev, "MAC address: %s\n", format_mac_addr);5209	dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps);5210	dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size);5211	dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size);5212	dev_info(priv->dev, "RX buffer length: %u\n", kinfo->rx_buf_len);5213	dev_info(priv->dev, "Desc num per TX queue: %u\n", kinfo->num_tx_desc);5214	dev_info(priv->dev, "Desc num per RX queue: %u\n", kinfo->num_rx_desc);5215	dev_info(priv->dev, "Total number of enabled TCs: %u\n",5216		 kinfo->tc_info.num_tc);5217	dev_info(priv->dev, "Max mtu size: %u\n", priv->netdev->max_mtu);5218}5219 5220static void hns3_set_cq_period_mode(struct hns3_nic_priv *priv,5221				    enum dim_cq_period_mode mode, bool is_tx)5222{5223	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(priv->ae_handle->pdev);5224	struct hnae3_handle *handle = priv->ae_handle;5225	int i;5226 5227	if (is_tx) {5228		priv->tx_cqe_mode = mode;5229 5230		for (i = 0; i < priv->vector_num; i++)5231			priv->tqp_vector[i].tx_group.dim.mode = mode;5232	} else {5233		priv->rx_cqe_mode = mode;5234 5235		for (i = 0; i < priv->vector_num; i++)5236			priv->tqp_vector[i].rx_group.dim.mode = mode;5237	}5238 5239	if (hnae3_ae_dev_cq_supported(ae_dev)) {5240		u32 new_mode;5241		u64 reg;5242 5243		new_mode = (mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE) ?5244			HNS3_CQ_MODE_CQE : HNS3_CQ_MODE_EQE;5245		reg = is_tx ? HNS3_GL1_CQ_MODE_REG : HNS3_GL0_CQ_MODE_REG;5246 5247		writel(new_mode, handle->kinfo.io_base + reg);5248	}5249}5250 5251void hns3_cq_period_mode_init(struct hns3_nic_priv *priv,5252			      enum dim_cq_period_mode tx_mode,5253			      enum dim_cq_period_mode rx_mode)5254{5255	hns3_set_cq_period_mode(priv, tx_mode, true);5256	hns3_set_cq_period_mode(priv, rx_mode, false);5257}5258 5259static void hns3_state_init(struct hnae3_handle *handle)5260{5261	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev);5262	struct net_device *netdev = handle->kinfo.netdev;5263	struct hns3_nic_priv *priv = netdev_priv(netdev);5264 5265	set_bit(HNS3_NIC_STATE_INITED, &priv->state);5266 5267	if (test_bit(HNAE3_DEV_SUPPORT_TX_PUSH_B, ae_dev->caps))5268		set_bit(HNS3_NIC_STATE_TX_PUSH_ENABLE, &priv->state);5269 5270	if (ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)5271		set_bit(HNAE3_PFLAG_LIMIT_PROMISC, &handle->supported_pflags);5272 5273	if (test_bit(HNAE3_DEV_SUPPORT_HW_TX_CSUM_B, ae_dev->caps))5274		set_bit(HNS3_NIC_STATE_HW_TX_CSUM_ENABLE, &priv->state);5275 5276	if (hnae3_ae_dev_rxd_adv_layout_supported(ae_dev))5277		set_bit(HNS3_NIC_STATE_RXD_ADV_LAYOUT_ENABLE, &priv->state);5278}5279 5280static void hns3_state_uninit(struct hnae3_handle *handle)5281{5282	struct hns3_nic_priv *priv  = handle->priv;5283 5284	clear_bit(HNS3_NIC_STATE_INITED, &priv->state);5285}5286 5287static int hns3_client_init(struct hnae3_handle *handle)5288{5289	struct pci_dev *pdev = handle->pdev;5290	struct hnae3_ae_dev *ae_dev = pci_get_drvdata(pdev);5291	u16 alloc_tqps, max_rss_size;5292	struct hns3_nic_priv *priv;5293	struct net_device *netdev;5294	int ret;5295 5296	handle->ae_algo->ops->get_tqps_and_rss_info(handle, &alloc_tqps,5297						    &max_rss_size);5298	netdev = alloc_etherdev_mq(sizeof(struct hns3_nic_priv), alloc_tqps);5299	if (!netdev)5300		return -ENOMEM;5301 5302	priv = netdev_priv(netdev);5303	priv->dev = &pdev->dev;5304	priv->netdev = netdev;5305	priv->ae_handle = handle;5306	priv->tx_timeout_count = 0;5307	priv->max_non_tso_bd_num = ae_dev->dev_specs.max_non_tso_bd_num;5308	set_bit(HNS3_NIC_STATE_DOWN, &priv->state);5309 5310	handle->msg_enable = netif_msg_init(debug, DEFAULT_MSG_LEVEL);5311 5312	handle->kinfo.netdev = netdev;5313	handle->priv = (void *)priv;5314 5315	hns3_init_mac_addr(netdev);5316 5317	hns3_set_default_feature(netdev);5318 5319	netdev->watchdog_timeo = HNS3_TX_TIMEOUT;5320	netdev->priv_flags |= IFF_UNICAST_FLT;5321	netdev->netdev_ops = &hns3_nic_netdev_ops;5322	SET_NETDEV_DEV(netdev, &pdev->dev);5323	hns3_ethtool_set_ops(netdev);5324 5325	/* Carrier off reporting is important to ethtool even BEFORE open */5326	netif_carrier_off(netdev);5327 5328	ret = hns3_get_ring_config(priv);5329	if (ret) {5330		ret = -ENOMEM;5331		goto out_get_ring_cfg;5332	}5333 5334	hns3_nic_init_coal_cfg(priv);5335 5336	ret = hns3_nic_alloc_vector_data(priv);5337	if (ret) {5338		ret = -ENOMEM;5339		goto out_alloc_vector_data;5340	}5341 5342	ret = hns3_nic_init_vector_data(priv);5343	if (ret) {5344		ret = -ENOMEM;5345		goto out_init_vector_data;5346	}5347 5348	ret = hns3_init_all_ring(priv);5349	if (ret) {5350		ret = -ENOMEM;5351		goto out_init_ring;5352	}5353 5354	hns3_cq_period_mode_init(priv, DIM_CQ_PERIOD_MODE_START_FROM_EQE,5355				 DIM_CQ_PERIOD_MODE_START_FROM_EQE);5356 5357	ret = hns3_init_phy(netdev);5358	if (ret)5359		goto out_init_phy;5360 5361	/* the device can work without cpu rmap, only aRFS needs it */5362	ret = hns3_set_rx_cpu_rmap(netdev);5363	if (ret)5364		dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret);5365 5366	ret = hns3_nic_init_irq(priv);5367	if (ret) {5368		dev_err(priv->dev, "init irq failed! ret=%d\n", ret);5369		hns3_free_rx_cpu_rmap(netdev);5370		goto out_init_irq_fail;5371	}5372 5373	ret = hns3_client_start(handle);5374	if (ret) {5375		dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);5376		goto out_client_start;5377	}5378 5379	hns3_dcbnl_setup(handle);5380 5381	ret = hns3_dbg_init(handle);5382	if (ret) {5383		dev_err(priv->dev, "failed to init debugfs, ret = %d\n",5384			ret);5385		goto out_client_start;5386	}5387 5388	netdev->max_mtu = HNS3_MAX_MTU(ae_dev->dev_specs.max_frm_size);5389 5390	hns3_state_init(handle);5391 5392	ret = register_netdev(netdev);5393	if (ret) {5394		dev_err(priv->dev, "probe register netdev fail!\n");5395		goto out_reg_netdev_fail;5396	}5397 5398	if (netif_msg_drv(handle))5399		hns3_info_show(priv);5400 5401	return ret;5402 5403out_reg_netdev_fail:5404	hns3_state_uninit(handle);5405	hns3_dbg_uninit(handle);5406	hns3_client_stop(handle);5407out_client_start:5408	hns3_free_rx_cpu_rmap(netdev);5409	hns3_nic_uninit_irq(priv);5410out_init_irq_fail:5411	hns3_uninit_phy(netdev);5412out_init_phy:5413	hns3_uninit_all_ring(priv);5414out_init_ring:5415	hns3_nic_uninit_vector_data(priv);5416out_init_vector_data:5417	hns3_nic_dealloc_vector_data(priv);5418out_alloc_vector_data:5419	priv->ring = NULL;5420out_get_ring_cfg:5421	priv->ae_handle = NULL;5422	free_netdev(netdev);5423	return ret;5424}5425 5426static void hns3_client_uninit(struct hnae3_handle *handle, bool reset)5427{5428	struct net_device *netdev = handle->kinfo.netdev;5429	struct hns3_nic_priv *priv = netdev_priv(netdev);5430 5431	if (netdev->reg_state != NETREG_UNINITIALIZED)5432		unregister_netdev(netdev);5433 5434	hns3_client_stop(handle);5435 5436	hns3_uninit_phy(netdev);5437 5438	if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {5439		netdev_warn(netdev, "already uninitialized\n");5440		goto out_netdev_free;5441	}5442 5443	hns3_free_rx_cpu_rmap(netdev);5444 5445	hns3_nic_uninit_irq(priv);5446 5447	hns3_clear_all_ring(handle, true);5448 5449	hns3_nic_uninit_vector_data(priv);5450 5451	hns3_nic_dealloc_vector_data(priv);5452 5453	hns3_uninit_all_ring(priv);5454 5455	hns3_put_ring_config(priv);5456 5457out_netdev_free:5458	hns3_dbg_uninit(handle);5459	free_netdev(netdev);5460}5461 5462static void hns3_link_status_change(struct hnae3_handle *handle, bool linkup)5463{5464	struct net_device *netdev = handle->kinfo.netdev;5465 5466	if (!netdev)5467		return;5468 5469	if (linkup) {5470		netif_tx_wake_all_queues(netdev);5471		netif_carrier_on(netdev);5472		if (netif_msg_link(handle))5473			netdev_info(netdev, "link up\n");5474	} else {5475		netif_carrier_off(netdev);5476		netif_tx_stop_all_queues(netdev);5477		if (netif_msg_link(handle))5478			netdev_info(netdev, "link down\n");5479	}5480}5481 5482static void hns3_clear_tx_ring(struct hns3_enet_ring *ring)5483{5484	while (ring->next_to_clean != ring->next_to_use) {5485		ring->desc[ring->next_to_clean].tx.bdtp_fe_sc_vld_ra_ri = 0;5486		hns3_free_buffer_detach(ring, ring->next_to_clean, 0);5487		ring_ptr_move_fw(ring, next_to_clean);5488	}5489 5490	ring->pending_buf = 0;5491}5492 5493static int hns3_clear_rx_ring(struct hns3_enet_ring *ring)5494{5495	struct hns3_desc_cb res_cbs;5496	int ret;5497 5498	while (ring->next_to_use != ring->next_to_clean) {5499		/* When a buffer is not reused, it's memory has been5500		 * freed in hns3_handle_rx_bd or will be freed by5501		 * stack, so we need to replace the buffer here.5502		 */5503		if (!ring->desc_cb[ring->next_to_use].reuse_flag) {5504			ret = hns3_alloc_and_map_buffer(ring, &res_cbs);5505			if (ret) {5506				hns3_ring_stats_update(ring, sw_err_cnt);5507				/* if alloc new buffer fail, exit directly5508				 * and reclear in up flow.5509				 */5510				netdev_warn(ring_to_netdev(ring),5511					    "reserve buffer map failed, ret = %d\n",5512					    ret);5513				return ret;5514			}5515			hns3_replace_buffer(ring, ring->next_to_use, &res_cbs);5516		}5517		ring_ptr_move_fw(ring, next_to_use);5518	}5519 5520	/* Free the pending skb in rx ring */5521	if (ring->skb) {5522		dev_kfree_skb_any(ring->skb);5523		ring->skb = NULL;5524		ring->pending_buf = 0;5525	}5526 5527	return 0;5528}5529 5530static void hns3_force_clear_rx_ring(struct hns3_enet_ring *ring)5531{5532	while (ring->next_to_use != ring->next_to_clean) {5533		/* When a buffer is not reused, it's memory has been5534		 * freed in hns3_handle_rx_bd or will be freed by5535		 * stack, so only need to unmap the buffer here.5536		 */5537		if (!ring->desc_cb[ring->next_to_use].reuse_flag) {5538			hns3_unmap_buffer(ring,5539					  &ring->desc_cb[ring->next_to_use]);5540			ring->desc_cb[ring->next_to_use].dma = 0;5541		}5542 5543		ring_ptr_move_fw(ring, next_to_use);5544	}5545}5546 5547static void hns3_clear_all_ring(struct hnae3_handle *h, bool force)5548{5549	struct net_device *ndev = h->kinfo.netdev;5550	struct hns3_nic_priv *priv = netdev_priv(ndev);5551	u32 i;5552 5553	for (i = 0; i < h->kinfo.num_tqps; i++) {5554		struct hns3_enet_ring *ring;5555 5556		ring = &priv->ring[i];5557		hns3_clear_tx_ring(ring);5558 5559		ring = &priv->ring[i + h->kinfo.num_tqps];5560		/* Continue to clear other rings even if clearing some5561		 * rings failed.5562		 */5563		if (force)5564			hns3_force_clear_rx_ring(ring);5565		else5566			hns3_clear_rx_ring(ring);5567	}5568}5569 5570int hns3_nic_reset_all_ring(struct hnae3_handle *h)5571{5572	struct net_device *ndev = h->kinfo.netdev;5573	struct hns3_nic_priv *priv = netdev_priv(ndev);5574	struct hns3_enet_ring *rx_ring;5575	int i, j;5576	int ret;5577 5578	ret = h->ae_algo->ops->reset_queue(h);5579	if (ret)5580		return ret;5581 5582	for (i = 0; i < h->kinfo.num_tqps; i++) {5583		hns3_init_ring_hw(&priv->ring[i]);5584 5585		/* We need to clear tx ring here because self test will5586		 * use the ring and will not run down before up5587		 */5588		hns3_clear_tx_ring(&priv->ring[i]);5589		priv->ring[i].next_to_clean = 0;5590		priv->ring[i].next_to_use = 0;5591		priv->ring[i].last_to_use = 0;5592 5593		rx_ring = &priv->ring[i + h->kinfo.num_tqps];5594		hns3_init_ring_hw(rx_ring);5595		ret = hns3_clear_rx_ring(rx_ring);5596		if (ret)5597			return ret;5598 5599		/* We can not know the hardware head and tail when this5600		 * function is called in reset flow, so we reuse all desc.5601		 */5602		for (j = 0; j < rx_ring->desc_num; j++)5603			hns3_reuse_buffer(rx_ring, j);5604 5605		rx_ring->next_to_clean = 0;5606		rx_ring->next_to_use = 0;5607	}5608 5609	hns3_init_tx_ring_tc(priv);5610 5611	return 0;5612}5613 5614static int hns3_reset_notify_down_enet(struct hnae3_handle *handle)5615{5616	struct hnae3_knic_private_info *kinfo = &handle->kinfo;5617	struct net_device *ndev = kinfo->netdev;5618	struct hns3_nic_priv *priv = netdev_priv(ndev);5619 5620	if (test_and_set_bit(HNS3_NIC_STATE_RESETTING, &priv->state))5621		return 0;5622 5623	if (!netif_running(ndev))5624		return 0;5625 5626	return hns3_nic_net_stop(ndev);5627}5628 5629static int hns3_reset_notify_up_enet(struct hnae3_handle *handle)5630{5631	struct hnae3_knic_private_info *kinfo = &handle->kinfo;5632	struct hns3_nic_priv *priv = netdev_priv(kinfo->netdev);5633	int ret = 0;5634 5635	if (!test_bit(HNS3_NIC_STATE_INITED, &priv->state)) {5636		netdev_err(kinfo->netdev, "device is not initialized yet\n");5637		return -EFAULT;5638	}5639 5640	clear_bit(HNS3_NIC_STATE_RESETTING, &priv->state);5641 5642	if (netif_running(kinfo->netdev)) {5643		ret = hns3_nic_net_open(kinfo->netdev);5644		if (ret) {5645			set_bit(HNS3_NIC_STATE_RESETTING, &priv->state);5646			netdev_err(kinfo->netdev,5647				   "net up fail, ret=%d!\n", ret);5648			return ret;5649		}5650	}5651 5652	return ret;5653}5654 5655static int hns3_reset_notify_init_enet(struct hnae3_handle *handle)5656{5657	struct net_device *netdev = handle->kinfo.netdev;5658	struct hns3_nic_priv *priv = netdev_priv(netdev);5659	int ret;5660 5661	/* Carrier off reporting is important to ethtool even BEFORE open */5662	netif_carrier_off(netdev);5663 5664	ret = hns3_get_ring_config(priv);5665	if (ret)5666		return ret;5667 5668	ret = hns3_nic_alloc_vector_data(priv);5669	if (ret)5670		goto err_put_ring;5671 5672	ret = hns3_nic_init_vector_data(priv);5673	if (ret)5674		goto err_dealloc_vector;5675 5676	ret = hns3_init_all_ring(priv);5677	if (ret)5678		goto err_uninit_vector;5679 5680	hns3_cq_period_mode_init(priv, priv->tx_cqe_mode, priv->rx_cqe_mode);5681 5682	/* the device can work without cpu rmap, only aRFS needs it */5683	ret = hns3_set_rx_cpu_rmap(netdev);5684	if (ret)5685		dev_warn(priv->dev, "set rx cpu rmap fail, ret=%d\n", ret);5686 5687	ret = hns3_nic_init_irq(priv);5688	if (ret) {5689		dev_err(priv->dev, "init irq failed! ret=%d\n", ret);5690		hns3_free_rx_cpu_rmap(netdev);5691		goto err_init_irq_fail;5692	}5693 5694	if (!hns3_is_phys_func(handle->pdev))5695		hns3_init_mac_addr(netdev);5696 5697	ret = hns3_client_start(handle);5698	if (ret) {5699		dev_err(priv->dev, "hns3_client_start fail! ret=%d\n", ret);5700		goto err_client_start_fail;5701	}5702 5703	set_bit(HNS3_NIC_STATE_INITED, &priv->state);5704 5705	return ret;5706 5707err_client_start_fail:5708	hns3_free_rx_cpu_rmap(netdev);5709	hns3_nic_uninit_irq(priv);5710err_init_irq_fail:5711	hns3_uninit_all_ring(priv);5712err_uninit_vector:5713	hns3_nic_uninit_vector_data(priv);5714err_dealloc_vector:5715	hns3_nic_dealloc_vector_data(priv);5716err_put_ring:5717	hns3_put_ring_config(priv);5718 5719	return ret;5720}5721 5722static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle)5723{5724	struct net_device *netdev = handle->kinfo.netdev;5725	struct hns3_nic_priv *priv = netdev_priv(netdev);5726 5727	if (!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))5728		hns3_nic_net_stop(netdev);5729 5730	if (!test_and_clear_bit(HNS3_NIC_STATE_INITED, &priv->state)) {5731		netdev_warn(netdev, "already uninitialized\n");5732		return 0;5733	}5734 5735	hns3_free_rx_cpu_rmap(netdev);5736	hns3_nic_uninit_irq(priv);5737	hns3_clear_all_ring(handle, true);5738	hns3_reset_tx_queue(priv->ae_handle);5739 5740	hns3_nic_uninit_vector_data(priv);5741 5742	hns3_nic_dealloc_vector_data(priv);5743 5744	hns3_uninit_all_ring(priv);5745 5746	hns3_put_ring_config(priv);5747 5748	return 0;5749}5750 5751int hns3_reset_notify(struct hnae3_handle *handle,5752		      enum hnae3_reset_notify_type type)5753{5754	int ret = 0;5755 5756	switch (type) {5757	case HNAE3_UP_CLIENT:5758		ret = hns3_reset_notify_up_enet(handle);5759		break;5760	case HNAE3_DOWN_CLIENT:5761		ret = hns3_reset_notify_down_enet(handle);5762		break;5763	case HNAE3_INIT_CLIENT:5764		ret = hns3_reset_notify_init_enet(handle);5765		break;5766	case HNAE3_UNINIT_CLIENT:5767		ret = hns3_reset_notify_uninit_enet(handle);5768		break;5769	default:5770		break;5771	}5772 5773	return ret;5774}5775 5776static int hns3_change_channels(struct hnae3_handle *handle, u32 new_tqp_num,5777				bool rxfh_configured)5778{5779	int ret;5780 5781	ret = handle->ae_algo->ops->set_channels(handle, new_tqp_num,5782						 rxfh_configured);5783	if (ret) {5784		dev_err(&handle->pdev->dev,5785			"Change tqp num(%u) fail.\n", new_tqp_num);5786		return ret;5787	}5788 5789	ret = hns3_reset_notify(handle, HNAE3_INIT_CLIENT);5790	if (ret)5791		return ret;5792 5793	ret =  hns3_reset_notify(handle, HNAE3_UP_CLIENT);5794	if (ret)5795		hns3_reset_notify(handle, HNAE3_UNINIT_CLIENT);5796 5797	return ret;5798}5799 5800int hns3_set_channels(struct net_device *netdev,5801		      struct ethtool_channels *ch)5802{5803	struct hnae3_handle *h = hns3_get_handle(netdev);5804	struct hnae3_knic_private_info *kinfo = &h->kinfo;5805	bool rxfh_configured = netif_is_rxfh_configured(netdev);5806	u32 new_tqp_num = ch->combined_count;5807	u16 org_tqp_num;5808	int ret;5809 5810	if (hns3_nic_resetting(netdev))5811		return -EBUSY;5812 5813	if (ch->rx_count || ch->tx_count)5814		return -EINVAL;5815 5816	if (kinfo->tc_info.mqprio_active) {5817		dev_err(&netdev->dev,5818			"it's not allowed to set channels via ethtool when MQPRIO mode is on\n");5819		return -EINVAL;5820	}5821 5822	if (new_tqp_num > hns3_get_max_available_channels(h) ||5823	    new_tqp_num < 1) {5824		dev_err(&netdev->dev,5825			"Change tqps fail, the tqp range is from 1 to %u",5826			hns3_get_max_available_channels(h));5827		return -EINVAL;5828	}5829 5830	if (kinfo->rss_size == new_tqp_num)5831		return 0;5832 5833	netif_dbg(h, drv, netdev,5834		  "set channels: tqp_num=%u, rxfh=%d\n",5835		  new_tqp_num, rxfh_configured);5836 5837	ret = hns3_reset_notify(h, HNAE3_DOWN_CLIENT);5838	if (ret)5839		return ret;5840 5841	ret = hns3_reset_notify(h, HNAE3_UNINIT_CLIENT);5842	if (ret)5843		return ret;5844 5845	org_tqp_num = h->kinfo.num_tqps;5846	ret = hns3_change_channels(h, new_tqp_num, rxfh_configured);5847	if (ret) {5848		int ret1;5849 5850		netdev_warn(netdev,5851			    "Change channels fail, revert to old value\n");5852		ret1 = hns3_change_channels(h, org_tqp_num, rxfh_configured);5853		if (ret1) {5854			netdev_err(netdev,5855				   "revert to old channel fail\n");5856			return ret1;5857		}5858 5859		return ret;5860	}5861 5862	return 0;5863}5864 5865void hns3_external_lb_prepare(struct net_device *ndev, bool if_running)5866{5867	struct hns3_nic_priv *priv = netdev_priv(ndev);5868	struct hnae3_handle *h = priv->ae_handle;5869	int i;5870 5871	if (!if_running)5872		return;5873 5874	if (test_and_set_bit(HNS3_NIC_STATE_DOWN, &priv->state))5875		return;5876 5877	netif_carrier_off(ndev);5878	netif_tx_disable(ndev);5879 5880	for (i = 0; i < priv->vector_num; i++)5881		hns3_vector_disable(&priv->tqp_vector[i]);5882 5883	for (i = 0; i < h->kinfo.num_tqps; i++)5884		hns3_tqp_disable(h->kinfo.tqp[i]);5885 5886	/* delay ring buffer clearing to hns3_reset_notify_uninit_enet5887	 * during reset process, because driver may not be able5888	 * to disable the ring through firmware when downing the netdev.5889	 */5890	if (!hns3_nic_resetting(ndev))5891		hns3_nic_reset_all_ring(priv->ae_handle);5892 5893	hns3_reset_tx_queue(priv->ae_handle);5894}5895 5896void hns3_external_lb_restore(struct net_device *ndev, bool if_running)5897{5898	struct hns3_nic_priv *priv = netdev_priv(ndev);5899	struct hnae3_handle *h = priv->ae_handle;5900	int i;5901 5902	if (!if_running)5903		return;5904 5905	if (hns3_nic_resetting(ndev))5906		return;5907 5908	if (!test_bit(HNS3_NIC_STATE_DOWN, &priv->state))5909		return;5910 5911	if (hns3_nic_reset_all_ring(priv->ae_handle))5912		return;5913 5914	clear_bit(HNS3_NIC_STATE_DOWN, &priv->state);5915 5916	for (i = 0; i < priv->vector_num; i++)5917		hns3_vector_enable(&priv->tqp_vector[i]);5918 5919	for (i = 0; i < h->kinfo.num_tqps; i++)5920		hns3_tqp_enable(h->kinfo.tqp[i]);5921 5922	netif_tx_wake_all_queues(ndev);5923 5924	if (h->ae_algo->ops->get_status(h))5925		netif_carrier_on(ndev);5926}5927 5928static const struct hns3_hw_error_info hns3_hw_err[] = {5929	{ .type = HNAE3_PPU_POISON_ERROR,5930	  .msg = "PPU poison" },5931	{ .type = HNAE3_CMDQ_ECC_ERROR,5932	  .msg = "IMP CMDQ error" },5933	{ .type = HNAE3_IMP_RD_POISON_ERROR,5934	  .msg = "IMP RD poison" },5935	{ .type = HNAE3_ROCEE_AXI_RESP_ERROR,5936	  .msg = "ROCEE AXI RESP error" },5937};5938 5939static void hns3_process_hw_error(struct hnae3_handle *handle,5940				  enum hnae3_hw_error_type type)5941{5942	int i;5943 5944	for (i = 0; i < ARRAY_SIZE(hns3_hw_err); i++) {5945		if (hns3_hw_err[i].type == type) {5946			dev_err(&handle->pdev->dev, "Detected %s!\n",5947				hns3_hw_err[i].msg);5948			break;5949		}5950	}5951}5952 5953static const struct hnae3_client_ops client_ops = {5954	.init_instance = hns3_client_init,5955	.uninit_instance = hns3_client_uninit,5956	.link_status_change = hns3_link_status_change,5957	.reset_notify = hns3_reset_notify,5958	.process_hw_error = hns3_process_hw_error,5959};5960 5961/* hns3_init_module - Driver registration routine5962 * hns3_init_module is the first routine called when the driver is5963 * loaded. All it does is register with the PCI subsystem.5964 */5965static int __init hns3_init_module(void)5966{5967	int ret;5968 5969	pr_info("%s: %s - version\n", hns3_driver_name, hns3_driver_string);5970	pr_info("%s: %s\n", hns3_driver_name, hns3_copyright);5971 5972	client.type = HNAE3_CLIENT_KNIC;5973	snprintf(client.name, HNAE3_CLIENT_NAME_LENGTH, "%s",5974		 hns3_driver_name);5975 5976	client.ops = &client_ops;5977 5978	INIT_LIST_HEAD(&client.node);5979 5980	hns3_dbg_register_debugfs(hns3_driver_name);5981 5982	ret = hnae3_register_client(&client);5983	if (ret)5984		goto err_reg_client;5985 5986	ret = pci_register_driver(&hns3_driver);5987	if (ret)5988		goto err_reg_driver;5989 5990	return ret;5991 5992err_reg_driver:5993	hnae3_unregister_client(&client);5994err_reg_client:5995	hns3_dbg_unregister_debugfs();5996	return ret;5997}5998module_init(hns3_init_module);5999 6000/* hns3_exit_module - Driver exit cleanup routine6001 * hns3_exit_module is called just before the driver is removed6002 * from memory.6003 */6004static void __exit hns3_exit_module(void)6005{6006	pci_unregister_driver(&hns3_driver);6007	hnae3_unregister_client(&client);6008	hns3_dbg_unregister_debugfs();6009}6010module_exit(hns3_exit_module);6011 6012MODULE_DESCRIPTION("HNS3: Hisilicon Ethernet Driver");6013MODULE_AUTHOR("Huawei Tech. Co., Ltd.");6014MODULE_LICENSE("GPL");6015MODULE_ALIAS("pci:hns-nic");6016