brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-or-later2/**************************************************************************/3/*                                                                        */4/*  IBM System i and System p Virtual NIC Device Driver                   */5/*  Copyright (C) 2014 IBM Corp.                                          */6/*  Santiago Leon (santi_leon@yahoo.com)                                  */7/*  Thomas Falcon (tlfalcon@linux.vnet.ibm.com)                           */8/*  John Allen (jallen@linux.vnet.ibm.com)                                */9/*                                                                        */10/*                                                                        */11/* This module contains the implementation of a virtual ethernet device   */12/* for use with IBM i/p Series LPAR Linux. It utilizes the logical LAN    */13/* option of the RS/6000 Platform Architecture to interface with virtual  */14/* ethernet NICs that are presented to the partition by the hypervisor.   */15/*									   */16/* Messages are passed between the VNIC driver and the VNIC server using  */17/* Command/Response Queues (CRQs) and sub CRQs (sCRQs). CRQs are used to  */18/* issue and receive commands that initiate communication with the server */19/* on driver initialization. Sub CRQs (sCRQs) are similar to CRQs, but    */20/* are used by the driver to notify the server that a packet is           */21/* ready for transmission or that a buffer has been added to receive a    */22/* packet. Subsequently, sCRQs are used by the server to notify the       */23/* driver that a packet transmission has been completed or that a packet  */24/* has been received and placed in a waiting buffer.                      */25/*                                                                        */26/* In lieu of a more conventional "on-the-fly" DMA mapping strategy in    */27/* which skbs are DMA mapped and immediately unmapped when the transmit   */28/* or receive has been completed, the VNIC driver is required to use      */29/* "long term mapping". This entails that large, continuous DMA mapped    */30/* buffers are allocated on driver initialization and these buffers are   */31/* then continuously reused to pass skbs to and from the VNIC server.     */32/*                                                                        */33/**************************************************************************/34 35#include <linux/module.h>36#include <linux/moduleparam.h>37#include <linux/types.h>38#include <linux/errno.h>39#include <linux/completion.h>40#include <linux/ioport.h>41#include <linux/dma-mapping.h>42#include <linux/kernel.h>43#include <linux/netdevice.h>44#include <linux/etherdevice.h>45#include <linux/skbuff.h>46#include <linux/init.h>47#include <linux/delay.h>48#include <linux/mm.h>49#include <linux/ethtool.h>50#include <linux/proc_fs.h>51#include <linux/if_arp.h>52#include <linux/in.h>53#include <linux/ip.h>54#include <linux/ipv6.h>55#include <linux/irq.h>56#include <linux/irqdomain.h>57#include <linux/kthread.h>58#include <linux/seq_file.h>59#include <linux/interrupt.h>60#include <net/net_namespace.h>61#include <asm/hvcall.h>62#include <linux/atomic.h>63#include <asm/vio.h>64#include <asm/xive.h>65#include <asm/iommu.h>66#include <linux/uaccess.h>67#include <asm/firmware.h>68#include <linux/workqueue.h>69#include <linux/if_vlan.h>70#include <linux/utsname.h>71#include <linux/cpu.h>72 73#include "ibmvnic.h"74 75static const char ibmvnic_driver_name[] = "ibmvnic";76static const char ibmvnic_driver_string[] = "IBM System i/p Virtual NIC Driver";77 78MODULE_AUTHOR("Santiago Leon");79MODULE_DESCRIPTION("IBM System i/p Virtual NIC Driver");80MODULE_LICENSE("GPL");81MODULE_VERSION(IBMVNIC_DRIVER_VERSION);82 83static int ibmvnic_version = IBMVNIC_INITIAL_VERSION;84static void release_sub_crqs(struct ibmvnic_adapter *, bool);85static int ibmvnic_reset_crq(struct ibmvnic_adapter *);86static int ibmvnic_send_crq_init(struct ibmvnic_adapter *);87static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *);88static int ibmvnic_send_crq(struct ibmvnic_adapter *, union ibmvnic_crq *);89static int send_subcrq_indirect(struct ibmvnic_adapter *, u64, u64, u64);90static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance);91static int enable_scrq_irq(struct ibmvnic_adapter *,92			   struct ibmvnic_sub_crq_queue *);93static int disable_scrq_irq(struct ibmvnic_adapter *,94			    struct ibmvnic_sub_crq_queue *);95static int pending_scrq(struct ibmvnic_adapter *,96			struct ibmvnic_sub_crq_queue *);97static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *,98					struct ibmvnic_sub_crq_queue *);99static int ibmvnic_poll(struct napi_struct *napi, int data);100static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter);101static inline void reinit_init_done(struct ibmvnic_adapter *adapter);102static void send_query_map(struct ibmvnic_adapter *adapter);103static int send_request_map(struct ibmvnic_adapter *, dma_addr_t, u32, u8);104static int send_request_unmap(struct ibmvnic_adapter *, u8);105static int send_login(struct ibmvnic_adapter *adapter);106static void send_query_cap(struct ibmvnic_adapter *adapter);107static int init_sub_crqs(struct ibmvnic_adapter *);108static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter);109static int ibmvnic_reset_init(struct ibmvnic_adapter *, bool reset);110static void release_crq_queue(struct ibmvnic_adapter *);111static int __ibmvnic_set_mac(struct net_device *, u8 *);112static int init_crq_queue(struct ibmvnic_adapter *adapter);113static int send_query_phys_parms(struct ibmvnic_adapter *adapter);114static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,115					 struct ibmvnic_sub_crq_queue *tx_scrq);116static void free_long_term_buff(struct ibmvnic_adapter *adapter,117				struct ibmvnic_long_term_buff *ltb);118static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter);119static void flush_reset_queue(struct ibmvnic_adapter *adapter);120static void print_subcrq_error(struct device *dev, int rc, const char *func);121 122struct ibmvnic_stat {123	char name[ETH_GSTRING_LEN];124	int offset;125};126 127#define IBMVNIC_STAT_OFF(stat) (offsetof(struct ibmvnic_adapter, stats) + \128			     offsetof(struct ibmvnic_statistics, stat))129#define IBMVNIC_GET_STAT(a, off) (*((u64 *)(((unsigned long)(a)) + (off))))130 131static const struct ibmvnic_stat ibmvnic_stats[] = {132	{"rx_packets", IBMVNIC_STAT_OFF(rx_packets)},133	{"rx_bytes", IBMVNIC_STAT_OFF(rx_bytes)},134	{"tx_packets", IBMVNIC_STAT_OFF(tx_packets)},135	{"tx_bytes", IBMVNIC_STAT_OFF(tx_bytes)},136	{"ucast_tx_packets", IBMVNIC_STAT_OFF(ucast_tx_packets)},137	{"ucast_rx_packets", IBMVNIC_STAT_OFF(ucast_rx_packets)},138	{"mcast_tx_packets", IBMVNIC_STAT_OFF(mcast_tx_packets)},139	{"mcast_rx_packets", IBMVNIC_STAT_OFF(mcast_rx_packets)},140	{"bcast_tx_packets", IBMVNIC_STAT_OFF(bcast_tx_packets)},141	{"bcast_rx_packets", IBMVNIC_STAT_OFF(bcast_rx_packets)},142	{"align_errors", IBMVNIC_STAT_OFF(align_errors)},143	{"fcs_errors", IBMVNIC_STAT_OFF(fcs_errors)},144	{"single_collision_frames", IBMVNIC_STAT_OFF(single_collision_frames)},145	{"multi_collision_frames", IBMVNIC_STAT_OFF(multi_collision_frames)},146	{"sqe_test_errors", IBMVNIC_STAT_OFF(sqe_test_errors)},147	{"deferred_tx", IBMVNIC_STAT_OFF(deferred_tx)},148	{"late_collisions", IBMVNIC_STAT_OFF(late_collisions)},149	{"excess_collisions", IBMVNIC_STAT_OFF(excess_collisions)},150	{"internal_mac_tx_errors", IBMVNIC_STAT_OFF(internal_mac_tx_errors)},151	{"carrier_sense", IBMVNIC_STAT_OFF(carrier_sense)},152	{"too_long_frames", IBMVNIC_STAT_OFF(too_long_frames)},153	{"internal_mac_rx_errors", IBMVNIC_STAT_OFF(internal_mac_rx_errors)},154};155 156static int send_crq_init_complete(struct ibmvnic_adapter *adapter)157{158	union ibmvnic_crq crq;159 160	memset(&crq, 0, sizeof(crq));161	crq.generic.first = IBMVNIC_CRQ_INIT_CMD;162	crq.generic.cmd = IBMVNIC_CRQ_INIT_COMPLETE;163 164	return ibmvnic_send_crq(adapter, &crq);165}166 167static int send_version_xchg(struct ibmvnic_adapter *adapter)168{169	union ibmvnic_crq crq;170 171	memset(&crq, 0, sizeof(crq));172	crq.version_exchange.first = IBMVNIC_CRQ_CMD;173	crq.version_exchange.cmd = VERSION_EXCHANGE;174	crq.version_exchange.version = cpu_to_be16(ibmvnic_version);175 176	return ibmvnic_send_crq(adapter, &crq);177}178 179static void ibmvnic_clean_queue_affinity(struct ibmvnic_adapter *adapter,180					 struct ibmvnic_sub_crq_queue *queue)181{182	if (!(queue && queue->irq))183		return;184 185	cpumask_clear(queue->affinity_mask);186 187	if (irq_set_affinity_and_hint(queue->irq, NULL))188		netdev_warn(adapter->netdev,189			    "%s: Clear affinity failed, queue addr = %p, IRQ = %d\n",190			    __func__, queue, queue->irq);191}192 193static void ibmvnic_clean_affinity(struct ibmvnic_adapter *adapter)194{195	struct ibmvnic_sub_crq_queue **rxqs;196	struct ibmvnic_sub_crq_queue **txqs;197	int num_rxqs, num_txqs;198	int i;199 200	rxqs = adapter->rx_scrq;201	txqs = adapter->tx_scrq;202	num_txqs = adapter->num_active_tx_scrqs;203	num_rxqs = adapter->num_active_rx_scrqs;204 205	netdev_dbg(adapter->netdev, "%s: Cleaning irq affinity hints", __func__);206	if (txqs) {207		for (i = 0; i < num_txqs; i++)208			ibmvnic_clean_queue_affinity(adapter, txqs[i]);209	}210	if (rxqs) {211		for (i = 0; i < num_rxqs; i++)212			ibmvnic_clean_queue_affinity(adapter, rxqs[i]);213	}214}215 216static int ibmvnic_set_queue_affinity(struct ibmvnic_sub_crq_queue *queue,217				      unsigned int *cpu, int *stragglers,218				      int stride)219{220	cpumask_var_t mask;221	int i;222	int rc = 0;223 224	if (!(queue && queue->irq))225		return rc;226 227	/* cpumask_var_t is either a pointer or array, allocation works here */228	if (!zalloc_cpumask_var(&mask, GFP_KERNEL))229		return -ENOMEM;230 231	/* while we have extra cpu give one extra to this irq */232	if (*stragglers) {233		stride++;234		(*stragglers)--;235	}236	/* atomic write is safer than writing bit by bit directly */237	for (i = 0; i < stride; i++) {238		cpumask_set_cpu(*cpu, mask);239		*cpu = cpumask_next_wrap(*cpu, cpu_online_mask,240					 nr_cpu_ids, false);241	}242	/* set queue affinity mask */243	cpumask_copy(queue->affinity_mask, mask);244	rc = irq_set_affinity_and_hint(queue->irq, queue->affinity_mask);245	free_cpumask_var(mask);246 247	return rc;248}249 250/* assumes cpu read lock is held */251static void ibmvnic_set_affinity(struct ibmvnic_adapter *adapter)252{253	struct ibmvnic_sub_crq_queue **rxqs = adapter->rx_scrq;254	struct ibmvnic_sub_crq_queue **txqs = adapter->tx_scrq;255	struct ibmvnic_sub_crq_queue *queue;256	int num_rxqs = adapter->num_active_rx_scrqs, i_rxqs = 0;257	int num_txqs = adapter->num_active_tx_scrqs, i_txqs = 0;258	int total_queues, stride, stragglers, i;259	unsigned int num_cpu, cpu;260	bool is_rx_queue;261	int rc = 0;262 263	netdev_dbg(adapter->netdev, "%s: Setting irq affinity hints", __func__);264	if (!(adapter->rx_scrq && adapter->tx_scrq)) {265		netdev_warn(adapter->netdev,266			    "%s: Set affinity failed, queues not allocated\n",267			    __func__);268		return;269	}270 271	total_queues = num_rxqs + num_txqs;272	num_cpu = num_online_cpus();273	/* number of cpu's assigned per irq */274	stride = max_t(int, num_cpu / total_queues, 1);275	/* number of leftover cpu's */276	stragglers = num_cpu >= total_queues ? num_cpu % total_queues : 0;277	/* next available cpu to assign irq to */278	cpu = cpumask_next(-1, cpu_online_mask);279 280	for (i = 0; i < total_queues; i++) {281		is_rx_queue = false;282		/* balance core load by alternating rx and tx assignments283		 * ex: TX0 -> RX0 -> TX1 -> RX1 etc.284		 */285		if ((i % 2 == 1 && i_rxqs < num_rxqs) || i_txqs == num_txqs) {286			queue = rxqs[i_rxqs++];287			is_rx_queue = true;288		} else {289			queue = txqs[i_txqs++];290		}291 292		rc = ibmvnic_set_queue_affinity(queue, &cpu, &stragglers,293						stride);294		if (rc)295			goto out;296 297		if (!queue || is_rx_queue)298			continue;299 300		rc = __netif_set_xps_queue(adapter->netdev,301					   cpumask_bits(queue->affinity_mask),302					   i_txqs - 1, XPS_CPUS);303		if (rc)304			netdev_warn(adapter->netdev, "%s: Set XPS on queue %d failed, rc = %d.\n",305				    __func__, i_txqs - 1, rc);306	}307 308out:309	if (rc) {310		netdev_warn(adapter->netdev,311			    "%s: Set affinity failed, queue addr = %p, IRQ = %d, rc = %d.\n",312			    __func__, queue, queue->irq, rc);313		ibmvnic_clean_affinity(adapter);314	}315}316 317static int ibmvnic_cpu_online(unsigned int cpu, struct hlist_node *node)318{319	struct ibmvnic_adapter *adapter;320 321	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);322	ibmvnic_set_affinity(adapter);323	return 0;324}325 326static int ibmvnic_cpu_dead(unsigned int cpu, struct hlist_node *node)327{328	struct ibmvnic_adapter *adapter;329 330	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node_dead);331	ibmvnic_set_affinity(adapter);332	return 0;333}334 335static int ibmvnic_cpu_down_prep(unsigned int cpu, struct hlist_node *node)336{337	struct ibmvnic_adapter *adapter;338 339	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);340	ibmvnic_clean_affinity(adapter);341	return 0;342}343 344static enum cpuhp_state ibmvnic_online;345 346static int ibmvnic_cpu_notif_add(struct ibmvnic_adapter *adapter)347{348	int ret;349 350	ret = cpuhp_state_add_instance_nocalls(ibmvnic_online, &adapter->node);351	if (ret)352		return ret;353	ret = cpuhp_state_add_instance_nocalls(CPUHP_IBMVNIC_DEAD,354					       &adapter->node_dead);355	if (!ret)356		return ret;357	cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);358	return ret;359}360 361static void ibmvnic_cpu_notif_remove(struct ibmvnic_adapter *adapter)362{363	cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);364	cpuhp_state_remove_instance_nocalls(CPUHP_IBMVNIC_DEAD,365					    &adapter->node_dead);366}367 368static long h_reg_sub_crq(unsigned long unit_address, unsigned long token,369			  unsigned long length, unsigned long *number,370			  unsigned long *irq)371{372	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];373	long rc;374 375	rc = plpar_hcall(H_REG_SUB_CRQ, retbuf, unit_address, token, length);376	*number = retbuf[0];377	*irq = retbuf[1];378 379	return rc;380}381 382/**383 * ibmvnic_wait_for_completion - Check device state and wait for completion384 * @adapter: private device data385 * @comp_done: completion structure to wait for386 * @timeout: time to wait in milliseconds387 *388 * Wait for a completion signal or until the timeout limit is reached389 * while checking that the device is still active.390 */391static int ibmvnic_wait_for_completion(struct ibmvnic_adapter *adapter,392				       struct completion *comp_done,393				       unsigned long timeout)394{395	struct net_device *netdev;396	unsigned long div_timeout;397	u8 retry;398 399	netdev = adapter->netdev;400	retry = 5;401	div_timeout = msecs_to_jiffies(timeout / retry);402	while (true) {403		if (!adapter->crq.active) {404			netdev_err(netdev, "Device down!\n");405			return -ENODEV;406		}407		if (!retry--)408			break;409		if (wait_for_completion_timeout(comp_done, div_timeout))410			return 0;411	}412	netdev_err(netdev, "Operation timed out.\n");413	return -ETIMEDOUT;414}415 416/**417 * reuse_ltb() - Check if a long term buffer can be reused418 * @ltb:  The long term buffer to be checked419 * @size: The size of the long term buffer.420 *421 * An LTB can be reused unless its size has changed.422 *423 * Return: Return true if the LTB can be reused, false otherwise.424 */425static bool reuse_ltb(struct ibmvnic_long_term_buff *ltb, int size)426{427	return (ltb->buff && ltb->size == size);428}429 430/**431 * alloc_long_term_buff() - Allocate a long term buffer (LTB)432 *433 * @adapter: ibmvnic adapter associated to the LTB434 * @ltb:     container object for the LTB435 * @size:    size of the LTB436 *437 * Allocate an LTB of the specified size and notify VIOS.438 *439 * If the given @ltb already has the correct size, reuse it. Otherwise if440 * its non-NULL, free it. Then allocate a new one of the correct size.441 * Notify the VIOS either way since we may now be working with a new VIOS.442 *443 * Allocating larger chunks of memory during resets, specially LPM or under444 * low memory situations can cause resets to fail/timeout and for LPAR to445 * lose connectivity. So hold onto the LTB even if we fail to communicate446 * with the VIOS and reuse it on next open. Free LTB when adapter is closed.447 *448 * Return: 0 if we were able to allocate the LTB and notify the VIOS and449 *	   a negative value otherwise.450 */451static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,452				struct ibmvnic_long_term_buff *ltb, int size)453{454	struct device *dev = &adapter->vdev->dev;455	u64 prev = 0;456	int rc;457 458	if (!reuse_ltb(ltb, size)) {459		dev_dbg(dev,460			"LTB size changed from 0x%llx to 0x%x, reallocating\n",461			 ltb->size, size);462		prev = ltb->size;463		free_long_term_buff(adapter, ltb);464	}465 466	if (ltb->buff) {467		dev_dbg(dev, "Reusing LTB [map %d, size 0x%llx]\n",468			ltb->map_id, ltb->size);469	} else {470		ltb->buff = dma_alloc_coherent(dev, size, &ltb->addr,471					       GFP_KERNEL);472		if (!ltb->buff) {473			dev_err(dev, "Couldn't alloc long term buffer\n");474			return -ENOMEM;475		}476		ltb->size = size;477 478		ltb->map_id = find_first_zero_bit(adapter->map_ids,479						  MAX_MAP_ID);480		bitmap_set(adapter->map_ids, ltb->map_id, 1);481 482		dev_dbg(dev,483			"Allocated new LTB [map %d, size 0x%llx was 0x%llx]\n",484			 ltb->map_id, ltb->size, prev);485	}486 487	/* Ensure ltb is zeroed - specially when reusing it. */488	memset(ltb->buff, 0, ltb->size);489 490	mutex_lock(&adapter->fw_lock);491	adapter->fw_done_rc = 0;492	reinit_completion(&adapter->fw_done);493 494	rc = send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id);495	if (rc) {496		dev_err(dev, "send_request_map failed, rc = %d\n", rc);497		goto out;498	}499 500	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);501	if (rc) {502		dev_err(dev, "LTB map request aborted or timed out, rc = %d\n",503			rc);504		goto out;505	}506 507	if (adapter->fw_done_rc) {508		dev_err(dev, "Couldn't map LTB, rc = %d\n",509			adapter->fw_done_rc);510		rc = -EIO;511		goto out;512	}513	rc = 0;514out:515	/* don't free LTB on communication error - see function header */516	mutex_unlock(&adapter->fw_lock);517	return rc;518}519 520static void free_long_term_buff(struct ibmvnic_adapter *adapter,521				struct ibmvnic_long_term_buff *ltb)522{523	struct device *dev = &adapter->vdev->dev;524 525	if (!ltb->buff)526		return;527 528	/* VIOS automatically unmaps the long term buffer at remote529	 * end for the following resets:530	 * FAILOVER, MOBILITY, TIMEOUT.531	 */532	if (adapter->reset_reason != VNIC_RESET_FAILOVER &&533	    adapter->reset_reason != VNIC_RESET_MOBILITY &&534	    adapter->reset_reason != VNIC_RESET_TIMEOUT)535		send_request_unmap(adapter, ltb->map_id);536 537	dma_free_coherent(dev, ltb->size, ltb->buff, ltb->addr);538 539	ltb->buff = NULL;540	/* mark this map_id free */541	bitmap_clear(adapter->map_ids, ltb->map_id, 1);542	ltb->map_id = 0;543}544 545/**546 * free_ltb_set - free the given set of long term buffers (LTBS)547 * @adapter: The ibmvnic adapter containing this ltb set548 * @ltb_set: The ltb_set to be freed549 *550 * Free the set of LTBs in the given set.551 */552 553static void free_ltb_set(struct ibmvnic_adapter *adapter,554			 struct ibmvnic_ltb_set *ltb_set)555{556	int i;557 558	for (i = 0; i < ltb_set->num_ltbs; i++)559		free_long_term_buff(adapter, &ltb_set->ltbs[i]);560 561	kfree(ltb_set->ltbs);562	ltb_set->ltbs = NULL;563	ltb_set->num_ltbs = 0;564}565 566/**567 * alloc_ltb_set() - Allocate a set of long term buffers (LTBs)568 *569 * @adapter: ibmvnic adapter associated to the LTB570 * @ltb_set: container object for the set of LTBs571 * @num_buffs: Number of buffers in the LTB572 * @buff_size: Size of each buffer in the LTB573 *574 * Allocate a set of LTBs to accommodate @num_buffs buffers of @buff_size575 * each. We currently cap size each LTB to IBMVNIC_ONE_LTB_SIZE. If the576 * new set of LTBs have fewer LTBs than the old set, free the excess LTBs.577 * If new set needs more than in old set, allocate the remaining ones.578 * Try and reuse as many LTBs as possible and avoid reallocation.579 *580 * Any changes to this allocation strategy must be reflected in581 * map_rxpool_buff_to_ltb() and map_txpool_buff_to_ltb().582 */583static int alloc_ltb_set(struct ibmvnic_adapter *adapter,584			 struct ibmvnic_ltb_set *ltb_set, int num_buffs,585			 int buff_size)586{587	struct device *dev = &adapter->vdev->dev;588	struct ibmvnic_ltb_set old_set;589	struct ibmvnic_ltb_set new_set;590	int rem_size;591	int tot_size;		/* size of all ltbs */592	int ltb_size;		/* size of one ltb */593	int nltbs;594	int rc;595	int n;596	int i;597 598	dev_dbg(dev, "%s() num_buffs %d, buff_size %d\n", __func__, num_buffs,599		buff_size);600 601	ltb_size = rounddown(IBMVNIC_ONE_LTB_SIZE, buff_size);602	tot_size = num_buffs * buff_size;603 604	if (ltb_size > tot_size)605		ltb_size = tot_size;606 607	nltbs = tot_size / ltb_size;608	if (tot_size % ltb_size)609		nltbs++;610 611	old_set = *ltb_set;612 613	if (old_set.num_ltbs == nltbs) {614		new_set = old_set;615	} else {616		int tmp = nltbs * sizeof(struct ibmvnic_long_term_buff);617 618		new_set.ltbs = kzalloc(tmp, GFP_KERNEL);619		if (!new_set.ltbs)620			return -ENOMEM;621 622		new_set.num_ltbs = nltbs;623 624		/* Free any excess ltbs in old set */625		for (i = new_set.num_ltbs; i < old_set.num_ltbs; i++)626			free_long_term_buff(adapter, &old_set.ltbs[i]);627 628		/* Copy remaining ltbs to new set. All LTBs except the629		 * last one are of the same size. alloc_long_term_buff()630		 * will realloc if the size changes.631		 */632		n = min(old_set.num_ltbs, new_set.num_ltbs);633		for (i = 0; i < n; i++)634			new_set.ltbs[i] = old_set.ltbs[i];635 636		/* Any additional ltbs in new set will have NULL ltbs for637		 * now and will be allocated in alloc_long_term_buff().638		 */639 640		/* We no longer need the old_set so free it. Note that we641		 * may have reused some ltbs from old set and freed excess642		 * ltbs above. So we only need to free the container now643		 * not the LTBs themselves. (i.e. dont free_ltb_set()!)644		 */645		kfree(old_set.ltbs);646		old_set.ltbs = NULL;647		old_set.num_ltbs = 0;648 649		/* Install the new set. If allocations fail below, we will650		 * retry later and know what size LTBs we need.651		 */652		*ltb_set = new_set;653	}654 655	i = 0;656	rem_size = tot_size;657	while (rem_size) {658		if (ltb_size > rem_size)659			ltb_size = rem_size;660 661		rem_size -= ltb_size;662 663		rc = alloc_long_term_buff(adapter, &new_set.ltbs[i], ltb_size);664		if (rc)665			goto out;666		i++;667	}668 669	WARN_ON(i != new_set.num_ltbs);670 671	return 0;672out:673	/* We may have allocated one/more LTBs before failing and we674	 * want to try and reuse on next reset. So don't free ltb set.675	 */676	return rc;677}678 679/**680 * map_rxpool_buf_to_ltb - Map given rxpool buffer to offset in an LTB.681 * @rxpool: The receive buffer pool containing buffer682 * @bufidx: Index of buffer in rxpool683 * @ltbp: (Output) pointer to the long term buffer containing the buffer684 * @offset: (Output) offset of buffer in the LTB from @ltbp685 *686 * Map the given buffer identified by [rxpool, bufidx] to an LTB in the687 * pool and its corresponding offset. Assume for now that each LTB is of688 * different size but could possibly be optimized based on the allocation689 * strategy in alloc_ltb_set().690 */691static void map_rxpool_buf_to_ltb(struct ibmvnic_rx_pool *rxpool,692				  unsigned int bufidx,693				  struct ibmvnic_long_term_buff **ltbp,694				  unsigned int *offset)695{696	struct ibmvnic_long_term_buff *ltb;697	int nbufs;	/* # of buffers in one ltb */698	int i;699 700	WARN_ON(bufidx >= rxpool->size);701 702	for (i = 0; i < rxpool->ltb_set.num_ltbs; i++) {703		ltb = &rxpool->ltb_set.ltbs[i];704		nbufs = ltb->size / rxpool->buff_size;705		if (bufidx < nbufs)706			break;707		bufidx -= nbufs;708	}709 710	*ltbp = ltb;711	*offset = bufidx * rxpool->buff_size;712}713 714/**715 * map_txpool_buf_to_ltb - Map given txpool buffer to offset in an LTB.716 * @txpool: The transmit buffer pool containing buffer717 * @bufidx: Index of buffer in txpool718 * @ltbp: (Output) pointer to the long term buffer (LTB) containing the buffer719 * @offset: (Output) offset of buffer in the LTB from @ltbp720 *721 * Map the given buffer identified by [txpool, bufidx] to an LTB in the722 * pool and its corresponding offset.723 */724static void map_txpool_buf_to_ltb(struct ibmvnic_tx_pool *txpool,725				  unsigned int bufidx,726				  struct ibmvnic_long_term_buff **ltbp,727				  unsigned int *offset)728{729	struct ibmvnic_long_term_buff *ltb;730	int nbufs;	/* # of buffers in one ltb */731	int i;732 733	WARN_ON_ONCE(bufidx >= txpool->num_buffers);734 735	for (i = 0; i < txpool->ltb_set.num_ltbs; i++) {736		ltb = &txpool->ltb_set.ltbs[i];737		nbufs = ltb->size / txpool->buf_size;738		if (bufidx < nbufs)739			break;740		bufidx -= nbufs;741	}742 743	*ltbp = ltb;744	*offset = bufidx * txpool->buf_size;745}746 747static void deactivate_rx_pools(struct ibmvnic_adapter *adapter)748{749	int i;750 751	for (i = 0; i < adapter->num_active_rx_pools; i++)752		adapter->rx_pool[i].active = 0;753}754 755static void replenish_rx_pool(struct ibmvnic_adapter *adapter,756			      struct ibmvnic_rx_pool *pool)757{758	int count = pool->size - atomic_read(&pool->available);759	u64 handle = adapter->rx_scrq[pool->index]->handle;760	struct device *dev = &adapter->vdev->dev;761	struct ibmvnic_ind_xmit_queue *ind_bufp;762	struct ibmvnic_sub_crq_queue *rx_scrq;763	struct ibmvnic_long_term_buff *ltb;764	union sub_crq *sub_crq;765	int buffers_added = 0;766	unsigned long lpar_rc;767	struct sk_buff *skb;768	unsigned int offset;769	dma_addr_t dma_addr;770	unsigned char *dst;771	int shift = 0;772	int bufidx;773	int i;774 775	if (!pool->active)776		return;777 778	rx_scrq = adapter->rx_scrq[pool->index];779	ind_bufp = &rx_scrq->ind_buf;780 781	/* netdev_skb_alloc() could have failed after we saved a few skbs782	 * in the indir_buf and we would not have sent them to VIOS yet.783	 * To account for them, start the loop at ind_bufp->index rather784	 * than 0. If we pushed all the skbs to VIOS, ind_bufp->index will785	 * be 0.786	 */787	for (i = ind_bufp->index; i < count; ++i) {788		bufidx = pool->free_map[pool->next_free];789 790		/* We maybe reusing the skb from earlier resets. Allocate791		 * only if necessary. But since the LTB may have changed792		 * during reset (see init_rx_pools()), update LTB below793		 * even if reusing skb.794		 */795		skb = pool->rx_buff[bufidx].skb;796		if (!skb) {797			skb = netdev_alloc_skb(adapter->netdev,798					       pool->buff_size);799			if (!skb) {800				dev_err(dev, "Couldn't replenish rx buff\n");801				adapter->replenish_no_mem++;802				break;803			}804		}805 806		pool->free_map[pool->next_free] = IBMVNIC_INVALID_MAP;807		pool->next_free = (pool->next_free + 1) % pool->size;808 809		/* Copy the skb to the long term mapped DMA buffer */810		map_rxpool_buf_to_ltb(pool, bufidx, &ltb, &offset);811		dst = ltb->buff + offset;812		memset(dst, 0, pool->buff_size);813		dma_addr = ltb->addr + offset;814 815		/* add the skb to an rx_buff in the pool */816		pool->rx_buff[bufidx].data = dst;817		pool->rx_buff[bufidx].dma = dma_addr;818		pool->rx_buff[bufidx].skb = skb;819		pool->rx_buff[bufidx].pool_index = pool->index;820		pool->rx_buff[bufidx].size = pool->buff_size;821 822		/* queue the rx_buff for the next send_subcrq_indirect */823		sub_crq = &ind_bufp->indir_arr[ind_bufp->index++];824		memset(sub_crq, 0, sizeof(*sub_crq));825		sub_crq->rx_add.first = IBMVNIC_CRQ_CMD;826		sub_crq->rx_add.correlator =827		    cpu_to_be64((u64)&pool->rx_buff[bufidx]);828		sub_crq->rx_add.ioba = cpu_to_be32(dma_addr);829		sub_crq->rx_add.map_id = ltb->map_id;830 831		/* The length field of the sCRQ is defined to be 24 bits so the832		 * buffer size needs to be left shifted by a byte before it is833		 * converted to big endian to prevent the last byte from being834		 * truncated.835		 */836#ifdef __LITTLE_ENDIAN__837		shift = 8;838#endif839		sub_crq->rx_add.len = cpu_to_be32(pool->buff_size << shift);840 841		/* if send_subcrq_indirect queue is full, flush to VIOS */842		if (ind_bufp->index == IBMVNIC_MAX_IND_DESCS ||843		    i == count - 1) {844			lpar_rc =845				send_subcrq_indirect(adapter, handle,846						     (u64)ind_bufp->indir_dma,847						     (u64)ind_bufp->index);848			if (lpar_rc != H_SUCCESS)849				goto failure;850			buffers_added += ind_bufp->index;851			adapter->replenish_add_buff_success += ind_bufp->index;852			ind_bufp->index = 0;853		}854	}855	atomic_add(buffers_added, &pool->available);856	return;857 858failure:859	if (lpar_rc != H_PARAMETER && lpar_rc != H_CLOSED)860		dev_err_ratelimited(dev, "rx: replenish packet buffer failed\n");861	for (i = ind_bufp->index - 1; i >= 0; --i) {862		struct ibmvnic_rx_buff *rx_buff;863 864		pool->next_free = pool->next_free == 0 ?865				  pool->size - 1 : pool->next_free - 1;866		sub_crq = &ind_bufp->indir_arr[i];867		rx_buff = (struct ibmvnic_rx_buff *)868				be64_to_cpu(sub_crq->rx_add.correlator);869		bufidx = (int)(rx_buff - pool->rx_buff);870		pool->free_map[pool->next_free] = bufidx;871		dev_kfree_skb_any(pool->rx_buff[bufidx].skb);872		pool->rx_buff[bufidx].skb = NULL;873	}874	adapter->replenish_add_buff_failure += ind_bufp->index;875	atomic_add(buffers_added, &pool->available);876	ind_bufp->index = 0;877	if (lpar_rc == H_CLOSED || adapter->failover_pending) {878		/* Disable buffer pool replenishment and report carrier off if879		 * queue is closed or pending failover.880		 * Firmware guarantees that a signal will be sent to the881		 * driver, triggering a reset.882		 */883		deactivate_rx_pools(adapter);884		netif_carrier_off(adapter->netdev);885	}886}887 888static void replenish_pools(struct ibmvnic_adapter *adapter)889{890	int i;891 892	adapter->replenish_task_cycles++;893	for (i = 0; i < adapter->num_active_rx_pools; i++) {894		if (adapter->rx_pool[i].active)895			replenish_rx_pool(adapter, &adapter->rx_pool[i]);896	}897 898	netdev_dbg(adapter->netdev, "Replenished %d pools\n", i);899}900 901static void release_stats_buffers(struct ibmvnic_adapter *adapter)902{903	kfree(adapter->tx_stats_buffers);904	kfree(adapter->rx_stats_buffers);905	adapter->tx_stats_buffers = NULL;906	adapter->rx_stats_buffers = NULL;907}908 909static int init_stats_buffers(struct ibmvnic_adapter *adapter)910{911	adapter->tx_stats_buffers =912				kcalloc(IBMVNIC_MAX_QUEUES,913					sizeof(struct ibmvnic_tx_queue_stats),914					GFP_KERNEL);915	if (!adapter->tx_stats_buffers)916		return -ENOMEM;917 918	adapter->rx_stats_buffers =919				kcalloc(IBMVNIC_MAX_QUEUES,920					sizeof(struct ibmvnic_rx_queue_stats),921					GFP_KERNEL);922	if (!adapter->rx_stats_buffers)923		return -ENOMEM;924 925	return 0;926}927 928static void release_stats_token(struct ibmvnic_adapter *adapter)929{930	struct device *dev = &adapter->vdev->dev;931 932	if (!adapter->stats_token)933		return;934 935	dma_unmap_single(dev, adapter->stats_token,936			 sizeof(struct ibmvnic_statistics),937			 DMA_FROM_DEVICE);938	adapter->stats_token = 0;939}940 941static int init_stats_token(struct ibmvnic_adapter *adapter)942{943	struct device *dev = &adapter->vdev->dev;944	dma_addr_t stok;945	int rc;946 947	stok = dma_map_single(dev, &adapter->stats,948			      sizeof(struct ibmvnic_statistics),949			      DMA_FROM_DEVICE);950	rc = dma_mapping_error(dev, stok);951	if (rc) {952		dev_err(dev, "Couldn't map stats buffer, rc = %d\n", rc);953		return rc;954	}955 956	adapter->stats_token = stok;957	netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok);958	return 0;959}960 961/**962 * release_rx_pools() - Release any rx pools attached to @adapter.963 * @adapter: ibmvnic adapter964 *965 * Safe to call this multiple times - even if no pools are attached.966 */967static void release_rx_pools(struct ibmvnic_adapter *adapter)968{969	struct ibmvnic_rx_pool *rx_pool;970	int i, j;971 972	if (!adapter->rx_pool)973		return;974 975	for (i = 0; i < adapter->num_active_rx_pools; i++) {976		rx_pool = &adapter->rx_pool[i];977 978		netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i);979 980		kfree(rx_pool->free_map);981 982		free_ltb_set(adapter, &rx_pool->ltb_set);983 984		if (!rx_pool->rx_buff)985			continue;986 987		for (j = 0; j < rx_pool->size; j++) {988			if (rx_pool->rx_buff[j].skb) {989				dev_kfree_skb_any(rx_pool->rx_buff[j].skb);990				rx_pool->rx_buff[j].skb = NULL;991			}992		}993 994		kfree(rx_pool->rx_buff);995	}996 997	kfree(adapter->rx_pool);998	adapter->rx_pool = NULL;999	adapter->num_active_rx_pools = 0;1000	adapter->prev_rx_pool_size = 0;1001}1002 1003/**1004 * reuse_rx_pools() - Check if the existing rx pools can be reused.1005 * @adapter: ibmvnic adapter1006 *1007 * Check if the existing rx pools in the adapter can be reused. The1008 * pools can be reused if the pool parameters (number of pools,1009 * number of buffers in the pool and size of each buffer) have not1010 * changed.1011 *1012 * NOTE: This assumes that all pools have the same number of buffers1013 *       which is the case currently. If that changes, we must fix this.1014 *1015 * Return: true if the rx pools can be reused, false otherwise.1016 */1017static bool reuse_rx_pools(struct ibmvnic_adapter *adapter)1018{1019	u64 old_num_pools, new_num_pools;1020	u64 old_pool_size, new_pool_size;1021	u64 old_buff_size, new_buff_size;1022 1023	if (!adapter->rx_pool)1024		return false;1025 1026	old_num_pools = adapter->num_active_rx_pools;1027	new_num_pools = adapter->req_rx_queues;1028 1029	old_pool_size = adapter->prev_rx_pool_size;1030	new_pool_size = adapter->req_rx_add_entries_per_subcrq;1031 1032	old_buff_size = adapter->prev_rx_buf_sz;1033	new_buff_size = adapter->cur_rx_buf_sz;1034 1035	if (old_buff_size != new_buff_size ||1036	    old_num_pools != new_num_pools ||1037	    old_pool_size != new_pool_size)1038		return false;1039 1040	return true;1041}1042 1043/**1044 * init_rx_pools(): Initialize the set of receiver pools in the adapter.1045 * @netdev: net device associated with the vnic interface1046 *1047 * Initialize the set of receiver pools in the ibmvnic adapter associated1048 * with the net_device @netdev. If possible, reuse the existing rx pools.1049 * Otherwise free any existing pools and  allocate a new set of pools1050 * before initializing them.1051 *1052 * Return: 0 on success and negative value on error.1053 */1054static int init_rx_pools(struct net_device *netdev)1055{1056	struct ibmvnic_adapter *adapter = netdev_priv(netdev);1057	struct device *dev = &adapter->vdev->dev;1058	struct ibmvnic_rx_pool *rx_pool;1059	u64 num_pools;1060	u64 pool_size;		/* # of buffers in one pool */1061	u64 buff_size;1062	int i, j, rc;1063 1064	pool_size = adapter->req_rx_add_entries_per_subcrq;1065	num_pools = adapter->req_rx_queues;1066	buff_size = adapter->cur_rx_buf_sz;1067 1068	if (reuse_rx_pools(adapter)) {1069		dev_dbg(dev, "Reusing rx pools\n");1070		goto update_ltb;1071	}1072 1073	/* Allocate/populate the pools. */1074	release_rx_pools(adapter);1075 1076	adapter->rx_pool = kcalloc(num_pools,1077				   sizeof(struct ibmvnic_rx_pool),1078				   GFP_KERNEL);1079	if (!adapter->rx_pool) {1080		dev_err(dev, "Failed to allocate rx pools\n");1081		return -ENOMEM;1082	}1083 1084	/* Set num_active_rx_pools early. If we fail below after partial1085	 * allocation, release_rx_pools() will know how many to look for.1086	 */1087	adapter->num_active_rx_pools = num_pools;1088 1089	for (i = 0; i < num_pools; i++) {1090		rx_pool = &adapter->rx_pool[i];1091 1092		netdev_dbg(adapter->netdev,1093			   "Initializing rx_pool[%d], %lld buffs, %lld bytes each\n",1094			   i, pool_size, buff_size);1095 1096		rx_pool->size = pool_size;1097		rx_pool->index = i;1098		rx_pool->buff_size = ALIGN(buff_size, L1_CACHE_BYTES);1099 1100		rx_pool->free_map = kcalloc(rx_pool->size, sizeof(int),1101					    GFP_KERNEL);1102		if (!rx_pool->free_map) {1103			dev_err(dev, "Couldn't alloc free_map %d\n", i);1104			rc = -ENOMEM;1105			goto out_release;1106		}1107 1108		rx_pool->rx_buff = kcalloc(rx_pool->size,1109					   sizeof(struct ibmvnic_rx_buff),1110					   GFP_KERNEL);1111		if (!rx_pool->rx_buff) {1112			dev_err(dev, "Couldn't alloc rx buffers\n");1113			rc = -ENOMEM;1114			goto out_release;1115		}1116	}1117 1118	adapter->prev_rx_pool_size = pool_size;1119	adapter->prev_rx_buf_sz = adapter->cur_rx_buf_sz;1120 1121update_ltb:1122	for (i = 0; i < num_pools; i++) {1123		rx_pool = &adapter->rx_pool[i];1124		dev_dbg(dev, "Updating LTB for rx pool %d [%d, %d]\n",1125			i, rx_pool->size, rx_pool->buff_size);1126 1127		rc = alloc_ltb_set(adapter, &rx_pool->ltb_set,1128				   rx_pool->size, rx_pool->buff_size);1129		if (rc)1130			goto out;1131 1132		for (j = 0; j < rx_pool->size; ++j) {1133			struct ibmvnic_rx_buff *rx_buff;1134 1135			rx_pool->free_map[j] = j;1136 1137			/* NOTE: Don't clear rx_buff->skb here - will leak1138			 * memory! replenish_rx_pool() will reuse skbs or1139			 * allocate as necessary.1140			 */1141			rx_buff = &rx_pool->rx_buff[j];1142			rx_buff->dma = 0;1143			rx_buff->data = 0;1144			rx_buff->size = 0;1145			rx_buff->pool_index = 0;1146		}1147 1148		/* Mark pool "empty" so replenish_rx_pools() will1149		 * update the LTB info for each buffer1150		 */1151		atomic_set(&rx_pool->available, 0);1152		rx_pool->next_alloc = 0;1153		rx_pool->next_free = 0;1154		/* replenish_rx_pool() may have called deactivate_rx_pools()1155		 * on failover. Ensure pool is active now.1156		 */1157		rx_pool->active = 1;1158	}1159	return 0;1160out_release:1161	release_rx_pools(adapter);1162out:1163	/* We failed to allocate one or more LTBs or map them on the VIOS.1164	 * Hold onto the pools and any LTBs that we did allocate/map.1165	 */1166	return rc;1167}1168 1169static void release_vpd_data(struct ibmvnic_adapter *adapter)1170{1171	if (!adapter->vpd)1172		return;1173 1174	kfree(adapter->vpd->buff);1175	kfree(adapter->vpd);1176 1177	adapter->vpd = NULL;1178}1179 1180static void release_one_tx_pool(struct ibmvnic_adapter *adapter,1181				struct ibmvnic_tx_pool *tx_pool)1182{1183	kfree(tx_pool->tx_buff);1184	kfree(tx_pool->free_map);1185	free_ltb_set(adapter, &tx_pool->ltb_set);1186}1187 1188/**1189 * release_tx_pools() - Release any tx pools attached to @adapter.1190 * @adapter: ibmvnic adapter1191 *1192 * Safe to call this multiple times - even if no pools are attached.1193 */1194static void release_tx_pools(struct ibmvnic_adapter *adapter)1195{1196	int i;1197 1198	/* init_tx_pools() ensures that ->tx_pool and ->tso_pool are1199	 * both NULL or both non-NULL. So we only need to check one.1200	 */1201	if (!adapter->tx_pool)1202		return;1203 1204	for (i = 0; i < adapter->num_active_tx_pools; i++) {1205		release_one_tx_pool(adapter, &adapter->tx_pool[i]);1206		release_one_tx_pool(adapter, &adapter->tso_pool[i]);1207	}1208 1209	kfree(adapter->tx_pool);1210	adapter->tx_pool = NULL;1211	kfree(adapter->tso_pool);1212	adapter->tso_pool = NULL;1213	adapter->num_active_tx_pools = 0;1214	adapter->prev_tx_pool_size = 0;1215}1216 1217static int init_one_tx_pool(struct net_device *netdev,1218			    struct ibmvnic_tx_pool *tx_pool,1219			    int pool_size, int buf_size)1220{1221	int i;1222 1223	tx_pool->tx_buff = kcalloc(pool_size,1224				   sizeof(struct ibmvnic_tx_buff),1225				   GFP_KERNEL);1226	if (!tx_pool->tx_buff)1227		return -ENOMEM;1228 1229	tx_pool->free_map = kcalloc(pool_size, sizeof(int), GFP_KERNEL);1230	if (!tx_pool->free_map) {1231		kfree(tx_pool->tx_buff);1232		tx_pool->tx_buff = NULL;1233		return -ENOMEM;1234	}1235 1236	for (i = 0; i < pool_size; i++)1237		tx_pool->free_map[i] = i;1238 1239	tx_pool->consumer_index = 0;1240	tx_pool->producer_index = 0;1241	tx_pool->num_buffers = pool_size;1242	tx_pool->buf_size = buf_size;1243 1244	return 0;1245}1246 1247/**1248 * reuse_tx_pools() - Check if the existing tx pools can be reused.1249 * @adapter: ibmvnic adapter1250 *1251 * Check if the existing tx pools in the adapter can be reused. The1252 * pools can be reused if the pool parameters (number of pools,1253 * number of buffers in the pool and mtu) have not changed.1254 *1255 * NOTE: This assumes that all pools have the same number of buffers1256 *       which is the case currently. If that changes, we must fix this.1257 *1258 * Return: true if the tx pools can be reused, false otherwise.1259 */1260static bool reuse_tx_pools(struct ibmvnic_adapter *adapter)1261{1262	u64 old_num_pools, new_num_pools;1263	u64 old_pool_size, new_pool_size;1264	u64 old_mtu, new_mtu;1265 1266	if (!adapter->tx_pool)1267		return false;1268 1269	old_num_pools = adapter->num_active_tx_pools;1270	new_num_pools = adapter->num_active_tx_scrqs;1271	old_pool_size = adapter->prev_tx_pool_size;1272	new_pool_size = adapter->req_tx_entries_per_subcrq;1273	old_mtu = adapter->prev_mtu;1274	new_mtu = adapter->req_mtu;1275 1276	if (old_mtu != new_mtu ||1277	    old_num_pools != new_num_pools ||1278	    old_pool_size != new_pool_size)1279		return false;1280 1281	return true;1282}1283 1284/**1285 * init_tx_pools(): Initialize the set of transmit pools in the adapter.1286 * @netdev: net device associated with the vnic interface1287 *1288 * Initialize the set of transmit pools in the ibmvnic adapter associated1289 * with the net_device @netdev. If possible, reuse the existing tx pools.1290 * Otherwise free any existing pools and  allocate a new set of pools1291 * before initializing them.1292 *1293 * Return: 0 on success and negative value on error.1294 */1295static int init_tx_pools(struct net_device *netdev)1296{1297	struct ibmvnic_adapter *adapter = netdev_priv(netdev);1298	struct device *dev = &adapter->vdev->dev;1299	int num_pools;1300	u64 pool_size;		/* # of buffers in pool */1301	u64 buff_size;1302	int i, j, rc;1303 1304	num_pools = adapter->req_tx_queues;1305 1306	/* We must notify the VIOS about the LTB on all resets - but we only1307	 * need to alloc/populate pools if either the number of buffers or1308	 * size of each buffer in the pool has changed.1309	 */1310	if (reuse_tx_pools(adapter)) {1311		netdev_dbg(netdev, "Reusing tx pools\n");1312		goto update_ltb;1313	}1314 1315	/* Allocate/populate the pools. */1316	release_tx_pools(adapter);1317 1318	pool_size = adapter->req_tx_entries_per_subcrq;1319	num_pools = adapter->num_active_tx_scrqs;1320 1321	adapter->tx_pool = kcalloc(num_pools,1322				   sizeof(struct ibmvnic_tx_pool), GFP_KERNEL);1323	if (!adapter->tx_pool)1324		return -ENOMEM;1325 1326	adapter->tso_pool = kcalloc(num_pools,1327				    sizeof(struct ibmvnic_tx_pool), GFP_KERNEL);1328	/* To simplify release_tx_pools() ensure that ->tx_pool and1329	 * ->tso_pool are either both NULL or both non-NULL.1330	 */1331	if (!adapter->tso_pool) {1332		kfree(adapter->tx_pool);1333		adapter->tx_pool = NULL;1334		return -ENOMEM;1335	}1336 1337	/* Set num_active_tx_pools early. If we fail below after partial1338	 * allocation, release_tx_pools() will know how many to look for.1339	 */1340	adapter->num_active_tx_pools = num_pools;1341 1342	buff_size = adapter->req_mtu + VLAN_HLEN;1343	buff_size = ALIGN(buff_size, L1_CACHE_BYTES);1344 1345	for (i = 0; i < num_pools; i++) {1346		dev_dbg(dev, "Init tx pool %d [%llu, %llu]\n",1347			i, adapter->req_tx_entries_per_subcrq, buff_size);1348 1349		rc = init_one_tx_pool(netdev, &adapter->tx_pool[i],1350				      pool_size, buff_size);1351		if (rc)1352			goto out_release;1353 1354		rc = init_one_tx_pool(netdev, &adapter->tso_pool[i],1355				      IBMVNIC_TSO_BUFS,1356				      IBMVNIC_TSO_BUF_SZ);1357		if (rc)1358			goto out_release;1359	}1360 1361	adapter->prev_tx_pool_size = pool_size;1362	adapter->prev_mtu = adapter->req_mtu;1363 1364update_ltb:1365	/* NOTE: All tx_pools have the same number of buffers (which is1366	 *       same as pool_size). All tso_pools have IBMVNIC_TSO_BUFS1367	 *       buffers (see calls init_one_tx_pool() for these).1368	 *       For consistency, we use tx_pool->num_buffers and1369	 *       tso_pool->num_buffers below.1370	 */1371	rc = -1;1372	for (i = 0; i < num_pools; i++) {1373		struct ibmvnic_tx_pool *tso_pool;1374		struct ibmvnic_tx_pool *tx_pool;1375 1376		tx_pool = &adapter->tx_pool[i];1377 1378		dev_dbg(dev, "Updating LTB for tx pool %d [%d, %d]\n",1379			i, tx_pool->num_buffers, tx_pool->buf_size);1380 1381		rc = alloc_ltb_set(adapter, &tx_pool->ltb_set,1382				   tx_pool->num_buffers, tx_pool->buf_size);1383		if (rc)1384			goto out;1385 1386		tx_pool->consumer_index = 0;1387		tx_pool->producer_index = 0;1388 1389		for (j = 0; j < tx_pool->num_buffers; j++)1390			tx_pool->free_map[j] = j;1391 1392		tso_pool = &adapter->tso_pool[i];1393 1394		dev_dbg(dev, "Updating LTB for tso pool %d [%d, %d]\n",1395			i, tso_pool->num_buffers, tso_pool->buf_size);1396 1397		rc = alloc_ltb_set(adapter, &tso_pool->ltb_set,1398				   tso_pool->num_buffers, tso_pool->buf_size);1399		if (rc)1400			goto out;1401 1402		tso_pool->consumer_index = 0;1403		tso_pool->producer_index = 0;1404 1405		for (j = 0; j < tso_pool->num_buffers; j++)1406			tso_pool->free_map[j] = j;1407	}1408 1409	return 0;1410out_release:1411	release_tx_pools(adapter);1412out:1413	/* We failed to allocate one or more LTBs or map them on the VIOS.1414	 * Hold onto the pools and any LTBs that we did allocate/map.1415	 */1416	return rc;1417}1418 1419static void ibmvnic_napi_enable(struct ibmvnic_adapter *adapter)1420{1421	int i;1422 1423	if (adapter->napi_enabled)1424		return;1425 1426	for (i = 0; i < adapter->req_rx_queues; i++)1427		napi_enable(&adapter->napi[i]);1428 1429	adapter->napi_enabled = true;1430}1431 1432static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter)1433{1434	int i;1435 1436	if (!adapter->napi_enabled)1437		return;1438 1439	for (i = 0; i < adapter->req_rx_queues; i++) {1440		netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i);1441		napi_disable(&adapter->napi[i]);1442	}1443 1444	adapter->napi_enabled = false;1445}1446 1447static int init_napi(struct ibmvnic_adapter *adapter)1448{1449	int i;1450 1451	adapter->napi = kcalloc(adapter->req_rx_queues,1452				sizeof(struct napi_struct), GFP_KERNEL);1453	if (!adapter->napi)1454		return -ENOMEM;1455 1456	for (i = 0; i < adapter->req_rx_queues; i++) {1457		netdev_dbg(adapter->netdev, "Adding napi[%d]\n", i);1458		netif_napi_add(adapter->netdev, &adapter->napi[i],1459			       ibmvnic_poll);1460	}1461 1462	adapter->num_active_rx_napi = adapter->req_rx_queues;1463	return 0;1464}1465 1466static void release_napi(struct ibmvnic_adapter *adapter)1467{1468	int i;1469 1470	if (!adapter->napi)1471		return;1472 1473	for (i = 0; i < adapter->num_active_rx_napi; i++) {1474		netdev_dbg(adapter->netdev, "Releasing napi[%d]\n", i);1475		netif_napi_del(&adapter->napi[i]);1476	}1477 1478	kfree(adapter->napi);1479	adapter->napi = NULL;1480	adapter->num_active_rx_napi = 0;1481	adapter->napi_enabled = false;1482}1483 1484static const char *adapter_state_to_string(enum vnic_state state)1485{1486	switch (state) {1487	case VNIC_PROBING:1488		return "PROBING";1489	case VNIC_PROBED:1490		return "PROBED";1491	case VNIC_OPENING:1492		return "OPENING";1493	case VNIC_OPEN:1494		return "OPEN";1495	case VNIC_CLOSING:1496		return "CLOSING";1497	case VNIC_CLOSED:1498		return "CLOSED";1499	case VNIC_REMOVING:1500		return "REMOVING";1501	case VNIC_REMOVED:1502		return "REMOVED";1503	case VNIC_DOWN:1504		return "DOWN";1505	}1506	return "UNKNOWN";1507}1508 1509static int ibmvnic_login(struct net_device *netdev)1510{1511	unsigned long flags, timeout = msecs_to_jiffies(20000);1512	struct ibmvnic_adapter *adapter = netdev_priv(netdev);1513	int retry_count = 0;1514	int retries = 10;1515	bool retry;1516	int rc;1517 1518	do {1519		retry = false;1520		if (retry_count > retries) {1521			netdev_warn(netdev, "Login attempts exceeded\n");1522			return -EACCES;1523		}1524 1525		adapter->init_done_rc = 0;1526		reinit_completion(&adapter->init_done);1527		rc = send_login(adapter);1528		if (rc)1529			return rc;1530 1531		if (!wait_for_completion_timeout(&adapter->init_done,1532						 timeout)) {1533			netdev_warn(netdev, "Login timed out\n");1534			adapter->login_pending = false;1535			goto partial_reset;1536		}1537 1538		if (adapter->init_done_rc == ABORTED) {1539			netdev_warn(netdev, "Login aborted, retrying...\n");1540			retry = true;1541			adapter->init_done_rc = 0;1542			retry_count++;1543			/* FW or device may be busy, so1544			 * wait a bit before retrying login1545			 */1546			msleep(500);1547		} else if (adapter->init_done_rc == PARTIALSUCCESS) {1548			retry_count++;1549			release_sub_crqs(adapter, 1);1550 1551			retry = true;1552			netdev_dbg(netdev,1553				   "Received partial success, retrying...\n");1554			adapter->init_done_rc = 0;1555			reinit_completion(&adapter->init_done);1556			send_query_cap(adapter);1557			if (!wait_for_completion_timeout(&adapter->init_done,1558							 timeout)) {1559				netdev_warn(netdev,1560					    "Capabilities query timed out\n");1561				return -ETIMEDOUT;1562			}1563 1564			rc = init_sub_crqs(adapter);1565			if (rc) {1566				netdev_warn(netdev,1567					    "SCRQ initialization failed\n");1568				return rc;1569			}1570 1571			rc = init_sub_crq_irqs(adapter);1572			if (rc) {1573				netdev_warn(netdev,1574					    "SCRQ irq initialization failed\n");1575				return rc;1576			}1577		/* Default/timeout error handling, reset and start fresh */1578		} else if (adapter->init_done_rc) {1579			netdev_warn(netdev, "Adapter login failed, init_done_rc = %d\n",1580				    adapter->init_done_rc);1581 1582partial_reset:1583			/* adapter login failed, so free any CRQs or sub-CRQs1584			 * and register again before attempting to login again.1585			 * If we don't do this then the VIOS may think that1586			 * we are already logged in and reject any subsequent1587			 * attempts1588			 */1589			netdev_warn(netdev,1590				    "Freeing and re-registering CRQs before attempting to login again\n");1591			retry = true;1592			adapter->init_done_rc = 0;1593			release_sub_crqs(adapter, true);1594			/* Much of this is similar logic as ibmvnic_probe(),1595			 * we are essentially re-initializing communication1596			 * with the server. We really should not run any1597			 * resets/failovers here because this is already a form1598			 * of reset and we do not want parallel resets occurring1599			 */1600			do {1601				reinit_init_done(adapter);1602				/* Clear any failovers we got in the previous1603				 * pass since we are re-initializing the CRQ1604				 */1605				adapter->failover_pending = false;1606				release_crq_queue(adapter);1607				/* If we don't sleep here then we risk an1608				 * unnecessary failover event from the VIOS.1609				 * This is a known VIOS issue caused by a vnic1610				 * device freeing and registering a CRQ too1611				 * quickly.1612				 */1613				msleep(1500);1614				/* Avoid any resets, since we are currently1615				 * resetting.1616				 */1617				spin_lock_irqsave(&adapter->rwi_lock, flags);1618				flush_reset_queue(adapter);1619				spin_unlock_irqrestore(&adapter->rwi_lock,1620						       flags);1621 1622				rc = init_crq_queue(adapter);1623				if (rc) {1624					netdev_err(netdev, "login recovery: init CRQ failed %d\n",1625						   rc);1626					return -EIO;1627				}1628 1629				rc = ibmvnic_reset_init(adapter, false);1630				if (rc)1631					netdev_err(netdev, "login recovery: Reset init failed %d\n",1632						   rc);1633				/* IBMVNIC_CRQ_INIT will return EAGAIN if it1634				 * fails, since ibmvnic_reset_init will free1635				 * irq's in failure, we won't be able to receive1636				 * new CRQs so we need to keep trying. probe()1637				 * handles this similarly.1638				 */1639			} while (rc == -EAGAIN && retry_count++ < retries);1640		}1641	} while (retry);1642 1643	__ibmvnic_set_mac(netdev, adapter->mac_addr);1644 1645	netdev_dbg(netdev, "[S:%s] Login succeeded\n", adapter_state_to_string(adapter->state));1646	return 0;1647}1648 1649static void release_login_buffer(struct ibmvnic_adapter *adapter)1650{1651	if (!adapter->login_buf)1652		return;1653 1654	dma_unmap_single(&adapter->vdev->dev, adapter->login_buf_token,1655			 adapter->login_buf_sz, DMA_TO_DEVICE);1656	kfree(adapter->login_buf);1657	adapter->login_buf = NULL;1658}1659 1660static void release_login_rsp_buffer(struct ibmvnic_adapter *adapter)1661{1662	if (!adapter->login_rsp_buf)1663		return;1664 1665	dma_unmap_single(&adapter->vdev->dev, adapter->login_rsp_buf_token,1666			 adapter->login_rsp_buf_sz, DMA_FROM_DEVICE);1667	kfree(adapter->login_rsp_buf);1668	adapter->login_rsp_buf = NULL;1669}1670 1671static void release_resources(struct ibmvnic_adapter *adapter)1672{1673	release_vpd_data(adapter);1674 1675	release_napi(adapter);1676	release_login_buffer(adapter);1677	release_login_rsp_buffer(adapter);1678}1679 1680static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)1681{1682	struct net_device *netdev = adapter->netdev;1683	unsigned long timeout = msecs_to_jiffies(20000);1684	union ibmvnic_crq crq;1685	bool resend;1686	int rc;1687 1688	netdev_dbg(netdev, "setting link state %d\n", link_state);1689 1690	memset(&crq, 0, sizeof(crq));1691	crq.logical_link_state.first = IBMVNIC_CRQ_CMD;1692	crq.logical_link_state.cmd = LOGICAL_LINK_STATE;1693	crq.logical_link_state.link_state = link_state;1694 1695	do {1696		resend = false;1697 1698		reinit_completion(&adapter->init_done);1699		rc = ibmvnic_send_crq(adapter, &crq);1700		if (rc) {1701			netdev_err(netdev, "Failed to set link state\n");1702			return rc;1703		}1704 1705		if (!wait_for_completion_timeout(&adapter->init_done,1706						 timeout)) {1707			netdev_err(netdev, "timeout setting link state\n");1708			return -ETIMEDOUT;1709		}1710 1711		if (adapter->init_done_rc == PARTIALSUCCESS) {1712			/* Partuial success, delay and re-send */1713			mdelay(1000);1714			resend = true;1715		} else if (adapter->init_done_rc) {1716			netdev_warn(netdev, "Unable to set link state, rc=%d\n",1717				    adapter->init_done_rc);1718			return adapter->init_done_rc;1719		}1720	} while (resend);1721 1722	return 0;1723}1724 1725static int set_real_num_queues(struct net_device *netdev)1726{1727	struct ibmvnic_adapter *adapter = netdev_priv(netdev);1728	int rc;1729 1730	netdev_dbg(netdev, "Setting real tx/rx queues (%llx/%llx)\n",1731		   adapter->req_tx_queues, adapter->req_rx_queues);1732 1733	rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues);1734	if (rc) {1735		netdev_err(netdev, "failed to set the number of tx queues\n");1736		return rc;1737	}1738 1739	rc = netif_set_real_num_rx_queues(netdev, adapter->req_rx_queues);1740	if (rc)1741		netdev_err(netdev, "failed to set the number of rx queues\n");1742 1743	return rc;1744}1745 1746static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)1747{1748	struct device *dev = &adapter->vdev->dev;1749	union ibmvnic_crq crq;1750	int len = 0;1751	int rc;1752 1753	if (adapter->vpd->buff)1754		len = adapter->vpd->len;1755 1756	mutex_lock(&adapter->fw_lock);1757	adapter->fw_done_rc = 0;1758	reinit_completion(&adapter->fw_done);1759 1760	crq.get_vpd_size.first = IBMVNIC_CRQ_CMD;1761	crq.get_vpd_size.cmd = GET_VPD_SIZE;1762	rc = ibmvnic_send_crq(adapter, &crq);1763	if (rc) {1764		mutex_unlock(&adapter->fw_lock);1765		return rc;1766	}1767 1768	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);1769	if (rc) {1770		dev_err(dev, "Could not retrieve VPD size, rc = %d\n", rc);1771		mutex_unlock(&adapter->fw_lock);1772		return rc;1773	}1774	mutex_unlock(&adapter->fw_lock);1775 1776	if (!adapter->vpd->len)1777		return -ENODATA;1778 1779	if (!adapter->vpd->buff)1780		adapter->vpd->buff = kzalloc(adapter->vpd->len, GFP_KERNEL);1781	else if (adapter->vpd->len != len)1782		adapter->vpd->buff =1783			krealloc(adapter->vpd->buff,1784				 adapter->vpd->len, GFP_KERNEL);1785 1786	if (!adapter->vpd->buff) {1787		dev_err(dev, "Could allocate VPD buffer\n");1788		return -ENOMEM;1789	}1790 1791	adapter->vpd->dma_addr =1792		dma_map_single(dev, adapter->vpd->buff, adapter->vpd->len,1793			       DMA_FROM_DEVICE);1794	if (dma_mapping_error(dev, adapter->vpd->dma_addr)) {1795		dev_err(dev, "Could not map VPD buffer\n");1796		kfree(adapter->vpd->buff);1797		adapter->vpd->buff = NULL;1798		return -ENOMEM;1799	}1800 1801	mutex_lock(&adapter->fw_lock);1802	adapter->fw_done_rc = 0;1803	reinit_completion(&adapter->fw_done);1804 1805	crq.get_vpd.first = IBMVNIC_CRQ_CMD;1806	crq.get_vpd.cmd = GET_VPD;1807	crq.get_vpd.ioba = cpu_to_be32(adapter->vpd->dma_addr);1808	crq.get_vpd.len = cpu_to_be32((u32)adapter->vpd->len);1809	rc = ibmvnic_send_crq(adapter, &crq);1810	if (rc) {1811		kfree(adapter->vpd->buff);1812		adapter->vpd->buff = NULL;1813		mutex_unlock(&adapter->fw_lock);1814		return rc;1815	}1816 1817	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);1818	if (rc) {1819		dev_err(dev, "Unable to retrieve VPD, rc = %d\n", rc);1820		kfree(adapter->vpd->buff);1821		adapter->vpd->buff = NULL;1822		mutex_unlock(&adapter->fw_lock);1823		return rc;1824	}1825 1826	mutex_unlock(&adapter->fw_lock);1827	return 0;1828}1829 1830static int init_resources(struct ibmvnic_adapter *adapter)1831{1832	struct net_device *netdev = adapter->netdev;1833	int rc;1834 1835	rc = set_real_num_queues(netdev);1836	if (rc)1837		return rc;1838 1839	adapter->vpd = kzalloc(sizeof(*adapter->vpd), GFP_KERNEL);1840	if (!adapter->vpd)1841		return -ENOMEM;1842 1843	/* Vital Product Data (VPD) */1844	rc = ibmvnic_get_vpd(adapter);1845	if (rc) {1846		netdev_err(netdev, "failed to initialize Vital Product Data (VPD)\n");1847		return rc;1848	}1849 1850	rc = init_napi(adapter);1851	if (rc)1852		return rc;1853 1854	send_query_map(adapter);1855 1856	rc = init_rx_pools(netdev);1857	if (rc)1858		return rc;1859 1860	rc = init_tx_pools(netdev);1861	return rc;1862}1863 1864static int __ibmvnic_open(struct net_device *netdev)1865{1866	struct ibmvnic_adapter *adapter = netdev_priv(netdev);1867	enum vnic_state prev_state = adapter->state;1868	int i, rc;1869 1870	adapter->state = VNIC_OPENING;1871	replenish_pools(adapter);1872	ibmvnic_napi_enable(adapter);1873 1874	/* We're ready to receive frames, enable the sub-crq interrupts and1875	 * set the logical link state to up1876	 */1877	for (i = 0; i < adapter->req_rx_queues; i++) {1878		netdev_dbg(netdev, "Enabling rx_scrq[%d] irq\n", i);1879		if (prev_state == VNIC_CLOSED)1880			enable_irq(adapter->rx_scrq[i]->irq);1881		enable_scrq_irq(adapter, adapter->rx_scrq[i]);1882	}1883 1884	for (i = 0; i < adapter->req_tx_queues; i++) {1885		netdev_dbg(netdev, "Enabling tx_scrq[%d] irq\n", i);1886		if (prev_state == VNIC_CLOSED)1887			enable_irq(adapter->tx_scrq[i]->irq);1888		enable_scrq_irq(adapter, adapter->tx_scrq[i]);1889		/* netdev_tx_reset_queue will reset dql stats. During NON_FATAL1890		 * resets, don't reset the stats because there could be batched1891		 * skb's waiting to be sent. If we reset dql stats, we risk1892		 * num_completed being greater than num_queued. This will cause1893		 * a BUG_ON in dql_completed().1894		 */1895		if (adapter->reset_reason != VNIC_RESET_NON_FATAL)1896			netdev_tx_reset_queue(netdev_get_tx_queue(netdev, i));1897	}1898 1899	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);1900	if (rc) {1901		ibmvnic_napi_disable(adapter);1902		ibmvnic_disable_irqs(adapter);1903		return rc;1904	}1905 1906	adapter->tx_queues_active = true;1907 1908	/* Since queues were stopped until now, there shouldn't be any1909	 * one in ibmvnic_complete_tx() or ibmvnic_xmit() so maybe we1910	 * don't need the synchronize_rcu()? Leaving it for consistency1911	 * with setting ->tx_queues_active = false.1912	 */1913	synchronize_rcu();1914 1915	netif_tx_start_all_queues(netdev);1916 1917	if (prev_state == VNIC_CLOSED) {1918		for (i = 0; i < adapter->req_rx_queues; i++)1919			napi_schedule(&adapter->napi[i]);1920	}1921 1922	adapter->state = VNIC_OPEN;1923	return rc;1924}1925 1926static int ibmvnic_open(struct net_device *netdev)1927{1928	struct ibmvnic_adapter *adapter = netdev_priv(netdev);1929	int rc;1930 1931	ASSERT_RTNL();1932 1933	/* If device failover is pending or we are about to reset, just set1934	 * device state and return. Device operation will be handled by reset1935	 * routine.1936	 *1937	 * It should be safe to overwrite the adapter->state here. Since1938	 * we hold the rtnl, either the reset has not actually started or1939	 * the rtnl got dropped during the set_link_state() in do_reset().1940	 * In the former case, no one else is changing the state (again we1941	 * have the rtnl) and in the latter case, do_reset() will detect and1942	 * honor our setting below.1943	 */1944	if (adapter->failover_pending || (test_bit(0, &adapter->resetting))) {1945		netdev_dbg(netdev, "[S:%s FOP:%d] Resetting, deferring open\n",1946			   adapter_state_to_string(adapter->state),1947			   adapter->failover_pending);1948		adapter->state = VNIC_OPEN;1949		rc = 0;1950		goto out;1951	}1952 1953	if (adapter->state != VNIC_CLOSED) {1954		rc = ibmvnic_login(netdev);1955		if (rc)1956			goto out;1957 1958		rc = init_resources(adapter);1959		if (rc) {1960			netdev_err(netdev, "failed to initialize resources\n");1961			goto out;1962		}1963	}1964 1965	rc = __ibmvnic_open(netdev);1966 1967out:1968	/* If open failed and there is a pending failover or in-progress reset,1969	 * set device state and return. Device operation will be handled by1970	 * reset routine. See also comments above regarding rtnl.1971	 */1972	if (rc &&1973	    (adapter->failover_pending || (test_bit(0, &adapter->resetting)))) {1974		adapter->state = VNIC_OPEN;1975		rc = 0;1976	}1977 1978	if (rc) {1979		release_resources(adapter);1980		release_rx_pools(adapter);1981		release_tx_pools(adapter);1982	}1983 1984	return rc;1985}1986 1987static void clean_rx_pools(struct ibmvnic_adapter *adapter)1988{1989	struct ibmvnic_rx_pool *rx_pool;1990	struct ibmvnic_rx_buff *rx_buff;1991	u64 rx_entries;1992	int rx_scrqs;1993	int i, j;1994 1995	if (!adapter->rx_pool)1996		return;1997 1998	rx_scrqs = adapter->num_active_rx_pools;1999	rx_entries = adapter->req_rx_add_entries_per_subcrq;2000 2001	/* Free any remaining skbs in the rx buffer pools */2002	for (i = 0; i < rx_scrqs; i++) {2003		rx_pool = &adapter->rx_pool[i];2004		if (!rx_pool || !rx_pool->rx_buff)2005			continue;2006 2007		netdev_dbg(adapter->netdev, "Cleaning rx_pool[%d]\n", i);2008		for (j = 0; j < rx_entries; j++) {2009			rx_buff = &rx_pool->rx_buff[j];2010			if (rx_buff && rx_buff->skb) {2011				dev_kfree_skb_any(rx_buff->skb);2012				rx_buff->skb = NULL;2013			}2014		}2015	}2016}2017 2018static void clean_one_tx_pool(struct ibmvnic_adapter *adapter,2019			      struct ibmvnic_tx_pool *tx_pool)2020{2021	struct ibmvnic_tx_buff *tx_buff;2022	u64 tx_entries;2023	int i;2024 2025	if (!tx_pool || !tx_pool->tx_buff)2026		return;2027 2028	tx_entries = tx_pool->num_buffers;2029 2030	for (i = 0; i < tx_entries; i++) {2031		tx_buff = &tx_pool->tx_buff[i];2032		if (tx_buff && tx_buff->skb) {2033			dev_kfree_skb_any(tx_buff->skb);2034			tx_buff->skb = NULL;2035		}2036	}2037}2038 2039static void clean_tx_pools(struct ibmvnic_adapter *adapter)2040{2041	int tx_scrqs;2042	int i;2043 2044	if (!adapter->tx_pool || !adapter->tso_pool)2045		return;2046 2047	tx_scrqs = adapter->num_active_tx_pools;2048 2049	/* Free any remaining skbs in the tx buffer pools */2050	for (i = 0; i < tx_scrqs; i++) {2051		netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i);2052		clean_one_tx_pool(adapter, &adapter->tx_pool[i]);2053		clean_one_tx_pool(adapter, &adapter->tso_pool[i]);2054	}2055}2056 2057static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter)2058{2059	struct net_device *netdev = adapter->netdev;2060	int i;2061 2062	if (adapter->tx_scrq) {2063		for (i = 0; i < adapter->req_tx_queues; i++)2064			if (adapter->tx_scrq[i]->irq) {2065				netdev_dbg(netdev,2066					   "Disabling tx_scrq[%d] irq\n", i);2067				disable_scrq_irq(adapter, adapter->tx_scrq[i]);2068				disable_irq(adapter->tx_scrq[i]->irq);2069			}2070	}2071 2072	if (adapter->rx_scrq) {2073		for (i = 0; i < adapter->req_rx_queues; i++) {2074			if (adapter->rx_scrq[i]->irq) {2075				netdev_dbg(netdev,2076					   "Disabling rx_scrq[%d] irq\n", i);2077				disable_scrq_irq(adapter, adapter->rx_scrq[i]);2078				disable_irq(adapter->rx_scrq[i]->irq);2079			}2080		}2081	}2082}2083 2084static void ibmvnic_cleanup(struct net_device *netdev)2085{2086	struct ibmvnic_adapter *adapter = netdev_priv(netdev);2087 2088	/* ensure that transmissions are stopped if called by do_reset */2089 2090	adapter->tx_queues_active = false;2091 2092	/* Ensure complete_tx() and ibmvnic_xmit() see ->tx_queues_active2093	 * update so they don't restart a queue after we stop it below.2094	 */2095	synchronize_rcu();2096 2097	if (test_bit(0, &adapter->resetting))2098		netif_tx_disable(netdev);2099	else2100		netif_tx_stop_all_queues(netdev);2101 2102	ibmvnic_napi_disable(adapter);2103	ibmvnic_disable_irqs(adapter);2104}2105 2106static int __ibmvnic_close(struct net_device *netdev)2107{2108	struct ibmvnic_adapter *adapter = netdev_priv(netdev);2109	int rc = 0;2110 2111	adapter->state = VNIC_CLOSING;2112	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);2113	adapter->state = VNIC_CLOSED;2114	return rc;2115}2116 2117static int ibmvnic_close(struct net_device *netdev)2118{2119	struct ibmvnic_adapter *adapter = netdev_priv(netdev);2120	int rc;2121 2122	netdev_dbg(netdev, "[S:%s FOP:%d FRR:%d] Closing\n",2123		   adapter_state_to_string(adapter->state),2124		   adapter->failover_pending,2125		   adapter->force_reset_recovery);2126 2127	/* If device failover is pending, just set device state and return.2128	 * Device operation will be handled by reset routine.2129	 */2130	if (adapter->failover_pending) {2131		adapter->state = VNIC_CLOSED;2132		return 0;2133	}2134 2135	rc = __ibmvnic_close(netdev);2136	ibmvnic_cleanup(netdev);2137	clean_rx_pools(adapter);2138	clean_tx_pools(adapter);2139 2140	return rc;2141}2142 2143/**2144 * get_hdr_lens - fills list of L2/L3/L4 hdr lens2145 * @hdr_field: bitfield determining needed headers2146 * @skb: socket buffer2147 * @hdr_len: array of header lengths to be filled2148 *2149 * Reads hdr_field to determine which headers are needed by firmware.2150 * Builds a buffer containing these headers.  Saves individual header2151 * lengths and total buffer length to be used to build descriptors.2152 *2153 * Return: total len of all headers2154 */2155static int get_hdr_lens(u8 hdr_field, struct sk_buff *skb,2156			int *hdr_len)2157{2158	int len = 0;2159 2160 2161	if ((hdr_field >> 6) & 1) {2162		hdr_len[0] = skb_mac_header_len(skb);2163		len += hdr_len[0];2164	}2165 2166	if ((hdr_field >> 5) & 1) {2167		hdr_len[1] = skb_network_header_len(skb);2168		len += hdr_len[1];2169	}2170 2171	if (!((hdr_field >> 4) & 1))2172		return len;2173 2174	if (skb->protocol == htons(ETH_P_IP)) {2175		if (ip_hdr(skb)->protocol == IPPROTO_TCP)2176			hdr_len[2] = tcp_hdrlen(skb);2177		else if (ip_hdr(skb)->protocol == IPPROTO_UDP)2178			hdr_len[2] = sizeof(struct udphdr);2179	} else if (skb->protocol == htons(ETH_P_IPV6)) {2180		if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)2181			hdr_len[2] = tcp_hdrlen(skb);2182		else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)2183			hdr_len[2] = sizeof(struct udphdr);2184	}2185 2186	return len + hdr_len[2];2187}2188 2189/**2190 * create_hdr_descs - create header and header extension descriptors2191 * @hdr_field: bitfield determining needed headers2192 * @hdr_data: buffer containing header data2193 * @len: length of data buffer2194 * @hdr_len: array of individual header lengths2195 * @scrq_arr: descriptor array2196 *2197 * Creates header and, if needed, header extension descriptors and2198 * places them in a descriptor array, scrq_arr2199 *2200 * Return: Number of header descs2201 */2202 2203static int create_hdr_descs(u8 hdr_field, u8 *hdr_data, int len, int *hdr_len,2204			    union sub_crq *scrq_arr)2205{2206	union sub_crq *hdr_desc;2207	int tmp_len = len;2208	int num_descs = 0;2209	u8 *data, *cur;2210	int tmp;2211 2212	while (tmp_len > 0) {2213		cur = hdr_data + len - tmp_len;2214 2215		hdr_desc = &scrq_arr[num_descs];2216		if (num_descs) {2217			data = hdr_desc->hdr_ext.data;2218			tmp = tmp_len > 29 ? 29 : tmp_len;2219			hdr_desc->hdr_ext.first = IBMVNIC_CRQ_CMD;2220			hdr_desc->hdr_ext.type = IBMVNIC_HDR_EXT_DESC;2221			hdr_desc->hdr_ext.len = tmp;2222		} else {2223			data = hdr_desc->hdr.data;2224			tmp = tmp_len > 24 ? 24 : tmp_len;2225			hdr_desc->hdr.first = IBMVNIC_CRQ_CMD;2226			hdr_desc->hdr.type = IBMVNIC_HDR_DESC;2227			hdr_desc->hdr.len = tmp;2228			hdr_desc->hdr.l2_len = (u8)hdr_len[0];2229			hdr_desc->hdr.l3_len = cpu_to_be16((u16)hdr_len[1]);2230			hdr_desc->hdr.l4_len = (u8)hdr_len[2];2231			hdr_desc->hdr.flag = hdr_field << 1;2232		}2233		memcpy(data, cur, tmp);2234		tmp_len -= tmp;2235		num_descs++;2236	}2237 2238	return num_descs;2239}2240 2241/**2242 * build_hdr_descs_arr - build a header descriptor array2243 * @skb: tx socket buffer2244 * @indir_arr: indirect array2245 * @num_entries: number of descriptors to be sent2246 * @hdr_field: bit field determining which headers will be sent2247 *2248 * This function will build a TX descriptor array with applicable2249 * L2/L3/L4 packet header descriptors to be sent by send_subcrq_indirect.2250 */2251 2252static void build_hdr_descs_arr(struct sk_buff *skb,2253				union sub_crq *indir_arr,2254				int *num_entries, u8 hdr_field)2255{2256	int hdr_len[3] = {0, 0, 0};2257	int tot_len;2258 2259	tot_len = get_hdr_lens(hdr_field, skb, hdr_len);2260	*num_entries += create_hdr_descs(hdr_field, skb_mac_header(skb),2261					 tot_len, hdr_len, indir_arr + 1);2262}2263 2264static int ibmvnic_xmit_workarounds(struct sk_buff *skb,2265				    struct net_device *netdev)2266{2267	/* For some backing devices, mishandling of small packets2268	 * can result in a loss of connection or TX stall. Device2269	 * architects recommend that no packet should be smaller2270	 * than the minimum MTU value provided to the driver, so2271	 * pad any packets to that length2272	 */2273	if (skb->len < netdev->min_mtu)2274		return skb_put_padto(skb, netdev->min_mtu);2275 2276	return 0;2277}2278 2279static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,2280					 struct ibmvnic_sub_crq_queue *tx_scrq)2281{2282	struct ibmvnic_ind_xmit_queue *ind_bufp;2283	struct ibmvnic_tx_buff *tx_buff;2284	struct ibmvnic_tx_pool *tx_pool;2285	union sub_crq tx_scrq_entry;2286	int queue_num;2287	int entries;2288	int index;2289	int i;2290 2291	ind_bufp = &tx_scrq->ind_buf;2292	entries = (u64)ind_bufp->index;2293	queue_num = tx_scrq->pool_index;2294 2295	for (i = entries - 1; i >= 0; --i) {2296		tx_scrq_entry = ind_bufp->indir_arr[i];2297		if (tx_scrq_entry.v1.type != IBMVNIC_TX_DESC)2298			continue;2299		index = be32_to_cpu(tx_scrq_entry.v1.correlator);2300		if (index & IBMVNIC_TSO_POOL_MASK) {2301			tx_pool = &adapter->tso_pool[queue_num];2302			index &= ~IBMVNIC_TSO_POOL_MASK;2303		} else {2304			tx_pool = &adapter->tx_pool[queue_num];2305		}2306		tx_pool->free_map[tx_pool->consumer_index] = index;2307		tx_pool->consumer_index = tx_pool->consumer_index == 0 ?2308					  tx_pool->num_buffers - 1 :2309					  tx_pool->consumer_index - 1;2310		tx_buff = &tx_pool->tx_buff[index];2311		adapter->netdev->stats.tx_packets--;2312		adapter->netdev->stats.tx_bytes -= tx_buff->skb->len;2313		adapter->tx_stats_buffers[queue_num].packets--;2314		adapter->tx_stats_buffers[queue_num].bytes -=2315						tx_buff->skb->len;2316		dev_kfree_skb_any(tx_buff->skb);2317		tx_buff->skb = NULL;2318		adapter->netdev->stats.tx_dropped++;2319	}2320 2321	ind_bufp->index = 0;2322 2323	if (atomic_sub_return(entries, &tx_scrq->used) <=2324	    (adapter->req_tx_entries_per_subcrq / 2) &&2325	    __netif_subqueue_stopped(adapter->netdev, queue_num)) {2326		rcu_read_lock();2327 2328		if (adapter->tx_queues_active) {2329			netif_wake_subqueue(adapter->netdev, queue_num);2330			netdev_dbg(adapter->netdev, "Started queue %d\n",2331				   queue_num);2332		}2333 2334		rcu_read_unlock();2335	}2336}2337 2338static int send_subcrq_direct(struct ibmvnic_adapter *adapter,2339			      u64 remote_handle, u64 *entry)2340{2341	unsigned int ua = adapter->vdev->unit_address;2342	struct device *dev = &adapter->vdev->dev;2343	int rc;2344 2345	/* Make sure the hypervisor sees the complete request */2346	dma_wmb();2347	rc = plpar_hcall_norets(H_SEND_SUB_CRQ, ua,2348				cpu_to_be64(remote_handle),2349				cpu_to_be64(entry[0]), cpu_to_be64(entry[1]),2350				cpu_to_be64(entry[2]), cpu_to_be64(entry[3]));2351 2352	if (rc)2353		print_subcrq_error(dev, rc, __func__);2354 2355	return rc;2356}2357 2358static int ibmvnic_tx_scrq_flush(struct ibmvnic_adapter *adapter,2359				 struct ibmvnic_sub_crq_queue *tx_scrq,2360				 bool indirect)2361{2362	struct ibmvnic_ind_xmit_queue *ind_bufp;2363	u64 dma_addr;2364	u64 entries;2365	u64 handle;2366	int rc;2367 2368	ind_bufp = &tx_scrq->ind_buf;2369	dma_addr = (u64)ind_bufp->indir_dma;2370	entries = (u64)ind_bufp->index;2371	handle = tx_scrq->handle;2372 2373	if (!entries)2374		return 0;2375 2376	if (indirect)2377		rc = send_subcrq_indirect(adapter, handle, dma_addr, entries);2378	else2379		rc = send_subcrq_direct(adapter, handle,2380					(u64 *)ind_bufp->indir_arr);2381 2382	if (rc)2383		ibmvnic_tx_scrq_clean_buffer(adapter, tx_scrq);2384	else2385		ind_bufp->index = 0;2386	return rc;2387}2388 2389static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev)2390{2391	struct ibmvnic_adapter *adapter = netdev_priv(netdev);2392	int queue_num = skb_get_queue_mapping(skb);2393	u8 *hdrs = (u8 *)&adapter->tx_rx_desc_req;2394	struct device *dev = &adapter->vdev->dev;2395	struct ibmvnic_ind_xmit_queue *ind_bufp;2396	struct ibmvnic_tx_buff *tx_buff = NULL;2397	struct ibmvnic_sub_crq_queue *tx_scrq;2398	struct ibmvnic_long_term_buff *ltb;2399	struct ibmvnic_tx_pool *tx_pool;2400	unsigned int tx_send_failed = 0;2401	netdev_tx_t ret = NETDEV_TX_OK;2402	unsigned int tx_map_failed = 0;2403	union sub_crq indir_arr[16];2404	unsigned int tx_dropped = 0;2405	unsigned int tx_packets = 0;2406	unsigned int tx_bytes = 0;2407	dma_addr_t data_dma_addr;2408	struct netdev_queue *txq;2409	unsigned long lpar_rc;2410	union sub_crq tx_crq;2411	unsigned int offset;2412	bool use_scrq_send_direct = false;2413	int num_entries = 1;2414	unsigned char *dst;2415	int bufidx = 0;2416	u8 proto = 0;2417 2418	/* If a reset is in progress, drop the packet since2419	 * the scrqs may get torn down. Otherwise use the2420	 * rcu to ensure reset waits for us to complete.2421	 */2422	rcu_read_lock();2423	if (!adapter->tx_queues_active) {2424		dev_kfree_skb_any(skb);2425 2426		tx_send_failed++;2427		tx_dropped++;2428		ret = NETDEV_TX_OK;2429		goto out;2430	}2431 2432	tx_scrq = adapter->tx_scrq[queue_num];2433	txq = netdev_get_tx_queue(netdev, queue_num);2434	ind_bufp = &tx_scrq->ind_buf;2435 2436	if (ibmvnic_xmit_workarounds(skb, netdev)) {2437		tx_dropped++;2438		tx_send_failed++;2439		ret = NETDEV_TX_OK;2440		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);2441		if (lpar_rc != H_SUCCESS)2442			goto tx_err;2443		goto out;2444	}2445 2446	if (skb_is_gso(skb))2447		tx_pool = &adapter->tso_pool[queue_num];2448	else2449		tx_pool = &adapter->tx_pool[queue_num];2450 2451	bufidx = tx_pool->free_map[tx_pool->consumer_index];2452 2453	if (bufidx == IBMVNIC_INVALID_MAP) {2454		dev_kfree_skb_any(skb);2455		tx_send_failed++;2456		tx_dropped++;2457		ret = NETDEV_TX_OK;2458		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);2459		if (lpar_rc != H_SUCCESS)2460			goto tx_err;2461		goto out;2462	}2463 2464	tx_pool->free_map[tx_pool->consumer_index] = IBMVNIC_INVALID_MAP;2465 2466	map_txpool_buf_to_ltb(tx_pool, bufidx, &ltb, &offset);2467 2468	dst = ltb->buff + offset;2469	memset(dst, 0, tx_pool->buf_size);2470	data_dma_addr = ltb->addr + offset;2471 2472	/* if we are going to send_subcrq_direct this then we need to2473	 * update the checksum before copying the data into ltb. Essentially2474	 * these packets force disable CSO so that we can guarantee that2475	 * FW does not need header info and we can send direct. Also, vnic2476	 * server must be able to xmit standard packets without header data2477	 */2478	if (*hdrs == 0 && !skb_is_gso(skb) &&2479	    !ind_bufp->index && !netdev_xmit_more()) {2480		use_scrq_send_direct = true;2481		if (skb->ip_summed == CHECKSUM_PARTIAL &&2482		    skb_checksum_help(skb))2483			use_scrq_send_direct = false;2484	}2485 2486	if (skb_shinfo(skb)->nr_frags) {2487		int cur, i;2488 2489		/* Copy the head */2490		skb_copy_from_linear_data(skb, dst, skb_headlen(skb));2491		cur = skb_headlen(skb);2492 2493		/* Copy the frags */2494		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {2495			const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];2496 2497			memcpy(dst + cur, skb_frag_address(frag),2498			       skb_frag_size(frag));2499			cur += skb_frag_size(frag);2500		}2501	} else {2502		skb_copy_from_linear_data(skb, dst, skb->len);2503	}2504 2505	tx_pool->consumer_index =2506	    (tx_pool->consumer_index + 1) % tx_pool->num_buffers;2507 2508	tx_buff = &tx_pool->tx_buff[bufidx];2509 2510	/* Sanity checks on our free map to make sure it points to an index2511	 * that is not being occupied by another skb. If skb memory is2512	 * not freed then we see congestion control kick in and halt tx.2513	 */2514	if (unlikely(tx_buff->skb)) {2515		dev_warn_ratelimited(dev, "TX free map points to untracked skb (%s %d idx=%d)\n",2516				     skb_is_gso(skb) ? "tso_pool" : "tx_pool",2517				     queue_num, bufidx);2518		dev_kfree_skb_any(tx_buff->skb);2519	}2520 2521	tx_buff->skb = skb;2522	tx_buff->index = bufidx;2523	tx_buff->pool_index = queue_num;2524 2525	memset(&tx_crq, 0, sizeof(tx_crq));2526	tx_crq.v1.first = IBMVNIC_CRQ_CMD;2527	tx_crq.v1.type = IBMVNIC_TX_DESC;2528	tx_crq.v1.n_crq_elem = 1;2529	tx_crq.v1.n_sge = 1;2530	tx_crq.v1.flags1 = IBMVNIC_TX_COMP_NEEDED;2531 2532	if (skb_is_gso(skb))2533		tx_crq.v1.correlator =2534			cpu_to_be32(bufidx | IBMVNIC_TSO_POOL_MASK);2535	else2536		tx_crq.v1.correlator = cpu_to_be32(bufidx);2537	tx_crq.v1.dma_reg = cpu_to_be16(ltb->map_id);2538	tx_crq.v1.sge_len = cpu_to_be32(skb->len);2539	tx_crq.v1.ioba = cpu_to_be64(data_dma_addr);2540 2541	if (adapter->vlan_header_insertion && skb_vlan_tag_present(skb)) {2542		tx_crq.v1.flags2 |= IBMVNIC_TX_VLAN_INSERT;2543		tx_crq.v1.vlan_id = cpu_to_be16(skb->vlan_tci);2544	}2545 2546	if (skb->protocol == htons(ETH_P_IP)) {2547		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV4;2548		proto = ip_hdr(skb)->protocol;2549	} else if (skb->protocol == htons(ETH_P_IPV6)) {2550		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_IPV6;2551		proto = ipv6_hdr(skb)->nexthdr;2552	}2553 2554	if (proto == IPPROTO_TCP)2555		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_TCP;2556	else if (proto == IPPROTO_UDP)2557		tx_crq.v1.flags1 |= IBMVNIC_TX_PROT_UDP;2558 2559	if (skb->ip_summed == CHECKSUM_PARTIAL) {2560		tx_crq.v1.flags1 |= IBMVNIC_TX_CHKSUM_OFFLOAD;2561		hdrs += 2;2562	}2563	if (skb_is_gso(skb)) {2564		tx_crq.v1.flags1 |= IBMVNIC_TX_LSO;2565		tx_crq.v1.mss = cpu_to_be16(skb_shinfo(skb)->gso_size);2566		hdrs += 2;2567	} else if (use_scrq_send_direct) {2568		/* See above comment, CSO disabled with direct xmit */2569		tx_crq.v1.flags1 &= ~(IBMVNIC_TX_CHKSUM_OFFLOAD);2570		ind_bufp->index = 1;2571		tx_buff->num_entries = 1;2572		netdev_tx_sent_queue(txq, skb->len);2573		ind_bufp->indir_arr[0] = tx_crq;2574		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, false);2575		if (lpar_rc != H_SUCCESS)2576			goto tx_err;2577 2578		goto early_exit;2579	}2580 2581	if ((*hdrs >> 7) & 1)2582		build_hdr_descs_arr(skb, indir_arr, &num_entries, *hdrs);2583 2584	tx_crq.v1.n_crq_elem = num_entries;2585	tx_buff->num_entries = num_entries;2586	/* flush buffer if current entry can not fit */2587	if (num_entries + ind_bufp->index > IBMVNIC_MAX_IND_DESCS) {2588		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);2589		if (lpar_rc != H_SUCCESS)2590			goto tx_flush_err;2591	}2592 2593	indir_arr[0] = tx_crq;2594	memcpy(&ind_bufp->indir_arr[ind_bufp->index], &indir_arr[0],2595	       num_entries * sizeof(struct ibmvnic_generic_scrq));2596 2597	ind_bufp->index += num_entries;2598	if (__netdev_tx_sent_queue(txq, skb->len,2599				   netdev_xmit_more() &&2600				   ind_bufp->index < IBMVNIC_MAX_IND_DESCS)) {2601		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);2602		if (lpar_rc != H_SUCCESS)2603			goto tx_err;2604	}2605 2606early_exit:2607	if (atomic_add_return(num_entries, &tx_scrq->used)2608					>= adapter->req_tx_entries_per_subcrq) {2609		netdev_dbg(netdev, "Stopping queue %d\n", queue_num);2610		netif_stop_subqueue(netdev, queue_num);2611	}2612 2613	tx_packets++;2614	tx_bytes += skb->len;2615	txq_trans_cond_update(txq);2616	ret = NETDEV_TX_OK;2617	goto out;2618 2619tx_flush_err:2620	dev_kfree_skb_any(skb);2621	tx_buff->skb = NULL;2622	tx_pool->consumer_index = tx_pool->consumer_index == 0 ?2623				  tx_pool->num_buffers - 1 :2624				  tx_pool->consumer_index - 1;2625	tx_dropped++;2626tx_err:2627	if (lpar_rc != H_CLOSED && lpar_rc != H_PARAMETER)2628		dev_err_ratelimited(dev, "tx: send failed\n");2629 2630	if (lpar_rc == H_CLOSED || adapter->failover_pending) {2631		/* Disable TX and report carrier off if queue is closed2632		 * or pending failover.2633		 * Firmware guarantees that a signal will be sent to the2634		 * driver, triggering a reset or some other action.2635		 */2636		netif_tx_stop_all_queues(netdev);2637		netif_carrier_off(netdev);2638	}2639out:2640	rcu_read_unlock();2641	netdev->stats.tx_dropped += tx_dropped;2642	netdev->stats.tx_bytes += tx_bytes;2643	netdev->stats.tx_packets += tx_packets;2644	adapter->tx_send_failed += tx_send_failed;2645	adapter->tx_map_failed += tx_map_failed;2646	adapter->tx_stats_buffers[queue_num].packets += tx_packets;2647	adapter->tx_stats_buffers[queue_num].bytes += tx_bytes;2648	adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped;2649 2650	return ret;2651}2652 2653static void ibmvnic_set_multi(struct net_device *netdev)2654{2655	struct ibmvnic_adapter *adapter = netdev_priv(netdev);2656	struct netdev_hw_addr *ha;2657	union ibmvnic_crq crq;2658 2659	memset(&crq, 0, sizeof(crq));2660	crq.request_capability.first = IBMVNIC_CRQ_CMD;2661	crq.request_capability.cmd = REQUEST_CAPABILITY;2662 2663	if (netdev->flags & IFF_PROMISC) {2664		if (!adapter->promisc_supported)2665			return;2666	} else {2667		if (netdev->flags & IFF_ALLMULTI) {2668			/* Accept all multicast */2669			memset(&crq, 0, sizeof(crq));2670			crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;2671			crq.multicast_ctrl.cmd = MULTICAST_CTRL;2672			crq.multicast_ctrl.flags = IBMVNIC_ENABLE_ALL;2673			ibmvnic_send_crq(adapter, &crq);2674		} else if (netdev_mc_empty(netdev)) {2675			/* Reject all multicast */2676			memset(&crq, 0, sizeof(crq));2677			crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;2678			crq.multicast_ctrl.cmd = MULTICAST_CTRL;2679			crq.multicast_ctrl.flags = IBMVNIC_DISABLE_ALL;2680			ibmvnic_send_crq(adapter, &crq);2681		} else {2682			/* Accept one or more multicast(s) */2683			netdev_for_each_mc_addr(ha, netdev) {2684				memset(&crq, 0, sizeof(crq));2685				crq.multicast_ctrl.first = IBMVNIC_CRQ_CMD;2686				crq.multicast_ctrl.cmd = MULTICAST_CTRL;2687				crq.multicast_ctrl.flags = IBMVNIC_ENABLE_MC;2688				ether_addr_copy(&crq.multicast_ctrl.mac_addr[0],2689						ha->addr);2690				ibmvnic_send_crq(adapter, &crq);2691			}2692		}2693	}2694}2695 2696static int __ibmvnic_set_mac(struct net_device *netdev, u8 *dev_addr)2697{2698	struct ibmvnic_adapter *adapter = netdev_priv(netdev);2699	union ibmvnic_crq crq;2700	int rc;2701 2702	if (!is_valid_ether_addr(dev_addr)) {2703		rc = -EADDRNOTAVAIL;2704		goto err;2705	}2706 2707	memset(&crq, 0, sizeof(crq));2708	crq.change_mac_addr.first = IBMVNIC_CRQ_CMD;2709	crq.change_mac_addr.cmd = CHANGE_MAC_ADDR;2710	ether_addr_copy(&crq.change_mac_addr.mac_addr[0], dev_addr);2711 2712	mutex_lock(&adapter->fw_lock);2713	adapter->fw_done_rc = 0;2714	reinit_completion(&adapter->fw_done);2715 2716	rc = ibmvnic_send_crq(adapter, &crq);2717	if (rc) {2718		rc = -EIO;2719		mutex_unlock(&adapter->fw_lock);2720		goto err;2721	}2722 2723	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);2724	/* netdev->dev_addr is changed in handle_change_mac_rsp function */2725	if (rc || adapter->fw_done_rc) {2726		rc = -EIO;2727		mutex_unlock(&adapter->fw_lock);2728		goto err;2729	}2730	mutex_unlock(&adapter->fw_lock);2731	return 0;2732err:2733	ether_addr_copy(adapter->mac_addr, netdev->dev_addr);2734	return rc;2735}2736 2737static int ibmvnic_set_mac(struct net_device *netdev, void *p)2738{2739	struct ibmvnic_adapter *adapter = netdev_priv(netdev);2740	struct sockaddr *addr = p;2741	int rc;2742 2743	rc = 0;2744	if (!is_valid_ether_addr(addr->sa_data))2745		return -EADDRNOTAVAIL;2746 2747	ether_addr_copy(adapter->mac_addr, addr->sa_data);2748	if (adapter->state != VNIC_PROBED)2749		rc = __ibmvnic_set_mac(netdev, addr->sa_data);2750 2751	return rc;2752}2753 2754static const char *reset_reason_to_string(enum ibmvnic_reset_reason reason)2755{2756	switch (reason) {2757	case VNIC_RESET_FAILOVER:2758		return "FAILOVER";2759	case VNIC_RESET_MOBILITY:2760		return "MOBILITY";2761	case VNIC_RESET_FATAL:2762		return "FATAL";2763	case VNIC_RESET_NON_FATAL:2764		return "NON_FATAL";2765	case VNIC_RESET_TIMEOUT:2766		return "TIMEOUT";2767	case VNIC_RESET_CHANGE_PARAM:2768		return "CHANGE_PARAM";2769	case VNIC_RESET_PASSIVE_INIT:2770		return "PASSIVE_INIT";2771	}2772	return "UNKNOWN";2773}2774 2775/*2776 * Initialize the init_done completion and return code values. We2777 * can get a transport event just after registering the CRQ and the2778 * tasklet will use this to communicate the transport event. To ensure2779 * we don't miss the notification/error, initialize these _before_2780 * regisering the CRQ.2781 */2782static inline void reinit_init_done(struct ibmvnic_adapter *adapter)2783{2784	reinit_completion(&adapter->init_done);2785	adapter->init_done_rc = 0;2786}2787 2788/*2789 * do_reset returns zero if we are able to keep processing reset events, or2790 * non-zero if we hit a fatal error and must halt.2791 */2792static int do_reset(struct ibmvnic_adapter *adapter,2793		    struct ibmvnic_rwi *rwi, u32 reset_state)2794{2795	struct net_device *netdev = adapter->netdev;2796	u64 old_num_rx_queues, old_num_tx_queues;2797	u64 old_num_rx_slots, old_num_tx_slots;2798	int rc;2799 2800	netdev_dbg(adapter->netdev,2801		   "[S:%s FOP:%d] Reset reason: %s, reset_state: %s\n",2802		   adapter_state_to_string(adapter->state),2803		   adapter->failover_pending,2804		   reset_reason_to_string(rwi->reset_reason),2805		   adapter_state_to_string(reset_state));2806 2807	adapter->reset_reason = rwi->reset_reason;2808	/* requestor of VNIC_RESET_CHANGE_PARAM already has the rtnl lock */2809	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))2810		rtnl_lock();2811 2812	/* Now that we have the rtnl lock, clear any pending failover.2813	 * This will ensure ibmvnic_open() has either completed or will2814	 * block until failover is complete.2815	 */2816	if (rwi->reset_reason == VNIC_RESET_FAILOVER)2817		adapter->failover_pending = false;2818 2819	/* read the state and check (again) after getting rtnl */2820	reset_state = adapter->state;2821 2822	if (reset_state == VNIC_REMOVING || reset_state == VNIC_REMOVED) {2823		rc = -EBUSY;2824		goto out;2825	}2826 2827	netif_carrier_off(netdev);2828 2829	old_num_rx_queues = adapter->req_rx_queues;2830	old_num_tx_queues = adapter->req_tx_queues;2831	old_num_rx_slots = adapter->req_rx_add_entries_per_subcrq;2832	old_num_tx_slots = adapter->req_tx_entries_per_subcrq;2833 2834	ibmvnic_cleanup(netdev);2835 2836	if (reset_state == VNIC_OPEN &&2837	    adapter->reset_reason != VNIC_RESET_MOBILITY &&2838	    adapter->reset_reason != VNIC_RESET_FAILOVER) {2839		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {2840			rc = __ibmvnic_close(netdev);2841			if (rc)2842				goto out;2843		} else {2844			adapter->state = VNIC_CLOSING;2845 2846			/* Release the RTNL lock before link state change and2847			 * re-acquire after the link state change to allow2848			 * linkwatch_event to grab the RTNL lock and run during2849			 * a reset.2850			 */2851			rtnl_unlock();2852			rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);2853			rtnl_lock();2854			if (rc)2855				goto out;2856 2857			if (adapter->state == VNIC_OPEN) {2858				/* When we dropped rtnl, ibmvnic_open() got2859				 * it and noticed that we are resetting and2860				 * set the adapter state to OPEN. Update our2861				 * new "target" state, and resume the reset2862				 * from VNIC_CLOSING state.2863				 */2864				netdev_dbg(netdev,2865					   "Open changed state from %s, updating.\n",2866					   adapter_state_to_string(reset_state));2867				reset_state = VNIC_OPEN;2868				adapter->state = VNIC_CLOSING;2869			}2870 2871			if (adapter->state != VNIC_CLOSING) {2872				/* If someone else changed the adapter state2873				 * when we dropped the rtnl, fail the reset2874				 */2875				rc = -EAGAIN;2876				goto out;2877			}2878			adapter->state = VNIC_CLOSED;2879		}2880	}2881 2882	if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {2883		release_resources(adapter);2884		release_sub_crqs(adapter, 1);2885		release_crq_queue(adapter);2886	}2887 2888	if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {2889		/* remove the closed state so when we call open it appears2890		 * we are coming from the probed state.2891		 */2892		adapter->state = VNIC_PROBED;2893 2894		reinit_init_done(adapter);2895 2896		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {2897			rc = init_crq_queue(adapter);2898		} else if (adapter->reset_reason == VNIC_RESET_MOBILITY) {2899			rc = ibmvnic_reenable_crq_queue(adapter);2900			release_sub_crqs(adapter, 1);2901		} else {2902			rc = ibmvnic_reset_crq(adapter);2903			if (rc == H_CLOSED || rc == H_SUCCESS) {2904				rc = vio_enable_interrupts(adapter->vdev);2905				if (rc)2906					netdev_err(adapter->netdev,2907						   "Reset failed to enable interrupts. rc=%d\n",2908						   rc);2909			}2910		}2911 2912		if (rc) {2913			netdev_err(adapter->netdev,2914				   "Reset couldn't initialize crq. rc=%d\n", rc);2915			goto out;2916		}2917 2918		rc = ibmvnic_reset_init(adapter, true);2919		if (rc)2920			goto out;2921 2922		/* If the adapter was in PROBE or DOWN state prior to the reset,2923		 * exit here.2924		 */2925		if (reset_state == VNIC_PROBED || reset_state == VNIC_DOWN) {2926			rc = 0;2927			goto out;2928		}2929 2930		rc = ibmvnic_login(netdev);2931		if (rc)2932			goto out;2933 2934		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {2935			rc = init_resources(adapter);2936			if (rc)2937				goto out;2938		} else if (adapter->req_rx_queues != old_num_rx_queues ||2939		    adapter->req_tx_queues != old_num_tx_queues ||2940		    adapter->req_rx_add_entries_per_subcrq !=2941		    old_num_rx_slots ||2942		    adapter->req_tx_entries_per_subcrq !=2943		    old_num_tx_slots ||2944		    !adapter->rx_pool ||2945		    !adapter->tso_pool ||2946		    !adapter->tx_pool) {2947			release_napi(adapter);2948			release_vpd_data(adapter);2949 2950			rc = init_resources(adapter);2951			if (rc)2952				goto out;2953 2954		} else {2955			rc = init_tx_pools(netdev);2956			if (rc) {2957				netdev_dbg(netdev,2958					   "init tx pools failed (%d)\n",2959					   rc);2960				goto out;2961			}2962 2963			rc = init_rx_pools(netdev);2964			if (rc) {2965				netdev_dbg(netdev,2966					   "init rx pools failed (%d)\n",2967					   rc);2968				goto out;2969			}2970		}2971		ibmvnic_disable_irqs(adapter);2972	}2973	adapter->state = VNIC_CLOSED;2974 2975	if (reset_state == VNIC_CLOSED) {2976		rc = 0;2977		goto out;2978	}2979 2980	rc = __ibmvnic_open(netdev);2981	if (rc) {2982		rc = IBMVNIC_OPEN_FAILED;2983		goto out;2984	}2985 2986	/* refresh device's multicast list */2987	ibmvnic_set_multi(netdev);2988 2989	if (adapter->reset_reason == VNIC_RESET_FAILOVER ||2990	    adapter->reset_reason == VNIC_RESET_MOBILITY)2991		__netdev_notify_peers(netdev);2992 2993	rc = 0;2994 2995out:2996	/* restore the adapter state if reset failed */2997	if (rc)2998		adapter->state = reset_state;2999	/* requestor of VNIC_RESET_CHANGE_PARAM should still hold the rtnl lock */3000	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))3001		rtnl_unlock();3002 3003	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Reset done, rc %d\n",3004		   adapter_state_to_string(adapter->state),3005		   adapter->failover_pending, rc);3006	return rc;3007}3008 3009static int do_hard_reset(struct ibmvnic_adapter *adapter,3010			 struct ibmvnic_rwi *rwi, u32 reset_state)3011{3012	struct net_device *netdev = adapter->netdev;3013	int rc;3014 3015	netdev_dbg(adapter->netdev, "Hard resetting driver (%s)\n",3016		   reset_reason_to_string(rwi->reset_reason));3017 3018	/* read the state and check (again) after getting rtnl */3019	reset_state = adapter->state;3020 3021	if (reset_state == VNIC_REMOVING || reset_state == VNIC_REMOVED) {3022		rc = -EBUSY;3023		goto out;3024	}3025 3026	netif_carrier_off(netdev);3027	adapter->reset_reason = rwi->reset_reason;3028 3029	ibmvnic_cleanup(netdev);3030	release_resources(adapter);3031	release_sub_crqs(adapter, 0);3032	release_crq_queue(adapter);3033 3034	/* remove the closed state so when we call open it appears3035	 * we are coming from the probed state.3036	 */3037	adapter->state = VNIC_PROBED;3038 3039	reinit_init_done(adapter);3040 3041	rc = init_crq_queue(adapter);3042	if (rc) {3043		netdev_err(adapter->netdev,3044			   "Couldn't initialize crq. rc=%d\n", rc);3045		goto out;3046	}3047 3048	rc = ibmvnic_reset_init(adapter, false);3049	if (rc)3050		goto out;3051 3052	/* If the adapter was in PROBE or DOWN state prior to the reset,3053	 * exit here.3054	 */3055	if (reset_state == VNIC_PROBED || reset_state == VNIC_DOWN)3056		goto out;3057 3058	rc = ibmvnic_login(netdev);3059	if (rc)3060		goto out;3061 3062	rc = init_resources(adapter);3063	if (rc)3064		goto out;3065 3066	ibmvnic_disable_irqs(adapter);3067	adapter->state = VNIC_CLOSED;3068 3069	if (reset_state == VNIC_CLOSED)3070		goto out;3071 3072	rc = __ibmvnic_open(netdev);3073	if (rc) {3074		rc = IBMVNIC_OPEN_FAILED;3075		goto out;3076	}3077 3078	__netdev_notify_peers(netdev);3079out:3080	/* restore adapter state if reset failed */3081	if (rc)3082		adapter->state = reset_state;3083	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Hard reset done, rc %d\n",3084		   adapter_state_to_string(adapter->state),3085		   adapter->failover_pending, rc);3086	return rc;3087}3088 3089static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)3090{3091	struct ibmvnic_rwi *rwi;3092	unsigned long flags;3093 3094	spin_lock_irqsave(&adapter->rwi_lock, flags);3095 3096	if (!list_empty(&adapter->rwi_list)) {3097		rwi = list_first_entry(&adapter->rwi_list, struct ibmvnic_rwi,3098				       list);3099		list_del(&rwi->list);3100	} else {3101		rwi = NULL;3102	}3103 3104	spin_unlock_irqrestore(&adapter->rwi_lock, flags);3105	return rwi;3106}3107 3108/**3109 * do_passive_init - complete probing when partner device is detected.3110 * @adapter: ibmvnic_adapter struct3111 *3112 * If the ibmvnic device does not have a partner device to communicate with at boot3113 * and that partner device comes online at a later time, this function is called3114 * to complete the initialization process of ibmvnic device.3115 * Caller is expected to hold rtnl_lock().3116 *3117 * Returns non-zero if sub-CRQs are not initialized properly leaving the device3118 * in the down state.3119 * Returns 0 upon success and the device is in PROBED state.3120 */3121 3122static int do_passive_init(struct ibmvnic_adapter *adapter)3123{3124	unsigned long timeout = msecs_to_jiffies(30000);3125	struct net_device *netdev = adapter->netdev;3126	struct device *dev = &adapter->vdev->dev;3127	int rc;3128 3129	netdev_dbg(netdev, "Partner device found, probing.\n");3130 3131	adapter->state = VNIC_PROBING;3132	reinit_completion(&adapter->init_done);3133	adapter->init_done_rc = 0;3134	adapter->crq.active = true;3135 3136	rc = send_crq_init_complete(adapter);3137	if (rc)3138		goto out;3139 3140	rc = send_version_xchg(adapter);3141	if (rc)3142		netdev_dbg(adapter->netdev, "send_version_xchg failed, rc=%d\n", rc);3143 3144	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {3145		dev_err(dev, "Initialization sequence timed out\n");3146		rc = -ETIMEDOUT;3147		goto out;3148	}3149 3150	rc = init_sub_crqs(adapter);3151	if (rc) {3152		dev_err(dev, "Initialization of sub crqs failed, rc=%d\n", rc);3153		goto out;3154	}3155 3156	rc = init_sub_crq_irqs(adapter);3157	if (rc) {3158		dev_err(dev, "Failed to initialize sub crq irqs\n, rc=%d", rc);3159		goto init_failed;3160	}3161 3162	netdev->mtu = adapter->req_mtu - ETH_HLEN;3163	netdev->min_mtu = adapter->min_mtu - ETH_HLEN;3164	netdev->max_mtu = adapter->max_mtu - ETH_HLEN;3165 3166	adapter->state = VNIC_PROBED;3167	netdev_dbg(netdev, "Probed successfully. Waiting for signal from partner device.\n");3168 3169	return 0;3170 3171init_failed:3172	release_sub_crqs(adapter, 1);3173out:3174	adapter->state = VNIC_DOWN;3175	return rc;3176}3177 3178static void __ibmvnic_reset(struct work_struct *work)3179{3180	struct ibmvnic_adapter *adapter;3181	unsigned int timeout = 5000;3182	struct ibmvnic_rwi *tmprwi;3183	bool saved_state = false;3184	struct ibmvnic_rwi *rwi;3185	unsigned long flags;3186	struct device *dev;3187	bool need_reset;3188	int num_fails = 0;3189	u32 reset_state;3190	int rc = 0;3191 3192	adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);3193		dev = &adapter->vdev->dev;3194 3195	/* Wait for ibmvnic_probe() to complete. If probe is taking too long3196	 * or if another reset is in progress, defer work for now. If probe3197	 * eventually fails it will flush and terminate our work.3198	 *3199	 * Three possibilities here:3200	 * 1. Adpater being removed  - just return3201	 * 2. Timed out on probe or another reset in progress - delay the work3202	 * 3. Completed probe - perform any resets in queue3203	 */3204	if (adapter->state == VNIC_PROBING &&3205	    !wait_for_completion_timeout(&adapter->probe_done, timeout)) {3206		dev_err(dev, "Reset thread timed out on probe");3207		queue_delayed_work(system_long_wq,3208				   &adapter->ibmvnic_delayed_reset,3209				   IBMVNIC_RESET_DELAY);3210		return;3211	}3212 3213	/* adapter is done with probe (i.e state is never VNIC_PROBING now) */3214	if (adapter->state == VNIC_REMOVING)3215		return;3216 3217	/* ->rwi_list is stable now (no one else is removing entries) */3218 3219	/* ibmvnic_probe() may have purged the reset queue after we were3220	 * scheduled to process a reset so there maybe no resets to process.3221	 * Before setting the ->resetting bit though, we have to make sure3222	 * that there is infact a reset to process. Otherwise we may race3223	 * with ibmvnic_open() and end up leaving the vnic down:3224	 *3225	 *	__ibmvnic_reset()	    ibmvnic_open()3226	 *	-----------------	    --------------3227	 *3228	 *  set ->resetting bit3229	 *  				find ->resetting bit is set3230	 *  				set ->state to IBMVNIC_OPEN (i.e3231	 *  				assume reset will open device)3232	 *  				return3233	 *  find reset queue empty3234	 *  return3235	 *3236	 *  	Neither performed vnic login/open and vnic stays down3237	 *3238	 * If we hold the lock and conditionally set the bit, either we3239	 * or ibmvnic_open() will complete the open.3240	 */3241	need_reset = false;3242	spin_lock(&adapter->rwi_lock);3243	if (!list_empty(&adapter->rwi_list)) {3244		if (test_and_set_bit_lock(0, &adapter->resetting)) {3245			queue_delayed_work(system_long_wq,3246					   &adapter->ibmvnic_delayed_reset,3247					   IBMVNIC_RESET_DELAY);3248		} else {3249			need_reset = true;3250		}3251	}3252	spin_unlock(&adapter->rwi_lock);3253 3254	if (!need_reset)3255		return;3256 3257	rwi = get_next_rwi(adapter);3258	while (rwi) {3259		spin_lock_irqsave(&adapter->state_lock, flags);3260 3261		if (adapter->state == VNIC_REMOVING ||3262		    adapter->state == VNIC_REMOVED) {3263			spin_unlock_irqrestore(&adapter->state_lock, flags);3264			kfree(rwi);3265			rc = EBUSY;3266			break;3267		}3268 3269		if (!saved_state) {3270			reset_state = adapter->state;3271			saved_state = true;3272		}3273		spin_unlock_irqrestore(&adapter->state_lock, flags);3274 3275		if (rwi->reset_reason == VNIC_RESET_PASSIVE_INIT) {3276			rtnl_lock();3277			rc = do_passive_init(adapter);3278			rtnl_unlock();3279			if (!rc)3280				netif_carrier_on(adapter->netdev);3281		} else if (adapter->force_reset_recovery) {3282			/* Since we are doing a hard reset now, clear the3283			 * failover_pending flag so we don't ignore any3284			 * future MOBILITY or other resets.3285			 */3286			adapter->failover_pending = false;3287 3288			/* Transport event occurred during previous reset */3289			if (adapter->wait_for_reset) {3290				/* Previous was CHANGE_PARAM; caller locked */3291				adapter->force_reset_recovery = false;3292				rc = do_hard_reset(adapter, rwi, reset_state);3293			} else {3294				rtnl_lock();3295				adapter->force_reset_recovery = false;3296				rc = do_hard_reset(adapter, rwi, reset_state);3297				rtnl_unlock();3298			}3299			if (rc)3300				num_fails++;3301			else3302				num_fails = 0;3303 3304			/* If auto-priority-failover is enabled we can get3305			 * back to back failovers during resets, resulting3306			 * in at least two failed resets (from high-priority3307			 * backing device to low-priority one and then back)3308			 * If resets continue to fail beyond that, give the3309			 * adapter some time to settle down before retrying.3310			 */3311			if (num_fails >= 3) {3312				netdev_dbg(adapter->netdev,3313					   "[S:%s] Hard reset failed %d times, waiting 60 secs\n",3314					   adapter_state_to_string(adapter->state),3315					   num_fails);3316				set_current_state(TASK_UNINTERRUPTIBLE);3317				schedule_timeout(60 * HZ);3318			}3319		} else {3320			rc = do_reset(adapter, rwi, reset_state);3321		}3322		tmprwi = rwi;3323		adapter->last_reset_time = jiffies;3324 3325		if (rc)3326			netdev_dbg(adapter->netdev, "Reset failed, rc=%d\n", rc);3327 3328		rwi = get_next_rwi(adapter);3329 3330		/*3331		 * If there are no resets queued and the previous reset failed,3332		 * the adapter would be in an undefined state. So retry the3333		 * previous reset as a hard reset.3334		 *3335		 * Else, free the previous rwi and, if there is another reset3336		 * queued, process the new reset even if previous reset failed3337		 * (the previous reset could have failed because of a fail3338		 * over for instance, so process the fail over).3339		 */3340		if (!rwi && rc)3341			rwi = tmprwi;3342		else3343			kfree(tmprwi);3344 3345		if (rwi && (rwi->reset_reason == VNIC_RESET_FAILOVER ||3346			    rwi->reset_reason == VNIC_RESET_MOBILITY || rc))3347			adapter->force_reset_recovery = true;3348	}3349 3350	if (adapter->wait_for_reset) {3351		adapter->reset_done_rc = rc;3352		complete(&adapter->reset_done);3353	}3354 3355	clear_bit_unlock(0, &adapter->resetting);3356 3357	netdev_dbg(adapter->netdev,3358		   "[S:%s FRR:%d WFR:%d] Done processing resets\n",3359		   adapter_state_to_string(adapter->state),3360		   adapter->force_reset_recovery,3361		   adapter->wait_for_reset);3362}3363 3364static void __ibmvnic_delayed_reset(struct work_struct *work)3365{3366	struct ibmvnic_adapter *adapter;3367 3368	adapter = container_of(work, struct ibmvnic_adapter,3369			       ibmvnic_delayed_reset.work);3370	__ibmvnic_reset(&adapter->ibmvnic_reset);3371}3372 3373static void flush_reset_queue(struct ibmvnic_adapter *adapter)3374{3375	struct list_head *entry, *tmp_entry;3376 3377	if (!list_empty(&adapter->rwi_list)) {3378		list_for_each_safe(entry, tmp_entry, &adapter->rwi_list) {3379			list_del(entry);3380			kfree(list_entry(entry, struct ibmvnic_rwi, list));3381		}3382	}3383}3384 3385static int ibmvnic_reset(struct ibmvnic_adapter *adapter,3386			 enum ibmvnic_reset_reason reason)3387{3388	struct net_device *netdev = adapter->netdev;3389	struct ibmvnic_rwi *rwi, *tmp;3390	unsigned long flags;3391	int ret;3392 3393	spin_lock_irqsave(&adapter->rwi_lock, flags);3394 3395	/* If failover is pending don't schedule any other reset.3396	 * Instead let the failover complete. If there is already a3397	 * a failover reset scheduled, we will detect and drop the3398	 * duplicate reset when walking the ->rwi_list below.3399	 */3400	if (adapter->state == VNIC_REMOVING ||3401	    adapter->state == VNIC_REMOVED ||3402	    (adapter->failover_pending && reason != VNIC_RESET_FAILOVER)) {3403		ret = EBUSY;3404		netdev_dbg(netdev, "Adapter removing or pending failover, skipping reset\n");3405		goto err;3406	}3407 3408	list_for_each_entry(tmp, &adapter->rwi_list, list) {3409		if (tmp->reset_reason == reason) {3410			netdev_dbg(netdev, "Skipping matching reset, reason=%s\n",3411				   reset_reason_to_string(reason));3412			ret = EBUSY;3413			goto err;3414		}3415	}3416 3417	rwi = kzalloc(sizeof(*rwi), GFP_ATOMIC);3418	if (!rwi) {3419		ret = ENOMEM;3420		goto err;3421	}3422	/* if we just received a transport event,3423	 * flush reset queue and process this reset3424	 */3425	if (adapter->force_reset_recovery)3426		flush_reset_queue(adapter);3427 3428	rwi->reset_reason = reason;3429	list_add_tail(&rwi->list, &adapter->rwi_list);3430	netdev_dbg(adapter->netdev, "Scheduling reset (reason %s)\n",3431		   reset_reason_to_string(reason));3432	queue_work(system_long_wq, &adapter->ibmvnic_reset);3433 3434	ret = 0;3435err:3436	/* ibmvnic_close() below can block, so drop the lock first */3437	spin_unlock_irqrestore(&adapter->rwi_lock, flags);3438 3439	if (ret == ENOMEM)3440		ibmvnic_close(netdev);3441 3442	return -ret;3443}3444 3445static void ibmvnic_tx_timeout(struct net_device *dev, unsigned int txqueue)3446{3447	struct ibmvnic_adapter *adapter = netdev_priv(dev);3448 3449	if (test_bit(0, &adapter->resetting)) {3450		netdev_err(adapter->netdev,3451			   "Adapter is resetting, skip timeout reset\n");3452		return;3453	}3454	/* No queuing up reset until at least 5 seconds (default watchdog val)3455	 * after last reset3456	 */3457	if (time_before(jiffies, (adapter->last_reset_time + dev->watchdog_timeo))) {3458		netdev_dbg(dev, "Not yet time to tx timeout.\n");3459		return;3460	}3461	ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT);3462}3463 3464static void remove_buff_from_pool(struct ibmvnic_adapter *adapter,3465				  struct ibmvnic_rx_buff *rx_buff)3466{3467	struct ibmvnic_rx_pool *pool = &adapter->rx_pool[rx_buff->pool_index];3468 3469	rx_buff->skb = NULL;3470 3471	pool->free_map[pool->next_alloc] = (int)(rx_buff - pool->rx_buff);3472	pool->next_alloc = (pool->next_alloc + 1) % pool->size;3473 3474	atomic_dec(&pool->available);3475}3476 3477static int ibmvnic_poll(struct napi_struct *napi, int budget)3478{3479	struct ibmvnic_sub_crq_queue *rx_scrq;3480	struct ibmvnic_adapter *adapter;3481	struct net_device *netdev;3482	int frames_processed;3483	int scrq_num;3484 3485	netdev = napi->dev;3486	adapter = netdev_priv(netdev);3487	scrq_num = (int)(napi - adapter->napi);3488	frames_processed = 0;3489	rx_scrq = adapter->rx_scrq[scrq_num];3490 3491restart_poll:3492	while (frames_processed < budget) {3493		struct sk_buff *skb;3494		struct ibmvnic_rx_buff *rx_buff;3495		union sub_crq *next;3496		u32 length;3497		u16 offset;3498		u8 flags = 0;3499 3500		if (unlikely(test_bit(0, &adapter->resetting) &&3501			     adapter->reset_reason != VNIC_RESET_NON_FATAL)) {3502			enable_scrq_irq(adapter, rx_scrq);3503			napi_complete_done(napi, frames_processed);3504			return frames_processed;3505		}3506 3507		if (!pending_scrq(adapter, rx_scrq))3508			break;3509		next = ibmvnic_next_scrq(adapter, rx_scrq);3510		rx_buff = (struct ibmvnic_rx_buff *)3511			  be64_to_cpu(next->rx_comp.correlator);3512		/* do error checking */3513		if (next->rx_comp.rc) {3514			netdev_dbg(netdev, "rx buffer returned with rc %x\n",3515				   be16_to_cpu(next->rx_comp.rc));3516			/* free the entry */3517			next->rx_comp.first = 0;3518			dev_kfree_skb_any(rx_buff->skb);3519			remove_buff_from_pool(adapter, rx_buff);3520			continue;3521		} else if (!rx_buff->skb) {3522			/* free the entry */3523			next->rx_comp.first = 0;3524			remove_buff_from_pool(adapter, rx_buff);3525			continue;3526		}3527 3528		length = be32_to_cpu(next->rx_comp.len);3529		offset = be16_to_cpu(next->rx_comp.off_frame_data);3530		flags = next->rx_comp.flags;3531		skb = rx_buff->skb;3532		/* load long_term_buff before copying to skb */3533		dma_rmb();3534		skb_copy_to_linear_data(skb, rx_buff->data + offset,3535					length);3536 3537		/* VLAN Header has been stripped by the system firmware and3538		 * needs to be inserted by the driver3539		 */3540		if (adapter->rx_vlan_header_insertion &&3541		    (flags & IBMVNIC_VLAN_STRIPPED))3542			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),3543					       ntohs(next->rx_comp.vlan_tci));3544 3545		/* free the entry */3546		next->rx_comp.first = 0;3547		remove_buff_from_pool(adapter, rx_buff);3548 3549		skb_put(skb, length);3550		skb->protocol = eth_type_trans(skb, netdev);3551		skb_record_rx_queue(skb, scrq_num);3552 3553		if (flags & IBMVNIC_IP_CHKSUM_GOOD &&3554		    flags & IBMVNIC_TCP_UDP_CHKSUM_GOOD) {3555			skb->ip_summed = CHECKSUM_UNNECESSARY;3556		}3557 3558		length = skb->len;3559		napi_gro_receive(napi, skb); /* send it up */3560		netdev->stats.rx_packets++;3561		netdev->stats.rx_bytes += length;3562		adapter->rx_stats_buffers[scrq_num].packets++;3563		adapter->rx_stats_buffers[scrq_num].bytes += length;3564		frames_processed++;3565	}3566 3567	if (adapter->state != VNIC_CLOSING &&3568	    (atomic_read(&adapter->rx_pool[scrq_num].available) <3569	      adapter->req_rx_add_entries_per_subcrq / 2))3570		replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]);3571	if (frames_processed < budget) {3572		if (napi_complete_done(napi, frames_processed)) {3573			enable_scrq_irq(adapter, rx_scrq);3574			if (pending_scrq(adapter, rx_scrq)) {3575				if (napi_schedule(napi)) {3576					disable_scrq_irq(adapter, rx_scrq);3577					goto restart_poll;3578				}3579			}3580		}3581	}3582	return frames_processed;3583}3584 3585static int wait_for_reset(struct ibmvnic_adapter *adapter)3586{3587	int rc, ret;3588 3589	adapter->fallback.mtu = adapter->req_mtu;3590	adapter->fallback.rx_queues = adapter->req_rx_queues;3591	adapter->fallback.tx_queues = adapter->req_tx_queues;3592	adapter->fallback.rx_entries = adapter->req_rx_add_entries_per_subcrq;3593	adapter->fallback.tx_entries = adapter->req_tx_entries_per_subcrq;3594 3595	reinit_completion(&adapter->reset_done);3596	adapter->wait_for_reset = true;3597	rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);3598 3599	if (rc) {3600		ret = rc;3601		goto out;3602	}3603	rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done, 60000);3604	if (rc) {3605		ret = -ENODEV;3606		goto out;3607	}3608 3609	ret = 0;3610	if (adapter->reset_done_rc) {3611		ret = -EIO;3612		adapter->desired.mtu = adapter->fallback.mtu;3613		adapter->desired.rx_queues = adapter->fallback.rx_queues;3614		adapter->desired.tx_queues = adapter->fallback.tx_queues;3615		adapter->desired.rx_entries = adapter->fallback.rx_entries;3616		adapter->desired.tx_entries = adapter->fallback.tx_entries;3617 3618		reinit_completion(&adapter->reset_done);3619		adapter->wait_for_reset = true;3620		rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);3621		if (rc) {3622			ret = rc;3623			goto out;3624		}3625		rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done,3626						 60000);3627		if (rc) {3628			ret = -ENODEV;3629			goto out;3630		}3631	}3632out:3633	adapter->wait_for_reset = false;3634 3635	return ret;3636}3637 3638static int ibmvnic_change_mtu(struct net_device *netdev, int new_mtu)3639{3640	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3641 3642	adapter->desired.mtu = new_mtu + ETH_HLEN;3643 3644	return wait_for_reset(adapter);3645}3646 3647static netdev_features_t ibmvnic_features_check(struct sk_buff *skb,3648						struct net_device *dev,3649						netdev_features_t features)3650{3651	/* Some backing hardware adapters can not3652	 * handle packets with a MSS less than 2243653	 * or with only one segment.3654	 */3655	if (skb_is_gso(skb)) {3656		if (skb_shinfo(skb)->gso_size < 224 ||3657		    skb_shinfo(skb)->gso_segs == 1)3658			features &= ~NETIF_F_GSO_MASK;3659	}3660 3661	return features;3662}3663 3664static const struct net_device_ops ibmvnic_netdev_ops = {3665	.ndo_open		= ibmvnic_open,3666	.ndo_stop		= ibmvnic_close,3667	.ndo_start_xmit		= ibmvnic_xmit,3668	.ndo_set_rx_mode	= ibmvnic_set_multi,3669	.ndo_set_mac_address	= ibmvnic_set_mac,3670	.ndo_validate_addr	= eth_validate_addr,3671	.ndo_tx_timeout		= ibmvnic_tx_timeout,3672	.ndo_change_mtu		= ibmvnic_change_mtu,3673	.ndo_features_check     = ibmvnic_features_check,3674};3675 3676/* ethtool functions */3677 3678static int ibmvnic_get_link_ksettings(struct net_device *netdev,3679				      struct ethtool_link_ksettings *cmd)3680{3681	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3682	int rc;3683 3684	rc = send_query_phys_parms(adapter);3685	if (rc) {3686		adapter->speed = SPEED_UNKNOWN;3687		adapter->duplex = DUPLEX_UNKNOWN;3688	}3689	cmd->base.speed = adapter->speed;3690	cmd->base.duplex = adapter->duplex;3691	cmd->base.port = PORT_FIBRE;3692	cmd->base.phy_address = 0;3693	cmd->base.autoneg = AUTONEG_ENABLE;3694 3695	return 0;3696}3697 3698static void ibmvnic_get_drvinfo(struct net_device *netdev,3699				struct ethtool_drvinfo *info)3700{3701	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3702 3703	strscpy(info->driver, ibmvnic_driver_name, sizeof(info->driver));3704	strscpy(info->version, IBMVNIC_DRIVER_VERSION, sizeof(info->version));3705	strscpy(info->fw_version, adapter->fw_version,3706		sizeof(info->fw_version));3707}3708 3709static u32 ibmvnic_get_msglevel(struct net_device *netdev)3710{3711	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3712 3713	return adapter->msg_enable;3714}3715 3716static void ibmvnic_set_msglevel(struct net_device *netdev, u32 data)3717{3718	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3719 3720	adapter->msg_enable = data;3721}3722 3723static u32 ibmvnic_get_link(struct net_device *netdev)3724{3725	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3726 3727	/* Don't need to send a query because we request a logical link up at3728	 * init and then we wait for link state indications3729	 */3730	return adapter->logical_link_state;3731}3732 3733static void ibmvnic_get_ringparam(struct net_device *netdev,3734				  struct ethtool_ringparam *ring,3735				  struct kernel_ethtool_ringparam *kernel_ring,3736				  struct netlink_ext_ack *extack)3737{3738	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3739 3740	ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;3741	ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;3742	ring->rx_mini_max_pending = 0;3743	ring->rx_jumbo_max_pending = 0;3744	ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;3745	ring->tx_pending = adapter->req_tx_entries_per_subcrq;3746	ring->rx_mini_pending = 0;3747	ring->rx_jumbo_pending = 0;3748}3749 3750static int ibmvnic_set_ringparam(struct net_device *netdev,3751				 struct ethtool_ringparam *ring,3752				 struct kernel_ethtool_ringparam *kernel_ring,3753				 struct netlink_ext_ack *extack)3754{3755	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3756 3757	if (ring->rx_pending > adapter->max_rx_add_entries_per_subcrq  ||3758	    ring->tx_pending > adapter->max_tx_entries_per_subcrq) {3759		netdev_err(netdev, "Invalid request.\n");3760		netdev_err(netdev, "Max tx buffers = %llu\n",3761			   adapter->max_rx_add_entries_per_subcrq);3762		netdev_err(netdev, "Max rx buffers = %llu\n",3763			   adapter->max_tx_entries_per_subcrq);3764		return -EINVAL;3765	}3766 3767	adapter->desired.rx_entries = ring->rx_pending;3768	adapter->desired.tx_entries = ring->tx_pending;3769 3770	return wait_for_reset(adapter);3771}3772 3773static void ibmvnic_get_channels(struct net_device *netdev,3774				 struct ethtool_channels *channels)3775{3776	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3777 3778	channels->max_rx = adapter->max_rx_queues;3779	channels->max_tx = adapter->max_tx_queues;3780	channels->max_other = 0;3781	channels->max_combined = 0;3782	channels->rx_count = adapter->req_rx_queues;3783	channels->tx_count = adapter->req_tx_queues;3784	channels->other_count = 0;3785	channels->combined_count = 0;3786}3787 3788static int ibmvnic_set_channels(struct net_device *netdev,3789				struct ethtool_channels *channels)3790{3791	struct ibmvnic_adapter *adapter = netdev_priv(netdev);3792 3793	adapter->desired.rx_queues = channels->rx_count;3794	adapter->desired.tx_queues = channels->tx_count;3795 3796	return wait_for_reset(adapter);3797}3798 3799static void ibmvnic_get_strings(struct net_device *dev, u32 stringset, u8 *data)3800{3801	struct ibmvnic_adapter *adapter = netdev_priv(dev);3802	int i;3803 3804	if (stringset != ETH_SS_STATS)3805		return;3806 3807	for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++, data += ETH_GSTRING_LEN)3808		memcpy(data, ibmvnic_stats[i].name, ETH_GSTRING_LEN);3809 3810	for (i = 0; i < adapter->req_tx_queues; i++) {3811		snprintf(data, ETH_GSTRING_LEN, "tx%d_packets", i);3812		data += ETH_GSTRING_LEN;3813 3814		snprintf(data, ETH_GSTRING_LEN, "tx%d_bytes", i);3815		data += ETH_GSTRING_LEN;3816 3817		snprintf(data, ETH_GSTRING_LEN, "tx%d_dropped_packets", i);3818		data += ETH_GSTRING_LEN;3819	}3820 3821	for (i = 0; i < adapter->req_rx_queues; i++) {3822		snprintf(data, ETH_GSTRING_LEN, "rx%d_packets", i);3823		data += ETH_GSTRING_LEN;3824 3825		snprintf(data, ETH_GSTRING_LEN, "rx%d_bytes", i);3826		data += ETH_GSTRING_LEN;3827 3828		snprintf(data, ETH_GSTRING_LEN, "rx%d_interrupts", i);3829		data += ETH_GSTRING_LEN;3830	}3831}3832 3833static int ibmvnic_get_sset_count(struct net_device *dev, int sset)3834{3835	struct ibmvnic_adapter *adapter = netdev_priv(dev);3836 3837	switch (sset) {3838	case ETH_SS_STATS:3839		return ARRAY_SIZE(ibmvnic_stats) +3840		       adapter->req_tx_queues * NUM_TX_STATS +3841		       adapter->req_rx_queues * NUM_RX_STATS;3842	default:3843		return -EOPNOTSUPP;3844	}3845}3846 3847static void ibmvnic_get_ethtool_stats(struct net_device *dev,3848				      struct ethtool_stats *stats, u64 *data)3849{3850	struct ibmvnic_adapter *adapter = netdev_priv(dev);3851	union ibmvnic_crq crq;3852	int i, j;3853	int rc;3854 3855	memset(&crq, 0, sizeof(crq));3856	crq.request_statistics.first = IBMVNIC_CRQ_CMD;3857	crq.request_statistics.cmd = REQUEST_STATISTICS;3858	crq.request_statistics.ioba = cpu_to_be32(adapter->stats_token);3859	crq.request_statistics.len =3860	    cpu_to_be32(sizeof(struct ibmvnic_statistics));3861 3862	/* Wait for data to be written */3863	reinit_completion(&adapter->stats_done);3864	rc = ibmvnic_send_crq(adapter, &crq);3865	if (rc)3866		return;3867	rc = ibmvnic_wait_for_completion(adapter, &adapter->stats_done, 10000);3868	if (rc)3869		return;3870 3871	for (i = 0; i < ARRAY_SIZE(ibmvnic_stats); i++)3872		data[i] = be64_to_cpu(IBMVNIC_GET_STAT3873				      (adapter, ibmvnic_stats[i].offset));3874 3875	for (j = 0; j < adapter->req_tx_queues; j++) {3876		data[i] = adapter->tx_stats_buffers[j].packets;3877		i++;3878		data[i] = adapter->tx_stats_buffers[j].bytes;3879		i++;3880		data[i] = adapter->tx_stats_buffers[j].dropped_packets;3881		i++;3882	}3883 3884	for (j = 0; j < adapter->req_rx_queues; j++) {3885		data[i] = adapter->rx_stats_buffers[j].packets;3886		i++;3887		data[i] = adapter->rx_stats_buffers[j].bytes;3888		i++;3889		data[i] = adapter->rx_stats_buffers[j].interrupts;3890		i++;3891	}3892}3893 3894static const struct ethtool_ops ibmvnic_ethtool_ops = {3895	.get_drvinfo		= ibmvnic_get_drvinfo,3896	.get_msglevel		= ibmvnic_get_msglevel,3897	.set_msglevel		= ibmvnic_set_msglevel,3898	.get_link		= ibmvnic_get_link,3899	.get_ringparam		= ibmvnic_get_ringparam,3900	.set_ringparam		= ibmvnic_set_ringparam,3901	.get_channels		= ibmvnic_get_channels,3902	.set_channels		= ibmvnic_set_channels,3903	.get_strings            = ibmvnic_get_strings,3904	.get_sset_count         = ibmvnic_get_sset_count,3905	.get_ethtool_stats	= ibmvnic_get_ethtool_stats,3906	.get_link_ksettings	= ibmvnic_get_link_ksettings,3907};3908 3909/* Routines for managing CRQs/sCRQs  */3910 3911static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter,3912				   struct ibmvnic_sub_crq_queue *scrq)3913{3914	int rc;3915 3916	if (!scrq) {3917		netdev_dbg(adapter->netdev, "Invalid scrq reset.\n");3918		return -EINVAL;3919	}3920 3921	if (scrq->irq) {3922		free_irq(scrq->irq, scrq);3923		irq_dispose_mapping(scrq->irq);3924		scrq->irq = 0;3925	}3926 3927	if (scrq->msgs) {3928		memset(scrq->msgs, 0, 4 * PAGE_SIZE);3929		atomic_set(&scrq->used, 0);3930		scrq->cur = 0;3931		scrq->ind_buf.index = 0;3932	} else {3933		netdev_dbg(adapter->netdev, "Invalid scrq reset\n");3934		return -EINVAL;3935	}3936 3937	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,3938			   4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);3939	return rc;3940}3941 3942static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)3943{3944	int i, rc;3945 3946	if (!adapter->tx_scrq || !adapter->rx_scrq)3947		return -EINVAL;3948 3949	ibmvnic_clean_affinity(adapter);3950 3951	for (i = 0; i < adapter->req_tx_queues; i++) {3952		netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);3953		rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);3954		if (rc)3955			return rc;3956	}3957 3958	for (i = 0; i < adapter->req_rx_queues; i++) {3959		netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i);3960		rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]);3961		if (rc)3962			return rc;3963	}3964 3965	return rc;3966}3967 3968static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,3969				  struct ibmvnic_sub_crq_queue *scrq,3970				  bool do_h_free)3971{3972	struct device *dev = &adapter->vdev->dev;3973	long rc;3974 3975	netdev_dbg(adapter->netdev, "Releasing sub-CRQ\n");3976 3977	if (do_h_free) {3978		/* Close the sub-crqs */3979		do {3980			rc = plpar_hcall_norets(H_FREE_SUB_CRQ,3981						adapter->vdev->unit_address,3982						scrq->crq_num);3983		} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));3984 3985		if (rc) {3986			netdev_err(adapter->netdev,3987				   "Failed to release sub-CRQ %16lx, rc = %ld\n",3988				   scrq->crq_num, rc);3989		}3990	}3991 3992	dma_free_coherent(dev,3993			  IBMVNIC_IND_ARR_SZ,3994			  scrq->ind_buf.indir_arr,3995			  scrq->ind_buf.indir_dma);3996 3997	dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,3998			 DMA_BIDIRECTIONAL);3999	free_pages((unsigned long)scrq->msgs, 2);4000	free_cpumask_var(scrq->affinity_mask);4001	kfree(scrq);4002}4003 4004static struct ibmvnic_sub_crq_queue *init_sub_crq_queue(struct ibmvnic_adapter4005							*adapter)4006{4007	struct device *dev = &adapter->vdev->dev;4008	struct ibmvnic_sub_crq_queue *scrq;4009	int rc;4010 4011	scrq = kzalloc(sizeof(*scrq), GFP_KERNEL);4012	if (!scrq)4013		return NULL;4014 4015	scrq->msgs =4016		(union sub_crq *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 2);4017	if (!scrq->msgs) {4018		dev_warn(dev, "Couldn't allocate crq queue messages page\n");4019		goto zero_page_failed;4020	}4021	if (!zalloc_cpumask_var(&scrq->affinity_mask, GFP_KERNEL))4022		goto cpumask_alloc_failed;4023 4024	scrq->msg_token = dma_map_single(dev, scrq->msgs, 4 * PAGE_SIZE,4025					 DMA_BIDIRECTIONAL);4026	if (dma_mapping_error(dev, scrq->msg_token)) {4027		dev_warn(dev, "Couldn't map crq queue messages page\n");4028		goto map_failed;4029	}4030 4031	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,4032			   4 * PAGE_SIZE, &scrq->crq_num, &scrq->hw_irq);4033 4034	if (rc == H_RESOURCE)4035		rc = ibmvnic_reset_crq(adapter);4036 4037	if (rc == H_CLOSED) {4038		dev_warn(dev, "Partner adapter not ready, waiting.\n");4039	} else if (rc) {4040		dev_warn(dev, "Error %d registering sub-crq\n", rc);4041		goto reg_failed;4042	}4043 4044	scrq->adapter = adapter;4045	scrq->size = 4 * PAGE_SIZE / sizeof(*scrq->msgs);4046	scrq->ind_buf.index = 0;4047 4048	scrq->ind_buf.indir_arr =4049		dma_alloc_coherent(dev,4050				   IBMVNIC_IND_ARR_SZ,4051				   &scrq->ind_buf.indir_dma,4052				   GFP_KERNEL);4053 4054	if (!scrq->ind_buf.indir_arr)4055		goto indir_failed;4056 4057	spin_lock_init(&scrq->lock);4058 4059	netdev_dbg(adapter->netdev,4060		   "sub-crq initialized, num %lx, hw_irq=%lx, irq=%x\n",4061		   scrq->crq_num, scrq->hw_irq, scrq->irq);4062 4063	return scrq;4064 4065indir_failed:4066	do {4067		rc = plpar_hcall_norets(H_FREE_SUB_CRQ,4068					adapter->vdev->unit_address,4069					scrq->crq_num);4070	} while (rc == H_BUSY || rc == H_IS_LONG_BUSY(rc));4071reg_failed:4072	dma_unmap_single(dev, scrq->msg_token, 4 * PAGE_SIZE,4073			 DMA_BIDIRECTIONAL);4074map_failed:4075	free_cpumask_var(scrq->affinity_mask);4076cpumask_alloc_failed:4077	free_pages((unsigned long)scrq->msgs, 2);4078zero_page_failed:4079	kfree(scrq);4080 4081	return NULL;4082}4083 4084static void release_sub_crqs(struct ibmvnic_adapter *adapter, bool do_h_free)4085{4086	int i;4087 4088	ibmvnic_clean_affinity(adapter);4089	if (adapter->tx_scrq) {4090		for (i = 0; i < adapter->num_active_tx_scrqs; i++) {4091			if (!adapter->tx_scrq[i])4092				continue;4093 4094			netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n",4095				   i);4096			ibmvnic_tx_scrq_clean_buffer(adapter, adapter->tx_scrq[i]);4097			if (adapter->tx_scrq[i]->irq) {4098				free_irq(adapter->tx_scrq[i]->irq,4099					 adapter->tx_scrq[i]);4100				irq_dispose_mapping(adapter->tx_scrq[i]->irq);4101				adapter->tx_scrq[i]->irq = 0;4102			}4103 4104			release_sub_crq_queue(adapter, adapter->tx_scrq[i],4105					      do_h_free);4106		}4107 4108		kfree(adapter->tx_scrq);4109		adapter->tx_scrq = NULL;4110		adapter->num_active_tx_scrqs = 0;4111	}4112 4113	/* Clean any remaining outstanding SKBs4114	 * we freed the irq so we won't be hearing4115	 * from them4116	 */4117	clean_tx_pools(adapter);4118 4119	if (adapter->rx_scrq) {4120		for (i = 0; i < adapter->num_active_rx_scrqs; i++) {4121			if (!adapter->rx_scrq[i])4122				continue;4123 4124			netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n",4125				   i);4126			if (adapter->rx_scrq[i]->irq) {4127				free_irq(adapter->rx_scrq[i]->irq,4128					 adapter->rx_scrq[i]);4129				irq_dispose_mapping(adapter->rx_scrq[i]->irq);4130				adapter->rx_scrq[i]->irq = 0;4131			}4132 4133			release_sub_crq_queue(adapter, adapter->rx_scrq[i],4134					      do_h_free);4135		}4136 4137		kfree(adapter->rx_scrq);4138		adapter->rx_scrq = NULL;4139		adapter->num_active_rx_scrqs = 0;4140	}4141}4142 4143static int disable_scrq_irq(struct ibmvnic_adapter *adapter,4144			    struct ibmvnic_sub_crq_queue *scrq)4145{4146	struct device *dev = &adapter->vdev->dev;4147	unsigned long rc;4148 4149	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,4150				H_DISABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);4151	if (rc)4152		dev_err(dev, "Couldn't disable scrq irq 0x%lx. rc=%ld\n",4153			scrq->hw_irq, rc);4154	return rc;4155}4156 4157/* We can not use the IRQ chip EOI handler because that has the4158 * unintended effect of changing the interrupt priority.4159 */4160static void ibmvnic_xics_eoi(struct device *dev, struct ibmvnic_sub_crq_queue *scrq)4161{4162	u64 val = 0xff000000 | scrq->hw_irq;4163	unsigned long rc;4164 4165	rc = plpar_hcall_norets(H_EOI, val);4166	if (rc)4167		dev_err(dev, "H_EOI FAILED irq 0x%llx. rc=%ld\n", val, rc);4168}4169 4170/* Due to a firmware bug, the hypervisor can send an interrupt to a4171 * transmit or receive queue just prior to a partition migration.4172 * Force an EOI after migration.4173 */4174static void ibmvnic_clear_pending_interrupt(struct device *dev,4175					    struct ibmvnic_sub_crq_queue *scrq)4176{4177	if (!xive_enabled())4178		ibmvnic_xics_eoi(dev, scrq);4179}4180 4181static int enable_scrq_irq(struct ibmvnic_adapter *adapter,4182			   struct ibmvnic_sub_crq_queue *scrq)4183{4184	struct device *dev = &adapter->vdev->dev;4185	unsigned long rc;4186 4187	if (scrq->hw_irq > 0x100000000ULL) {4188		dev_err(dev, "bad hw_irq = %lx\n", scrq->hw_irq);4189		return 1;4190	}4191 4192	if (test_bit(0, &adapter->resetting) &&4193	    adapter->reset_reason == VNIC_RESET_MOBILITY) {4194		ibmvnic_clear_pending_interrupt(dev, scrq);4195	}4196 4197	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,4198				H_ENABLE_VIO_INTERRUPT, scrq->hw_irq, 0, 0);4199	if (rc)4200		dev_err(dev, "Couldn't enable scrq irq 0x%lx. rc=%ld\n",4201			scrq->hw_irq, rc);4202	return rc;4203}4204 4205static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter,4206			       struct ibmvnic_sub_crq_queue *scrq)4207{4208	struct device *dev = &adapter->vdev->dev;4209	int num_packets = 0, total_bytes = 0;4210	struct ibmvnic_tx_pool *tx_pool;4211	struct ibmvnic_tx_buff *txbuff;4212	struct netdev_queue *txq;4213	union sub_crq *next;4214	int index, i;4215 4216restart_loop:4217	while (pending_scrq(adapter, scrq)) {4218		unsigned int pool = scrq->pool_index;4219		int num_entries = 0;4220		next = ibmvnic_next_scrq(adapter, scrq);4221		for (i = 0; i < next->tx_comp.num_comps; i++) {4222			index = be32_to_cpu(next->tx_comp.correlators[i]);4223			if (index & IBMVNIC_TSO_POOL_MASK) {4224				tx_pool = &adapter->tso_pool[pool];4225				index &= ~IBMVNIC_TSO_POOL_MASK;4226			} else {4227				tx_pool = &adapter->tx_pool[pool];4228			}4229 4230			txbuff = &tx_pool->tx_buff[index];4231			num_packets++;4232			num_entries += txbuff->num_entries;4233			if (txbuff->skb) {4234				total_bytes += txbuff->skb->len;4235				if (next->tx_comp.rcs[i]) {4236					dev_err(dev, "tx error %x\n",4237						next->tx_comp.rcs[i]);4238					dev_kfree_skb_irq(txbuff->skb);4239				} else {4240					dev_consume_skb_irq(txbuff->skb);4241				}4242				txbuff->skb = NULL;4243			} else {4244				netdev_warn(adapter->netdev,4245					    "TX completion received with NULL socket buffer\n");4246			}4247			tx_pool->free_map[tx_pool->producer_index] = index;4248			tx_pool->producer_index =4249				(tx_pool->producer_index + 1) %4250					tx_pool->num_buffers;4251		}4252		/* remove tx_comp scrq*/4253		next->tx_comp.first = 0;4254 4255 4256		if (atomic_sub_return(num_entries, &scrq->used) <=4257		    (adapter->req_tx_entries_per_subcrq / 2) &&4258		    __netif_subqueue_stopped(adapter->netdev,4259					     scrq->pool_index)) {4260			rcu_read_lock();4261			if (adapter->tx_queues_active) {4262				netif_wake_subqueue(adapter->netdev,4263						    scrq->pool_index);4264				netdev_dbg(adapter->netdev,4265					   "Started queue %d\n",4266					   scrq->pool_index);4267			}4268			rcu_read_unlock();4269		}4270	}4271 4272	enable_scrq_irq(adapter, scrq);4273 4274	if (pending_scrq(adapter, scrq)) {4275		disable_scrq_irq(adapter, scrq);4276		goto restart_loop;4277	}4278 4279	txq = netdev_get_tx_queue(adapter->netdev, scrq->pool_index);4280	netdev_tx_completed_queue(txq, num_packets, total_bytes);4281 4282	return 0;4283}4284 4285static irqreturn_t ibmvnic_interrupt_tx(int irq, void *instance)4286{4287	struct ibmvnic_sub_crq_queue *scrq = instance;4288	struct ibmvnic_adapter *adapter = scrq->adapter;4289 4290	disable_scrq_irq(adapter, scrq);4291	ibmvnic_complete_tx(adapter, scrq);4292 4293	return IRQ_HANDLED;4294}4295 4296static irqreturn_t ibmvnic_interrupt_rx(int irq, void *instance)4297{4298	struct ibmvnic_sub_crq_queue *scrq = instance;4299	struct ibmvnic_adapter *adapter = scrq->adapter;4300 4301	/* When booting a kdump kernel we can hit pending interrupts4302	 * prior to completing driver initialization.4303	 */4304	if (unlikely(adapter->state != VNIC_OPEN))4305		return IRQ_NONE;4306 4307	adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;4308 4309	if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {4310		disable_scrq_irq(adapter, scrq);4311		__napi_schedule(&adapter->napi[scrq->scrq_num]);4312	}4313 4314	return IRQ_HANDLED;4315}4316 4317static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)4318{4319	struct device *dev = &adapter->vdev->dev;4320	struct ibmvnic_sub_crq_queue *scrq;4321	int i = 0, j = 0;4322	int rc = 0;4323 4324	for (i = 0; i < adapter->req_tx_queues; i++) {4325		netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n",4326			   i);4327		scrq = adapter->tx_scrq[i];4328		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);4329 4330		if (!scrq->irq) {4331			rc = -EINVAL;4332			dev_err(dev, "Error mapping irq\n");4333			goto req_tx_irq_failed;4334		}4335 4336		snprintf(scrq->name, sizeof(scrq->name), "ibmvnic-%x-tx%d",4337			 adapter->vdev->unit_address, i);4338		rc = request_irq(scrq->irq, ibmvnic_interrupt_tx,4339				 0, scrq->name, scrq);4340 4341		if (rc) {4342			dev_err(dev, "Couldn't register tx irq 0x%x. rc=%d\n",4343				scrq->irq, rc);4344			irq_dispose_mapping(scrq->irq);4345			goto req_tx_irq_failed;4346		}4347	}4348 4349	for (i = 0; i < adapter->req_rx_queues; i++) {4350		netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n",4351			   i);4352		scrq = adapter->rx_scrq[i];4353		scrq->irq = irq_create_mapping(NULL, scrq->hw_irq);4354		if (!scrq->irq) {4355			rc = -EINVAL;4356			dev_err(dev, "Error mapping irq\n");4357			goto req_rx_irq_failed;4358		}4359		snprintf(scrq->name, sizeof(scrq->name), "ibmvnic-%x-rx%d",4360			 adapter->vdev->unit_address, i);4361		rc = request_irq(scrq->irq, ibmvnic_interrupt_rx,4362				 0, scrq->name, scrq);4363		if (rc) {4364			dev_err(dev, "Couldn't register rx irq 0x%x. rc=%d\n",4365				scrq->irq, rc);4366			irq_dispose_mapping(scrq->irq);4367			goto req_rx_irq_failed;4368		}4369	}4370 4371	cpus_read_lock();4372	ibmvnic_set_affinity(adapter);4373	cpus_read_unlock();4374 4375	return rc;4376 4377req_rx_irq_failed:4378	for (j = 0; j < i; j++) {4379		free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]);4380		irq_dispose_mapping(adapter->rx_scrq[j]->irq);4381	}4382	i = adapter->req_tx_queues;4383req_tx_irq_failed:4384	for (j = 0; j < i; j++) {4385		free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]);4386		irq_dispose_mapping(adapter->tx_scrq[j]->irq);4387	}4388	release_sub_crqs(adapter, 1);4389	return rc;4390}4391 4392static int init_sub_crqs(struct ibmvnic_adapter *adapter)4393{4394	struct device *dev = &adapter->vdev->dev;4395	struct ibmvnic_sub_crq_queue **allqueues;4396	int registered_queues = 0;4397	int total_queues;4398	int more = 0;4399	int i;4400 4401	total_queues = adapter->req_tx_queues + adapter->req_rx_queues;4402 4403	allqueues = kcalloc(total_queues, sizeof(*allqueues), GFP_KERNEL);4404	if (!allqueues)4405		return -ENOMEM;4406 4407	for (i = 0; i < total_queues; i++) {4408		allqueues[i] = init_sub_crq_queue(adapter);4409		if (!allqueues[i]) {4410			dev_warn(dev, "Couldn't allocate all sub-crqs\n");4411			break;4412		}4413		registered_queues++;4414	}4415 4416	/* Make sure we were able to register the minimum number of queues */4417	if (registered_queues <4418	    adapter->min_tx_queues + adapter->min_rx_queues) {4419		dev_err(dev, "Fatal: Couldn't init  min number of sub-crqs\n");4420		goto tx_failed;4421	}4422 4423	/* Distribute the failed allocated queues*/4424	for (i = 0; i < total_queues - registered_queues + more ; i++) {4425		netdev_dbg(adapter->netdev, "Reducing number of queues\n");4426		switch (i % 3) {4427		case 0:4428			if (adapter->req_rx_queues > adapter->min_rx_queues)4429				adapter->req_rx_queues--;4430			else4431				more++;4432			break;4433		case 1:4434			if (adapter->req_tx_queues > adapter->min_tx_queues)4435				adapter->req_tx_queues--;4436			else4437				more++;4438			break;4439		}4440	}4441 4442	adapter->tx_scrq = kcalloc(adapter->req_tx_queues,4443				   sizeof(*adapter->tx_scrq), GFP_KERNEL);4444	if (!adapter->tx_scrq)4445		goto tx_failed;4446 4447	for (i = 0; i < adapter->req_tx_queues; i++) {4448		adapter->tx_scrq[i] = allqueues[i];4449		adapter->tx_scrq[i]->pool_index = i;4450		adapter->num_active_tx_scrqs++;4451	}4452 4453	adapter->rx_scrq = kcalloc(adapter->req_rx_queues,4454				   sizeof(*adapter->rx_scrq), GFP_KERNEL);4455	if (!adapter->rx_scrq)4456		goto rx_failed;4457 4458	for (i = 0; i < adapter->req_rx_queues; i++) {4459		adapter->rx_scrq[i] = allqueues[i + adapter->req_tx_queues];4460		adapter->rx_scrq[i]->scrq_num = i;4461		adapter->num_active_rx_scrqs++;4462	}4463 4464	kfree(allqueues);4465	return 0;4466 4467rx_failed:4468	kfree(adapter->tx_scrq);4469	adapter->tx_scrq = NULL;4470tx_failed:4471	for (i = 0; i < registered_queues; i++)4472		release_sub_crq_queue(adapter, allqueues[i], 1);4473	kfree(allqueues);4474	return -ENOMEM;4475}4476 4477static void send_request_cap(struct ibmvnic_adapter *adapter, int retry)4478{4479	struct device *dev = &adapter->vdev->dev;4480	union ibmvnic_crq crq;4481	int max_entries;4482	int cap_reqs;4483 4484	/* We send out 6 or 7 REQUEST_CAPABILITY CRQs below (depending on4485	 * the PROMISC flag). Initialize this count upfront. When the tasklet4486	 * receives a response to all of these, it will send the next protocol4487	 * message (QUERY_IP_OFFLOAD).4488	 */4489	if (!(adapter->netdev->flags & IFF_PROMISC) ||4490	    adapter->promisc_supported)4491		cap_reqs = 7;4492	else4493		cap_reqs = 6;4494 4495	if (!retry) {4496		/* Sub-CRQ entries are 32 byte long */4497		int entries_page = 4 * PAGE_SIZE / (sizeof(u64) * 4);4498 4499		atomic_set(&adapter->running_cap_crqs, cap_reqs);4500 4501		if (adapter->min_tx_entries_per_subcrq > entries_page ||4502		    adapter->min_rx_add_entries_per_subcrq > entries_page) {4503			dev_err(dev, "Fatal, invalid entries per sub-crq\n");4504			return;4505		}4506 4507		if (adapter->desired.mtu)4508			adapter->req_mtu = adapter->desired.mtu;4509		else4510			adapter->req_mtu = adapter->netdev->mtu + ETH_HLEN;4511 4512		if (!adapter->desired.tx_entries)4513			adapter->desired.tx_entries =4514					adapter->max_tx_entries_per_subcrq;4515		if (!adapter->desired.rx_entries)4516			adapter->desired.rx_entries =4517					adapter->max_rx_add_entries_per_subcrq;4518 4519		max_entries = IBMVNIC_LTB_SET_SIZE /4520			      (adapter->req_mtu + IBMVNIC_BUFFER_HLEN);4521 4522		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *4523			adapter->desired.tx_entries > IBMVNIC_LTB_SET_SIZE) {4524			adapter->desired.tx_entries = max_entries;4525		}4526 4527		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *4528			adapter->desired.rx_entries > IBMVNIC_LTB_SET_SIZE) {4529			adapter->desired.rx_entries = max_entries;4530		}4531 4532		if (adapter->desired.tx_entries)4533			adapter->req_tx_entries_per_subcrq =4534					adapter->desired.tx_entries;4535		else4536			adapter->req_tx_entries_per_subcrq =4537					adapter->max_tx_entries_per_subcrq;4538 4539		if (adapter->desired.rx_entries)4540			adapter->req_rx_add_entries_per_subcrq =4541					adapter->desired.rx_entries;4542		else4543			adapter->req_rx_add_entries_per_subcrq =4544					adapter->max_rx_add_entries_per_subcrq;4545 4546		if (adapter->desired.tx_queues)4547			adapter->req_tx_queues =4548					adapter->desired.tx_queues;4549		else4550			adapter->req_tx_queues =4551					adapter->opt_tx_comp_sub_queues;4552 4553		if (adapter->desired.rx_queues)4554			adapter->req_rx_queues =4555					adapter->desired.rx_queues;4556		else4557			adapter->req_rx_queues =4558					adapter->opt_rx_comp_queues;4559 4560		adapter->req_rx_add_queues = adapter->max_rx_add_queues;4561	} else {4562		atomic_add(cap_reqs, &adapter->running_cap_crqs);4563	}4564	memset(&crq, 0, sizeof(crq));4565	crq.request_capability.first = IBMVNIC_CRQ_CMD;4566	crq.request_capability.cmd = REQUEST_CAPABILITY;4567 4568	crq.request_capability.capability = cpu_to_be16(REQ_TX_QUEUES);4569	crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues);4570	cap_reqs--;4571	ibmvnic_send_crq(adapter, &crq);4572 4573	crq.request_capability.capability = cpu_to_be16(REQ_RX_QUEUES);4574	crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues);4575	cap_reqs--;4576	ibmvnic_send_crq(adapter, &crq);4577 4578	crq.request_capability.capability = cpu_to_be16(REQ_RX_ADD_QUEUES);4579	crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues);4580	cap_reqs--;4581	ibmvnic_send_crq(adapter, &crq);4582 4583	crq.request_capability.capability =4584	    cpu_to_be16(REQ_TX_ENTRIES_PER_SUBCRQ);4585	crq.request_capability.number =4586	    cpu_to_be64(adapter->req_tx_entries_per_subcrq);4587	cap_reqs--;4588	ibmvnic_send_crq(adapter, &crq);4589 4590	crq.request_capability.capability =4591	    cpu_to_be16(REQ_RX_ADD_ENTRIES_PER_SUBCRQ);4592	crq.request_capability.number =4593	    cpu_to_be64(adapter->req_rx_add_entries_per_subcrq);4594	cap_reqs--;4595	ibmvnic_send_crq(adapter, &crq);4596 4597	crq.request_capability.capability = cpu_to_be16(REQ_MTU);4598	crq.request_capability.number = cpu_to_be64(adapter->req_mtu);4599	cap_reqs--;4600	ibmvnic_send_crq(adapter, &crq);4601 4602	if (adapter->netdev->flags & IFF_PROMISC) {4603		if (adapter->promisc_supported) {4604			crq.request_capability.capability =4605			    cpu_to_be16(PROMISC_REQUESTED);4606			crq.request_capability.number = cpu_to_be64(1);4607			cap_reqs--;4608			ibmvnic_send_crq(adapter, &crq);4609		}4610	} else {4611		crq.request_capability.capability =4612		    cpu_to_be16(PROMISC_REQUESTED);4613		crq.request_capability.number = cpu_to_be64(0);4614		cap_reqs--;4615		ibmvnic_send_crq(adapter, &crq);4616	}4617 4618	/* Keep at end to catch any discrepancy between expected and actual4619	 * CRQs sent.4620	 */4621	WARN_ON(cap_reqs != 0);4622}4623 4624static int pending_scrq(struct ibmvnic_adapter *adapter,4625			struct ibmvnic_sub_crq_queue *scrq)4626{4627	union sub_crq *entry = &scrq->msgs[scrq->cur];4628	int rc;4629 4630	rc = !!(entry->generic.first & IBMVNIC_CRQ_CMD_RSP);4631 4632	/* Ensure that the SCRQ valid flag is loaded prior to loading the4633	 * contents of the SCRQ descriptor4634	 */4635	dma_rmb();4636 4637	return rc;4638}4639 4640static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *adapter,4641					struct ibmvnic_sub_crq_queue *scrq)4642{4643	union sub_crq *entry;4644	unsigned long flags;4645 4646	spin_lock_irqsave(&scrq->lock, flags);4647	entry = &scrq->msgs[scrq->cur];4648	if (entry->generic.first & IBMVNIC_CRQ_CMD_RSP) {4649		if (++scrq->cur == scrq->size)4650			scrq->cur = 0;4651	} else {4652		entry = NULL;4653	}4654	spin_unlock_irqrestore(&scrq->lock, flags);4655 4656	/* Ensure that the SCRQ valid flag is loaded prior to loading the4657	 * contents of the SCRQ descriptor4658	 */4659	dma_rmb();4660 4661	return entry;4662}4663 4664static union ibmvnic_crq *ibmvnic_next_crq(struct ibmvnic_adapter *adapter)4665{4666	struct ibmvnic_crq_queue *queue = &adapter->crq;4667	union ibmvnic_crq *crq;4668 4669	crq = &queue->msgs[queue->cur];4670	if (crq->generic.first & IBMVNIC_CRQ_CMD_RSP) {4671		if (++queue->cur == queue->size)4672			queue->cur = 0;4673	} else {4674		crq = NULL;4675	}4676 4677	return crq;4678}4679 4680static void print_subcrq_error(struct device *dev, int rc, const char *func)4681{4682	switch (rc) {4683	case H_PARAMETER:4684		dev_warn_ratelimited(dev,4685				     "%s failed: Send request is malformed or adapter failover pending. (rc=%d)\n",4686				     func, rc);4687		break;4688	case H_CLOSED:4689		dev_warn_ratelimited(dev,4690				     "%s failed: Backing queue closed. Adapter is down or failover pending. (rc=%d)\n",4691				     func, rc);4692		break;4693	default:4694		dev_err_ratelimited(dev, "%s failed: (rc=%d)\n", func, rc);4695		break;4696	}4697}4698 4699static int send_subcrq_indirect(struct ibmvnic_adapter *adapter,4700				u64 remote_handle, u64 ioba, u64 num_entries)4701{4702	unsigned int ua = adapter->vdev->unit_address;4703	struct device *dev = &adapter->vdev->dev;4704	int rc;4705 4706	/* Make sure the hypervisor sees the complete request */4707	dma_wmb();4708	rc = plpar_hcall_norets(H_SEND_SUB_CRQ_INDIRECT, ua,4709				cpu_to_be64(remote_handle),4710				ioba, num_entries);4711 4712	if (rc)4713		print_subcrq_error(dev, rc, __func__);4714 4715	return rc;4716}4717 4718static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,4719			    union ibmvnic_crq *crq)4720{4721	unsigned int ua = adapter->vdev->unit_address;4722	struct device *dev = &adapter->vdev->dev;4723	u64 *u64_crq = (u64 *)crq;4724	int rc;4725 4726	netdev_dbg(adapter->netdev, "Sending CRQ: %016lx %016lx\n",4727		   (unsigned long)cpu_to_be64(u64_crq[0]),4728		   (unsigned long)cpu_to_be64(u64_crq[1]));4729 4730	if (!adapter->crq.active &&4731	    crq->generic.first != IBMVNIC_CRQ_INIT_CMD) {4732		dev_warn(dev, "Invalid request detected while CRQ is inactive, possible device state change during reset\n");4733		return -EINVAL;4734	}4735 4736	/* Make sure the hypervisor sees the complete request */4737	dma_wmb();4738 4739	rc = plpar_hcall_norets(H_SEND_CRQ, ua,4740				cpu_to_be64(u64_crq[0]),4741				cpu_to_be64(u64_crq[1]));4742 4743	if (rc) {4744		if (rc == H_CLOSED) {4745			dev_warn(dev, "CRQ Queue closed\n");4746			/* do not reset, report the fail, wait for passive init from server */4747		}4748 4749		dev_warn(dev, "Send error (rc=%d)\n", rc);4750	}4751 4752	return rc;4753}4754 4755static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter)4756{4757	struct device *dev = &adapter->vdev->dev;4758	union ibmvnic_crq crq;4759	int retries = 100;4760	int rc;4761 4762	memset(&crq, 0, sizeof(crq));4763	crq.generic.first = IBMVNIC_CRQ_INIT_CMD;4764	crq.generic.cmd = IBMVNIC_CRQ_INIT;4765	netdev_dbg(adapter->netdev, "Sending CRQ init\n");4766 4767	do {4768		rc = ibmvnic_send_crq(adapter, &crq);4769		if (rc != H_CLOSED)4770			break;4771		retries--;4772		msleep(50);4773 4774	} while (retries > 0);4775 4776	if (rc) {4777		dev_err(dev, "Failed to send init request, rc = %d\n", rc);4778		return rc;4779	}4780 4781	return 0;4782}4783 4784struct vnic_login_client_data {4785	u8	type;4786	__be16	len;4787	char	name[];4788} __packed;4789 4790static int vnic_client_data_len(struct ibmvnic_adapter *adapter)4791{4792	int len;4793 4794	/* Calculate the amount of buffer space needed for the4795	 * vnic client data in the login buffer. There are four entries,4796	 * OS name, LPAR name, device name, and a null last entry.4797	 */4798	len = 4 * sizeof(struct vnic_login_client_data);4799	len += 6; /* "Linux" plus NULL */4800	len += strlen(utsname()->nodename) + 1;4801	len += strlen(adapter->netdev->name) + 1;4802 4803	return len;4804}4805 4806static void vnic_add_client_data(struct ibmvnic_adapter *adapter,4807				 struct vnic_login_client_data *vlcd)4808{4809	const char *os_name = "Linux";4810	int len;4811 4812	/* Type 1 - LPAR OS */4813	vlcd->type = 1;4814	len = strlen(os_name) + 1;4815	vlcd->len = cpu_to_be16(len);4816	strscpy(vlcd->name, os_name, len);4817	vlcd = (struct vnic_login_client_data *)(vlcd->name + len);4818 4819	/* Type 2 - LPAR name */4820	vlcd->type = 2;4821	len = strlen(utsname()->nodename) + 1;4822	vlcd->len = cpu_to_be16(len);4823	strscpy(vlcd->name, utsname()->nodename, len);4824	vlcd = (struct vnic_login_client_data *)(vlcd->name + len);4825 4826	/* Type 3 - device name */4827	vlcd->type = 3;4828	len = strlen(adapter->netdev->name) + 1;4829	vlcd->len = cpu_to_be16(len);4830	strscpy(vlcd->name, adapter->netdev->name, len);4831}4832 4833static int send_login(struct ibmvnic_adapter *adapter)4834{4835	struct ibmvnic_login_rsp_buffer *login_rsp_buffer;4836	struct ibmvnic_login_buffer *login_buffer;4837	struct device *dev = &adapter->vdev->dev;4838	struct vnic_login_client_data *vlcd;4839	dma_addr_t rsp_buffer_token;4840	dma_addr_t buffer_token;4841	size_t rsp_buffer_size;4842	union ibmvnic_crq crq;4843	int client_data_len;4844	size_t buffer_size;4845	__be64 *tx_list_p;4846	__be64 *rx_list_p;4847	int rc;4848	int i;4849 4850	if (!adapter->tx_scrq || !adapter->rx_scrq) {4851		netdev_err(adapter->netdev,4852			   "RX or TX queues are not allocated, device login failed\n");4853		return -ENOMEM;4854	}4855 4856	release_login_buffer(adapter);4857	release_login_rsp_buffer(adapter);4858 4859	client_data_len = vnic_client_data_len(adapter);4860 4861	buffer_size =4862	    sizeof(struct ibmvnic_login_buffer) +4863	    sizeof(u64) * (adapter->req_tx_queues + adapter->req_rx_queues) +4864	    client_data_len;4865 4866	login_buffer = kzalloc(buffer_size, GFP_ATOMIC);4867	if (!login_buffer)4868		goto buf_alloc_failed;4869 4870	buffer_token = dma_map_single(dev, login_buffer, buffer_size,4871				      DMA_TO_DEVICE);4872	if (dma_mapping_error(dev, buffer_token)) {4873		dev_err(dev, "Couldn't map login buffer\n");4874		goto buf_map_failed;4875	}4876 4877	rsp_buffer_size = sizeof(struct ibmvnic_login_rsp_buffer) +4878			  sizeof(u64) * adapter->req_tx_queues +4879			  sizeof(u64) * adapter->req_rx_queues +4880			  sizeof(u64) * adapter->req_rx_queues +4881			  sizeof(u8) * IBMVNIC_TX_DESC_VERSIONS;4882 4883	login_rsp_buffer = kmalloc(rsp_buffer_size, GFP_ATOMIC);4884	if (!login_rsp_buffer)4885		goto buf_rsp_alloc_failed;4886 4887	rsp_buffer_token = dma_map_single(dev, login_rsp_buffer,4888					  rsp_buffer_size, DMA_FROM_DEVICE);4889	if (dma_mapping_error(dev, rsp_buffer_token)) {4890		dev_err(dev, "Couldn't map login rsp buffer\n");4891		goto buf_rsp_map_failed;4892	}4893 4894	adapter->login_buf = login_buffer;4895	adapter->login_buf_token = buffer_token;4896	adapter->login_buf_sz = buffer_size;4897	adapter->login_rsp_buf = login_rsp_buffer;4898	adapter->login_rsp_buf_token = rsp_buffer_token;4899	adapter->login_rsp_buf_sz = rsp_buffer_size;4900 4901	login_buffer->len = cpu_to_be32(buffer_size);4902	login_buffer->version = cpu_to_be32(INITIAL_VERSION_LB);4903	login_buffer->num_txcomp_subcrqs = cpu_to_be32(adapter->req_tx_queues);4904	login_buffer->off_txcomp_subcrqs =4905	    cpu_to_be32(sizeof(struct ibmvnic_login_buffer));4906	login_buffer->num_rxcomp_subcrqs = cpu_to_be32(adapter->req_rx_queues);4907	login_buffer->off_rxcomp_subcrqs =4908	    cpu_to_be32(sizeof(struct ibmvnic_login_buffer) +4909			sizeof(u64) * adapter->req_tx_queues);4910	login_buffer->login_rsp_ioba = cpu_to_be32(rsp_buffer_token);4911	login_buffer->login_rsp_len = cpu_to_be32(rsp_buffer_size);4912 4913	tx_list_p = (__be64 *)((char *)login_buffer +4914				      sizeof(struct ibmvnic_login_buffer));4915	rx_list_p = (__be64 *)((char *)login_buffer +4916				      sizeof(struct ibmvnic_login_buffer) +4917				      sizeof(u64) * adapter->req_tx_queues);4918 4919	for (i = 0; i < adapter->req_tx_queues; i++) {4920		if (adapter->tx_scrq[i]) {4921			tx_list_p[i] =4922				cpu_to_be64(adapter->tx_scrq[i]->crq_num);4923		}4924	}4925 4926	for (i = 0; i < adapter->req_rx_queues; i++) {4927		if (adapter->rx_scrq[i]) {4928			rx_list_p[i] =4929				cpu_to_be64(adapter->rx_scrq[i]->crq_num);4930		}4931	}4932 4933	/* Insert vNIC login client data */4934	vlcd = (struct vnic_login_client_data *)4935		((char *)rx_list_p + (sizeof(u64) * adapter->req_rx_queues));4936	login_buffer->client_data_offset =4937			cpu_to_be32((char *)vlcd - (char *)login_buffer);4938	login_buffer->client_data_len = cpu_to_be32(client_data_len);4939 4940	vnic_add_client_data(adapter, vlcd);4941 4942	netdev_dbg(adapter->netdev, "Login Buffer:\n");4943	for (i = 0; i < (adapter->login_buf_sz - 1) / 8 + 1; i++) {4944		netdev_dbg(adapter->netdev, "%016lx\n",4945			   ((unsigned long *)(adapter->login_buf))[i]);4946	}4947 4948	memset(&crq, 0, sizeof(crq));4949	crq.login.first = IBMVNIC_CRQ_CMD;4950	crq.login.cmd = LOGIN;4951	crq.login.ioba = cpu_to_be32(buffer_token);4952	crq.login.len = cpu_to_be32(buffer_size);4953 4954	adapter->login_pending = true;4955	rc = ibmvnic_send_crq(adapter, &crq);4956	if (rc) {4957		adapter->login_pending = false;4958		netdev_err(adapter->netdev, "Failed to send login, rc=%d\n", rc);4959		goto buf_send_failed;4960	}4961 4962	return 0;4963 4964buf_send_failed:4965	dma_unmap_single(dev, rsp_buffer_token, rsp_buffer_size,4966			 DMA_FROM_DEVICE);4967buf_rsp_map_failed:4968	kfree(login_rsp_buffer);4969	adapter->login_rsp_buf = NULL;4970buf_rsp_alloc_failed:4971	dma_unmap_single(dev, buffer_token, buffer_size, DMA_TO_DEVICE);4972buf_map_failed:4973	kfree(login_buffer);4974	adapter->login_buf = NULL;4975buf_alloc_failed:4976	return -ENOMEM;4977}4978 4979static int send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,4980			    u32 len, u8 map_id)4981{4982	union ibmvnic_crq crq;4983 4984	memset(&crq, 0, sizeof(crq));4985	crq.request_map.first = IBMVNIC_CRQ_CMD;4986	crq.request_map.cmd = REQUEST_MAP;4987	crq.request_map.map_id = map_id;4988	crq.request_map.ioba = cpu_to_be32(addr);4989	crq.request_map.len = cpu_to_be32(len);4990	return ibmvnic_send_crq(adapter, &crq);4991}4992 4993static int send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)4994{4995	union ibmvnic_crq crq;4996 4997	memset(&crq, 0, sizeof(crq));4998	crq.request_unmap.first = IBMVNIC_CRQ_CMD;4999	crq.request_unmap.cmd = REQUEST_UNMAP;5000	crq.request_unmap.map_id = map_id;5001	return ibmvnic_send_crq(adapter, &crq);5002}5003 5004static void send_query_map(struct ibmvnic_adapter *adapter)5005{5006	union ibmvnic_crq crq;5007 5008	memset(&crq, 0, sizeof(crq));5009	crq.query_map.first = IBMVNIC_CRQ_CMD;5010	crq.query_map.cmd = QUERY_MAP;5011	ibmvnic_send_crq(adapter, &crq);5012}5013 5014/* Send a series of CRQs requesting various capabilities of the VNIC server */5015static void send_query_cap(struct ibmvnic_adapter *adapter)5016{5017	union ibmvnic_crq crq;5018	int cap_reqs;5019 5020	/* We send out 25 QUERY_CAPABILITY CRQs below.  Initialize this count5021	 * upfront. When the tasklet receives a response to all of these, it5022	 * can send out the next protocol messaage (REQUEST_CAPABILITY).5023	 */5024	cap_reqs = 25;5025 5026	atomic_set(&adapter->running_cap_crqs, cap_reqs);5027 5028	memset(&crq, 0, sizeof(crq));5029	crq.query_capability.first = IBMVNIC_CRQ_CMD;5030	crq.query_capability.cmd = QUERY_CAPABILITY;5031 5032	crq.query_capability.capability = cpu_to_be16(MIN_TX_QUEUES);5033	ibmvnic_send_crq(adapter, &crq);5034	cap_reqs--;5035 5036	crq.query_capability.capability = cpu_to_be16(MIN_RX_QUEUES);5037	ibmvnic_send_crq(adapter, &crq);5038	cap_reqs--;5039 5040	crq.query_capability.capability = cpu_to_be16(MIN_RX_ADD_QUEUES);5041	ibmvnic_send_crq(adapter, &crq);5042	cap_reqs--;5043 5044	crq.query_capability.capability = cpu_to_be16(MAX_TX_QUEUES);5045	ibmvnic_send_crq(adapter, &crq);5046	cap_reqs--;5047 5048	crq.query_capability.capability = cpu_to_be16(MAX_RX_QUEUES);5049	ibmvnic_send_crq(adapter, &crq);5050	cap_reqs--;5051 5052	crq.query_capability.capability = cpu_to_be16(MAX_RX_ADD_QUEUES);5053	ibmvnic_send_crq(adapter, &crq);5054	cap_reqs--;5055 5056	crq.query_capability.capability =5057	    cpu_to_be16(MIN_TX_ENTRIES_PER_SUBCRQ);5058	ibmvnic_send_crq(adapter, &crq);5059	cap_reqs--;5060 5061	crq.query_capability.capability =5062	    cpu_to_be16(MIN_RX_ADD_ENTRIES_PER_SUBCRQ);5063	ibmvnic_send_crq(adapter, &crq);5064	cap_reqs--;5065 5066	crq.query_capability.capability =5067	    cpu_to_be16(MAX_TX_ENTRIES_PER_SUBCRQ);5068	ibmvnic_send_crq(adapter, &crq);5069	cap_reqs--;5070 5071	crq.query_capability.capability =5072	    cpu_to_be16(MAX_RX_ADD_ENTRIES_PER_SUBCRQ);5073	ibmvnic_send_crq(adapter, &crq);5074	cap_reqs--;5075 5076	crq.query_capability.capability = cpu_to_be16(TCP_IP_OFFLOAD);5077	ibmvnic_send_crq(adapter, &crq);5078	cap_reqs--;5079 5080	crq.query_capability.capability = cpu_to_be16(PROMISC_SUPPORTED);5081	ibmvnic_send_crq(adapter, &crq);5082	cap_reqs--;5083 5084	crq.query_capability.capability = cpu_to_be16(MIN_MTU);5085	ibmvnic_send_crq(adapter, &crq);5086	cap_reqs--;5087 5088	crq.query_capability.capability = cpu_to_be16(MAX_MTU);5089	ibmvnic_send_crq(adapter, &crq);5090	cap_reqs--;5091 5092	crq.query_capability.capability = cpu_to_be16(MAX_MULTICAST_FILTERS);5093	ibmvnic_send_crq(adapter, &crq);5094	cap_reqs--;5095 5096	crq.query_capability.capability = cpu_to_be16(VLAN_HEADER_INSERTION);5097	ibmvnic_send_crq(adapter, &crq);5098	cap_reqs--;5099 5100	crq.query_capability.capability = cpu_to_be16(RX_VLAN_HEADER_INSERTION);5101	ibmvnic_send_crq(adapter, &crq);5102	cap_reqs--;5103 5104	crq.query_capability.capability = cpu_to_be16(MAX_TX_SG_ENTRIES);5105	ibmvnic_send_crq(adapter, &crq);5106	cap_reqs--;5107 5108	crq.query_capability.capability = cpu_to_be16(RX_SG_SUPPORTED);5109	ibmvnic_send_crq(adapter, &crq);5110	cap_reqs--;5111 5112	crq.query_capability.capability = cpu_to_be16(OPT_TX_COMP_SUB_QUEUES);5113	ibmvnic_send_crq(adapter, &crq);5114	cap_reqs--;5115 5116	crq.query_capability.capability = cpu_to_be16(OPT_RX_COMP_QUEUES);5117	ibmvnic_send_crq(adapter, &crq);5118	cap_reqs--;5119 5120	crq.query_capability.capability =5121			cpu_to_be16(OPT_RX_BUFADD_Q_PER_RX_COMP_Q);5122	ibmvnic_send_crq(adapter, &crq);5123	cap_reqs--;5124 5125	crq.query_capability.capability =5126			cpu_to_be16(OPT_TX_ENTRIES_PER_SUBCRQ);5127	ibmvnic_send_crq(adapter, &crq);5128	cap_reqs--;5129 5130	crq.query_capability.capability =5131			cpu_to_be16(OPT_RXBA_ENTRIES_PER_SUBCRQ);5132	ibmvnic_send_crq(adapter, &crq);5133	cap_reqs--;5134 5135	crq.query_capability.capability = cpu_to_be16(TX_RX_DESC_REQ);5136 5137	ibmvnic_send_crq(adapter, &crq);5138	cap_reqs--;5139 5140	/* Keep at end to catch any discrepancy between expected and actual5141	 * CRQs sent.5142	 */5143	WARN_ON(cap_reqs != 0);5144}5145 5146static void send_query_ip_offload(struct ibmvnic_adapter *adapter)5147{5148	int buf_sz = sizeof(struct ibmvnic_query_ip_offload_buffer);5149	struct device *dev = &adapter->vdev->dev;5150	union ibmvnic_crq crq;5151 5152	adapter->ip_offload_tok =5153		dma_map_single(dev,5154			       &adapter->ip_offload_buf,5155			       buf_sz,5156			       DMA_FROM_DEVICE);5157 5158	if (dma_mapping_error(dev, adapter->ip_offload_tok)) {5159		if (!firmware_has_feature(FW_FEATURE_CMO))5160			dev_err(dev, "Couldn't map offload buffer\n");5161		return;5162	}5163 5164	memset(&crq, 0, sizeof(crq));5165	crq.query_ip_offload.first = IBMVNIC_CRQ_CMD;5166	crq.query_ip_offload.cmd = QUERY_IP_OFFLOAD;5167	crq.query_ip_offload.len = cpu_to_be32(buf_sz);5168	crq.query_ip_offload.ioba =5169	    cpu_to_be32(adapter->ip_offload_tok);5170 5171	ibmvnic_send_crq(adapter, &crq);5172}5173 5174static void send_control_ip_offload(struct ibmvnic_adapter *adapter)5175{5176	struct ibmvnic_control_ip_offload_buffer *ctrl_buf = &adapter->ip_offload_ctrl;5177	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;5178	struct device *dev = &adapter->vdev->dev;5179	netdev_features_t old_hw_features = 0;5180	union ibmvnic_crq crq;5181 5182	adapter->ip_offload_ctrl_tok =5183		dma_map_single(dev,5184			       ctrl_buf,5185			       sizeof(adapter->ip_offload_ctrl),5186			       DMA_TO_DEVICE);5187 5188	if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) {5189		dev_err(dev, "Couldn't map ip offload control buffer\n");5190		return;5191	}5192 5193	ctrl_buf->len = cpu_to_be32(sizeof(adapter->ip_offload_ctrl));5194	ctrl_buf->version = cpu_to_be32(INITIAL_VERSION_IOB);5195	ctrl_buf->ipv4_chksum = buf->ipv4_chksum;5196	ctrl_buf->ipv6_chksum = buf->ipv6_chksum;5197	ctrl_buf->tcp_ipv4_chksum = buf->tcp_ipv4_chksum;5198	ctrl_buf->udp_ipv4_chksum = buf->udp_ipv4_chksum;5199	ctrl_buf->tcp_ipv6_chksum = buf->tcp_ipv6_chksum;5200	ctrl_buf->udp_ipv6_chksum = buf->udp_ipv6_chksum;5201	ctrl_buf->large_tx_ipv4 = buf->large_tx_ipv4;5202	ctrl_buf->large_tx_ipv6 = buf->large_tx_ipv6;5203 5204	/* large_rx disabled for now, additional features needed */5205	ctrl_buf->large_rx_ipv4 = 0;5206	ctrl_buf->large_rx_ipv6 = 0;5207 5208	if (adapter->state != VNIC_PROBING) {5209		old_hw_features = adapter->netdev->hw_features;5210		adapter->netdev->hw_features = 0;5211	}5212 5213	adapter->netdev->hw_features = NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;5214 5215	if (buf->tcp_ipv4_chksum || buf->udp_ipv4_chksum)5216		adapter->netdev->hw_features |= NETIF_F_IP_CSUM;5217 5218	if (buf->tcp_ipv6_chksum || buf->udp_ipv6_chksum)5219		adapter->netdev->hw_features |= NETIF_F_IPV6_CSUM;5220 5221	if ((adapter->netdev->features &5222	    (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)))5223		adapter->netdev->hw_features |= NETIF_F_RXCSUM;5224 5225	if (buf->large_tx_ipv4)5226		adapter->netdev->hw_features |= NETIF_F_TSO;5227	if (buf->large_tx_ipv6)5228		adapter->netdev->hw_features |= NETIF_F_TSO6;5229 5230	if (adapter->state == VNIC_PROBING) {5231		adapter->netdev->features |= adapter->netdev->hw_features;5232	} else if (old_hw_features != adapter->netdev->hw_features) {5233		netdev_features_t tmp = 0;5234 5235		/* disable features no longer supported */5236		adapter->netdev->features &= adapter->netdev->hw_features;5237		/* turn on features now supported if previously enabled */5238		tmp = (old_hw_features ^ adapter->netdev->hw_features) &5239			adapter->netdev->hw_features;5240		adapter->netdev->features |=5241				tmp & adapter->netdev->wanted_features;5242	}5243 5244	memset(&crq, 0, sizeof(crq));5245	crq.control_ip_offload.first = IBMVNIC_CRQ_CMD;5246	crq.control_ip_offload.cmd = CONTROL_IP_OFFLOAD;5247	crq.control_ip_offload.len =5248	    cpu_to_be32(sizeof(adapter->ip_offload_ctrl));5249	crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok);5250	ibmvnic_send_crq(adapter, &crq);5251}5252 5253static void handle_vpd_size_rsp(union ibmvnic_crq *crq,5254				struct ibmvnic_adapter *adapter)5255{5256	struct device *dev = &adapter->vdev->dev;5257 5258	if (crq->get_vpd_size_rsp.rc.code) {5259		dev_err(dev, "Error retrieving VPD size, rc=%x\n",5260			crq->get_vpd_size_rsp.rc.code);5261		complete(&adapter->fw_done);5262		return;5263	}5264 5265	adapter->vpd->len = be64_to_cpu(crq->get_vpd_size_rsp.len);5266	complete(&adapter->fw_done);5267}5268 5269static void handle_vpd_rsp(union ibmvnic_crq *crq,5270			   struct ibmvnic_adapter *adapter)5271{5272	struct device *dev = &adapter->vdev->dev;5273	unsigned char *substr = NULL;5274	u8 fw_level_len = 0;5275 5276	memset(adapter->fw_version, 0, 32);5277 5278	dma_unmap_single(dev, adapter->vpd->dma_addr, adapter->vpd->len,5279			 DMA_FROM_DEVICE);5280 5281	if (crq->get_vpd_rsp.rc.code) {5282		dev_err(dev, "Error retrieving VPD from device, rc=%x\n",5283			crq->get_vpd_rsp.rc.code);5284		goto complete;5285	}5286 5287	/* get the position of the firmware version info5288	 * located after the ASCII 'RM' substring in the buffer5289	 */5290	substr = strnstr(adapter->vpd->buff, "RM", adapter->vpd->len);5291	if (!substr) {5292		dev_info(dev, "Warning - No FW level has been provided in the VPD buffer by the VIOS Server\n");5293		goto complete;5294	}5295 5296	/* get length of firmware level ASCII substring */5297	if ((substr + 2) < (adapter->vpd->buff + adapter->vpd->len)) {5298		fw_level_len = *(substr + 2);5299	} else {5300		dev_info(dev, "Length of FW substr extrapolated VDP buff\n");5301		goto complete;5302	}5303 5304	/* copy firmware version string from vpd into adapter */5305	if ((substr + 3 + fw_level_len) <5306	    (adapter->vpd->buff + adapter->vpd->len)) {5307		strscpy(adapter->fw_version, substr + 3,5308			sizeof(adapter->fw_version));5309	} else {5310		dev_info(dev, "FW substr extrapolated VPD buff\n");5311	}5312 5313complete:5314	if (adapter->fw_version[0] == '\0')5315		strscpy((char *)adapter->fw_version, "N/A", sizeof(adapter->fw_version));5316	complete(&adapter->fw_done);5317}5318 5319static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter)5320{5321	struct device *dev = &adapter->vdev->dev;5322	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;5323	int i;5324 5325	dma_unmap_single(dev, adapter->ip_offload_tok,5326			 sizeof(adapter->ip_offload_buf), DMA_FROM_DEVICE);5327 5328	netdev_dbg(adapter->netdev, "Query IP Offload Buffer:\n");5329	for (i = 0; i < (sizeof(adapter->ip_offload_buf) - 1) / 8 + 1; i++)5330		netdev_dbg(adapter->netdev, "%016lx\n",5331			   ((unsigned long *)(buf))[i]);5332 5333	netdev_dbg(adapter->netdev, "ipv4_chksum = %d\n", buf->ipv4_chksum);5334	netdev_dbg(adapter->netdev, "ipv6_chksum = %d\n", buf->ipv6_chksum);5335	netdev_dbg(adapter->netdev, "tcp_ipv4_chksum = %d\n",5336		   buf->tcp_ipv4_chksum);5337	netdev_dbg(adapter->netdev, "tcp_ipv6_chksum = %d\n",5338		   buf->tcp_ipv6_chksum);5339	netdev_dbg(adapter->netdev, "udp_ipv4_chksum = %d\n",5340		   buf->udp_ipv4_chksum);5341	netdev_dbg(adapter->netdev, "udp_ipv6_chksum = %d\n",5342		   buf->udp_ipv6_chksum);5343	netdev_dbg(adapter->netdev, "large_tx_ipv4 = %d\n",5344		   buf->large_tx_ipv4);5345	netdev_dbg(adapter->netdev, "large_tx_ipv6 = %d\n",5346		   buf->large_tx_ipv6);5347	netdev_dbg(adapter->netdev, "large_rx_ipv4 = %d\n",5348		   buf->large_rx_ipv4);5349	netdev_dbg(adapter->netdev, "large_rx_ipv6 = %d\n",5350		   buf->large_rx_ipv6);5351	netdev_dbg(adapter->netdev, "max_ipv4_hdr_sz = %d\n",5352		   buf->max_ipv4_header_size);5353	netdev_dbg(adapter->netdev, "max_ipv6_hdr_sz = %d\n",5354		   buf->max_ipv6_header_size);5355	netdev_dbg(adapter->netdev, "max_tcp_hdr_size = %d\n",5356		   buf->max_tcp_header_size);5357	netdev_dbg(adapter->netdev, "max_udp_hdr_size = %d\n",5358		   buf->max_udp_header_size);5359	netdev_dbg(adapter->netdev, "max_large_tx_size = %d\n",5360		   buf->max_large_tx_size);5361	netdev_dbg(adapter->netdev, "max_large_rx_size = %d\n",5362		   buf->max_large_rx_size);5363	netdev_dbg(adapter->netdev, "ipv6_ext_hdr = %d\n",5364		   buf->ipv6_extension_header);5365	netdev_dbg(adapter->netdev, "tcp_pseudosum_req = %d\n",5366		   buf->tcp_pseudosum_req);5367	netdev_dbg(adapter->netdev, "num_ipv6_ext_hd = %d\n",5368		   buf->num_ipv6_ext_headers);5369	netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n",5370		   buf->off_ipv6_ext_headers);5371 5372	send_control_ip_offload(adapter);5373}5374 5375static const char *ibmvnic_fw_err_cause(u16 cause)5376{5377	switch (cause) {5378	case ADAPTER_PROBLEM:5379		return "adapter problem";5380	case BUS_PROBLEM:5381		return "bus problem";5382	case FW_PROBLEM:5383		return "firmware problem";5384	case DD_PROBLEM:5385		return "device driver problem";5386	case EEH_RECOVERY:5387		return "EEH recovery";5388	case FW_UPDATED:5389		return "firmware updated";5390	case LOW_MEMORY:5391		return "low Memory";5392	default:5393		return "unknown";5394	}5395}5396 5397static void handle_error_indication(union ibmvnic_crq *crq,5398				    struct ibmvnic_adapter *adapter)5399{5400	struct device *dev = &adapter->vdev->dev;5401	u16 cause;5402 5403	cause = be16_to_cpu(crq->error_indication.error_cause);5404 5405	dev_warn_ratelimited(dev,5406			     "Firmware reports %serror, cause: %s. Starting recovery...\n",5407			     crq->error_indication.flags5408				& IBMVNIC_FATAL_ERROR ? "FATAL " : "",5409			     ibmvnic_fw_err_cause(cause));5410 5411	if (crq->error_indication.flags & IBMVNIC_FATAL_ERROR)5412		ibmvnic_reset(adapter, VNIC_RESET_FATAL);5413	else5414		ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL);5415}5416 5417static int handle_change_mac_rsp(union ibmvnic_crq *crq,5418				 struct ibmvnic_adapter *adapter)5419{5420	struct net_device *netdev = adapter->netdev;5421	struct device *dev = &adapter->vdev->dev;5422	long rc;5423 5424	rc = crq->change_mac_addr_rsp.rc.code;5425	if (rc) {5426		dev_err(dev, "Error %ld in CHANGE_MAC_ADDR_RSP\n", rc);5427		goto out;5428	}5429	/* crq->change_mac_addr.mac_addr is the requested one5430	 * crq->change_mac_addr_rsp.mac_addr is the returned valid one.5431	 */5432	eth_hw_addr_set(netdev, &crq->change_mac_addr_rsp.mac_addr[0]);5433	ether_addr_copy(adapter->mac_addr,5434			&crq->change_mac_addr_rsp.mac_addr[0]);5435out:5436	complete(&adapter->fw_done);5437	return rc;5438}5439 5440static void handle_request_cap_rsp(union ibmvnic_crq *crq,5441				   struct ibmvnic_adapter *adapter)5442{5443	struct device *dev = &adapter->vdev->dev;5444	u64 *req_value;5445	char *name;5446 5447	atomic_dec(&adapter->running_cap_crqs);5448	netdev_dbg(adapter->netdev, "Outstanding request-caps: %d\n",5449		   atomic_read(&adapter->running_cap_crqs));5450	switch (be16_to_cpu(crq->request_capability_rsp.capability)) {5451	case REQ_TX_QUEUES:5452		req_value = &adapter->req_tx_queues;5453		name = "tx";5454		break;5455	case REQ_RX_QUEUES:5456		req_value = &adapter->req_rx_queues;5457		name = "rx";5458		break;5459	case REQ_RX_ADD_QUEUES:5460		req_value = &adapter->req_rx_add_queues;5461		name = "rx_add";5462		break;5463	case REQ_TX_ENTRIES_PER_SUBCRQ:5464		req_value = &adapter->req_tx_entries_per_subcrq;5465		name = "tx_entries_per_subcrq";5466		break;5467	case REQ_RX_ADD_ENTRIES_PER_SUBCRQ:5468		req_value = &adapter->req_rx_add_entries_per_subcrq;5469		name = "rx_add_entries_per_subcrq";5470		break;5471	case REQ_MTU:5472		req_value = &adapter->req_mtu;5473		name = "mtu";5474		break;5475	case PROMISC_REQUESTED:5476		req_value = &adapter->promisc;5477		name = "promisc";5478		break;5479	default:5480		dev_err(dev, "Got invalid cap request rsp %d\n",5481			crq->request_capability.capability);5482		return;5483	}5484 5485	switch (crq->request_capability_rsp.rc.code) {5486	case SUCCESS:5487		break;5488	case PARTIALSUCCESS:5489		dev_info(dev, "req=%lld, rsp=%ld in %s queue, retrying.\n",5490			 *req_value,5491			 (long)be64_to_cpu(crq->request_capability_rsp.number),5492			 name);5493 5494		if (be16_to_cpu(crq->request_capability_rsp.capability) ==5495		    REQ_MTU) {5496			pr_err("mtu of %llu is not supported. Reverting.\n",5497			       *req_value);5498			*req_value = adapter->fallback.mtu;5499		} else {5500			*req_value =5501				be64_to_cpu(crq->request_capability_rsp.number);5502		}5503 5504		send_request_cap(adapter, 1);5505		return;5506	default:5507		dev_err(dev, "Error %d in request cap rsp\n",5508			crq->request_capability_rsp.rc.code);5509		return;5510	}5511 5512	/* Done receiving requested capabilities, query IP offload support */5513	if (atomic_read(&adapter->running_cap_crqs) == 0)5514		send_query_ip_offload(adapter);5515}5516 5517static int handle_login_rsp(union ibmvnic_crq *login_rsp_crq,5518			    struct ibmvnic_adapter *adapter)5519{5520	struct device *dev = &adapter->vdev->dev;5521	struct net_device *netdev = adapter->netdev;5522	struct ibmvnic_login_rsp_buffer *login_rsp = adapter->login_rsp_buf;5523	struct ibmvnic_login_buffer *login = adapter->login_buf;5524	u64 *tx_handle_array;5525	u64 *rx_handle_array;5526	int num_tx_pools;5527	int num_rx_pools;5528	u64 *size_array;5529	u32 rsp_len;5530	int i;5531 5532	/* CHECK: Test/set of login_pending does not need to be atomic5533	 * because only ibmvnic_tasklet tests/clears this.5534	 */5535	if (!adapter->login_pending) {5536		netdev_warn(netdev, "Ignoring unexpected login response\n");5537		return 0;5538	}5539	adapter->login_pending = false;5540 5541	/* If the number of queues requested can't be allocated by the5542	 * server, the login response will return with code 1. We will need5543	 * to resend the login buffer with fewer queues requested.5544	 */5545	if (login_rsp_crq->generic.rc.code) {5546		adapter->init_done_rc = login_rsp_crq->generic.rc.code;5547		complete(&adapter->init_done);5548		return 0;5549	}5550 5551	if (adapter->failover_pending) {5552		adapter->init_done_rc = -EAGAIN;5553		netdev_dbg(netdev, "Failover pending, ignoring login response\n");5554		complete(&adapter->init_done);5555		/* login response buffer will be released on reset */5556		return 0;5557	}5558 5559	netdev->mtu = adapter->req_mtu - ETH_HLEN;5560 5561	netdev_dbg(adapter->netdev, "Login Response Buffer:\n");5562	for (i = 0; i < (adapter->login_rsp_buf_sz - 1) / 8 + 1; i++) {5563		netdev_dbg(adapter->netdev, "%016lx\n",5564			   ((unsigned long *)(adapter->login_rsp_buf))[i]);5565	}5566 5567	/* Sanity checks */5568	if (login->num_txcomp_subcrqs != login_rsp->num_txsubm_subcrqs ||5569	    (be32_to_cpu(login->num_rxcomp_subcrqs) *5570	     adapter->req_rx_add_queues !=5571	     be32_to_cpu(login_rsp->num_rxadd_subcrqs))) {5572		dev_err(dev, "FATAL: Inconsistent login and login rsp\n");5573		ibmvnic_reset(adapter, VNIC_RESET_FATAL);5574		return -EIO;5575	}5576 5577	rsp_len = be32_to_cpu(login_rsp->len);5578	if (be32_to_cpu(login->login_rsp_len) < rsp_len ||5579	    rsp_len <= be32_to_cpu(login_rsp->off_txsubm_subcrqs) ||5580	    rsp_len <= be32_to_cpu(login_rsp->off_rxadd_subcrqs) ||5581	    rsp_len <= be32_to_cpu(login_rsp->off_rxadd_buff_size) ||5582	    rsp_len <= be32_to_cpu(login_rsp->off_supp_tx_desc)) {5583		/* This can happen if a login request times out and there are5584		 * 2 outstanding login requests sent, the LOGIN_RSP crq5585		 * could have been for the older login request. So we are5586		 * parsing the newer response buffer which may be incomplete5587		 */5588		dev_err(dev, "FATAL: Login rsp offsets/lengths invalid\n");5589		ibmvnic_reset(adapter, VNIC_RESET_FATAL);5590		return -EIO;5591	}5592 5593	size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +5594		be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size));5595	/* variable buffer sizes are not supported, so just read the5596	 * first entry.5597	 */5598	adapter->cur_rx_buf_sz = be64_to_cpu(size_array[0]);5599 5600	num_tx_pools = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);5601	num_rx_pools = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);5602 5603	tx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +5604				  be32_to_cpu(adapter->login_rsp_buf->off_txsubm_subcrqs));5605	rx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +5606				  be32_to_cpu(adapter->login_rsp_buf->off_rxadd_subcrqs));5607 5608	for (i = 0; i < num_tx_pools; i++)5609		adapter->tx_scrq[i]->handle = tx_handle_array[i];5610 5611	for (i = 0; i < num_rx_pools; i++)5612		adapter->rx_scrq[i]->handle = rx_handle_array[i];5613 5614	adapter->num_active_tx_scrqs = num_tx_pools;5615	adapter->num_active_rx_scrqs = num_rx_pools;5616	release_login_rsp_buffer(adapter);5617	release_login_buffer(adapter);5618	complete(&adapter->init_done);5619 5620	return 0;5621}5622 5623static void handle_request_unmap_rsp(union ibmvnic_crq *crq,5624				     struct ibmvnic_adapter *adapter)5625{5626	struct device *dev = &adapter->vdev->dev;5627	long rc;5628 5629	rc = crq->request_unmap_rsp.rc.code;5630	if (rc)5631		dev_err(dev, "Error %ld in REQUEST_UNMAP_RSP\n", rc);5632}5633 5634static void handle_query_map_rsp(union ibmvnic_crq *crq,5635				 struct ibmvnic_adapter *adapter)5636{5637	struct net_device *netdev = adapter->netdev;5638	struct device *dev = &adapter->vdev->dev;5639	long rc;5640 5641	rc = crq->query_map_rsp.rc.code;5642	if (rc) {5643		dev_err(dev, "Error %ld in QUERY_MAP_RSP\n", rc);5644		return;5645	}5646	netdev_dbg(netdev, "page_size = %d\ntot_pages = %u\nfree_pages = %u\n",5647		   crq->query_map_rsp.page_size,5648		   __be32_to_cpu(crq->query_map_rsp.tot_pages),5649		   __be32_to_cpu(crq->query_map_rsp.free_pages));5650}5651 5652static void handle_query_cap_rsp(union ibmvnic_crq *crq,5653				 struct ibmvnic_adapter *adapter)5654{5655	struct net_device *netdev = adapter->netdev;5656	struct device *dev = &adapter->vdev->dev;5657	long rc;5658 5659	atomic_dec(&adapter->running_cap_crqs);5660	netdev_dbg(netdev, "Outstanding queries: %d\n",5661		   atomic_read(&adapter->running_cap_crqs));5662	rc = crq->query_capability.rc.code;5663	if (rc) {5664		dev_err(dev, "Error %ld in QUERY_CAP_RSP\n", rc);5665		goto out;5666	}5667 5668	switch (be16_to_cpu(crq->query_capability.capability)) {5669	case MIN_TX_QUEUES:5670		adapter->min_tx_queues =5671		    be64_to_cpu(crq->query_capability.number);5672		netdev_dbg(netdev, "min_tx_queues = %lld\n",5673			   adapter->min_tx_queues);5674		break;5675	case MIN_RX_QUEUES:5676		adapter->min_rx_queues =5677		    be64_to_cpu(crq->query_capability.number);5678		netdev_dbg(netdev, "min_rx_queues = %lld\n",5679			   adapter->min_rx_queues);5680		break;5681	case MIN_RX_ADD_QUEUES:5682		adapter->min_rx_add_queues =5683		    be64_to_cpu(crq->query_capability.number);5684		netdev_dbg(netdev, "min_rx_add_queues = %lld\n",5685			   adapter->min_rx_add_queues);5686		break;5687	case MAX_TX_QUEUES:5688		adapter->max_tx_queues =5689		    be64_to_cpu(crq->query_capability.number);5690		netdev_dbg(netdev, "max_tx_queues = %lld\n",5691			   adapter->max_tx_queues);5692		break;5693	case MAX_RX_QUEUES:5694		adapter->max_rx_queues =5695		    be64_to_cpu(crq->query_capability.number);5696		netdev_dbg(netdev, "max_rx_queues = %lld\n",5697			   adapter->max_rx_queues);5698		break;5699	case MAX_RX_ADD_QUEUES:5700		adapter->max_rx_add_queues =5701		    be64_to_cpu(crq->query_capability.number);5702		netdev_dbg(netdev, "max_rx_add_queues = %lld\n",5703			   adapter->max_rx_add_queues);5704		break;5705	case MIN_TX_ENTRIES_PER_SUBCRQ:5706		adapter->min_tx_entries_per_subcrq =5707		    be64_to_cpu(crq->query_capability.number);5708		netdev_dbg(netdev, "min_tx_entries_per_subcrq = %lld\n",5709			   adapter->min_tx_entries_per_subcrq);5710		break;5711	case MIN_RX_ADD_ENTRIES_PER_SUBCRQ:5712		adapter->min_rx_add_entries_per_subcrq =5713		    be64_to_cpu(crq->query_capability.number);5714		netdev_dbg(netdev, "min_rx_add_entrs_per_subcrq = %lld\n",5715			   adapter->min_rx_add_entries_per_subcrq);5716		break;5717	case MAX_TX_ENTRIES_PER_SUBCRQ:5718		adapter->max_tx_entries_per_subcrq =5719		    be64_to_cpu(crq->query_capability.number);5720		netdev_dbg(netdev, "max_tx_entries_per_subcrq = %lld\n",5721			   adapter->max_tx_entries_per_subcrq);5722		break;5723	case MAX_RX_ADD_ENTRIES_PER_SUBCRQ:5724		adapter->max_rx_add_entries_per_subcrq =5725		    be64_to_cpu(crq->query_capability.number);5726		netdev_dbg(netdev, "max_rx_add_entrs_per_subcrq = %lld\n",5727			   adapter->max_rx_add_entries_per_subcrq);5728		break;5729	case TCP_IP_OFFLOAD:5730		adapter->tcp_ip_offload =5731		    be64_to_cpu(crq->query_capability.number);5732		netdev_dbg(netdev, "tcp_ip_offload = %lld\n",5733			   adapter->tcp_ip_offload);5734		break;5735	case PROMISC_SUPPORTED:5736		adapter->promisc_supported =5737		    be64_to_cpu(crq->query_capability.number);5738		netdev_dbg(netdev, "promisc_supported = %lld\n",5739			   adapter->promisc_supported);5740		break;5741	case MIN_MTU:5742		adapter->min_mtu = be64_to_cpu(crq->query_capability.number);5743		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;5744		netdev_dbg(netdev, "min_mtu = %lld\n", adapter->min_mtu);5745		break;5746	case MAX_MTU:5747		adapter->max_mtu = be64_to_cpu(crq->query_capability.number);5748		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;5749		netdev_dbg(netdev, "max_mtu = %lld\n", adapter->max_mtu);5750		break;5751	case MAX_MULTICAST_FILTERS:5752		adapter->max_multicast_filters =5753		    be64_to_cpu(crq->query_capability.number);5754		netdev_dbg(netdev, "max_multicast_filters = %lld\n",5755			   adapter->max_multicast_filters);5756		break;5757	case VLAN_HEADER_INSERTION:5758		adapter->vlan_header_insertion =5759		    be64_to_cpu(crq->query_capability.number);5760		if (adapter->vlan_header_insertion)5761			netdev->features |= NETIF_F_HW_VLAN_STAG_TX;5762		netdev_dbg(netdev, "vlan_header_insertion = %lld\n",5763			   adapter->vlan_header_insertion);5764		break;5765	case RX_VLAN_HEADER_INSERTION:5766		adapter->rx_vlan_header_insertion =5767		    be64_to_cpu(crq->query_capability.number);5768		netdev_dbg(netdev, "rx_vlan_header_insertion = %lld\n",5769			   adapter->rx_vlan_header_insertion);5770		break;5771	case MAX_TX_SG_ENTRIES:5772		adapter->max_tx_sg_entries =5773		    be64_to_cpu(crq->query_capability.number);5774		netdev_dbg(netdev, "max_tx_sg_entries = %lld\n",5775			   adapter->max_tx_sg_entries);5776		break;5777	case RX_SG_SUPPORTED:5778		adapter->rx_sg_supported =5779		    be64_to_cpu(crq->query_capability.number);5780		netdev_dbg(netdev, "rx_sg_supported = %lld\n",5781			   adapter->rx_sg_supported);5782		break;5783	case OPT_TX_COMP_SUB_QUEUES:5784		adapter->opt_tx_comp_sub_queues =5785		    be64_to_cpu(crq->query_capability.number);5786		netdev_dbg(netdev, "opt_tx_comp_sub_queues = %lld\n",5787			   adapter->opt_tx_comp_sub_queues);5788		break;5789	case OPT_RX_COMP_QUEUES:5790		adapter->opt_rx_comp_queues =5791		    be64_to_cpu(crq->query_capability.number);5792		netdev_dbg(netdev, "opt_rx_comp_queues = %lld\n",5793			   adapter->opt_rx_comp_queues);5794		break;5795	case OPT_RX_BUFADD_Q_PER_RX_COMP_Q:5796		adapter->opt_rx_bufadd_q_per_rx_comp_q =5797		    be64_to_cpu(crq->query_capability.number);5798		netdev_dbg(netdev, "opt_rx_bufadd_q_per_rx_comp_q = %lld\n",5799			   adapter->opt_rx_bufadd_q_per_rx_comp_q);5800		break;5801	case OPT_TX_ENTRIES_PER_SUBCRQ:5802		adapter->opt_tx_entries_per_subcrq =5803		    be64_to_cpu(crq->query_capability.number);5804		netdev_dbg(netdev, "opt_tx_entries_per_subcrq = %lld\n",5805			   adapter->opt_tx_entries_per_subcrq);5806		break;5807	case OPT_RXBA_ENTRIES_PER_SUBCRQ:5808		adapter->opt_rxba_entries_per_subcrq =5809		    be64_to_cpu(crq->query_capability.number);5810		netdev_dbg(netdev, "opt_rxba_entries_per_subcrq = %lld\n",5811			   adapter->opt_rxba_entries_per_subcrq);5812		break;5813	case TX_RX_DESC_REQ:5814		adapter->tx_rx_desc_req = crq->query_capability.number;5815		netdev_dbg(netdev, "tx_rx_desc_req = %llx\n",5816			   adapter->tx_rx_desc_req);5817		break;5818 5819	default:5820		netdev_err(netdev, "Got invalid cap rsp %d\n",5821			   crq->query_capability.capability);5822	}5823 5824out:5825	if (atomic_read(&adapter->running_cap_crqs) == 0)5826		send_request_cap(adapter, 0);5827}5828 5829static int send_query_phys_parms(struct ibmvnic_adapter *adapter)5830{5831	union ibmvnic_crq crq;5832	int rc;5833 5834	memset(&crq, 0, sizeof(crq));5835	crq.query_phys_parms.first = IBMVNIC_CRQ_CMD;5836	crq.query_phys_parms.cmd = QUERY_PHYS_PARMS;5837 5838	mutex_lock(&adapter->fw_lock);5839	adapter->fw_done_rc = 0;5840	reinit_completion(&adapter->fw_done);5841 5842	rc = ibmvnic_send_crq(adapter, &crq);5843	if (rc) {5844		mutex_unlock(&adapter->fw_lock);5845		return rc;5846	}5847 5848	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);5849	if (rc) {5850		mutex_unlock(&adapter->fw_lock);5851		return rc;5852	}5853 5854	mutex_unlock(&adapter->fw_lock);5855	return adapter->fw_done_rc ? -EIO : 0;5856}5857 5858static int handle_query_phys_parms_rsp(union ibmvnic_crq *crq,5859				       struct ibmvnic_adapter *adapter)5860{5861	struct net_device *netdev = adapter->netdev;5862	int rc;5863	__be32 rspeed = cpu_to_be32(crq->query_phys_parms_rsp.speed);5864 5865	rc = crq->query_phys_parms_rsp.rc.code;5866	if (rc) {5867		netdev_err(netdev, "Error %d in QUERY_PHYS_PARMS\n", rc);5868		return rc;5869	}5870	switch (rspeed) {5871	case IBMVNIC_10MBPS:5872		adapter->speed = SPEED_10;5873		break;5874	case IBMVNIC_100MBPS:5875		adapter->speed = SPEED_100;5876		break;5877	case IBMVNIC_1GBPS:5878		adapter->speed = SPEED_1000;5879		break;5880	case IBMVNIC_10GBPS:5881		adapter->speed = SPEED_10000;5882		break;5883	case IBMVNIC_25GBPS:5884		adapter->speed = SPEED_25000;5885		break;5886	case IBMVNIC_40GBPS:5887		adapter->speed = SPEED_40000;5888		break;5889	case IBMVNIC_50GBPS:5890		adapter->speed = SPEED_50000;5891		break;5892	case IBMVNIC_100GBPS:5893		adapter->speed = SPEED_100000;5894		break;5895	case IBMVNIC_200GBPS:5896		adapter->speed = SPEED_200000;5897		break;5898	default:5899		if (netif_carrier_ok(netdev))5900			netdev_warn(netdev, "Unknown speed 0x%08x\n", rspeed);5901		adapter->speed = SPEED_UNKNOWN;5902	}5903	if (crq->query_phys_parms_rsp.flags1 & IBMVNIC_FULL_DUPLEX)5904		adapter->duplex = DUPLEX_FULL;5905	else if (crq->query_phys_parms_rsp.flags1 & IBMVNIC_HALF_DUPLEX)5906		adapter->duplex = DUPLEX_HALF;5907	else5908		adapter->duplex = DUPLEX_UNKNOWN;5909 5910	return rc;5911}5912 5913static void ibmvnic_handle_crq(union ibmvnic_crq *crq,5914			       struct ibmvnic_adapter *adapter)5915{5916	struct ibmvnic_generic_crq *gen_crq = &crq->generic;5917	struct net_device *netdev = adapter->netdev;5918	struct device *dev = &adapter->vdev->dev;5919	u64 *u64_crq = (u64 *)crq;5920	long rc;5921 5922	netdev_dbg(netdev, "Handling CRQ: %016lx %016lx\n",5923		   (unsigned long)cpu_to_be64(u64_crq[0]),5924		   (unsigned long)cpu_to_be64(u64_crq[1]));5925	switch (gen_crq->first) {5926	case IBMVNIC_CRQ_INIT_RSP:5927		switch (gen_crq->cmd) {5928		case IBMVNIC_CRQ_INIT:5929			dev_info(dev, "Partner initialized\n");5930			adapter->from_passive_init = true;5931			/* Discard any stale login responses from prev reset.5932			 * CHECK: should we clear even on INIT_COMPLETE?5933			 */5934			adapter->login_pending = false;5935 5936			if (adapter->state == VNIC_DOWN)5937				rc = ibmvnic_reset(adapter, VNIC_RESET_PASSIVE_INIT);5938			else5939				rc = ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);5940 5941			if (rc && rc != -EBUSY) {5942				/* We were unable to schedule the failover5943				 * reset either because the adapter was still5944				 * probing (eg: during kexec) or we could not5945				 * allocate memory. Clear the failover_pending5946				 * flag since no one else will. We ignore5947				 * EBUSY because it means either FAILOVER reset5948				 * is already scheduled or the adapter is5949				 * being removed.5950				 */5951				netdev_err(netdev,5952					   "Error %ld scheduling failover reset\n",5953					   rc);5954				adapter->failover_pending = false;5955			}5956 5957			if (!completion_done(&adapter->init_done)) {5958				if (!adapter->init_done_rc)5959					adapter->init_done_rc = -EAGAIN;5960				complete(&adapter->init_done);5961			}5962 5963			break;5964		case IBMVNIC_CRQ_INIT_COMPLETE:5965			dev_info(dev, "Partner initialization complete\n");5966			adapter->crq.active = true;5967			send_version_xchg(adapter);5968			break;5969		default:5970			dev_err(dev, "Unknown crq cmd: %d\n", gen_crq->cmd);5971		}5972		return;5973	case IBMVNIC_CRQ_XPORT_EVENT:5974		netif_carrier_off(netdev);5975		adapter->crq.active = false;5976		/* terminate any thread waiting for a response5977		 * from the device5978		 */5979		if (!completion_done(&adapter->fw_done)) {5980			adapter->fw_done_rc = -EIO;5981			complete(&adapter->fw_done);5982		}5983 5984		/* if we got here during crq-init, retry crq-init */5985		if (!completion_done(&adapter->init_done)) {5986			adapter->init_done_rc = -EAGAIN;5987			complete(&adapter->init_done);5988		}5989 5990		if (!completion_done(&adapter->stats_done))5991			complete(&adapter->stats_done);5992		if (test_bit(0, &adapter->resetting))5993			adapter->force_reset_recovery = true;5994		if (gen_crq->cmd == IBMVNIC_PARTITION_MIGRATED) {5995			dev_info(dev, "Migrated, re-enabling adapter\n");5996			ibmvnic_reset(adapter, VNIC_RESET_MOBILITY);5997		} else if (gen_crq->cmd == IBMVNIC_DEVICE_FAILOVER) {5998			dev_info(dev, "Backing device failover detected\n");5999			adapter->failover_pending = true;6000		} else {6001			/* The adapter lost the connection */6002			dev_err(dev, "Virtual Adapter failed (rc=%d)\n",6003				gen_crq->cmd);6004			ibmvnic_reset(adapter, VNIC_RESET_FATAL);6005		}6006		return;6007	case IBMVNIC_CRQ_CMD_RSP:6008		break;6009	default:6010		dev_err(dev, "Got an invalid msg type 0x%02x\n",6011			gen_crq->first);6012		return;6013	}6014 6015	switch (gen_crq->cmd) {6016	case VERSION_EXCHANGE_RSP:6017		rc = crq->version_exchange_rsp.rc.code;6018		if (rc) {6019			dev_err(dev, "Error %ld in VERSION_EXCHG_RSP\n", rc);6020			break;6021		}6022		ibmvnic_version =6023			    be16_to_cpu(crq->version_exchange_rsp.version);6024		dev_info(dev, "Partner protocol version is %d\n",6025			 ibmvnic_version);6026		send_query_cap(adapter);6027		break;6028	case QUERY_CAPABILITY_RSP:6029		handle_query_cap_rsp(crq, adapter);6030		break;6031	case QUERY_MAP_RSP:6032		handle_query_map_rsp(crq, adapter);6033		break;6034	case REQUEST_MAP_RSP:6035		adapter->fw_done_rc = crq->request_map_rsp.rc.code;6036		complete(&adapter->fw_done);6037		break;6038	case REQUEST_UNMAP_RSP:6039		handle_request_unmap_rsp(crq, adapter);6040		break;6041	case REQUEST_CAPABILITY_RSP:6042		handle_request_cap_rsp(crq, adapter);6043		break;6044	case LOGIN_RSP:6045		netdev_dbg(netdev, "Got Login Response\n");6046		handle_login_rsp(crq, adapter);6047		break;6048	case LOGICAL_LINK_STATE_RSP:6049		netdev_dbg(netdev,6050			   "Got Logical Link State Response, state: %d rc: %d\n",6051			   crq->logical_link_state_rsp.link_state,6052			   crq->logical_link_state_rsp.rc.code);6053		adapter->logical_link_state =6054		    crq->logical_link_state_rsp.link_state;6055		adapter->init_done_rc = crq->logical_link_state_rsp.rc.code;6056		complete(&adapter->init_done);6057		break;6058	case LINK_STATE_INDICATION:6059		netdev_dbg(netdev, "Got Logical Link State Indication\n");6060		adapter->phys_link_state =6061		    crq->link_state_indication.phys_link_state;6062		adapter->logical_link_state =6063		    crq->link_state_indication.logical_link_state;6064		if (adapter->phys_link_state && adapter->logical_link_state)6065			netif_carrier_on(netdev);6066		else6067			netif_carrier_off(netdev);6068		break;6069	case CHANGE_MAC_ADDR_RSP:6070		netdev_dbg(netdev, "Got MAC address change Response\n");6071		adapter->fw_done_rc = handle_change_mac_rsp(crq, adapter);6072		break;6073	case ERROR_INDICATION:6074		netdev_dbg(netdev, "Got Error Indication\n");6075		handle_error_indication(crq, adapter);6076		break;6077	case REQUEST_STATISTICS_RSP:6078		netdev_dbg(netdev, "Got Statistics Response\n");6079		complete(&adapter->stats_done);6080		break;6081	case QUERY_IP_OFFLOAD_RSP:6082		netdev_dbg(netdev, "Got Query IP offload Response\n");6083		handle_query_ip_offload_rsp(adapter);6084		break;6085	case MULTICAST_CTRL_RSP:6086		netdev_dbg(netdev, "Got multicast control Response\n");6087		break;6088	case CONTROL_IP_OFFLOAD_RSP:6089		netdev_dbg(netdev, "Got Control IP offload Response\n");6090		dma_unmap_single(dev, adapter->ip_offload_ctrl_tok,6091				 sizeof(adapter->ip_offload_ctrl),6092				 DMA_TO_DEVICE);6093		complete(&adapter->init_done);6094		break;6095	case COLLECT_FW_TRACE_RSP:6096		netdev_dbg(netdev, "Got Collect firmware trace Response\n");6097		complete(&adapter->fw_done);6098		break;6099	case GET_VPD_SIZE_RSP:6100		handle_vpd_size_rsp(crq, adapter);6101		break;6102	case GET_VPD_RSP:6103		handle_vpd_rsp(crq, adapter);6104		break;6105	case QUERY_PHYS_PARMS_RSP:6106		adapter->fw_done_rc = handle_query_phys_parms_rsp(crq, adapter);6107		complete(&adapter->fw_done);6108		break;6109	default:6110		netdev_err(netdev, "Got an invalid cmd type 0x%02x\n",6111			   gen_crq->cmd);6112	}6113}6114 6115static irqreturn_t ibmvnic_interrupt(int irq, void *instance)6116{6117	struct ibmvnic_adapter *adapter = instance;6118 6119	tasklet_schedule(&adapter->tasklet);6120	return IRQ_HANDLED;6121}6122 6123static void ibmvnic_tasklet(struct tasklet_struct *t)6124{6125	struct ibmvnic_adapter *adapter = from_tasklet(adapter, t, tasklet);6126	struct ibmvnic_crq_queue *queue = &adapter->crq;6127	union ibmvnic_crq *crq;6128	unsigned long flags;6129 6130	spin_lock_irqsave(&queue->lock, flags);6131 6132	/* Pull all the valid messages off the CRQ */6133	while ((crq = ibmvnic_next_crq(adapter)) != NULL) {6134		/* This barrier makes sure ibmvnic_next_crq()'s6135		 * crq->generic.first & IBMVNIC_CRQ_CMD_RSP is loaded6136		 * before ibmvnic_handle_crq()'s6137		 * switch(gen_crq->first) and switch(gen_crq->cmd).6138		 */6139		dma_rmb();6140		ibmvnic_handle_crq(crq, adapter);6141		crq->generic.first = 0;6142	}6143 6144	spin_unlock_irqrestore(&queue->lock, flags);6145}6146 6147static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *adapter)6148{6149	struct vio_dev *vdev = adapter->vdev;6150	int rc;6151 6152	do {6153		rc = plpar_hcall_norets(H_ENABLE_CRQ, vdev->unit_address);6154	} while (rc == H_IN_PROGRESS || rc == H_BUSY || H_IS_LONG_BUSY(rc));6155 6156	if (rc)6157		dev_err(&vdev->dev, "Error enabling adapter (rc=%d)\n", rc);6158 6159	return rc;6160}6161 6162static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter)6163{6164	struct ibmvnic_crq_queue *crq = &adapter->crq;6165	struct device *dev = &adapter->vdev->dev;6166	struct vio_dev *vdev = adapter->vdev;6167	int rc;6168 6169	/* Close the CRQ */6170	do {6171		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);6172	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));6173 6174	/* Clean out the queue */6175	if (!crq->msgs)6176		return -EINVAL;6177 6178	memset(crq->msgs, 0, PAGE_SIZE);6179	crq->cur = 0;6180	crq->active = false;6181 6182	/* And re-open it again */6183	rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,6184				crq->msg_token, PAGE_SIZE);6185 6186	if (rc == H_CLOSED)6187		/* Adapter is good, but other end is not ready */6188		dev_warn(dev, "Partner adapter not ready\n");6189	else if (rc != 0)6190		dev_warn(dev, "Couldn't register crq (rc=%d)\n", rc);6191 6192	return rc;6193}6194 6195static void release_crq_queue(struct ibmvnic_adapter *adapter)6196{6197	struct ibmvnic_crq_queue *crq = &adapter->crq;6198	struct vio_dev *vdev = adapter->vdev;6199	long rc;6200 6201	if (!crq->msgs)6202		return;6203 6204	netdev_dbg(adapter->netdev, "Releasing CRQ\n");6205	free_irq(vdev->irq, adapter);6206	tasklet_kill(&adapter->tasklet);6207	do {6208		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);6209	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));6210 6211	dma_unmap_single(&vdev->dev, crq->msg_token, PAGE_SIZE,6212			 DMA_BIDIRECTIONAL);6213	free_page((unsigned long)crq->msgs);6214	crq->msgs = NULL;6215	crq->active = false;6216}6217 6218static int init_crq_queue(struct ibmvnic_adapter *adapter)6219{6220	struct ibmvnic_crq_queue *crq = &adapter->crq;6221	struct device *dev = &adapter->vdev->dev;6222	struct vio_dev *vdev = adapter->vdev;6223	int rc, retrc = -ENOMEM;6224 6225	if (crq->msgs)6226		return 0;6227 6228	crq->msgs = (union ibmvnic_crq *)get_zeroed_page(GFP_KERNEL);6229	/* Should we allocate more than one page? */6230 6231	if (!crq->msgs)6232		return -ENOMEM;6233 6234	crq->size = PAGE_SIZE / sizeof(*crq->msgs);6235	crq->msg_token = dma_map_single(dev, crq->msgs, PAGE_SIZE,6236					DMA_BIDIRECTIONAL);6237	if (dma_mapping_error(dev, crq->msg_token))6238		goto map_failed;6239 6240	rc = plpar_hcall_norets(H_REG_CRQ, vdev->unit_address,6241				crq->msg_token, PAGE_SIZE);6242 6243	if (rc == H_RESOURCE)6244		/* maybe kexecing and resource is busy. try a reset */6245		rc = ibmvnic_reset_crq(adapter);6246	retrc = rc;6247 6248	if (rc == H_CLOSED) {6249		dev_warn(dev, "Partner adapter not ready\n");6250	} else if (rc) {6251		dev_warn(dev, "Error %d opening adapter\n", rc);6252		goto reg_crq_failed;6253	}6254 6255	retrc = 0;6256 6257	tasklet_setup(&adapter->tasklet, (void *)ibmvnic_tasklet);6258 6259	netdev_dbg(adapter->netdev, "registering irq 0x%x\n", vdev->irq);6260	snprintf(crq->name, sizeof(crq->name), "ibmvnic-%x",6261		 adapter->vdev->unit_address);6262	rc = request_irq(vdev->irq, ibmvnic_interrupt, 0, crq->name, adapter);6263	if (rc) {6264		dev_err(dev, "Couldn't register irq 0x%x. rc=%d\n",6265			vdev->irq, rc);6266		goto req_irq_failed;6267	}6268 6269	rc = vio_enable_interrupts(vdev);6270	if (rc) {6271		dev_err(dev, "Error %d enabling interrupts\n", rc);6272		goto req_irq_failed;6273	}6274 6275	crq->cur = 0;6276	spin_lock_init(&crq->lock);6277 6278	/* process any CRQs that were queued before we enabled interrupts */6279	tasklet_schedule(&adapter->tasklet);6280 6281	return retrc;6282 6283req_irq_failed:6284	tasklet_kill(&adapter->tasklet);6285	do {6286		rc = plpar_hcall_norets(H_FREE_CRQ, vdev->unit_address);6287	} while (rc == H_BUSY || H_IS_LONG_BUSY(rc));6288reg_crq_failed:6289	dma_unmap_single(dev, crq->msg_token, PAGE_SIZE, DMA_BIDIRECTIONAL);6290map_failed:6291	free_page((unsigned long)crq->msgs);6292	crq->msgs = NULL;6293	return retrc;6294}6295 6296static int ibmvnic_reset_init(struct ibmvnic_adapter *adapter, bool reset)6297{6298	struct device *dev = &adapter->vdev->dev;6299	unsigned long timeout = msecs_to_jiffies(20000);6300	u64 old_num_rx_queues = adapter->req_rx_queues;6301	u64 old_num_tx_queues = adapter->req_tx_queues;6302	int rc;6303 6304	adapter->from_passive_init = false;6305 6306	rc = ibmvnic_send_crq_init(adapter);6307	if (rc) {6308		dev_err(dev, "Send crq init failed with error %d\n", rc);6309		return rc;6310	}6311 6312	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {6313		dev_err(dev, "Initialization sequence timed out\n");6314		return -ETIMEDOUT;6315	}6316 6317	if (adapter->init_done_rc) {6318		release_crq_queue(adapter);6319		dev_err(dev, "CRQ-init failed, %d\n", adapter->init_done_rc);6320		return adapter->init_done_rc;6321	}6322 6323	if (adapter->from_passive_init) {6324		adapter->state = VNIC_OPEN;6325		adapter->from_passive_init = false;6326		dev_err(dev, "CRQ-init failed, passive-init\n");6327		return -EINVAL;6328	}6329 6330	if (reset &&6331	    test_bit(0, &adapter->resetting) && !adapter->wait_for_reset &&6332	    adapter->reset_reason != VNIC_RESET_MOBILITY) {6333		if (adapter->req_rx_queues != old_num_rx_queues ||6334		    adapter->req_tx_queues != old_num_tx_queues) {6335			release_sub_crqs(adapter, 0);6336			rc = init_sub_crqs(adapter);6337		} else {6338			/* no need to reinitialize completely, but we do6339			 * need to clean up transmits that were in flight6340			 * when we processed the reset.  Failure to do so6341			 * will confound the upper layer, usually TCP, by6342			 * creating the illusion of transmits that are6343			 * awaiting completion.6344			 */6345			clean_tx_pools(adapter);6346 6347			rc = reset_sub_crq_queues(adapter);6348		}6349	} else {6350		rc = init_sub_crqs(adapter);6351	}6352 6353	if (rc) {6354		dev_err(dev, "Initialization of sub crqs failed\n");6355		release_crq_queue(adapter);6356		return rc;6357	}6358 6359	rc = init_sub_crq_irqs(adapter);6360	if (rc) {6361		dev_err(dev, "Failed to initialize sub crq irqs\n");6362		release_crq_queue(adapter);6363	}6364 6365	return rc;6366}6367 6368static struct device_attribute dev_attr_failover;6369 6370static int ibmvnic_probe(struct vio_dev *dev, const struct vio_device_id *id)6371{6372	struct ibmvnic_adapter *adapter;6373	struct net_device *netdev;6374	unsigned char *mac_addr_p;6375	unsigned long flags;6376	bool init_success;6377	int rc;6378 6379	dev_dbg(&dev->dev, "entering ibmvnic_probe for UA 0x%x\n",6380		dev->unit_address);6381 6382	mac_addr_p = (unsigned char *)vio_get_attribute(dev,6383							VETH_MAC_ADDR, NULL);6384	if (!mac_addr_p) {6385		dev_err(&dev->dev,6386			"(%s:%3.3d) ERROR: Can't find MAC_ADDR attribute\n",6387			__FILE__, __LINE__);6388		return 0;6389	}6390 6391	netdev = alloc_etherdev_mq(sizeof(struct ibmvnic_adapter),6392				   IBMVNIC_MAX_QUEUES);6393	if (!netdev)6394		return -ENOMEM;6395 6396	adapter = netdev_priv(netdev);6397	adapter->state = VNIC_PROBING;6398	dev_set_drvdata(&dev->dev, netdev);6399	adapter->vdev = dev;6400	adapter->netdev = netdev;6401	adapter->login_pending = false;6402	memset(&adapter->map_ids, 0, sizeof(adapter->map_ids));6403	/* map_ids start at 1, so ensure map_id 0 is always "in-use" */6404	bitmap_set(adapter->map_ids, 0, 1);6405 6406	ether_addr_copy(adapter->mac_addr, mac_addr_p);6407	eth_hw_addr_set(netdev, adapter->mac_addr);6408	netdev->irq = dev->irq;6409	netdev->netdev_ops = &ibmvnic_netdev_ops;6410	netdev->ethtool_ops = &ibmvnic_ethtool_ops;6411	SET_NETDEV_DEV(netdev, &dev->dev);6412 6413	INIT_WORK(&adapter->ibmvnic_reset, __ibmvnic_reset);6414	INIT_DELAYED_WORK(&adapter->ibmvnic_delayed_reset,6415			  __ibmvnic_delayed_reset);6416	INIT_LIST_HEAD(&adapter->rwi_list);6417	spin_lock_init(&adapter->rwi_lock);6418	spin_lock_init(&adapter->state_lock);6419	mutex_init(&adapter->fw_lock);6420	init_completion(&adapter->probe_done);6421	init_completion(&adapter->init_done);6422	init_completion(&adapter->fw_done);6423	init_completion(&adapter->reset_done);6424	init_completion(&adapter->stats_done);6425	clear_bit(0, &adapter->resetting);6426	adapter->prev_rx_buf_sz = 0;6427	adapter->prev_mtu = 0;6428 6429	init_success = false;6430	do {6431		reinit_init_done(adapter);6432 6433		/* clear any failovers we got in the previous pass6434		 * since we are reinitializing the CRQ6435		 */6436		adapter->failover_pending = false;6437 6438		/* If we had already initialized CRQ, we may have one or6439		 * more resets queued already. Discard those and release6440		 * the CRQ before initializing the CRQ again.6441		 */6442		release_crq_queue(adapter);6443 6444		/* Since we are still in PROBING state, __ibmvnic_reset()6445		 * will not access the ->rwi_list and since we released CRQ,6446		 * we won't get _new_ transport events. But there maybe an6447		 * ongoing ibmvnic_reset() call. So serialize access to6448		 * rwi_list. If we win the race, ibvmnic_reset() could add6449		 * a reset after we purged but thats ok - we just may end6450		 * up with an extra reset (i.e similar to having two or more6451		 * resets in the queue at once).6452		 * CHECK.6453		 */6454		spin_lock_irqsave(&adapter->rwi_lock, flags);6455		flush_reset_queue(adapter);6456		spin_unlock_irqrestore(&adapter->rwi_lock, flags);6457 6458		rc = init_crq_queue(adapter);6459		if (rc) {6460			dev_err(&dev->dev, "Couldn't initialize crq. rc=%d\n",6461				rc);6462			goto ibmvnic_init_fail;6463		}6464 6465		rc = ibmvnic_reset_init(adapter, false);6466	} while (rc == -EAGAIN);6467 6468	/* We are ignoring the error from ibmvnic_reset_init() assuming that the6469	 * partner is not ready. CRQ is not active. When the partner becomes6470	 * ready, we will do the passive init reset.6471	 */6472 6473	if (!rc)6474		init_success = true;6475 6476	rc = init_stats_buffers(adapter);6477	if (rc)6478		goto ibmvnic_init_fail;6479 6480	rc = init_stats_token(adapter);6481	if (rc)6482		goto ibmvnic_stats_fail;6483 6484	rc = device_create_file(&dev->dev, &dev_attr_failover);6485	if (rc)6486		goto ibmvnic_dev_file_err;6487 6488	netif_carrier_off(netdev);6489 6490	if (init_success) {6491		adapter->state = VNIC_PROBED;6492		netdev->mtu = adapter->req_mtu - ETH_HLEN;6493		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;6494		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;6495	} else {6496		adapter->state = VNIC_DOWN;6497	}6498 6499	adapter->wait_for_reset = false;6500	adapter->last_reset_time = jiffies;6501 6502	rc = register_netdev(netdev);6503	if (rc) {6504		dev_err(&dev->dev, "failed to register netdev rc=%d\n", rc);6505		goto ibmvnic_register_fail;6506	}6507	dev_info(&dev->dev, "ibmvnic registered\n");6508 6509	rc = ibmvnic_cpu_notif_add(adapter);6510	if (rc) {6511		netdev_err(netdev, "Registering cpu notifier failed\n");6512		goto cpu_notif_add_failed;6513	}6514 6515	complete(&adapter->probe_done);6516 6517	return 0;6518 6519cpu_notif_add_failed:6520	unregister_netdev(netdev);6521 6522ibmvnic_register_fail:6523	device_remove_file(&dev->dev, &dev_attr_failover);6524 6525ibmvnic_dev_file_err:6526	release_stats_token(adapter);6527 6528ibmvnic_stats_fail:6529	release_stats_buffers(adapter);6530 6531ibmvnic_init_fail:6532	release_sub_crqs(adapter, 1);6533	release_crq_queue(adapter);6534 6535	/* cleanup worker thread after releasing CRQ so we don't get6536	 * transport events (i.e new work items for the worker thread).6537	 */6538	adapter->state = VNIC_REMOVING;6539	complete(&adapter->probe_done);6540	flush_work(&adapter->ibmvnic_reset);6541	flush_delayed_work(&adapter->ibmvnic_delayed_reset);6542 6543	flush_reset_queue(adapter);6544 6545	mutex_destroy(&adapter->fw_lock);6546	free_netdev(netdev);6547 6548	return rc;6549}6550 6551static void ibmvnic_remove(struct vio_dev *dev)6552{6553	struct net_device *netdev = dev_get_drvdata(&dev->dev);6554	struct ibmvnic_adapter *adapter = netdev_priv(netdev);6555	unsigned long flags;6556 6557	spin_lock_irqsave(&adapter->state_lock, flags);6558 6559	/* If ibmvnic_reset() is scheduling a reset, wait for it to6560	 * finish. Then, set the state to REMOVING to prevent it from6561	 * scheduling any more work and to have reset functions ignore6562	 * any resets that have already been scheduled. Drop the lock6563	 * after setting state, so __ibmvnic_reset() which is called6564	 * from the flush_work() below, can make progress.6565	 */6566	spin_lock(&adapter->rwi_lock);6567	adapter->state = VNIC_REMOVING;6568	spin_unlock(&adapter->rwi_lock);6569 6570	spin_unlock_irqrestore(&adapter->state_lock, flags);6571 6572	ibmvnic_cpu_notif_remove(adapter);6573 6574	flush_work(&adapter->ibmvnic_reset);6575	flush_delayed_work(&adapter->ibmvnic_delayed_reset);6576 6577	rtnl_lock();6578	unregister_netdevice(netdev);6579 6580	release_resources(adapter);6581	release_rx_pools(adapter);6582	release_tx_pools(adapter);6583	release_sub_crqs(adapter, 1);6584	release_crq_queue(adapter);6585 6586	release_stats_token(adapter);6587	release_stats_buffers(adapter);6588 6589	adapter->state = VNIC_REMOVED;6590 6591	rtnl_unlock();6592	mutex_destroy(&adapter->fw_lock);6593	device_remove_file(&dev->dev, &dev_attr_failover);6594	free_netdev(netdev);6595	dev_set_drvdata(&dev->dev, NULL);6596}6597 6598static ssize_t failover_store(struct device *dev, struct device_attribute *attr,6599			      const char *buf, size_t count)6600{6601	struct net_device *netdev = dev_get_drvdata(dev);6602	struct ibmvnic_adapter *adapter = netdev_priv(netdev);6603	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];6604	__be64 session_token;6605	long rc;6606 6607	if (!sysfs_streq(buf, "1"))6608		return -EINVAL;6609 6610	rc = plpar_hcall(H_VIOCTL, retbuf, adapter->vdev->unit_address,6611			 H_GET_SESSION_TOKEN, 0, 0, 0);6612	if (rc) {6613		netdev_err(netdev, "Couldn't retrieve session token, rc %ld\n",6614			   rc);6615		goto last_resort;6616	}6617 6618	session_token = (__be64)retbuf[0];6619	netdev_dbg(netdev, "Initiating client failover, session id %llx\n",6620		   be64_to_cpu(session_token));6621	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,6622				H_SESSION_ERR_DETECTED, session_token, 0, 0);6623	if (rc) {6624		netdev_err(netdev,6625			   "H_VIOCTL initiated failover failed, rc %ld\n",6626			   rc);6627		goto last_resort;6628	}6629 6630	return count;6631 6632last_resort:6633	netdev_dbg(netdev, "Trying to send CRQ_CMD, the last resort\n");6634	ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);6635 6636	return count;6637}6638static DEVICE_ATTR_WO(failover);6639 6640static unsigned long ibmvnic_get_desired_dma(struct vio_dev *vdev)6641{6642	struct net_device *netdev = dev_get_drvdata(&vdev->dev);6643	struct ibmvnic_adapter *adapter;6644	struct iommu_table *tbl;6645	unsigned long ret = 0;6646	int i;6647 6648	tbl = get_iommu_table_base(&vdev->dev);6649 6650	/* netdev inits at probe time along with the structures we need below*/6651	if (!netdev)6652		return IOMMU_PAGE_ALIGN(IBMVNIC_IO_ENTITLEMENT_DEFAULT, tbl);6653 6654	adapter = netdev_priv(netdev);6655 6656	ret += PAGE_SIZE; /* the crq message queue */6657	ret += IOMMU_PAGE_ALIGN(sizeof(struct ibmvnic_statistics), tbl);6658 6659	for (i = 0; i < adapter->req_tx_queues + adapter->req_rx_queues; i++)6660		ret += 4 * PAGE_SIZE; /* the scrq message queue */6661 6662	for (i = 0; i < adapter->num_active_rx_pools; i++)6663		ret += adapter->rx_pool[i].size *6664		    IOMMU_PAGE_ALIGN(adapter->rx_pool[i].buff_size, tbl);6665 6666	return ret;6667}6668 6669static int ibmvnic_resume(struct device *dev)6670{6671	struct net_device *netdev = dev_get_drvdata(dev);6672	struct ibmvnic_adapter *adapter = netdev_priv(netdev);6673 6674	if (adapter->state != VNIC_OPEN)6675		return 0;6676 6677	tasklet_schedule(&adapter->tasklet);6678 6679	return 0;6680}6681 6682static const struct vio_device_id ibmvnic_device_table[] = {6683	{"network", "IBM,vnic"},6684	{"", "" }6685};6686MODULE_DEVICE_TABLE(vio, ibmvnic_device_table);6687 6688static const struct dev_pm_ops ibmvnic_pm_ops = {6689	.resume = ibmvnic_resume6690};6691 6692static struct vio_driver ibmvnic_driver = {6693	.id_table       = ibmvnic_device_table,6694	.probe          = ibmvnic_probe,6695	.remove         = ibmvnic_remove,6696	.get_desired_dma = ibmvnic_get_desired_dma,6697	.name		= ibmvnic_driver_name,6698	.pm		= &ibmvnic_pm_ops,6699};6700 6701/* module functions */6702static int __init ibmvnic_module_init(void)6703{6704	int ret;6705 6706	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/ibmvnic:online",6707				      ibmvnic_cpu_online,6708				      ibmvnic_cpu_down_prep);6709	if (ret < 0)6710		goto out;6711	ibmvnic_online = ret;6712	ret = cpuhp_setup_state_multi(CPUHP_IBMVNIC_DEAD, "net/ibmvnic:dead",6713				      NULL, ibmvnic_cpu_dead);6714	if (ret)6715		goto err_dead;6716 6717	ret = vio_register_driver(&ibmvnic_driver);6718	if (ret)6719		goto err_vio_register;6720 6721	pr_info("%s: %s %s\n", ibmvnic_driver_name, ibmvnic_driver_string,6722		IBMVNIC_DRIVER_VERSION);6723 6724	return 0;6725err_vio_register:6726	cpuhp_remove_multi_state(CPUHP_IBMVNIC_DEAD);6727err_dead:6728	cpuhp_remove_multi_state(ibmvnic_online);6729out:6730	return ret;6731}6732 6733static void __exit ibmvnic_module_exit(void)6734{6735	vio_unregister_driver(&ibmvnic_driver);6736	cpuhp_remove_multi_state(CPUHP_IBMVNIC_DEAD);6737	cpuhp_remove_multi_state(ibmvnic_online);6738}6739 6740module_init(ibmvnic_module_init);6741module_exit(ibmvnic_module_exit);6742