brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.0-only2/****************************************************************************3 * Driver for Solarflare network controllers and boards4 * Copyright 2019 Solarflare Communications Inc.5 * Copyright 2020-2022 Xilinx Inc.6 *7 * This program is free software; you can redistribute it and/or modify it8 * under the terms of the GNU General Public License version 2 as published9 * by the Free Software Foundation, incorporated herein by reference.10 */11 12#include <net/pkt_cls.h>13#include <net/vxlan.h>14#include <net/geneve.h>15#include <net/tc_act/tc_ct.h>16#include "tc.h"17#include "tc_bindings.h"18#include "tc_encap_actions.h"19#include "tc_conntrack.h"20#include "mae.h"21#include "ef100_rep.h"22#include "efx.h"23 24enum efx_encap_type efx_tc_indr_netdev_type(struct net_device *net_dev)25{26	if (netif_is_vxlan(net_dev))27		return EFX_ENCAP_TYPE_VXLAN;28	if (netif_is_geneve(net_dev))29		return EFX_ENCAP_TYPE_GENEVE;30 31	return EFX_ENCAP_TYPE_NONE;32}33 34#define EFX_TC_HDR_TYPE_TTL_MASK ((u32)0xff)35/* Hoplimit is stored in the most significant byte in the pedit ipv6 header action */36#define EFX_TC_HDR_TYPE_HLIMIT_MASK ~((u32)0xff000000)37#define EFX_EFV_PF	NULL38/* Look up the representor information (efv) for a device.39 * May return NULL for the PF (us), or an error pointer for a device that40 * isn't supported as a TC offload endpoint41 */42struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx,43					 struct net_device *dev)44{45	struct efx_rep *efv;46 47	if (!dev)48		return ERR_PTR(-EOPNOTSUPP);49	/* Is it us (the PF)? */50	if (dev == efx->net_dev)51		return EFX_EFV_PF;52	/* Is it an efx vfrep at all? */53	if (dev->netdev_ops != &efx_ef100_rep_netdev_ops)54		return ERR_PTR(-EOPNOTSUPP);55	/* Is it ours?  We don't support TC rules that include another56	 * EF100's netdevices (not even on another port of the same NIC).57	 */58	efv = netdev_priv(dev);59	if (efv->parent != efx)60		return ERR_PTR(-EOPNOTSUPP);61	return efv;62}63 64/* Convert a driver-internal vport ID into an internal device (PF or VF) */65static s64 efx_tc_flower_internal_mport(struct efx_nic *efx, struct efx_rep *efv)66{67	u32 mport;68 69	if (IS_ERR(efv))70		return PTR_ERR(efv);71	if (!efv) /* device is PF (us) */72		efx_mae_mport_uplink(efx, &mport);73	else /* device is repr */74		efx_mae_mport_mport(efx, efv->mport, &mport);75	return mport;76}77 78/* Convert a driver-internal vport ID into an external device (wire or VF) */79s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv)80{81	u32 mport;82 83	if (IS_ERR(efv))84		return PTR_ERR(efv);85	if (!efv) /* device is PF (us) */86		efx_mae_mport_wire(efx, &mport);87	else /* device is repr */88		efx_mae_mport_mport(efx, efv->mport, &mport);89	return mport;90}91 92static const struct rhashtable_params efx_tc_mac_ht_params = {93	.key_len	= offsetofend(struct efx_tc_mac_pedit_action, h_addr),94	.key_offset	= 0,95	.head_offset	= offsetof(struct efx_tc_mac_pedit_action, linkage),96};97 98static const struct rhashtable_params efx_tc_encap_match_ht_params = {99	.key_len	= offsetof(struct efx_tc_encap_match, linkage),100	.key_offset	= 0,101	.head_offset	= offsetof(struct efx_tc_encap_match, linkage),102};103 104static const struct rhashtable_params efx_tc_match_action_ht_params = {105	.key_len	= sizeof(unsigned long),106	.key_offset	= offsetof(struct efx_tc_flow_rule, cookie),107	.head_offset	= offsetof(struct efx_tc_flow_rule, linkage),108};109 110static const struct rhashtable_params efx_tc_lhs_rule_ht_params = {111	.key_len	= sizeof(unsigned long),112	.key_offset	= offsetof(struct efx_tc_lhs_rule, cookie),113	.head_offset	= offsetof(struct efx_tc_lhs_rule, linkage),114};115 116static const struct rhashtable_params efx_tc_recirc_ht_params = {117	.key_len	= offsetof(struct efx_tc_recirc_id, linkage),118	.key_offset	= 0,119	.head_offset	= offsetof(struct efx_tc_recirc_id, linkage),120};121 122static struct efx_tc_mac_pedit_action *efx_tc_flower_get_mac(struct efx_nic *efx,123							     unsigned char h_addr[ETH_ALEN],124							     struct netlink_ext_ack *extack)125{126	struct efx_tc_mac_pedit_action *ped, *old;127	int rc;128 129	ped = kzalloc(sizeof(*ped), GFP_USER);130	if (!ped)131		return ERR_PTR(-ENOMEM);132	memcpy(ped->h_addr, h_addr, ETH_ALEN);133	old = rhashtable_lookup_get_insert_fast(&efx->tc->mac_ht,134						&ped->linkage,135						efx_tc_mac_ht_params);136	if (old) {137		/* don't need our new entry */138		kfree(ped);139		if (IS_ERR(old)) /* oh dear, it's actually an error */140			return ERR_CAST(old);141		if (!refcount_inc_not_zero(&old->ref))142			return ERR_PTR(-EAGAIN);143		/* existing entry found, ref taken */144		return old;145	}146 147	rc = efx_mae_allocate_pedit_mac(efx, ped);148	if (rc < 0) {149		NL_SET_ERR_MSG_MOD(extack, "Failed to store pedit MAC address in hw");150		goto out_remove;151	}152 153	/* ref and return */154	refcount_set(&ped->ref, 1);155	return ped;156out_remove:157	rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,158			       efx_tc_mac_ht_params);159	kfree(ped);160	return ERR_PTR(rc);161}162 163static void efx_tc_flower_put_mac(struct efx_nic *efx,164				  struct efx_tc_mac_pedit_action *ped)165{166	if (!refcount_dec_and_test(&ped->ref))167		return; /* still in use */168	rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,169			       efx_tc_mac_ht_params);170	efx_mae_free_pedit_mac(efx, ped);171	kfree(ped);172}173 174static void efx_tc_free_action_set(struct efx_nic *efx,175				   struct efx_tc_action_set *act, bool in_hw)176{177	/* Failure paths calling this on the 'cursor' action set in_hw=false,178	 * because if the alloc had succeeded we'd've put it in acts.list and179	 * not still have it in act.180	 */181	if (in_hw) {182		efx_mae_free_action_set(efx, act->fw_id);183		/* in_hw is true iff we are on an acts.list; make sure to184		 * remove ourselves from that list before we are freed.185		 */186		list_del(&act->list);187	}188	if (act->count) {189		spin_lock_bh(&act->count->cnt->lock);190		if (!list_empty(&act->count_user))191			list_del(&act->count_user);192		spin_unlock_bh(&act->count->cnt->lock);193		efx_tc_flower_put_counter_index(efx, act->count);194	}195	if (act->encap_md) {196		list_del(&act->encap_user);197		efx_tc_flower_release_encap_md(efx, act->encap_md);198	}199	if (act->src_mac)200		efx_tc_flower_put_mac(efx, act->src_mac);201	if (act->dst_mac)202		efx_tc_flower_put_mac(efx, act->dst_mac);203	kfree(act);204}205 206static void efx_tc_free_action_set_list(struct efx_nic *efx,207					struct efx_tc_action_set_list *acts,208					bool in_hw)209{210	struct efx_tc_action_set *act, *next;211 212	/* Failure paths set in_hw=false, because usually the acts didn't get213	 * to efx_mae_alloc_action_set_list(); if they did, the failure tree214	 * has a separate efx_mae_free_action_set_list() before calling us.215	 */216	if (in_hw)217		efx_mae_free_action_set_list(efx, acts);218	/* Any act that's on the list will be in_hw even if the list isn't */219	list_for_each_entry_safe(act, next, &acts->list, list)220		efx_tc_free_action_set(efx, act, true);221	/* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */222}223 224/* Boilerplate for the simple 'copy a field' cases */225#define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field)	\226if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) {		\227	struct flow_match_##_type fm;					\228									\229	flow_rule_match_##_tcget(rule, &fm);				\230	match->value._field = fm.key->_tcfield;				\231	match->mask._field = fm.mask->_tcfield;				\232}233#define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field)	\234	_MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field)235#define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field)	\236	_MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field)237 238static int efx_tc_flower_parse_match(struct efx_nic *efx,239				     struct flow_rule *rule,240				     struct efx_tc_match *match,241				     struct netlink_ext_ack *extack)242{243	struct flow_dissector *dissector = rule->match.dissector;244	unsigned char ipv = 0;245 246	/* Owing to internal TC infelicities, the IPV6_ADDRS key might be set247	 * even on IPv4 filters; so rather than relying on dissector->used_keys248	 * we check the addr_type in the CONTROL key.  If we don't find it (or249	 * it's masked, which should never happen), we treat both IPV4_ADDRS250	 * and IPV6_ADDRS as absent.251	 */252	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {253		struct flow_match_control fm;254 255		flow_rule_match_control(rule, &fm);256		if (IS_ALL_ONES(fm.mask->addr_type))257			switch (fm.key->addr_type) {258			case FLOW_DISSECTOR_KEY_IPV4_ADDRS:259				ipv = 4;260				break;261			case FLOW_DISSECTOR_KEY_IPV6_ADDRS:262				ipv = 6;263				break;264			default:265				break;266			}267 268		if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) {269			match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT;270			match->mask.ip_frag = true;271		}272		if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) {273			match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG;274			match->mask.ip_firstfrag = true;275		}276		if (!flow_rule_is_supp_control_flags(FLOW_DIS_IS_FRAGMENT |277						     FLOW_DIS_FIRST_FRAG,278						     fm.mask->flags, extack))279			return -EOPNOTSUPP;280	}281	if (dissector->used_keys &282	    ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |283	      BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |284	      BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |285	      BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |286	      BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) |287	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |288	      BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |289	      BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |290	      BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |291	      BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |292	      BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |293	      BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |294	      BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) |295	      BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) |296	      BIT_ULL(FLOW_DISSECTOR_KEY_CT) |297	      BIT_ULL(FLOW_DISSECTOR_KEY_TCP) |298	      BIT_ULL(FLOW_DISSECTOR_KEY_IP))) {299		NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#llx",300				       dissector->used_keys);301		return -EOPNOTSUPP;302	}303 304	MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto);305	/* Make sure we're IP if any L3/L4 keys used. */306	if (!IS_ALL_ONES(match->mask.eth_proto) ||307	    !(match->value.eth_proto == htons(ETH_P_IP) ||308	      match->value.eth_proto == htons(ETH_P_IPV6)))309		if (dissector->used_keys &310		    (BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |311		     BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |312		     BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |313		     BIT_ULL(FLOW_DISSECTOR_KEY_IP) |314		     BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {315			NL_SET_ERR_MSG_FMT_MOD(extack,316					       "L3/L4 flower keys %#llx require protocol ipv[46]",317					       dissector->used_keys);318			return -EINVAL;319		}320 321	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {322		struct flow_match_vlan fm;323 324		flow_rule_match_vlan(rule, &fm);325		if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {326			match->value.vlan_proto[0] = fm.key->vlan_tpid;327			match->mask.vlan_proto[0] = fm.mask->vlan_tpid;328			match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 |329							       fm.key->vlan_id);330			match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 |331							      fm.mask->vlan_id);332		}333	}334 335	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) {336		struct flow_match_vlan fm;337 338		flow_rule_match_cvlan(rule, &fm);339		if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {340			match->value.vlan_proto[1] = fm.key->vlan_tpid;341			match->mask.vlan_proto[1] = fm.mask->vlan_tpid;342			match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 |343							       fm.key->vlan_id);344			match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 |345							      fm.mask->vlan_id);346		}347	}348 349	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {350		struct flow_match_eth_addrs fm;351 352		flow_rule_match_eth_addrs(rule, &fm);353		ether_addr_copy(match->value.eth_saddr, fm.key->src);354		ether_addr_copy(match->value.eth_daddr, fm.key->dst);355		ether_addr_copy(match->mask.eth_saddr, fm.mask->src);356		ether_addr_copy(match->mask.eth_daddr, fm.mask->dst);357	}358 359	MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto);360	/* Make sure we're TCP/UDP if any L4 keys used. */361	if ((match->value.ip_proto != IPPROTO_UDP &&362	     match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto))363		if (dissector->used_keys &364		    (BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |365		     BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {366			NL_SET_ERR_MSG_FMT_MOD(extack,367					       "L4 flower keys %#llx require ipproto udp or tcp",368					       dissector->used_keys);369			return -EINVAL;370		}371	MAP_KEY_AND_MASK(IP, ip, tos, ip_tos);372	MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl);373	if (ipv == 4) {374		MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip);375		MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip);376	}377#ifdef CONFIG_IPV6378	else if (ipv == 6) {379		MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6);380		MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6);381	}382#endif383	MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport);384	MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport);385	MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags);386	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {387		struct flow_match_control fm;388 389		flow_rule_match_enc_control(rule, &fm);390		if (flow_rule_has_enc_control_flags(fm.mask->flags, extack))391			return -EOPNOTSUPP;392		if (!IS_ALL_ONES(fm.mask->addr_type)) {393			NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)",394					       fm.mask->addr_type,395					       fm.key->addr_type);396			return -EOPNOTSUPP;397		}398		switch (fm.key->addr_type) {399		case FLOW_DISSECTOR_KEY_IPV4_ADDRS:400			MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,401					     src, enc_src_ip);402			MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,403					     dst, enc_dst_ip);404			break;405#ifdef CONFIG_IPV6406		case FLOW_DISSECTOR_KEY_IPV6_ADDRS:407			MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,408					     src, enc_src_ip6);409			MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,410					     dst, enc_dst_ip6);411			break;412#endif413		default:414			NL_SET_ERR_MSG_FMT_MOD(extack,415					       "Unsupported enc addr_type %u (supported are IPv4, IPv6)",416					       fm.key->addr_type);417			return -EOPNOTSUPP;418		}419		MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, tos, enc_ip_tos);420		MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, ttl, enc_ip_ttl);421		MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, src, enc_sport);422		MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport);423		MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid);424	} else if (dissector->used_keys &425		   (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |426		    BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |427		    BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |428		    BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |429		    BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS))) {430		NL_SET_ERR_MSG_FMT_MOD(extack,431				       "Flower enc keys require enc_control (keys: %#llx)",432				       dissector->used_keys);433		return -EOPNOTSUPP;434	}435	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT)) {436		struct flow_match_ct fm;437 438		flow_rule_match_ct(rule, &fm);439		match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);440		match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);441		match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);442		match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);443		if (fm.mask->ct_state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |444					  TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)) {445			NL_SET_ERR_MSG_FMT_MOD(extack,446					       "Unsupported ct_state match %#x",447					       fm.mask->ct_state);448			return -EOPNOTSUPP;449		}450		match->value.ct_mark = fm.key->ct_mark;451		match->mask.ct_mark = fm.mask->ct_mark;452		match->value.ct_zone = fm.key->ct_zone;453		match->mask.ct_zone = fm.mask->ct_zone;454 455		if (memchr_inv(fm.mask->ct_labels, 0, sizeof(fm.mask->ct_labels))) {456			NL_SET_ERR_MSG_MOD(extack, "Matching on ct_label not supported");457			return -EOPNOTSUPP;458		}459	}460 461	return 0;462}463 464static void efx_tc_flower_release_encap_match(struct efx_nic *efx,465					      struct efx_tc_encap_match *encap)466{467	int rc;468 469	if (!refcount_dec_and_test(&encap->ref))470		return; /* still in use */471 472	if (encap->type == EFX_TC_EM_DIRECT) {473		rc = efx_mae_unregister_encap_match(efx, encap);474		if (rc)475			/* Display message but carry on and remove entry from our476			 * SW tables, because there's not much we can do about it.477			 */478			netif_err(efx, drv, efx->net_dev,479				  "Failed to release encap match %#x, rc %d\n",480				  encap->fw_id, rc);481	}482	rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,483			       efx_tc_encap_match_ht_params);484	if (encap->pseudo)485		efx_tc_flower_release_encap_match(efx, encap->pseudo);486	kfree(encap);487}488 489static int efx_tc_flower_record_encap_match(struct efx_nic *efx,490					    struct efx_tc_match *match,491					    enum efx_encap_type type,492					    enum efx_tc_em_pseudo_type em_type,493					    u8 child_ip_tos_mask,494					    __be16 child_udp_sport_mask,495					    struct netlink_ext_ack *extack)496{497	struct efx_tc_encap_match *encap, *old, *pseudo = NULL;498	bool ipv6 = false;499	int rc;500 501	/* We require that the socket-defining fields (IP addrs and UDP dest502	 * port) are present and exact-match.  Other fields may only be used503	 * if the field-set (and any masks) are the same for all encap504	 * matches on the same <sip,dip,dport> tuple; this is enforced by505	 * pseudo encap matches.506	 */507	if (match->mask.enc_dst_ip | match->mask.enc_src_ip) {508		if (!IS_ALL_ONES(match->mask.enc_dst_ip)) {509			NL_SET_ERR_MSG_MOD(extack,510					   "Egress encap match is not exact on dst IP address");511			return -EOPNOTSUPP;512		}513		if (!IS_ALL_ONES(match->mask.enc_src_ip)) {514			NL_SET_ERR_MSG_MOD(extack,515					   "Egress encap match is not exact on src IP address");516			return -EOPNOTSUPP;517		}518#ifdef CONFIG_IPV6519		if (!ipv6_addr_any(&match->mask.enc_dst_ip6) ||520		    !ipv6_addr_any(&match->mask.enc_src_ip6)) {521			NL_SET_ERR_MSG_MOD(extack,522					   "Egress encap match on both IPv4 and IPv6, don't understand");523			return -EOPNOTSUPP;524		}525	} else {526		ipv6 = true;527		if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) {528			NL_SET_ERR_MSG_MOD(extack,529					   "Egress encap match is not exact on dst IP address");530			return -EOPNOTSUPP;531		}532		if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) {533			NL_SET_ERR_MSG_MOD(extack,534					   "Egress encap match is not exact on src IP address");535			return -EOPNOTSUPP;536		}537#endif538	}539	if (!IS_ALL_ONES(match->mask.enc_dport)) {540		NL_SET_ERR_MSG_MOD(extack, "Egress encap match is not exact on dst UDP port");541		return -EOPNOTSUPP;542	}543	if (match->mask.enc_sport || match->mask.enc_ip_tos) {544		struct efx_tc_match pmatch = *match;545 546		if (em_type == EFX_TC_EM_PSEUDO_MASK) { /* can't happen */547			NL_SET_ERR_MSG_MOD(extack, "Bad recursion in egress encap match handler");548			return -EOPNOTSUPP;549		}550		pmatch.value.enc_ip_tos = 0;551		pmatch.mask.enc_ip_tos = 0;552		pmatch.value.enc_sport = 0;553		pmatch.mask.enc_sport = 0;554		rc = efx_tc_flower_record_encap_match(efx, &pmatch, type,555						      EFX_TC_EM_PSEUDO_MASK,556						      match->mask.enc_ip_tos,557						      match->mask.enc_sport,558						      extack);559		if (rc)560			return rc;561		pseudo = pmatch.encap;562	}563	if (match->mask.enc_ip_ttl) {564		NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP TTL not supported");565		rc = -EOPNOTSUPP;566		goto fail_pseudo;567	}568 569	rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos,570					    match->mask.enc_sport, extack);571	if (rc)572		goto fail_pseudo;573 574	encap = kzalloc(sizeof(*encap), GFP_USER);575	if (!encap) {576		rc = -ENOMEM;577		goto fail_pseudo;578	}579	encap->src_ip = match->value.enc_src_ip;580	encap->dst_ip = match->value.enc_dst_ip;581#ifdef CONFIG_IPV6582	encap->src_ip6 = match->value.enc_src_ip6;583	encap->dst_ip6 = match->value.enc_dst_ip6;584#endif585	encap->udp_dport = match->value.enc_dport;586	encap->tun_type = type;587	encap->ip_tos = match->value.enc_ip_tos;588	encap->ip_tos_mask = match->mask.enc_ip_tos;589	encap->child_ip_tos_mask = child_ip_tos_mask;590	encap->udp_sport = match->value.enc_sport;591	encap->udp_sport_mask = match->mask.enc_sport;592	encap->child_udp_sport_mask = child_udp_sport_mask;593	encap->type = em_type;594	encap->pseudo = pseudo;595	old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht,596						&encap->linkage,597						efx_tc_encap_match_ht_params);598	if (old) {599		/* don't need our new entry */600		kfree(encap);601		if (pseudo) /* don't need our new pseudo either */602			efx_tc_flower_release_encap_match(efx, pseudo);603		if (IS_ERR(old)) /* oh dear, it's actually an error */604			return PTR_ERR(old);605		/* check old and new em_types are compatible */606		switch (old->type) {607		case EFX_TC_EM_DIRECT:608			/* old EM is in hardware, so mustn't overlap with a609			 * pseudo, but may be shared with another direct EM610			 */611			if (em_type == EFX_TC_EM_DIRECT)612				break;613			NL_SET_ERR_MSG_MOD(extack, "Pseudo encap match conflicts with existing direct entry");614			return -EEXIST;615		case EFX_TC_EM_PSEUDO_MASK:616			/* old EM is protecting a ToS- or src port-qualified617			 * filter, so may only be shared with another pseudo618			 * for the same ToS and src port masks.619			 */620			if (em_type != EFX_TC_EM_PSEUDO_MASK) {621				NL_SET_ERR_MSG_FMT_MOD(extack,622						       "%s encap match conflicts with existing pseudo(MASK) entry",623						       em_type ? "Pseudo" : "Direct");624				return -EEXIST;625			}626			if (child_ip_tos_mask != old->child_ip_tos_mask) {627				NL_SET_ERR_MSG_FMT_MOD(extack,628						       "Pseudo encap match for TOS mask %#04x conflicts with existing mask %#04x",629						       child_ip_tos_mask,630						       old->child_ip_tos_mask);631				return -EEXIST;632			}633			if (child_udp_sport_mask != old->child_udp_sport_mask) {634				NL_SET_ERR_MSG_FMT_MOD(extack,635						       "Pseudo encap match for UDP src port mask %#x conflicts with existing mask %#x",636						       child_udp_sport_mask,637						       old->child_udp_sport_mask);638				return -EEXIST;639			}640			break;641		case EFX_TC_EM_PSEUDO_OR:642			/* old EM corresponds to an OR that has to be unique643			 * (it must not overlap with any other OR, whether644			 * direct-EM or pseudo).645			 */646			NL_SET_ERR_MSG_FMT_MOD(extack,647					       "%s encap match conflicts with existing pseudo(OR) entry",648					       em_type ? "Pseudo" : "Direct");649			return -EEXIST;650		default: /* Unrecognised pseudo-type.  Just say no */651			NL_SET_ERR_MSG_FMT_MOD(extack,652					       "%s encap match conflicts with existing pseudo(%d) entry",653					       em_type ? "Pseudo" : "Direct",654					       old->type);655			return -EEXIST;656		}657		/* check old and new tun_types are compatible */658		if (old->tun_type != type) {659			NL_SET_ERR_MSG_FMT_MOD(extack,660					       "Egress encap match with conflicting tun_type %u != %u",661					       old->tun_type, type);662			return -EEXIST;663		}664		if (!refcount_inc_not_zero(&old->ref))665			return -EAGAIN;666		/* existing entry found */667		encap = old;668	} else {669		if (em_type == EFX_TC_EM_DIRECT) {670			rc = efx_mae_register_encap_match(efx, encap);671			if (rc) {672				NL_SET_ERR_MSG_MOD(extack, "Failed to record egress encap match in HW");673				goto fail;674			}675		}676		refcount_set(&encap->ref, 1);677	}678	match->encap = encap;679	return 0;680fail:681	rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,682			       efx_tc_encap_match_ht_params);683	kfree(encap);684fail_pseudo:685	if (pseudo)686		efx_tc_flower_release_encap_match(efx, pseudo);687	return rc;688}689 690static struct efx_tc_recirc_id *efx_tc_get_recirc_id(struct efx_nic *efx,691						     u32 chain_index,692						     struct net_device *net_dev)693{694	struct efx_tc_recirc_id *rid, *old;695	int rc;696 697	rid = kzalloc(sizeof(*rid), GFP_USER);698	if (!rid)699		return ERR_PTR(-ENOMEM);700	rid->chain_index = chain_index;701	/* We don't take a reference here, because it's implied - if there's702	 * a rule on the net_dev that's been offloaded to us, then the net_dev703	 * can't go away until the rule has been deoffloaded.704	 */705	rid->net_dev = net_dev;706	old = rhashtable_lookup_get_insert_fast(&efx->tc->recirc_ht,707						&rid->linkage,708						efx_tc_recirc_ht_params);709	if (old) {710		/* don't need our new entry */711		kfree(rid);712		if (IS_ERR(old)) /* oh dear, it's actually an error */713			return ERR_CAST(old);714		if (!refcount_inc_not_zero(&old->ref))715			return ERR_PTR(-EAGAIN);716		/* existing entry found */717		rid = old;718	} else {719		rc = ida_alloc_range(&efx->tc->recirc_ida, 1, U8_MAX, GFP_USER);720		if (rc < 0) {721			rhashtable_remove_fast(&efx->tc->recirc_ht,722					       &rid->linkage,723					       efx_tc_recirc_ht_params);724			kfree(rid);725			return ERR_PTR(rc);726		}727		rid->fw_id = rc;728		refcount_set(&rid->ref, 1);729	}730	return rid;731}732 733static void efx_tc_put_recirc_id(struct efx_nic *efx, struct efx_tc_recirc_id *rid)734{735	if (!refcount_dec_and_test(&rid->ref))736		return; /* still in use */737	rhashtable_remove_fast(&efx->tc->recirc_ht, &rid->linkage,738			       efx_tc_recirc_ht_params);739	ida_free(&efx->tc->recirc_ida, rid->fw_id);740	kfree(rid);741}742 743static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule)744{745	efx_mae_delete_rule(efx, rule->fw_id);746 747	/* Release entries in subsidiary tables */748	efx_tc_free_action_set_list(efx, &rule->acts, true);749	if (rule->match.rid)750		efx_tc_put_recirc_id(efx, rule->match.rid);751	if (rule->match.encap)752		efx_tc_flower_release_encap_match(efx, rule->match.encap);753	rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;754}755 756static const char *efx_tc_encap_type_name(enum efx_encap_type typ)757{758	switch (typ) {759	case EFX_ENCAP_TYPE_NONE:760		return "none";761	case EFX_ENCAP_TYPE_VXLAN:762		return "vxlan";763	case EFX_ENCAP_TYPE_GENEVE:764		return "geneve";765	default:766		pr_warn_once("Unknown efx_encap_type %d encountered\n", typ);767		return "unknown";768	}769}770 771/* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */772enum efx_tc_action_order {773	EFX_TC_AO_DECAP,774	EFX_TC_AO_DEC_TTL,775	EFX_TC_AO_PEDIT_MAC_ADDRS,776	EFX_TC_AO_VLAN_POP,777	EFX_TC_AO_VLAN_PUSH,778	EFX_TC_AO_COUNT,779	EFX_TC_AO_ENCAP,780	EFX_TC_AO_DELIVER781};782/* Determine whether we can add @new action without violating order */783static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,784					  enum efx_tc_action_order new)785{786	switch (new) {787	case EFX_TC_AO_DECAP:788		if (act->decap)789			return false;790		/* PEDIT_MAC_ADDRS must not happen before DECAP, though it791		 * can wait until much later792		 */793		if (act->dst_mac || act->src_mac)794			return false;795 796		/* Decrementing ttl must not happen before DECAP */797		if (act->do_ttl_dec)798			return false;799		fallthrough;800	case EFX_TC_AO_VLAN_POP:801		if (act->vlan_pop >= 2)802			return false;803		/* If we've already pushed a VLAN, we can't then pop it;804		 * the hardware would instead try to pop an existing VLAN805		 * before pushing the new one.806		 */807		if (act->vlan_push)808			return false;809		fallthrough;810	case EFX_TC_AO_VLAN_PUSH:811		if (act->vlan_push >= 2)812			return false;813		fallthrough;814	case EFX_TC_AO_COUNT:815		if (act->count)816			return false;817		fallthrough;818	case EFX_TC_AO_PEDIT_MAC_ADDRS:819	case EFX_TC_AO_ENCAP:820		if (act->encap_md)821			return false;822		fallthrough;823	case EFX_TC_AO_DELIVER:824		return !act->deliver;825	case EFX_TC_AO_DEC_TTL:826		if (act->encap_md)827			return false;828		return !act->do_ttl_dec;829	default:830		/* Bad caller.  Whatever they wanted to do, say they can't. */831		WARN_ON_ONCE(1);832		return false;833	}834}835 836/**837 * DOC: TC conntrack sequences838 *839 * The MAE hardware can handle at most two rounds of action rule matching,840 * consequently we support conntrack through the notion of a "left-hand side841 * rule".  This is a rule which typically contains only the actions "ct" and842 * "goto chain N", and corresponds to one or more "right-hand side rules" in843 * chain N, which typically match on +trk+est, and may perform ct(nat) actions.844 * RHS rules go in the Action Rule table as normal but with a nonzero recirc_id845 * (the hardware equivalent of chain_index), while LHS rules may go in either846 * the Action Rule or the Outer Rule table, the latter being preferred for847 * performance reasons, and set both DO_CT and a recirc_id in their response.848 *849 * Besides the RHS rules, there are often also similar rules matching on850 * +trk+new which perform the ct(commit) action.  These are not offloaded.851 */852 853static bool efx_tc_rule_is_lhs_rule(struct flow_rule *fr,854				    struct efx_tc_match *match)855{856	const struct flow_action_entry *fa;857	int i;858 859	flow_action_for_each(i, fa, &fr->action) {860		switch (fa->id) {861		case FLOW_ACTION_GOTO:862			return true;863		case FLOW_ACTION_CT:864			/* If rule is -trk, or doesn't mention trk at all, then865			 * a CT action implies a conntrack lookup (hence it's an866			 * LHS rule).  If rule is +trk, then a CT action could867			 * just be ct(nat) or even ct(commit) (though the latter868			 * can't be offloaded).869			 */870			if (!match->mask.ct_state_trk || !match->value.ct_state_trk)871				return true;872			break;873		default:874			break;875		}876	}877	return false;878}879 880/* A foreign LHS rule has matches on enc_ keys at the TC layer (including an881 * implied match on enc_ip_proto UDP).  Translate these into non-enc_ keys,882 * so that we can use the same MAE machinery as local LHS rules (and so that883 * the lhs_rules entries have uniform semantics).  It may seem odd to do it884 * this way round, given that the corresponding fields in the MAE MCDIs are885 * all ENC_, but (a) we don't have enc_L2 or enc_ip_proto in struct886 * efx_tc_match_fields and (b) semantically an LHS rule doesn't have inner887 * fields so it's just matching on *the* header rather than the outer header.888 * Make sure that the non-enc_ keys were not already being matched on, as that889 * would imply a rule that needed a triple lookup.  (Hardware can do that,890 * with OR-AR-CT-AR, but it halves packet rate so we avoid it where possible;891 * see efx_tc_flower_flhs_needs_ar().)892 */893static int efx_tc_flower_translate_flhs_match(struct efx_tc_match *match)894{895	int rc = 0;896 897#define COPY_MASK_AND_VALUE(_key, _ekey)	({	\898	if (match->mask._key) {				\899		rc = -EOPNOTSUPP;			\900	} else {					\901		match->mask._key = match->mask._ekey;	\902		match->mask._ekey = 0;			\903		match->value._key = match->value._ekey;	\904		match->value._ekey = 0;			\905	}						\906	rc;						\907})908#define COPY_FROM_ENC(_key)	COPY_MASK_AND_VALUE(_key, enc_##_key)909	if (match->mask.ip_proto)910		return -EOPNOTSUPP;911	match->mask.ip_proto = ~0;912	match->value.ip_proto = IPPROTO_UDP;913	if (COPY_FROM_ENC(src_ip) || COPY_FROM_ENC(dst_ip))914		return rc;915#ifdef CONFIG_IPV6916	if (!ipv6_addr_any(&match->mask.src_ip6))917		return -EOPNOTSUPP;918	match->mask.src_ip6 = match->mask.enc_src_ip6;919	memset(&match->mask.enc_src_ip6, 0, sizeof(struct in6_addr));920	if (!ipv6_addr_any(&match->mask.dst_ip6))921		return -EOPNOTSUPP;922	match->mask.dst_ip6 = match->mask.enc_dst_ip6;923	memset(&match->mask.enc_dst_ip6, 0, sizeof(struct in6_addr));924#endif925	if (COPY_FROM_ENC(ip_tos) || COPY_FROM_ENC(ip_ttl))926		return rc;927	/* should really copy enc_ip_frag but we don't have that in928	 * parse_match yet929	 */930	if (COPY_MASK_AND_VALUE(l4_sport, enc_sport) ||931	    COPY_MASK_AND_VALUE(l4_dport, enc_dport))932		return rc;933	return 0;934#undef COPY_FROM_ENC935#undef COPY_MASK_AND_VALUE936}937 938/* If a foreign LHS rule wants to match on keys that are only available after939 * encap header identification and parsing, then it can't be done in the Outer940 * Rule lookup, because that lookup determines the encap type used to parse941 * beyond the outer headers.  Thus, such rules must use the OR-AR-CT-AR lookup942 * sequence, with an EM (struct efx_tc_encap_match) in the OR step.943 * Return true iff the passed match requires this.944 */945static bool efx_tc_flower_flhs_needs_ar(struct efx_tc_match *match)946{947	/* matches on inner-header keys can't be done in OR */948	return match->mask.eth_proto ||949	       match->mask.vlan_tci[0] || match->mask.vlan_tci[1] ||950	       match->mask.vlan_proto[0] || match->mask.vlan_proto[1] ||951	       memchr_inv(match->mask.eth_saddr, 0, ETH_ALEN) ||952	       memchr_inv(match->mask.eth_daddr, 0, ETH_ALEN) ||953	       match->mask.ip_proto ||954	       match->mask.ip_tos || match->mask.ip_ttl ||955	       match->mask.src_ip || match->mask.dst_ip ||956#ifdef CONFIG_IPV6957	       !ipv6_addr_any(&match->mask.src_ip6) ||958	       !ipv6_addr_any(&match->mask.dst_ip6) ||959#endif960	       match->mask.ip_frag || match->mask.ip_firstfrag ||961	       match->mask.l4_sport || match->mask.l4_dport ||962	       match->mask.tcp_flags ||963	/* nor can VNI */964	       match->mask.enc_keyid;965}966 967static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx,968					    struct flow_cls_offload *tc,969					    struct flow_rule *fr,970					    struct net_device *net_dev,971					    struct efx_tc_lhs_rule *rule)972 973{974	struct netlink_ext_ack *extack = tc->common.extack;975	struct efx_tc_lhs_action *act = &rule->lhs_act;976	const struct flow_action_entry *fa;977	enum efx_tc_counter_type ctype;978	bool pipe = true;979	int i;980 981	ctype = rule->is_ar ? EFX_TC_COUNTER_TYPE_AR : EFX_TC_COUNTER_TYPE_OR;982 983	flow_action_for_each(i, fa, &fr->action) {984		struct efx_tc_ct_zone *ct_zone;985		struct efx_tc_recirc_id *rid;986 987		if (!pipe) {988			/* more actions after a non-pipe action */989			NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");990			return -EINVAL;991		}992		switch (fa->id) {993		case FLOW_ACTION_GOTO:994			if (!fa->chain_index) {995				NL_SET_ERR_MSG_MOD(extack, "Can't goto chain 0, no looping in hw");996				return -EOPNOTSUPP;997			}998			rid = efx_tc_get_recirc_id(efx, fa->chain_index,999						   net_dev);1000			if (IS_ERR(rid)) {1001				NL_SET_ERR_MSG_MOD(extack, "Failed to allocate a hardware recirculation ID for this chain_index");1002				return PTR_ERR(rid);1003			}1004			act->rid = rid;1005			if (fa->hw_stats) {1006				struct efx_tc_counter_index *cnt;1007 1008				if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {1009					NL_SET_ERR_MSG_FMT_MOD(extack,1010							       "hw_stats_type %u not supported (only 'delayed')",1011							       fa->hw_stats);1012					return -EOPNOTSUPP;1013				}1014				cnt = efx_tc_flower_get_counter_index(efx, tc->cookie,1015								      ctype);1016				if (IS_ERR(cnt)) {1017					NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");1018					return PTR_ERR(cnt);1019				}1020				WARN_ON(act->count); /* can't happen */1021				act->count = cnt;1022			}1023			pipe = false;1024			break;1025		case FLOW_ACTION_CT:1026			if (act->zone) {1027				NL_SET_ERR_MSG_MOD(extack, "Can't offload multiple ct actions");1028				return -EOPNOTSUPP;1029			}1030			if (fa->ct.action & (TCA_CT_ACT_COMMIT |1031					     TCA_CT_ACT_FORCE)) {1032				NL_SET_ERR_MSG_MOD(extack, "Can't offload ct commit/force");1033				return -EOPNOTSUPP;1034			}1035			if (fa->ct.action & TCA_CT_ACT_CLEAR) {1036				NL_SET_ERR_MSG_MOD(extack, "Can't clear ct in LHS rule");1037				return -EOPNOTSUPP;1038			}1039			if (fa->ct.action & (TCA_CT_ACT_NAT |1040					     TCA_CT_ACT_NAT_SRC |1041					     TCA_CT_ACT_NAT_DST)) {1042				NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet");1043				return -EOPNOTSUPP;1044			}1045			if (fa->ct.action) {1046				NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled ct.action %u for LHS rule\n",1047						       fa->ct.action);1048				return -EOPNOTSUPP;1049			}1050			ct_zone = efx_tc_ct_register_zone(efx, fa->ct.zone,1051							  fa->ct.flow_table);1052			if (IS_ERR(ct_zone)) {1053				NL_SET_ERR_MSG_MOD(extack, "Failed to register for CT updates");1054				return PTR_ERR(ct_zone);1055			}1056			act->zone = ct_zone;1057			break;1058		default:1059			NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u for LHS rule\n",1060					       fa->id);1061			return -EOPNOTSUPP;1062		}1063	}1064 1065	if (pipe) {1066		NL_SET_ERR_MSG_MOD(extack, "Missing goto chain in LHS rule");1067		return -EOPNOTSUPP;1068	}1069	return 0;1070}1071 1072static void efx_tc_flower_release_lhs_actions(struct efx_nic *efx,1073					      struct efx_tc_lhs_action *act)1074{1075	if (act->rid)1076		efx_tc_put_recirc_id(efx, act->rid);1077	if (act->zone)1078		efx_tc_ct_unregister_zone(efx, act->zone);1079	if (act->count)1080		efx_tc_flower_put_counter_index(efx, act->count);1081}1082 1083/**1084 * struct efx_tc_mangler_state - accumulates 32-bit pedits into fields1085 *1086 * @dst_mac_32:	dst_mac[0:3] has been populated1087 * @dst_mac_16:	dst_mac[4:5] has been populated1088 * @src_mac_16:	src_mac[0:1] has been populated1089 * @src_mac_32:	src_mac[2:5] has been populated1090 * @dst_mac:	h_dest field of ethhdr1091 * @src_mac:	h_source field of ethhdr1092 *1093 * Since FLOW_ACTION_MANGLE comes in 32-bit chunks that do not1094 * necessarily equate to whole fields of the packet header, this1095 * structure is used to hold the cumulative effect of the partial1096 * field pedits that have been processed so far.1097 */1098struct efx_tc_mangler_state {1099	u8 dst_mac_32:1; /* eth->h_dest[0:3] */1100	u8 dst_mac_16:1; /* eth->h_dest[4:5] */1101	u8 src_mac_16:1; /* eth->h_source[0:1] */1102	u8 src_mac_32:1; /* eth->h_source[2:5] */1103	unsigned char dst_mac[ETH_ALEN];1104	unsigned char src_mac[ETH_ALEN];1105};1106 1107/** efx_tc_complete_mac_mangle() - pull complete field pedits out of @mung1108 * @efx:	NIC we're installing a flow rule on1109 * @act:	action set (cursor) to update1110 * @mung:	accumulated partial mangles1111 * @extack:	netlink extended ack for reporting errors1112 *1113 * Check @mung to find any combinations of partial mangles that can be1114 * combined into a complete packet field edit, add that edit to @act,1115 * and consume the partial mangles from @mung.1116 */1117 1118static int efx_tc_complete_mac_mangle(struct efx_nic *efx,1119				      struct efx_tc_action_set *act,1120				      struct efx_tc_mangler_state *mung,1121				      struct netlink_ext_ack *extack)1122{1123	struct efx_tc_mac_pedit_action *ped;1124 1125	if (mung->dst_mac_32 && mung->dst_mac_16) {1126		ped = efx_tc_flower_get_mac(efx, mung->dst_mac, extack);1127		if (IS_ERR(ped))1128			return PTR_ERR(ped);1129 1130		/* Check that we have not already populated dst_mac */1131		if (act->dst_mac)1132			efx_tc_flower_put_mac(efx, act->dst_mac);1133 1134		act->dst_mac = ped;1135 1136		/* consume the incomplete state */1137		mung->dst_mac_32 = 0;1138		mung->dst_mac_16 = 0;1139	}1140	if (mung->src_mac_16 && mung->src_mac_32) {1141		ped = efx_tc_flower_get_mac(efx, mung->src_mac, extack);1142		if (IS_ERR(ped))1143			return PTR_ERR(ped);1144 1145		/* Check that we have not already populated src_mac */1146		if (act->src_mac)1147			efx_tc_flower_put_mac(efx, act->src_mac);1148 1149		act->src_mac = ped;1150 1151		/* consume the incomplete state */1152		mung->src_mac_32 = 0;1153		mung->src_mac_16 = 0;1154	}1155	return 0;1156}1157 1158static int efx_tc_pedit_add(struct efx_nic *efx, struct efx_tc_action_set *act,1159			    const struct flow_action_entry *fa,1160			    struct netlink_ext_ack *extack)1161{1162	switch (fa->mangle.htype) {1163	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:1164		switch (fa->mangle.offset) {1165		case offsetof(struct iphdr, ttl):1166			/* check that pedit applies to ttl only */1167			if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK)1168				break;1169 1170			/* Adding 0xff is equivalent to decrementing the ttl.1171			 * Other added values are not supported.1172			 */1173			if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) != U8_MAX)1174				break;1175 1176			/* check that we do not decrement ttl twice */1177			if (!efx_tc_flower_action_order_ok(act,1178							   EFX_TC_AO_DEC_TTL)) {1179				NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");1180				return -EOPNOTSUPP;1181			}1182			act->do_ttl_dec = 1;1183			return 0;1184		default:1185			break;1186		}1187		break;1188	case FLOW_ACT_MANGLE_HDR_TYPE_IP6:1189		switch (fa->mangle.offset) {1190		case round_down(offsetof(struct ipv6hdr, hop_limit), 4):1191			/* check that pedit applies to hoplimit only */1192			if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK)1193				break;1194 1195			/* Adding 0xff is equivalent to decrementing the hoplimit.1196			 * Other added values are not supported.1197			 */1198			if ((fa->mangle.val >> 24) != U8_MAX)1199				break;1200 1201			/* check that we do not decrement hoplimit twice */1202			if (!efx_tc_flower_action_order_ok(act,1203							   EFX_TC_AO_DEC_TTL)) {1204				NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");1205				return -EOPNOTSUPP;1206			}1207			act->do_ttl_dec = 1;1208			return 0;1209		default:1210			break;1211		}1212		break;1213	default:1214		break;1215	}1216 1217	NL_SET_ERR_MSG_FMT_MOD(extack,1218			       "ttl add action type %x %x %x/%x is not supported",1219			       fa->mangle.htype, fa->mangle.offset,1220			       fa->mangle.val, fa->mangle.mask);1221	return -EOPNOTSUPP;1222}1223 1224/**1225 * efx_tc_mangle() - handle a single 32-bit (or less) pedit1226 * @efx:	NIC we're installing a flow rule on1227 * @act:	action set (cursor) to update1228 * @fa:		FLOW_ACTION_MANGLE action metadata1229 * @mung:	accumulator for partial mangles1230 * @extack:	netlink extended ack for reporting errors1231 * @match:	original match used along with the mangle action1232 *1233 * Identify the fields written by a FLOW_ACTION_MANGLE, and record1234 * the partial mangle state in @mung.  If this mangle completes an1235 * earlier partial mangle, consume and apply to @act by calling1236 * efx_tc_complete_mac_mangle().1237 */1238 1239static int efx_tc_mangle(struct efx_nic *efx, struct efx_tc_action_set *act,1240			 const struct flow_action_entry *fa,1241			 struct efx_tc_mangler_state *mung,1242			 struct netlink_ext_ack *extack,1243			 struct efx_tc_match *match)1244{1245	__le32 mac32;1246	__le16 mac16;1247	u8 tr_ttl;1248 1249	switch (fa->mangle.htype) {1250	case FLOW_ACT_MANGLE_HDR_TYPE_ETH:1251		BUILD_BUG_ON(offsetof(struct ethhdr, h_dest) != 0);1252		BUILD_BUG_ON(offsetof(struct ethhdr, h_source) != 6);1253		if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_PEDIT_MAC_ADDRS)) {1254			NL_SET_ERR_MSG_MOD(extack,1255					   "Pedit mangle mac action violates action order");1256			return -EOPNOTSUPP;1257		}1258		switch (fa->mangle.offset) {1259		case 0:1260			if (fa->mangle.mask) {1261				NL_SET_ERR_MSG_FMT_MOD(extack,1262						       "mask (%#x) of eth.dst32 mangle is not supported",1263						       fa->mangle.mask);1264				return -EOPNOTSUPP;1265			}1266			/* Ethernet address is little-endian */1267			mac32 = cpu_to_le32(fa->mangle.val);1268			memcpy(mung->dst_mac, &mac32, sizeof(mac32));1269			mung->dst_mac_32 = 1;1270			return efx_tc_complete_mac_mangle(efx, act, mung, extack);1271		case 4:1272			if (fa->mangle.mask == 0xffff) {1273				mac16 = cpu_to_le16(fa->mangle.val >> 16);1274				memcpy(mung->src_mac, &mac16, sizeof(mac16));1275				mung->src_mac_16 = 1;1276			} else if (fa->mangle.mask == 0xffff0000) {1277				mac16 = cpu_to_le16((u16)fa->mangle.val);1278				memcpy(mung->dst_mac + 4, &mac16, sizeof(mac16));1279				mung->dst_mac_16 = 1;1280			} else {1281				NL_SET_ERR_MSG_FMT_MOD(extack,1282						       "mask (%#x) of eth+4 mangle is not high or low 16b",1283						       fa->mangle.mask);1284				return -EOPNOTSUPP;1285			}1286			return efx_tc_complete_mac_mangle(efx, act, mung, extack);1287		case 8:1288			if (fa->mangle.mask) {1289				NL_SET_ERR_MSG_FMT_MOD(extack,1290						       "mask (%#x) of eth.src32 mangle is not supported",1291						       fa->mangle.mask);1292				return -EOPNOTSUPP;1293			}1294			mac32 = cpu_to_le32(fa->mangle.val);1295			memcpy(mung->src_mac + 2, &mac32, sizeof(mac32));1296			mung->src_mac_32 = 1;1297			return efx_tc_complete_mac_mangle(efx, act, mung, extack);1298		default:1299			NL_SET_ERR_MSG_FMT_MOD(extack, "mangle eth+%u %x/%x is not supported",1300					       fa->mangle.offset, fa->mangle.val, fa->mangle.mask);1301			return -EOPNOTSUPP;1302		}1303		break;1304	case FLOW_ACT_MANGLE_HDR_TYPE_IP4:1305		switch (fa->mangle.offset) {1306		case offsetof(struct iphdr, ttl):1307			/* we currently only support pedit IP4 when it applies1308			 * to TTL and then only when it can be achieved with a1309			 * decrement ttl action1310			 */1311 1312			/* check that pedit applies to ttl only */1313			if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) {1314				NL_SET_ERR_MSG_FMT_MOD(extack,1315						       "mask (%#x) out of range, only support mangle action on ipv4.ttl",1316						       fa->mangle.mask);1317				return -EOPNOTSUPP;1318			}1319 1320			/* we can only convert to a dec ttl when we have an1321			 * exact match on the ttl field1322			 */1323			if (match->mask.ip_ttl != U8_MAX) {1324				NL_SET_ERR_MSG_FMT_MOD(extack,1325						       "only support mangle ttl when we have an exact match, current mask (%#x)",1326						       match->mask.ip_ttl);1327				return -EOPNOTSUPP;1328			}1329 1330			/* check that we don't try to decrement 0, which equates1331			 * to setting the ttl to 0xff1332			 */1333			if (match->value.ip_ttl == 0) {1334				NL_SET_ERR_MSG_MOD(extack,1335						   "decrement ttl past 0 is not supported");1336				return -EOPNOTSUPP;1337			}1338 1339			/* check that we do not decrement ttl twice */1340			if (!efx_tc_flower_action_order_ok(act,1341							   EFX_TC_AO_DEC_TTL)) {1342				NL_SET_ERR_MSG_MOD(extack,1343						   "multiple dec ttl is not supported");1344				return -EOPNOTSUPP;1345			}1346 1347			/* check pedit can be achieved with decrement action */1348			tr_ttl = match->value.ip_ttl - 1;1349			if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) == tr_ttl) {1350				act->do_ttl_dec = 1;1351				return 0;1352			}1353 1354			fallthrough;1355		default:1356			NL_SET_ERR_MSG_FMT_MOD(extack,1357					       "only support mangle on the ttl field (offset is %u)",1358					       fa->mangle.offset);1359			return -EOPNOTSUPP;1360		}1361		break;1362	case FLOW_ACT_MANGLE_HDR_TYPE_IP6:1363		switch (fa->mangle.offset) {1364		case round_down(offsetof(struct ipv6hdr, hop_limit), 4):1365			/* we currently only support pedit IP6 when it applies1366			 * to the hoplimit and then only when it can be achieved1367			 * with a decrement hoplimit action1368			 */1369 1370			/* check that pedit applies to ttl only */1371			if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) {1372				NL_SET_ERR_MSG_FMT_MOD(extack,1373						       "mask (%#x) out of range, only support mangle action on ipv6.hop_limit",1374						       fa->mangle.mask);1375 1376				return -EOPNOTSUPP;1377			}1378 1379			/* we can only convert to a dec ttl when we have an1380			 * exact match on the ttl field1381			 */1382			if (match->mask.ip_ttl != U8_MAX) {1383				NL_SET_ERR_MSG_FMT_MOD(extack,1384						       "only support hop_limit when we have an exact match, current mask (%#x)",1385						       match->mask.ip_ttl);1386				return -EOPNOTSUPP;1387			}1388 1389			/* check that we don't try to decrement 0, which equates1390			 * to setting the ttl to 0xff1391			 */1392			if (match->value.ip_ttl == 0) {1393				NL_SET_ERR_MSG_MOD(extack,1394						   "decrementing hop_limit past 0 is not supported");1395				return -EOPNOTSUPP;1396			}1397 1398			/* check that we do not decrement hoplimit twice */1399			if (!efx_tc_flower_action_order_ok(act,1400							   EFX_TC_AO_DEC_TTL)) {1401				NL_SET_ERR_MSG_MOD(extack,1402						   "multiple dec ttl is not supported");1403				return -EOPNOTSUPP;1404			}1405 1406			/* check pedit can be achieved with decrement action */1407			tr_ttl = match->value.ip_ttl - 1;1408			if ((fa->mangle.val >> 24) == tr_ttl) {1409				act->do_ttl_dec = 1;1410				return 0;1411			}1412 1413			fallthrough;1414		default:1415			NL_SET_ERR_MSG_FMT_MOD(extack,1416					       "only support mangle on the hop_limit field");1417			return -EOPNOTSUPP;1418		}1419	default:1420		NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled mangle htype %u for action rule",1421				       fa->mangle.htype);1422		return -EOPNOTSUPP;1423	}1424	return 0;1425}1426 1427/**1428 * efx_tc_incomplete_mangle() - check for leftover partial pedits1429 * @mung:	accumulator for partial mangles1430 * @extack:	netlink extended ack for reporting errors1431 *1432 * Since the MAE can only overwrite whole fields, any partial1433 * field mangle left over on reaching packet delivery (mirred or1434 * end of TC actions) cannot be offloaded.  Check for any such1435 * and reject them with -%EOPNOTSUPP.1436 */1437 1438static int efx_tc_incomplete_mangle(struct efx_tc_mangler_state *mung,1439				    struct netlink_ext_ack *extack)1440{1441	if (mung->dst_mac_32 || mung->dst_mac_16) {1442		NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of destination MAC address");1443		return -EOPNOTSUPP;1444	}1445	if (mung->src_mac_16 || mung->src_mac_32) {1446		NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of source MAC address");1447		return -EOPNOTSUPP;1448	}1449	return 0;1450}1451 1452static int efx_tc_flower_replace_foreign_lhs_ar(struct efx_nic *efx,1453						struct flow_cls_offload *tc,1454						struct flow_rule *fr,1455						struct efx_tc_match *match,1456						struct net_device *net_dev)1457{1458	struct netlink_ext_ack *extack = tc->common.extack;1459	struct efx_tc_lhs_rule *rule, *old;1460	enum efx_encap_type type;1461	int rc;1462 1463	type = efx_tc_indr_netdev_type(net_dev);1464	if (type == EFX_ENCAP_TYPE_NONE) {1465		NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device");1466		return -EOPNOTSUPP;1467	}1468 1469	rc = efx_mae_check_encap_type_supported(efx, type);1470	if (rc) {1471		NL_SET_ERR_MSG_FMT_MOD(extack,1472				       "Firmware reports no support for %s encap match",1473				       efx_tc_encap_type_name(type));1474		return rc;1475	}1476	/* This is an Action Rule, so it needs a separate Encap Match in the1477	 * Outer Rule table.  Insert that now.1478	 */1479	rc = efx_tc_flower_record_encap_match(efx, match, type,1480					      EFX_TC_EM_DIRECT, 0, 0, extack);1481	if (rc)1482		return rc;1483 1484	match->mask.recirc_id = 0xff;1485	if (match->mask.ct_state_trk && match->value.ct_state_trk) {1486		NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");1487		rc = -EOPNOTSUPP;1488		goto release_encap_match;1489	}1490	/* LHS rules are always -trk, so we don't need to match on that */1491	match->mask.ct_state_trk = 0;1492	match->value.ct_state_trk = 0;1493	/* We must inhibit match on TCP SYN/FIN/RST, so that SW can see1494	 * the packet and update the conntrack table.1495	 * Outer Rules will do that with CT_TCP_FLAGS_INHIBIT, but Action1496	 * Rules don't have that; instead they support matching on1497	 * TCP_SYN_FIN_RST (aka TCP_INTERESTING_FLAGS), so use that.1498	 * This is only strictly needed if there will be a DO_CT action,1499	 * which we don't know yet, but typically there will be and it's1500	 * simpler not to bother checking here.1501	 */1502	match->mask.tcp_syn_fin_rst = true;1503 1504	rc = efx_mae_match_check_caps(efx, &match->mask, extack);1505	if (rc)1506		goto release_encap_match;1507 1508	rule = kzalloc(sizeof(*rule), GFP_USER);1509	if (!rule) {1510		rc = -ENOMEM;1511		goto release_encap_match;1512	}1513	rule->cookie = tc->cookie;1514	rule->is_ar = true;1515	old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,1516						&rule->linkage,1517						efx_tc_lhs_rule_ht_params);1518	if (old) {1519		netif_dbg(efx, drv, efx->net_dev,1520			  "Already offloaded rule (cookie %lx)\n", tc->cookie);1521		rc = -EEXIST;1522		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");1523		goto release;1524	}1525 1526	/* Parse actions */1527	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);1528	if (rc)1529		goto release;1530 1531	rule->match = *match;1532	rule->lhs_act.tun_type = type;1533 1534	rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);1535	if (rc) {1536		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");1537		goto release;1538	}1539	netif_dbg(efx, drv, efx->net_dev,1540		  "Successfully parsed lhs rule (cookie %lx)\n",1541		  tc->cookie);1542	return 0;1543 1544release:1545	efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);1546	if (!old)1547		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,1548				       efx_tc_lhs_rule_ht_params);1549	kfree(rule);1550release_encap_match:1551	if (match->encap)1552		efx_tc_flower_release_encap_match(efx, match->encap);1553	return rc;1554}1555 1556static int efx_tc_flower_replace_foreign_lhs(struct efx_nic *efx,1557					     struct flow_cls_offload *tc,1558					     struct flow_rule *fr,1559					     struct efx_tc_match *match,1560					     struct net_device *net_dev)1561{1562	struct netlink_ext_ack *extack = tc->common.extack;1563	struct efx_tc_lhs_rule *rule, *old;1564	enum efx_encap_type type;1565	int rc;1566 1567	if (tc->common.chain_index) {1568		NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");1569		return -EOPNOTSUPP;1570	}1571 1572	if (!efx_tc_match_is_encap(&match->mask)) {1573		/* This is not a tunnel decap rule, ignore it */1574		netif_dbg(efx, drv, efx->net_dev, "Ignoring foreign LHS filter without encap match\n");1575		return -EOPNOTSUPP;1576	}1577 1578	if (efx_tc_flower_flhs_needs_ar(match))1579		return efx_tc_flower_replace_foreign_lhs_ar(efx, tc, fr, match,1580							    net_dev);1581 1582	type = efx_tc_indr_netdev_type(net_dev);1583	if (type == EFX_ENCAP_TYPE_NONE) {1584		NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device\n");1585		return -EOPNOTSUPP;1586	}1587 1588	rc = efx_mae_check_encap_type_supported(efx, type);1589	if (rc) {1590		NL_SET_ERR_MSG_FMT_MOD(extack,1591				       "Firmware reports no support for %s encap match",1592				       efx_tc_encap_type_name(type));1593		return rc;1594	}1595	/* Reserve the outer tuple with a pseudo Encap Match */1596	rc = efx_tc_flower_record_encap_match(efx, match, type,1597					      EFX_TC_EM_PSEUDO_OR, 0, 0,1598					      extack);1599	if (rc)1600		return rc;1601 1602	if (match->mask.ct_state_trk && match->value.ct_state_trk) {1603		NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");1604		rc = -EOPNOTSUPP;1605		goto release_encap_match;1606	}1607	/* LHS rules are always -trk, so we don't need to match on that */1608	match->mask.ct_state_trk = 0;1609	match->value.ct_state_trk = 0;1610 1611	rc = efx_tc_flower_translate_flhs_match(match);1612	if (rc) {1613		NL_SET_ERR_MSG_MOD(extack, "LHS rule cannot match on inner fields");1614		goto release_encap_match;1615	}1616 1617	rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);1618	if (rc)1619		goto release_encap_match;1620 1621	rule = kzalloc(sizeof(*rule), GFP_USER);1622	if (!rule) {1623		rc = -ENOMEM;1624		goto release_encap_match;1625	}1626	rule->cookie = tc->cookie;1627	old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,1628						&rule->linkage,1629						efx_tc_lhs_rule_ht_params);1630	if (old) {1631		netif_dbg(efx, drv, efx->net_dev,1632			  "Already offloaded rule (cookie %lx)\n", tc->cookie);1633		rc = -EEXIST;1634		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");1635		goto release;1636	}1637 1638	/* Parse actions */1639	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);1640	if (rc)1641		goto release;1642 1643	rule->match = *match;1644	rule->lhs_act.tun_type = type;1645 1646	rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);1647	if (rc) {1648		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");1649		goto release;1650	}1651	netif_dbg(efx, drv, efx->net_dev,1652		  "Successfully parsed lhs rule (cookie %lx)\n",1653		  tc->cookie);1654	return 0;1655 1656release:1657	efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);1658	if (!old)1659		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,1660				       efx_tc_lhs_rule_ht_params);1661	kfree(rule);1662release_encap_match:1663	if (match->encap)1664		efx_tc_flower_release_encap_match(efx, match->encap);1665	return rc;1666}1667 1668static int efx_tc_flower_replace_foreign(struct efx_nic *efx,1669					 struct net_device *net_dev,1670					 struct flow_cls_offload *tc)1671{1672	struct flow_rule *fr = flow_cls_offload_flow_rule(tc);1673	struct netlink_ext_ack *extack = tc->common.extack;1674	struct efx_tc_flow_rule *rule = NULL, *old = NULL;1675	struct efx_tc_action_set *act = NULL;1676	bool found = false, uplinked = false;1677	const struct flow_action_entry *fa;1678	struct efx_tc_match match;1679	struct efx_rep *to_efv;1680	s64 rc;1681	int i;1682 1683	/* Parse match */1684	memset(&match, 0, sizeof(match));1685	rc = efx_tc_flower_parse_match(efx, fr, &match, extack);1686	if (rc)1687		return rc;1688	/* The rule as given to us doesn't specify a source netdevice.1689	 * But, determining whether packets from a VF should match it is1690	 * complicated, so leave those to the software slowpath: qualify1691	 * the filter with source m-port == wire.1692	 */1693	rc = efx_tc_flower_external_mport(efx, EFX_EFV_PF);1694	if (rc < 0) {1695		NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter");1696		return rc;1697	}1698	match.value.ingress_port = rc;1699	match.mask.ingress_port = ~0;1700 1701	if (efx_tc_rule_is_lhs_rule(fr, &match))1702		return efx_tc_flower_replace_foreign_lhs(efx, tc, fr, &match,1703							 net_dev);1704 1705	if (tc->common.chain_index) {1706		struct efx_tc_recirc_id *rid;1707 1708		rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, net_dev);1709		if (IS_ERR(rid)) {1710			NL_SET_ERR_MSG_FMT_MOD(extack,1711					       "Failed to allocate a hardware recirculation ID for chain_index %u",1712					       tc->common.chain_index);1713			return PTR_ERR(rid);1714		}1715		match.rid = rid;1716		match.value.recirc_id = rid->fw_id;1717	}1718	match.mask.recirc_id = 0xff;1719 1720	/* AR table can't match on DO_CT (+trk).  But a commonly used pattern is1721	 * +trk+est, which is strictly implied by +est, so rewrite it to that.1722	 */1723	if (match.mask.ct_state_trk && match.value.ct_state_trk &&1724	    match.mask.ct_state_est && match.value.ct_state_est)1725		match.mask.ct_state_trk = 0;1726	/* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could1727	 * match +trk-est (CT_HIT=0) despite being on an established connection.1728	 * So make -est imply -tcp_syn_fin_rst match to ensure these packets1729	 * still hit the software path.1730	 */1731	if (match.mask.ct_state_est && !match.value.ct_state_est) {1732		if (match.value.tcp_syn_fin_rst) {1733			/* Can't offload this combination */1734			NL_SET_ERR_MSG_MOD(extack, "TCP flags and -est conflict for offload");1735			rc = -EOPNOTSUPP;1736			goto release;1737		}1738		match.mask.tcp_syn_fin_rst = true;1739	}1740 1741	flow_action_for_each(i, fa, &fr->action) {1742		switch (fa->id) {1743		case FLOW_ACTION_REDIRECT:1744		case FLOW_ACTION_MIRRED: /* mirred means mirror here */1745			to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);1746			if (IS_ERR(to_efv))1747				continue;1748			found = true;1749			break;1750		default:1751			break;1752		}1753	}1754	if (!found) { /* We don't care. */1755		netif_dbg(efx, drv, efx->net_dev,1756			  "Ignoring foreign filter that doesn't egdev us\n");1757		rc = -EOPNOTSUPP;1758		goto release;1759	}1760 1761	rc = efx_mae_match_check_caps(efx, &match.mask, extack);1762	if (rc)1763		goto release;1764 1765	if (efx_tc_match_is_encap(&match.mask)) {1766		enum efx_encap_type type;1767 1768		type = efx_tc_indr_netdev_type(net_dev);1769		if (type == EFX_ENCAP_TYPE_NONE) {1770			NL_SET_ERR_MSG_MOD(extack,1771					   "Egress encap match on unsupported tunnel device");1772			rc = -EOPNOTSUPP;1773			goto release;1774		}1775 1776		rc = efx_mae_check_encap_type_supported(efx, type);1777		if (rc) {1778			NL_SET_ERR_MSG_FMT_MOD(extack,1779					       "Firmware reports no support for %s encap match",1780					       efx_tc_encap_type_name(type));1781			goto release;1782		}1783 1784		rc = efx_tc_flower_record_encap_match(efx, &match, type,1785						      EFX_TC_EM_DIRECT, 0, 0,1786						      extack);1787		if (rc)1788			goto release;1789	} else if (!tc->common.chain_index) {1790		/* This is not a tunnel decap rule, ignore it */1791		netif_dbg(efx, drv, efx->net_dev,1792			  "Ignoring foreign filter without encap match\n");1793		rc = -EOPNOTSUPP;1794		goto release;1795	}1796 1797	rule = kzalloc(sizeof(*rule), GFP_USER);1798	if (!rule) {1799		rc = -ENOMEM;1800		goto release;1801	}1802	INIT_LIST_HEAD(&rule->acts.list);1803	rule->cookie = tc->cookie;1804	old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,1805						&rule->linkage,1806						efx_tc_match_action_ht_params);1807	if (IS_ERR(old)) {1808		rc = PTR_ERR(old);1809		goto release;1810	} else if (old) {1811		netif_dbg(efx, drv, efx->net_dev,1812			  "Ignoring already-offloaded rule (cookie %lx)\n",1813			  tc->cookie);1814		rc = -EEXIST;1815		goto release;1816	}1817 1818	act = kzalloc(sizeof(*act), GFP_USER);1819	if (!act) {1820		rc = -ENOMEM;1821		goto release;1822	}1823 1824	/* Parse actions.  For foreign rules we only support decap & redirect.1825	 * See corresponding code in efx_tc_flower_replace() for theory of1826	 * operation & how 'act' cursor is used.1827	 */1828	flow_action_for_each(i, fa, &fr->action) {1829		struct efx_tc_action_set save;1830 1831		switch (fa->id) {1832		case FLOW_ACTION_REDIRECT:1833		case FLOW_ACTION_MIRRED:1834			/* See corresponding code in efx_tc_flower_replace() for1835			 * long explanations of what's going on here.1836			 */1837			save = *act;1838			if (fa->hw_stats) {1839				struct efx_tc_counter_index *ctr;1840 1841				if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {1842					NL_SET_ERR_MSG_FMT_MOD(extack,1843							       "hw_stats_type %u not supported (only 'delayed')",1844							       fa->hw_stats);1845					rc = -EOPNOTSUPP;1846					goto release;1847				}1848				if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {1849					NL_SET_ERR_MSG_MOD(extack, "Count action violates action order (can't happen)");1850					rc = -EOPNOTSUPP;1851					goto release;1852				}1853 1854				ctr = efx_tc_flower_get_counter_index(efx,1855								      tc->cookie,1856								      EFX_TC_COUNTER_TYPE_AR);1857				if (IS_ERR(ctr)) {1858					rc = PTR_ERR(ctr);1859					NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");1860					goto release;1861				}1862				act->count = ctr;1863				INIT_LIST_HEAD(&act->count_user);1864			}1865 1866			if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {1867				/* can't happen */1868				rc = -EOPNOTSUPP;1869				NL_SET_ERR_MSG_MOD(extack,1870						   "Deliver action violates action order (can't happen)");1871				goto release;1872			}1873			to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);1874			/* PF implies egdev is us, in which case we really1875			 * want to deliver to the uplink (because this is an1876			 * ingress filter).  If we don't recognise the egdev1877			 * at all, then we'd better trap so SW can handle it.1878			 */1879			if (IS_ERR(to_efv))1880				to_efv = EFX_EFV_PF;1881			if (to_efv == EFX_EFV_PF) {1882				if (uplinked)1883					break;1884				uplinked = true;1885			}1886			rc = efx_tc_flower_internal_mport(efx, to_efv);1887			if (rc < 0) {1888				NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");1889				goto release;1890			}1891			act->dest_mport = rc;1892			act->deliver = 1;1893			rc = efx_mae_alloc_action_set(efx, act);1894			if (rc) {1895				NL_SET_ERR_MSG_MOD(extack,1896						   "Failed to write action set to hw (mirred)");1897				goto release;1898			}1899			list_add_tail(&act->list, &rule->acts.list);1900			act = NULL;1901			if (fa->id == FLOW_ACTION_REDIRECT)1902				break; /* end of the line */1903			/* Mirror, so continue on with saved act */1904			act = kzalloc(sizeof(*act), GFP_USER);1905			if (!act) {1906				rc = -ENOMEM;1907				goto release;1908			}1909			*act = save;1910			break;1911		case FLOW_ACTION_TUNNEL_DECAP:1912			if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DECAP)) {1913				rc = -EINVAL;1914				NL_SET_ERR_MSG_MOD(extack, "Decap action violates action order");1915				goto release;1916			}1917			act->decap = 1;1918			/* If we previously delivered/trapped to uplink, now1919			 * that we've decapped we'll want another copy if we1920			 * try to deliver/trap to uplink again.1921			 */1922			uplinked = false;1923			break;1924		default:1925			NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",1926					       fa->id);1927			rc = -EOPNOTSUPP;1928			goto release;1929		}1930	}1931 1932	if (act) {1933		if (!uplinked) {1934			/* Not shot/redirected, so deliver to default dest (which is1935			 * the uplink, as this is an ingress filter)1936			 */1937			efx_mae_mport_uplink(efx, &act->dest_mport);1938			act->deliver = 1;1939		}1940		rc = efx_mae_alloc_action_set(efx, act);1941		if (rc) {1942			NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");1943			goto release;1944		}1945		list_add_tail(&act->list, &rule->acts.list);1946		act = NULL; /* Prevent double-free in error path */1947	}1948 1949	rule->match = match;1950 1951	netif_dbg(efx, drv, efx->net_dev,1952		  "Successfully parsed foreign filter (cookie %lx)\n",1953		  tc->cookie);1954 1955	rc = efx_mae_alloc_action_set_list(efx, &rule->acts);1956	if (rc) {1957		NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");1958		goto release;1959	}1960	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,1961				 rule->acts.fw_id, &rule->fw_id);1962	if (rc) {1963		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");1964		goto release_acts;1965	}1966	return 0;1967 1968release_acts:1969	efx_mae_free_action_set_list(efx, &rule->acts);1970release:1971	/* We failed to insert the rule, so free up any entries we created in1972	 * subsidiary tables.1973	 */1974	if (match.rid)1975		efx_tc_put_recirc_id(efx, match.rid);1976	if (act)1977		efx_tc_free_action_set(efx, act, false);1978	if (rule) {1979		if (!old)1980			rhashtable_remove_fast(&efx->tc->match_action_ht,1981					       &rule->linkage,1982					       efx_tc_match_action_ht_params);1983		efx_tc_free_action_set_list(efx, &rule->acts, false);1984	}1985	kfree(rule);1986	if (match.encap)1987		efx_tc_flower_release_encap_match(efx, match.encap);1988	return rc;1989}1990 1991static int efx_tc_flower_replace_lhs(struct efx_nic *efx,1992				     struct flow_cls_offload *tc,1993				     struct flow_rule *fr,1994				     struct efx_tc_match *match,1995				     struct efx_rep *efv,1996				     struct net_device *net_dev)1997{1998	struct netlink_ext_ack *extack = tc->common.extack;1999	struct efx_tc_lhs_rule *rule, *old;2000	int rc;2001 2002	if (tc->common.chain_index) {2003		NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");2004		return -EOPNOTSUPP;2005	}2006 2007	if (match->mask.ct_state_trk && match->value.ct_state_trk) {2008		NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");2009		return -EOPNOTSUPP;2010	}2011	/* LHS rules are always -trk, so we don't need to match on that */2012	match->mask.ct_state_trk = 0;2013	match->value.ct_state_trk = 0;2014 2015	rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);2016	if (rc)2017		return rc;2018 2019	rule = kzalloc(sizeof(*rule), GFP_USER);2020	if (!rule)2021		return -ENOMEM;2022	rule->cookie = tc->cookie;2023	old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,2024						&rule->linkage,2025						efx_tc_lhs_rule_ht_params);2026	if (IS_ERR(old)) {2027		rc = PTR_ERR(old);2028		goto release;2029	} else if (old) {2030		netif_dbg(efx, drv, efx->net_dev,2031			  "Already offloaded rule (cookie %lx)\n", tc->cookie);2032		rc = -EEXIST;2033		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");2034		goto release;2035	}2036 2037	/* Parse actions */2038	/* See note in efx_tc_flower_replace() regarding passed net_dev2039	 * (used for efx_tc_get_recirc_id()).2040	 */2041	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule);2042	if (rc)2043		goto release;2044 2045	rule->match = *match;2046 2047	rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);2048	if (rc) {2049		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");2050		goto release;2051	}2052	netif_dbg(efx, drv, efx->net_dev,2053		  "Successfully parsed lhs rule (cookie %lx)\n",2054		  tc->cookie);2055	return 0;2056 2057release:2058	efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);2059	if (!old)2060		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,2061				       efx_tc_lhs_rule_ht_params);2062	kfree(rule);2063	return rc;2064}2065 2066static int efx_tc_flower_replace(struct efx_nic *efx,2067				 struct net_device *net_dev,2068				 struct flow_cls_offload *tc,2069				 struct efx_rep *efv)2070{2071	struct flow_rule *fr = flow_cls_offload_flow_rule(tc);2072	struct netlink_ext_ack *extack = tc->common.extack;2073	const struct ip_tunnel_info *encap_info = NULL;2074	struct efx_tc_flow_rule *rule = NULL, *old;2075	struct efx_tc_mangler_state mung = {};2076	struct efx_tc_action_set *act = NULL;2077	const struct flow_action_entry *fa;2078	struct efx_rep *from_efv, *to_efv;2079	struct efx_tc_match match;2080	u32 acts_id;2081	s64 rc;2082	int i;2083 2084	if (!tc_can_offload_extack(efx->net_dev, extack))2085		return -EOPNOTSUPP;2086	if (WARN_ON(!efx->tc))2087		return -ENETDOWN;2088	if (WARN_ON(!efx->tc->up))2089		return -ENETDOWN;2090 2091	from_efv = efx_tc_flower_lookup_efv(efx, net_dev);2092	if (IS_ERR(from_efv)) {2093		/* Not from our PF or representors, so probably a tunnel dev */2094		return efx_tc_flower_replace_foreign(efx, net_dev, tc);2095	}2096 2097	if (efv != from_efv) {2098		/* can't happen */2099		NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)",2100				       netdev_name(net_dev), efv ? "non-" : "",2101				       from_efv ? "non-" : "");2102		return -EINVAL;2103	}2104 2105	/* Parse match */2106	memset(&match, 0, sizeof(match));2107	rc = efx_tc_flower_external_mport(efx, from_efv);2108	if (rc < 0) {2109		NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port");2110		return rc;2111	}2112	match.value.ingress_port = rc;2113	match.mask.ingress_port = ~0;2114	rc = efx_tc_flower_parse_match(efx, fr, &match, extack);2115	if (rc)2116		return rc;2117	if (efx_tc_match_is_encap(&match.mask)) {2118		NL_SET_ERR_MSG_MOD(extack, "Ingress enc_key matches not supported");2119		return -EOPNOTSUPP;2120	}2121 2122	if (efx_tc_rule_is_lhs_rule(fr, &match))2123		return efx_tc_flower_replace_lhs(efx, tc, fr, &match, efv,2124						 net_dev);2125 2126	/* chain_index 0 is always recirc_id 0 (and does not appear in recirc_ht).2127	 * Conveniently, match.rid == NULL and match.value.recirc_id == 0 owing2128	 * to the initial memset(), so we don't need to do anything in that case.2129	 */2130	if (tc->common.chain_index) {2131		struct efx_tc_recirc_id *rid;2132 2133		/* Note regarding passed net_dev:2134		 * VFreps and PF can share chain namespace, as they have2135		 * distinct ingress_mports.  So we don't need to burn an2136		 * extra recirc_id if both use the same chain_index.2137		 * (Strictly speaking, we could give each VFrep its own2138		 * recirc_id namespace that doesn't take IDs away from the2139		 * PF, but that would require a bunch of additional IDAs -2140		 * one for each representor - and that's not likely to be2141		 * the main cause of recirc_id exhaustion anyway.)2142		 */2143		rid = efx_tc_get_recirc_id(efx, tc->common.chain_index,2144					   efx->net_dev);2145		if (IS_ERR(rid)) {2146			NL_SET_ERR_MSG_FMT_MOD(extack,2147					       "Failed to allocate a hardware recirculation ID for chain_index %u",2148					       tc->common.chain_index);2149			return PTR_ERR(rid);2150		}2151		match.rid = rid;2152		match.value.recirc_id = rid->fw_id;2153	}2154	match.mask.recirc_id = 0xff;2155 2156	/* AR table can't match on DO_CT (+trk).  But a commonly used pattern is2157	 * +trk+est, which is strictly implied by +est, so rewrite it to that.2158	 */2159	if (match.mask.ct_state_trk && match.value.ct_state_trk &&2160	    match.mask.ct_state_est && match.value.ct_state_est)2161		match.mask.ct_state_trk = 0;2162	/* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could2163	 * match +trk-est (CT_HIT=0) despite being on an established connection.2164	 * So make -est imply -tcp_syn_fin_rst match to ensure these packets2165	 * still hit the software path.2166	 */2167	if (match.mask.ct_state_est && !match.value.ct_state_est) {2168		if (match.value.tcp_syn_fin_rst) {2169			/* Can't offload this combination */2170			rc = -EOPNOTSUPP;2171			goto release;2172		}2173		match.mask.tcp_syn_fin_rst = true;2174	}2175 2176	rc = efx_mae_match_check_caps(efx, &match.mask, extack);2177	if (rc)2178		goto release;2179 2180	rule = kzalloc(sizeof(*rule), GFP_USER);2181	if (!rule) {2182		rc = -ENOMEM;2183		goto release;2184	}2185	INIT_LIST_HEAD(&rule->acts.list);2186	rule->cookie = tc->cookie;2187	old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,2188						&rule->linkage,2189						efx_tc_match_action_ht_params);2190	if (IS_ERR(old)) {2191		rc = PTR_ERR(old);2192		goto release;2193	} else if (old) {2194		netif_dbg(efx, drv, efx->net_dev,2195			  "Already offloaded rule (cookie %lx)\n", tc->cookie);2196		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");2197		rc = -EEXIST;2198		goto release;2199	}2200 2201	/* Parse actions */2202	act = kzalloc(sizeof(*act), GFP_USER);2203	if (!act) {2204		rc = -ENOMEM;2205		goto release;2206	}2207 2208	/**2209	 * DOC: TC action translation2210	 *2211	 * Actions in TC are sequential and cumulative, with delivery actions2212	 * potentially anywhere in the order.  The EF100 MAE, however, takes2213	 * an 'action set list' consisting of 'action sets', each of which is2214	 * applied to the _original_ packet, and consists of a set of optional2215	 * actions in a fixed order with delivery at the end.2216	 * To translate between these two models, we maintain a 'cursor', @act,2217	 * which describes the cumulative effect of all the packet-mutating2218	 * actions encountered so far; on handling a delivery (mirred or drop)2219	 * action, once the action-set has been inserted into hardware, we2220	 * append @act to the action-set list (@rule->acts); if this is a pipe2221	 * action (mirred mirror) we then allocate a new @act with a copy of2222	 * the cursor state _before_ the delivery action, otherwise we set @act2223	 * to %NULL.2224	 * This ensures that every allocated action-set is either attached to2225	 * @rule->acts or pointed to by @act (and never both), and that only2226	 * those action-sets in @rule->acts exist in hardware.  Consequently,2227	 * in the failure path, @act only needs to be freed in memory, whereas2228	 * for @rule->acts we remove each action-set from hardware before2229	 * freeing it (efx_tc_free_action_set_list()), even if the action-set2230	 * list itself is not in hardware.2231	 */2232	flow_action_for_each(i, fa, &fr->action) {2233		struct efx_tc_action_set save;2234		u16 tci;2235 2236		if (!act) {2237			/* more actions after a non-pipe action */2238			NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");2239			rc = -EINVAL;2240			goto release;2241		}2242 2243		if ((fa->id == FLOW_ACTION_REDIRECT ||2244		     fa->id == FLOW_ACTION_MIRRED ||2245		     fa->id == FLOW_ACTION_DROP) && fa->hw_stats) {2246			struct efx_tc_counter_index *ctr;2247 2248			/* Currently the only actions that want stats are2249			 * mirred and gact (ok, shot, trap, goto-chain), which2250			 * means we want stats just before delivery.  Also,2251			 * note that tunnel_key set shouldn't change the length2252			 * — it's only the subsequent mirred that does that,2253			 * and the stats are taken _before_ the mirred action2254			 * happens.2255			 */2256			if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {2257				/* All supported actions that count either steal2258				 * (gact shot, mirred redirect) or clone act2259				 * (mirred mirror), so we should never get two2260				 * count actions on one action_set.2261				 */2262				NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)");2263				rc = -EOPNOTSUPP;2264				goto release;2265			}2266 2267			if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {2268				NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')",2269						       fa->hw_stats);2270				rc = -EOPNOTSUPP;2271				goto release;2272			}2273 2274			ctr = efx_tc_flower_get_counter_index(efx, tc->cookie,2275							      EFX_TC_COUNTER_TYPE_AR);2276			if (IS_ERR(ctr)) {2277				rc = PTR_ERR(ctr);2278				NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");2279				goto release;2280			}2281			act->count = ctr;2282			INIT_LIST_HEAD(&act->count_user);2283		}2284 2285		switch (fa->id) {2286		case FLOW_ACTION_DROP:2287			rc = efx_mae_alloc_action_set(efx, act);2288			if (rc) {2289				NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)");2290				goto release;2291			}2292			list_add_tail(&act->list, &rule->acts.list);2293			act = NULL; /* end of the line */2294			break;2295		case FLOW_ACTION_REDIRECT:2296		case FLOW_ACTION_MIRRED:2297			save = *act;2298 2299			if (encap_info) {2300				struct efx_tc_encap_action *encap;2301 2302				if (!efx_tc_flower_action_order_ok(act,2303								   EFX_TC_AO_ENCAP)) {2304					rc = -EOPNOTSUPP;2305					NL_SET_ERR_MSG_MOD(extack, "Encap action violates action order");2306					goto release;2307				}2308				encap = efx_tc_flower_create_encap_md(2309						efx, encap_info, fa->dev, extack);2310				if (IS_ERR_OR_NULL(encap)) {2311					rc = PTR_ERR(encap);2312					if (!rc)2313						rc = -EIO; /* arbitrary */2314					goto release;2315				}2316				act->encap_md = encap;2317				list_add_tail(&act->encap_user, &encap->users);2318				act->dest_mport = encap->dest_mport;2319				act->deliver = 1;2320				if (act->count && !WARN_ON(!act->count->cnt)) {2321					/* This counter is used by an encap2322					 * action, which needs a reference back2323					 * so it can prod neighbouring whenever2324					 * traffic is seen.2325					 */2326					spin_lock_bh(&act->count->cnt->lock);2327					list_add_tail(&act->count_user,2328						      &act->count->cnt->users);2329					spin_unlock_bh(&act->count->cnt->lock);2330				}2331				rc = efx_mae_alloc_action_set(efx, act);2332				if (rc) {2333					NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (encap)");2334					goto release;2335				}2336				list_add_tail(&act->list, &rule->acts.list);2337				act->user = &rule->acts;2338				act = NULL;2339				if (fa->id == FLOW_ACTION_REDIRECT)2340					break; /* end of the line */2341				/* Mirror, so continue on with saved act */2342				save.count = NULL;2343				act = kzalloc(sizeof(*act), GFP_USER);2344				if (!act) {2345					rc = -ENOMEM;2346					goto release;2347				}2348				*act = save;2349				break;2350			}2351 2352			if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {2353				/* can't happen */2354				rc = -EOPNOTSUPP;2355				NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)");2356				goto release;2357			}2358 2359			to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);2360			if (IS_ERR(to_efv)) {2361				NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch");2362				rc = PTR_ERR(to_efv);2363				goto release;2364			}2365			rc = efx_tc_flower_external_mport(efx, to_efv);2366			if (rc < 0) {2367				NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");2368				goto release;2369			}2370			act->dest_mport = rc;2371			act->deliver = 1;2372			rc = efx_mae_alloc_action_set(efx, act);2373			if (rc) {2374				NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)");2375				goto release;2376			}2377			list_add_tail(&act->list, &rule->acts.list);2378			act = NULL;2379			if (fa->id == FLOW_ACTION_REDIRECT)2380				break; /* end of the line */2381			/* Mirror, so continue on with saved act */2382			save.count = NULL;2383			act = kzalloc(sizeof(*act), GFP_USER);2384			if (!act) {2385				rc = -ENOMEM;2386				goto release;2387			}2388			*act = save;2389			break;2390		case FLOW_ACTION_VLAN_POP:2391			if (act->vlan_push) {2392				act->vlan_push--;2393			} else if (efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_POP)) {2394				act->vlan_pop++;2395			} else {2396				NL_SET_ERR_MSG_MOD(extack,2397						   "More than two VLAN pops, or action order violated");2398				rc = -EINVAL;2399				goto release;2400			}2401			break;2402		case FLOW_ACTION_VLAN_PUSH:2403			if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_PUSH)) {2404				rc = -EINVAL;2405				NL_SET_ERR_MSG_MOD(extack,2406						   "More than two VLAN pushes, or action order violated");2407				goto release;2408			}2409			tci = fa->vlan.vid & VLAN_VID_MASK;2410			tci |= fa->vlan.prio << VLAN_PRIO_SHIFT;2411			act->vlan_tci[act->vlan_push] = cpu_to_be16(tci);2412			act->vlan_proto[act->vlan_push] = fa->vlan.proto;2413			act->vlan_push++;2414			break;2415		case FLOW_ACTION_ADD:2416			rc = efx_tc_pedit_add(efx, act, fa, extack);2417			if (rc < 0)2418				goto release;2419			break;2420		case FLOW_ACTION_MANGLE:2421			rc = efx_tc_mangle(efx, act, fa, &mung, extack, &match);2422			if (rc < 0)2423				goto release;2424			break;2425		case FLOW_ACTION_TUNNEL_ENCAP:2426			if (encap_info) {2427				/* Can't specify encap multiple times.2428				 * If you want to overwrite an existing2429				 * encap_info, use an intervening2430				 * FLOW_ACTION_TUNNEL_DECAP to clear it.2431				 */2432				NL_SET_ERR_MSG_MOD(extack, "Tunnel key set when already set");2433				rc = -EINVAL;2434				goto release;2435			}2436			if (!fa->tunnel) {2437				NL_SET_ERR_MSG_MOD(extack, "Tunnel key set is missing key");2438				rc = -EOPNOTSUPP;2439				goto release;2440			}2441			encap_info = fa->tunnel;2442			break;2443		case FLOW_ACTION_TUNNEL_DECAP:2444			if (encap_info) {2445				encap_info = NULL;2446				break;2447			}2448			/* Since we don't support enc_key matches on ingress2449			 * (and if we did there'd be no tunnel-device to give2450			 * us a type), we can't offload a decap that's not2451			 * just undoing a previous encap action.2452			 */2453			NL_SET_ERR_MSG_MOD(extack, "Cannot offload tunnel decap action without tunnel device");2454			rc = -EOPNOTSUPP;2455			goto release;2456		case FLOW_ACTION_CT:2457			if (fa->ct.action != TCA_CT_ACT_NAT) {2458				rc = -EOPNOTSUPP;2459				NL_SET_ERR_MSG_FMT_MOD(extack, "Can only offload CT 'nat' action in RHS rules, not %d", fa->ct.action);2460				goto release;2461			}2462			act->do_nat = 1;2463			break;2464		default:2465			NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",2466					       fa->id);2467			rc = -EOPNOTSUPP;2468			goto release;2469		}2470	}2471 2472	rc = efx_tc_incomplete_mangle(&mung, extack);2473	if (rc < 0)2474		goto release;2475	if (act) {2476		/* Not shot/redirected, so deliver to default dest */2477		if (from_efv == EFX_EFV_PF)2478			/* Rule applies to traffic from the wire,2479			 * and default dest is thus the PF2480			 */2481			efx_mae_mport_uplink(efx, &act->dest_mport);2482		else2483			/* Representor, so rule applies to traffic from2484			 * representee, and default dest is thus the rep.2485			 * All reps use the same mport for delivery2486			 */2487			efx_mae_mport_mport(efx, efx->tc->reps_mport_id,2488					    &act->dest_mport);2489		act->deliver = 1;2490		rc = efx_mae_alloc_action_set(efx, act);2491		if (rc) {2492			NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");2493			goto release;2494		}2495		list_add_tail(&act->list, &rule->acts.list);2496		act = NULL; /* Prevent double-free in error path */2497	}2498 2499	netif_dbg(efx, drv, efx->net_dev,2500		  "Successfully parsed filter (cookie %lx)\n",2501		  tc->cookie);2502 2503	rule->match = match;2504 2505	rc = efx_mae_alloc_action_set_list(efx, &rule->acts);2506	if (rc) {2507		NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");2508		goto release;2509	}2510	if (from_efv == EFX_EFV_PF)2511		/* PF netdev, so rule applies to traffic from wire */2512		rule->fallback = &efx->tc->facts.pf;2513	else2514		/* repdev, so rule applies to traffic from representee */2515		rule->fallback = &efx->tc->facts.reps;2516	if (!efx_tc_check_ready(efx, rule)) {2517		netif_dbg(efx, drv, efx->net_dev, "action not ready for hw\n");2518		acts_id = rule->fallback->fw_id;2519	} else {2520		netif_dbg(efx, drv, efx->net_dev, "ready for hw\n");2521		acts_id = rule->acts.fw_id;2522	}2523	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,2524				 acts_id, &rule->fw_id);2525	if (rc) {2526		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");2527		goto release_acts;2528	}2529	return 0;2530 2531release_acts:2532	efx_mae_free_action_set_list(efx, &rule->acts);2533release:2534	/* We failed to insert the rule, so free up any entries we created in2535	 * subsidiary tables.2536	 */2537	if (match.rid)2538		efx_tc_put_recirc_id(efx, match.rid);2539	if (act)2540		efx_tc_free_action_set(efx, act, false);2541	if (rule) {2542		if (!old)2543			rhashtable_remove_fast(&efx->tc->match_action_ht,2544					       &rule->linkage,2545					       efx_tc_match_action_ht_params);2546		efx_tc_free_action_set_list(efx, &rule->acts, false);2547	}2548	kfree(rule);2549	return rc;2550}2551 2552static int efx_tc_flower_destroy(struct efx_nic *efx,2553				 struct net_device *net_dev,2554				 struct flow_cls_offload *tc)2555{2556	struct netlink_ext_ack *extack = tc->common.extack;2557	struct efx_tc_lhs_rule *lhs_rule;2558	struct efx_tc_flow_rule *rule;2559 2560	lhs_rule = rhashtable_lookup_fast(&efx->tc->lhs_rule_ht, &tc->cookie,2561					  efx_tc_lhs_rule_ht_params);2562	if (lhs_rule) {2563		/* Remove it from HW */2564		efx_mae_remove_lhs_rule(efx, lhs_rule);2565		/* Delete it from SW */2566		efx_tc_flower_release_lhs_actions(efx, &lhs_rule->lhs_act);2567		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &lhs_rule->linkage,2568				       efx_tc_lhs_rule_ht_params);2569		if (lhs_rule->match.encap)2570			efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap);2571		netif_dbg(efx, drv, efx->net_dev, "Removed (lhs) filter %lx\n",2572			  lhs_rule->cookie);2573		kfree(lhs_rule);2574		return 0;2575	}2576 2577	rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie,2578				      efx_tc_match_action_ht_params);2579	if (!rule) {2580		/* Only log a message if we're the ingress device.  Otherwise2581		 * it's a foreign filter and we might just not have been2582		 * interested (e.g. we might not have been the egress device2583		 * either).2584		 */2585		if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))2586			netif_warn(efx, drv, efx->net_dev,2587				   "Filter %lx not found to remove\n", tc->cookie);2588		NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");2589		return -ENOENT;2590	}2591 2592	/* Remove it from HW */2593	efx_tc_delete_rule(efx, rule);2594	/* Delete it from SW */2595	rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage,2596			       efx_tc_match_action_ht_params);2597	netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie);2598	kfree(rule);2599	return 0;2600}2601 2602static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev,2603			       struct flow_cls_offload *tc)2604{2605	struct netlink_ext_ack *extack = tc->common.extack;2606	struct efx_tc_counter_index *ctr;2607	struct efx_tc_counter *cnt;2608	u64 packets, bytes;2609 2610	ctr = efx_tc_flower_find_counter_index(efx, tc->cookie);2611	if (!ctr) {2612		/* See comment in efx_tc_flower_destroy() */2613		if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))2614			if (net_ratelimit())2615				netif_warn(efx, drv, efx->net_dev,2616					   "Filter %lx not found for stats\n",2617					   tc->cookie);2618		NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");2619		return -ENOENT;2620	}2621	if (WARN_ON(!ctr->cnt)) /* can't happen */2622		return -EIO;2623	cnt = ctr->cnt;2624 2625	spin_lock_bh(&cnt->lock);2626	/* Report only new pkts/bytes since last time TC asked */2627	packets = cnt->packets;2628	bytes = cnt->bytes;2629	flow_stats_update(&tc->stats, bytes - cnt->old_bytes,2630			  packets - cnt->old_packets, 0, cnt->touched,2631			  FLOW_ACTION_HW_STATS_DELAYED);2632	cnt->old_packets = packets;2633	cnt->old_bytes = bytes;2634	spin_unlock_bh(&cnt->lock);2635	return 0;2636}2637 2638int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,2639		  struct flow_cls_offload *tc, struct efx_rep *efv)2640{2641	int rc;2642 2643	if (!efx->tc)2644		return -EOPNOTSUPP;2645 2646	mutex_lock(&efx->tc->mutex);2647	switch (tc->command) {2648	case FLOW_CLS_REPLACE:2649		rc = efx_tc_flower_replace(efx, net_dev, tc, efv);2650		break;2651	case FLOW_CLS_DESTROY:2652		rc = efx_tc_flower_destroy(efx, net_dev, tc);2653		break;2654	case FLOW_CLS_STATS:2655		rc = efx_tc_flower_stats(efx, net_dev, tc);2656		break;2657	default:2658		rc = -EOPNOTSUPP;2659		break;2660	}2661	mutex_unlock(&efx->tc->mutex);2662	return rc;2663}2664 2665static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port,2666					 u32 eg_port, struct efx_tc_flow_rule *rule)2667{2668	struct efx_tc_action_set_list *acts = &rule->acts;2669	struct efx_tc_match *match = &rule->match;2670	struct efx_tc_action_set *act;2671	int rc;2672 2673	match->value.ingress_port = ing_port;2674	match->mask.ingress_port = ~0;2675	act = kzalloc(sizeof(*act), GFP_KERNEL);2676	if (!act)2677		return -ENOMEM;2678	act->deliver = 1;2679	act->dest_mport = eg_port;2680	rc = efx_mae_alloc_action_set(efx, act);2681	if (rc)2682		goto fail1;2683	EFX_WARN_ON_PARANOID(!list_empty(&acts->list));2684	list_add_tail(&act->list, &acts->list);2685	rc = efx_mae_alloc_action_set_list(efx, acts);2686	if (rc)2687		goto fail2;2688	rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT,2689				 acts->fw_id, &rule->fw_id);2690	if (rc)2691		goto fail3;2692	return 0;2693fail3:2694	efx_mae_free_action_set_list(efx, acts);2695fail2:2696	list_del(&act->list);2697	efx_mae_free_action_set(efx, act->fw_id);2698fail1:2699	kfree(act);2700	return rc;2701}2702 2703static int efx_tc_configure_default_rule_pf(struct efx_nic *efx)2704{2705	struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf;2706	u32 ing_port, eg_port;2707 2708	efx_mae_mport_uplink(efx, &ing_port);2709	efx_mae_mport_wire(efx, &eg_port);2710	return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);2711}2712 2713static int efx_tc_configure_default_rule_wire(struct efx_nic *efx)2714{2715	struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire;2716	u32 ing_port, eg_port;2717 2718	efx_mae_mport_wire(efx, &ing_port);2719	efx_mae_mport_uplink(efx, &eg_port);2720	return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);2721}2722 2723int efx_tc_configure_default_rule_rep(struct efx_rep *efv)2724{2725	struct efx_tc_flow_rule *rule = &efv->dflt;2726	struct efx_nic *efx = efv->parent;2727	u32 ing_port, eg_port;2728 2729	efx_mae_mport_mport(efx, efv->mport, &ing_port);2730	efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);2731	return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);2732}2733 2734void efx_tc_deconfigure_default_rule(struct efx_nic *efx,2735				     struct efx_tc_flow_rule *rule)2736{2737	if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL)2738		efx_tc_delete_rule(efx, rule);2739	rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;2740}2741 2742static int efx_tc_configure_fallback_acts(struct efx_nic *efx, u32 eg_port,2743					  struct efx_tc_action_set_list *acts)2744{2745	struct efx_tc_action_set *act;2746	int rc;2747 2748	act = kzalloc(sizeof(*act), GFP_KERNEL);2749	if (!act)2750		return -ENOMEM;2751	act->deliver = 1;2752	act->dest_mport = eg_port;2753	rc = efx_mae_alloc_action_set(efx, act);2754	if (rc)2755		goto fail1;2756	EFX_WARN_ON_PARANOID(!list_empty(&acts->list));2757	list_add_tail(&act->list, &acts->list);2758	rc = efx_mae_alloc_action_set_list(efx, acts);2759	if (rc)2760		goto fail2;2761	return 0;2762fail2:2763	list_del(&act->list);2764	efx_mae_free_action_set(efx, act->fw_id);2765fail1:2766	kfree(act);2767	return rc;2768}2769 2770static int efx_tc_configure_fallback_acts_pf(struct efx_nic *efx)2771{2772	struct efx_tc_action_set_list *acts = &efx->tc->facts.pf;2773	u32 eg_port;2774 2775	efx_mae_mport_uplink(efx, &eg_port);2776	return efx_tc_configure_fallback_acts(efx, eg_port, acts);2777}2778 2779static int efx_tc_configure_fallback_acts_reps(struct efx_nic *efx)2780{2781	struct efx_tc_action_set_list *acts = &efx->tc->facts.reps;2782	u32 eg_port;2783 2784	efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);2785	return efx_tc_configure_fallback_acts(efx, eg_port, acts);2786}2787 2788static void efx_tc_deconfigure_fallback_acts(struct efx_nic *efx,2789					     struct efx_tc_action_set_list *acts)2790{2791	efx_tc_free_action_set_list(efx, acts, true);2792}2793 2794static int efx_tc_configure_rep_mport(struct efx_nic *efx)2795{2796	u32 rep_mport_label;2797	int rc;2798 2799	rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label);2800	if (rc)2801		return rc;2802	pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n",2803		efx->tc->reps_mport_id, rep_mport_label);2804	/* Use mport *selector* as vport ID */2805	efx_mae_mport_mport(efx, efx->tc->reps_mport_id,2806			    &efx->tc->reps_mport_vport_id);2807	return 0;2808}2809 2810static void efx_tc_deconfigure_rep_mport(struct efx_nic *efx)2811{2812	efx_mae_free_mport(efx, efx->tc->reps_mport_id);2813	efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL;2814}2815 2816int efx_tc_insert_rep_filters(struct efx_nic *efx)2817{2818	struct efx_filter_spec promisc, allmulti;2819	int rc;2820 2821	if (efx->type->is_vf)2822		return 0;2823	if (!efx->tc)2824		return 0;2825	efx_filter_init_rx(&promisc, EFX_FILTER_PRI_REQUIRED, 0, 0);2826	efx_filter_set_uc_def(&promisc);2827	efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id);2828	rc = efx_filter_insert_filter(efx, &promisc, false);2829	if (rc < 0)2830		return rc;2831	efx->tc->reps_filter_uc = rc;2832	efx_filter_init_rx(&allmulti, EFX_FILTER_PRI_REQUIRED, 0, 0);2833	efx_filter_set_mc_def(&allmulti);2834	efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id);2835	rc = efx_filter_insert_filter(efx, &allmulti, false);2836	if (rc < 0)2837		return rc;2838	efx->tc->reps_filter_mc = rc;2839	return 0;2840}2841 2842void efx_tc_remove_rep_filters(struct efx_nic *efx)2843{2844	if (efx->type->is_vf)2845		return;2846	if (!efx->tc)2847		return;2848	if (efx->tc->reps_filter_mc >= 0)2849		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc);2850	efx->tc->reps_filter_mc = -1;2851	if (efx->tc->reps_filter_uc >= 0)2852		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc);2853	efx->tc->reps_filter_uc = -1;2854}2855 2856int efx_init_tc(struct efx_nic *efx)2857{2858	int rc;2859 2860	rc = efx_mae_get_caps(efx, efx->tc->caps);2861	if (rc)2862		return rc;2863	if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS)2864		/* Firmware supports some match fields the driver doesn't know2865		 * about.  Not fatal, unless any of those fields are required2866		 * (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know.2867		 */2868		netif_warn(efx, probe, efx->net_dev,2869			   "FW reports additional match fields %u\n",2870			   efx->tc->caps->match_field_count);2871	if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) {2872		netif_err(efx, probe, efx->net_dev,2873			  "Too few action prios supported (have %u, need %u)\n",2874			  efx->tc->caps->action_prios, EFX_TC_PRIO__NUM);2875		return -EIO;2876	}2877	rc = efx_tc_configure_default_rule_pf(efx);2878	if (rc)2879		return rc;2880	rc = efx_tc_configure_default_rule_wire(efx);2881	if (rc)2882		return rc;2883	rc = efx_tc_configure_rep_mport(efx);2884	if (rc)2885		return rc;2886	rc = efx_tc_configure_fallback_acts_pf(efx);2887	if (rc)2888		return rc;2889	rc = efx_tc_configure_fallback_acts_reps(efx);2890	if (rc)2891		return rc;2892	rc = efx_mae_get_tables(efx);2893	if (rc)2894		return rc;2895	rc = flow_indr_dev_register(efx_tc_indr_setup_cb, efx);2896	if (rc)2897		goto out_free;2898	efx->tc->up = true;2899	return 0;2900out_free:2901	efx_mae_free_tables(efx);2902	return rc;2903}2904 2905void efx_fini_tc(struct efx_nic *efx)2906{2907	/* We can get called even if efx_init_struct_tc() failed */2908	if (!efx->tc)2909		return;2910	if (efx->tc->up)2911		flow_indr_dev_unregister(efx_tc_indr_setup_cb, efx, efx_tc_block_unbind);2912	efx_tc_deconfigure_rep_mport(efx);2913	efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf);2914	efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire);2915	efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.pf);2916	efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.reps);2917	efx->tc->up = false;2918	efx_mae_free_tables(efx);2919}2920 2921/* At teardown time, all TC filter rules (and thus all resources they created)2922 * should already have been removed.  If we find any in our hashtables, make a2923 * cursory attempt to clean up the software side.2924 */2925static void efx_tc_encap_match_free(void *ptr, void *__unused)2926{2927	struct efx_tc_encap_match *encap = ptr;2928 2929	WARN_ON(refcount_read(&encap->ref));2930	kfree(encap);2931}2932 2933static void efx_tc_recirc_free(void *ptr, void *arg)2934{2935	struct efx_tc_recirc_id *rid = ptr;2936	struct efx_nic *efx = arg;2937 2938	WARN_ON(refcount_read(&rid->ref));2939	ida_free(&efx->tc->recirc_ida, rid->fw_id);2940	kfree(rid);2941}2942 2943static void efx_tc_lhs_free(void *ptr, void *arg)2944{2945	struct efx_tc_lhs_rule *rule = ptr;2946	struct efx_nic *efx = arg;2947 2948	netif_err(efx, drv, efx->net_dev,2949		  "tc lhs_rule %lx still present at teardown, removing\n",2950		  rule->cookie);2951 2952	if (rule->lhs_act.zone)2953		efx_tc_ct_unregister_zone(efx, rule->lhs_act.zone);2954	if (rule->lhs_act.count)2955		efx_tc_flower_put_counter_index(efx, rule->lhs_act.count);2956	efx_mae_remove_lhs_rule(efx, rule);2957 2958	kfree(rule);2959}2960 2961static void efx_tc_mac_free(void *ptr, void *__unused)2962{2963	struct efx_tc_mac_pedit_action *ped = ptr;2964 2965	WARN_ON(refcount_read(&ped->ref));2966	kfree(ped);2967}2968 2969static void efx_tc_flow_free(void *ptr, void *arg)2970{2971	struct efx_tc_flow_rule *rule = ptr;2972	struct efx_nic *efx = arg;2973 2974	netif_err(efx, drv, efx->net_dev,2975		  "tc rule %lx still present at teardown, removing\n",2976		  rule->cookie);2977 2978	/* Also releases entries in subsidiary tables */2979	efx_tc_delete_rule(efx, rule);2980 2981	kfree(rule);2982}2983 2984int efx_init_struct_tc(struct efx_nic *efx)2985{2986	int rc;2987 2988	if (efx->type->is_vf)2989		return 0;2990 2991	efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL);2992	if (!efx->tc)2993		return -ENOMEM;2994	efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL);2995	if (!efx->tc->caps) {2996		rc = -ENOMEM;2997		goto fail_alloc_caps;2998	}2999	INIT_LIST_HEAD(&efx->tc->block_list);3000 3001	mutex_init(&efx->tc->mutex);3002	init_waitqueue_head(&efx->tc->flush_wq);3003	rc = efx_tc_init_encap_actions(efx);3004	if (rc < 0)3005		goto fail_encap_actions;3006	rc = efx_tc_init_counters(efx);3007	if (rc < 0)3008		goto fail_counters;3009	rc = rhashtable_init(&efx->tc->mac_ht, &efx_tc_mac_ht_params);3010	if (rc < 0)3011		goto fail_mac_ht;3012	rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params);3013	if (rc < 0)3014		goto fail_encap_match_ht;3015	rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);3016	if (rc < 0)3017		goto fail_match_action_ht;3018	rc = rhashtable_init(&efx->tc->lhs_rule_ht, &efx_tc_lhs_rule_ht_params);3019	if (rc < 0)3020		goto fail_lhs_rule_ht;3021	rc = efx_tc_init_conntrack(efx);3022	if (rc < 0)3023		goto fail_conntrack;3024	rc = rhashtable_init(&efx->tc->recirc_ht, &efx_tc_recirc_ht_params);3025	if (rc < 0)3026		goto fail_recirc_ht;3027	ida_init(&efx->tc->recirc_ida);3028	efx->tc->reps_filter_uc = -1;3029	efx->tc->reps_filter_mc = -1;3030	INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list);3031	efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;3032	INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list);3033	efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;3034	INIT_LIST_HEAD(&efx->tc->facts.pf.list);3035	efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;3036	INIT_LIST_HEAD(&efx->tc->facts.reps.list);3037	efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;3038	efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type;3039	return 0;3040fail_recirc_ht:3041	efx_tc_destroy_conntrack(efx);3042fail_conntrack:3043	rhashtable_destroy(&efx->tc->lhs_rule_ht);3044fail_lhs_rule_ht:3045	rhashtable_destroy(&efx->tc->match_action_ht);3046fail_match_action_ht:3047	rhashtable_destroy(&efx->tc->encap_match_ht);3048fail_encap_match_ht:3049	rhashtable_destroy(&efx->tc->mac_ht);3050fail_mac_ht:3051	efx_tc_destroy_counters(efx);3052fail_counters:3053	efx_tc_destroy_encap_actions(efx);3054fail_encap_actions:3055	mutex_destroy(&efx->tc->mutex);3056	kfree(efx->tc->caps);3057fail_alloc_caps:3058	kfree(efx->tc);3059	efx->tc = NULL;3060	return rc;3061}3062 3063void efx_fini_struct_tc(struct efx_nic *efx)3064{3065	if (!efx->tc)3066		return;3067 3068	mutex_lock(&efx->tc->mutex);3069	EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id !=3070			     MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);3071	EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id !=3072			     MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);3073	EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id !=3074			     MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);3075	EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id !=3076			     MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);3077	rhashtable_free_and_destroy(&efx->tc->lhs_rule_ht, efx_tc_lhs_free, efx);3078	rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,3079				    efx);3080	rhashtable_free_and_destroy(&efx->tc->encap_match_ht,3081				    efx_tc_encap_match_free, NULL);3082	efx_tc_fini_conntrack(efx);3083	rhashtable_free_and_destroy(&efx->tc->recirc_ht, efx_tc_recirc_free, efx);3084	WARN_ON(!ida_is_empty(&efx->tc->recirc_ida));3085	ida_destroy(&efx->tc->recirc_ida);3086	rhashtable_free_and_destroy(&efx->tc->mac_ht, efx_tc_mac_free, NULL);3087	efx_tc_fini_counters(efx);3088	efx_tc_fini_encap_actions(efx);3089	mutex_unlock(&efx->tc->mutex);3090	mutex_destroy(&efx->tc->mutex);3091	kfree(efx->tc->caps);3092	kfree(efx->tc);3093	efx->tc = NULL;3094}3095