brintos / linux-shallow public Read only

1// SPDX-License-Identifier: GPL-2.02/*3 * Thunderbolt driver - bus logic (NHI independent)4 *5 * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com>6 * Copyright (C) 2019, Intel Corporation7 */8 9#include <linux/slab.h>10#include <linux/errno.h>11#include <linux/delay.h>12#include <linux/pm_runtime.h>13#include <linux/platform_data/x86/apple.h>14 15#include "tb.h"16#include "tb_regs.h"17#include "tunnel.h"18 19#define TB_TIMEOUT		100	/* ms */20#define TB_RELEASE_BW_TIMEOUT	10000	/* ms */21 22/*23 * Minimum bandwidth (in Mb/s) that is needed in the single transmitter/receiver24 * direction. This is 40G - 10% guard band bandwidth.25 */26#define TB_ASYM_MIN		(40000 * 90 / 100)27 28/*29 * Threshold bandwidth (in Mb/s) that is used to switch the links to30 * asymmetric and back. This is selected as 45G which means when the31 * request is higher than this, we switch the link to asymmetric, and32 * when it is less than this we switch it back. The 45G is selected so33 * that we still have 27G (of the total 72G) for bulk PCIe traffic when34 * switching back to symmetric.35 */36#define TB_ASYM_THRESHOLD	4500037 38#define MAX_GROUPS		7	/* max Group_ID is 7 */39 40static unsigned int asym_threshold = TB_ASYM_THRESHOLD;41module_param_named(asym_threshold, asym_threshold, uint, 0444);42MODULE_PARM_DESC(asym_threshold,43		"threshold (Mb/s) when to Gen 4 switch link symmetry. 0 disables. (default: "44		__MODULE_STRING(TB_ASYM_THRESHOLD) ")");45 46/**47 * struct tb_cm - Simple Thunderbolt connection manager48 * @tunnel_list: List of active tunnels49 * @dp_resources: List of available DP resources for DP tunneling50 * @hotplug_active: tb_handle_hotplug will stop progressing plug51 *		    events and exit if this is not set (it needs to52 *		    acquire the lock one more time). Used to drain wq53 *		    after cfg has been paused.54 * @remove_work: Work used to remove any unplugged routers after55 *		 runtime resume56 * @groups: Bandwidth groups used in this domain.57 */58struct tb_cm {59	struct list_head tunnel_list;60	struct list_head dp_resources;61	bool hotplug_active;62	struct delayed_work remove_work;63	struct tb_bandwidth_group groups[MAX_GROUPS];64};65 66static inline struct tb *tcm_to_tb(struct tb_cm *tcm)67{68	return ((void *)tcm - sizeof(struct tb));69}70 71struct tb_hotplug_event {72	struct work_struct work;73	struct tb *tb;74	u64 route;75	u8 port;76	bool unplug;77};78 79static void tb_handle_hotplug(struct work_struct *work);80 81static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug)82{83	struct tb_hotplug_event *ev;84 85	ev = kmalloc(sizeof(*ev), GFP_KERNEL);86	if (!ev)87		return;88 89	ev->tb = tb;90	ev->route = route;91	ev->port = port;92	ev->unplug = unplug;93	INIT_WORK(&ev->work, tb_handle_hotplug);94	queue_work(tb->wq, &ev->work);95}96 97/* enumeration & hot plug handling */98 99static void tb_add_dp_resources(struct tb_switch *sw)100{101	struct tb_cm *tcm = tb_priv(sw->tb);102	struct tb_port *port;103 104	tb_switch_for_each_port(sw, port) {105		if (!tb_port_is_dpin(port))106			continue;107 108		if (!tb_switch_query_dp_resource(sw, port))109			continue;110 111		/*112		 * If DP IN on device router exist, position it at the113		 * beginning of the DP resources list, so that it is used114		 * before DP IN of the host router. This way external GPU(s)115		 * will be prioritized when pairing DP IN to a DP OUT.116		 */117		if (tb_route(sw))118			list_add(&port->list, &tcm->dp_resources);119		else120			list_add_tail(&port->list, &tcm->dp_resources);121 122		tb_port_dbg(port, "DP IN resource available\n");123	}124}125 126static void tb_remove_dp_resources(struct tb_switch *sw)127{128	struct tb_cm *tcm = tb_priv(sw->tb);129	struct tb_port *port, *tmp;130 131	/* Clear children resources first */132	tb_switch_for_each_port(sw, port) {133		if (tb_port_has_remote(port))134			tb_remove_dp_resources(port->remote->sw);135	}136 137	list_for_each_entry_safe(port, tmp, &tcm->dp_resources, list) {138		if (port->sw == sw) {139			tb_port_dbg(port, "DP OUT resource unavailable\n");140			list_del_init(&port->list);141		}142	}143}144 145static void tb_discover_dp_resource(struct tb *tb, struct tb_port *port)146{147	struct tb_cm *tcm = tb_priv(tb);148	struct tb_port *p;149 150	list_for_each_entry(p, &tcm->dp_resources, list) {151		if (p == port)152			return;153	}154 155	tb_port_dbg(port, "DP %s resource available discovered\n",156		    tb_port_is_dpin(port) ? "IN" : "OUT");157	list_add_tail(&port->list, &tcm->dp_resources);158}159 160static void tb_discover_dp_resources(struct tb *tb)161{162	struct tb_cm *tcm = tb_priv(tb);163	struct tb_tunnel *tunnel;164 165	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {166		if (tb_tunnel_is_dp(tunnel))167			tb_discover_dp_resource(tb, tunnel->dst_port);168	}169}170 171/* Enables CL states up to host router */172static int tb_enable_clx(struct tb_switch *sw)173{174	struct tb_cm *tcm = tb_priv(sw->tb);175	unsigned int clx = TB_CL0S | TB_CL1;176	const struct tb_tunnel *tunnel;177	int ret;178 179	/*180	 * Currently only enable CLx for the first link. This is enough181	 * to allow the CPU to save energy at least on Intel hardware182	 * and makes it slightly simpler to implement. We may change183	 * this in the future to cover the whole topology if it turns184	 * out to be beneficial.185	 */186	while (sw && tb_switch_depth(sw) > 1)187		sw = tb_switch_parent(sw);188 189	if (!sw)190		return 0;191 192	if (tb_switch_depth(sw) != 1)193		return 0;194 195	/*196	 * If we are re-enabling then check if there is an active DMA197	 * tunnel and in that case bail out.198	 */199	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {200		if (tb_tunnel_is_dma(tunnel)) {201			if (tb_tunnel_port_on_path(tunnel, tb_upstream_port(sw)))202				return 0;203		}204	}205 206	/*207	 * Initially try with CL2. If that's not supported by the208	 * topology try with CL0s and CL1 and then give up.209	 */210	ret = tb_switch_clx_enable(sw, clx | TB_CL2);211	if (ret == -EOPNOTSUPP)212		ret = tb_switch_clx_enable(sw, clx);213	return ret == -EOPNOTSUPP ? 0 : ret;214}215 216/**217 * tb_disable_clx() - Disable CL states up to host router218 * @sw: Router to start219 *220 * Disables CL states from @sw up to the host router. Returns true if221 * any CL state were disabled. This can be used to figure out whether222 * the link was setup by us or the boot firmware so we don't223 * accidentally enable them if they were not enabled during discovery.224 */225static bool tb_disable_clx(struct tb_switch *sw)226{227	bool disabled = false;228 229	do {230		int ret;231 232		ret = tb_switch_clx_disable(sw);233		if (ret > 0)234			disabled = true;235		else if (ret < 0)236			tb_sw_warn(sw, "failed to disable CL states\n");237 238		sw = tb_switch_parent(sw);239	} while (sw);240 241	return disabled;242}243 244static int tb_increase_switch_tmu_accuracy(struct device *dev, void *data)245{246	struct tb_switch *sw;247 248	sw = tb_to_switch(dev);249	if (!sw)250		return 0;251 252	if (tb_switch_tmu_is_configured(sw, TB_SWITCH_TMU_MODE_LOWRES)) {253		enum tb_switch_tmu_mode mode;254		int ret;255 256		if (tb_switch_clx_is_enabled(sw, TB_CL1))257			mode = TB_SWITCH_TMU_MODE_HIFI_UNI;258		else259			mode = TB_SWITCH_TMU_MODE_HIFI_BI;260 261		ret = tb_switch_tmu_configure(sw, mode);262		if (ret)263			return ret;264 265		return tb_switch_tmu_enable(sw);266	}267 268	return 0;269}270 271static void tb_increase_tmu_accuracy(struct tb_tunnel *tunnel)272{273	struct tb_switch *sw;274 275	if (!tunnel)276		return;277 278	/*279	 * Once first DP tunnel is established we change the TMU280	 * accuracy of first depth child routers (and the host router)281	 * to the highest. This is needed for the DP tunneling to work282	 * but also allows CL0s.283	 *284	 * If both routers are v2 then we don't need to do anything as285	 * they are using enhanced TMU mode that allows all CLx.286	 */287	sw = tunnel->tb->root_switch;288	device_for_each_child(&sw->dev, NULL, tb_increase_switch_tmu_accuracy);289}290 291static int tb_switch_tmu_hifi_uni_required(struct device *dev, void *not_used)292{293	struct tb_switch *sw = tb_to_switch(dev);294 295	if (sw && tb_switch_tmu_is_enabled(sw) &&296	    tb_switch_tmu_is_configured(sw, TB_SWITCH_TMU_MODE_HIFI_UNI))297		return 1;298 299	return device_for_each_child(dev, NULL,300				     tb_switch_tmu_hifi_uni_required);301}302 303static bool tb_tmu_hifi_uni_required(struct tb *tb)304{305	return device_for_each_child(&tb->dev, NULL,306				     tb_switch_tmu_hifi_uni_required) == 1;307}308 309static int tb_enable_tmu(struct tb_switch *sw)310{311	int ret;312 313	/*314	 * If both routers at the end of the link are v2 we simply315	 * enable the enhanched uni-directional mode. That covers all316	 * the CL states. For v1 and before we need to use the normal317	 * rate to allow CL1 (when supported). Otherwise we keep the TMU318	 * running at the highest accuracy.319	 */320	ret = tb_switch_tmu_configure(sw,321			TB_SWITCH_TMU_MODE_MEDRES_ENHANCED_UNI);322	if (ret == -EOPNOTSUPP) {323		if (tb_switch_clx_is_enabled(sw, TB_CL1)) {324			/*325			 * Figure out uni-directional HiFi TMU requirements326			 * currently in the domain. If there are no327			 * uni-directional HiFi requirements we can put the TMU328			 * into LowRes mode.329			 *330			 * Deliberately skip bi-directional HiFi links331			 * as these work independently of other links332			 * (and they do not allow any CL states anyway).333			 */334			if (tb_tmu_hifi_uni_required(sw->tb))335				ret = tb_switch_tmu_configure(sw,336						TB_SWITCH_TMU_MODE_HIFI_UNI);337			else338				ret = tb_switch_tmu_configure(sw,339						TB_SWITCH_TMU_MODE_LOWRES);340		} else {341			ret = tb_switch_tmu_configure(sw, TB_SWITCH_TMU_MODE_HIFI_BI);342		}343 344		/* If not supported, fallback to bi-directional HiFi */345		if (ret == -EOPNOTSUPP)346			ret = tb_switch_tmu_configure(sw, TB_SWITCH_TMU_MODE_HIFI_BI);347	}348	if (ret)349		return ret;350 351	/* If it is already enabled in correct mode, don't touch it */352	if (tb_switch_tmu_is_enabled(sw))353		return 0;354 355	ret = tb_switch_tmu_disable(sw);356	if (ret)357		return ret;358 359	ret = tb_switch_tmu_post_time(sw);360	if (ret)361		return ret;362 363	return tb_switch_tmu_enable(sw);364}365 366static void tb_switch_discover_tunnels(struct tb_switch *sw,367				       struct list_head *list,368				       bool alloc_hopids)369{370	struct tb *tb = sw->tb;371	struct tb_port *port;372 373	tb_switch_for_each_port(sw, port) {374		struct tb_tunnel *tunnel = NULL;375 376		switch (port->config.type) {377		case TB_TYPE_DP_HDMI_IN:378			tunnel = tb_tunnel_discover_dp(tb, port, alloc_hopids);379			tb_increase_tmu_accuracy(tunnel);380			break;381 382		case TB_TYPE_PCIE_DOWN:383			tunnel = tb_tunnel_discover_pci(tb, port, alloc_hopids);384			break;385 386		case TB_TYPE_USB3_DOWN:387			tunnel = tb_tunnel_discover_usb3(tb, port, alloc_hopids);388			break;389 390		default:391			break;392		}393 394		if (tunnel)395			list_add_tail(&tunnel->list, list);396	}397 398	tb_switch_for_each_port(sw, port) {399		if (tb_port_has_remote(port)) {400			tb_switch_discover_tunnels(port->remote->sw, list,401						   alloc_hopids);402		}403	}404}405 406static int tb_port_configure_xdomain(struct tb_port *port, struct tb_xdomain *xd)407{408	if (tb_switch_is_usb4(port->sw))409		return usb4_port_configure_xdomain(port, xd);410	return tb_lc_configure_xdomain(port);411}412 413static void tb_port_unconfigure_xdomain(struct tb_port *port)414{415	if (tb_switch_is_usb4(port->sw))416		usb4_port_unconfigure_xdomain(port);417	else418		tb_lc_unconfigure_xdomain(port);419}420 421static void tb_scan_xdomain(struct tb_port *port)422{423	struct tb_switch *sw = port->sw;424	struct tb *tb = sw->tb;425	struct tb_xdomain *xd;426	u64 route;427 428	if (!tb_is_xdomain_enabled())429		return;430 431	route = tb_downstream_route(port);432	xd = tb_xdomain_find_by_route(tb, route);433	if (xd) {434		tb_xdomain_put(xd);435		return;436	}437 438	xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid,439			      NULL);440	if (xd) {441		tb_port_at(route, sw)->xdomain = xd;442		tb_port_configure_xdomain(port, xd);443		tb_xdomain_add(xd);444	}445}446 447/**448 * tb_find_unused_port() - return the first inactive port on @sw449 * @sw: Switch to find the port on450 * @type: Port type to look for451 */452static struct tb_port *tb_find_unused_port(struct tb_switch *sw,453					   enum tb_port_type type)454{455	struct tb_port *port;456 457	tb_switch_for_each_port(sw, port) {458		if (tb_is_upstream_port(port))459			continue;460		if (port->config.type != type)461			continue;462		if (!port->cap_adap)463			continue;464		if (tb_port_is_enabled(port))465			continue;466		return port;467	}468	return NULL;469}470 471static struct tb_port *tb_find_usb3_down(struct tb_switch *sw,472					 const struct tb_port *port)473{474	struct tb_port *down;475 476	down = usb4_switch_map_usb3_down(sw, port);477	if (down && !tb_usb3_port_is_enabled(down))478		return down;479	return NULL;480}481 482static struct tb_tunnel *tb_find_tunnel(struct tb *tb, enum tb_tunnel_type type,483					struct tb_port *src_port,484					struct tb_port *dst_port)485{486	struct tb_cm *tcm = tb_priv(tb);487	struct tb_tunnel *tunnel;488 489	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {490		if (tunnel->type == type &&491		    ((src_port && src_port == tunnel->src_port) ||492		     (dst_port && dst_port == tunnel->dst_port))) {493			return tunnel;494		}495	}496 497	return NULL;498}499 500static struct tb_tunnel *tb_find_first_usb3_tunnel(struct tb *tb,501						   struct tb_port *src_port,502						   struct tb_port *dst_port)503{504	struct tb_port *port, *usb3_down;505	struct tb_switch *sw;506 507	/* Pick the router that is deepest in the topology */508	if (tb_port_path_direction_downstream(src_port, dst_port))509		sw = dst_port->sw;510	else511		sw = src_port->sw;512 513	/* Can't be the host router */514	if (sw == tb->root_switch)515		return NULL;516 517	/* Find the downstream USB4 port that leads to this router */518	port = tb_port_at(tb_route(sw), tb->root_switch);519	/* Find the corresponding host router USB3 downstream port */520	usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port);521	if (!usb3_down)522		return NULL;523 524	return tb_find_tunnel(tb, TB_TUNNEL_USB3, usb3_down, NULL);525}526 527/**528 * tb_consumed_usb3_pcie_bandwidth() - Consumed USB3/PCIe bandwidth over a single link529 * @tb: Domain structure530 * @src_port: Source protocol adapter531 * @dst_port: Destination protocol adapter532 * @port: USB4 port the consumed bandwidth is calculated533 * @consumed_up: Consumed upsream bandwidth (Mb/s)534 * @consumed_down: Consumed downstream bandwidth (Mb/s)535 *536 * Calculates consumed USB3 and PCIe bandwidth at @port between path537 * from @src_port to @dst_port. Does not take USB3 tunnel starting from538 * @src_port and ending on @src_port into account because that bandwidth is539 * already included in as part of the "first hop" USB3 tunnel.540 */541static int tb_consumed_usb3_pcie_bandwidth(struct tb *tb,542					   struct tb_port *src_port,543					   struct tb_port *dst_port,544					   struct tb_port *port,545					   int *consumed_up,546					   int *consumed_down)547{548	int pci_consumed_up, pci_consumed_down;549	struct tb_tunnel *tunnel;550 551	*consumed_up = *consumed_down = 0;552 553	tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);554	if (tunnel && !tb_port_is_usb3_down(src_port) &&555	    !tb_port_is_usb3_up(dst_port)) {556		int ret;557 558		ret = tb_tunnel_consumed_bandwidth(tunnel, consumed_up,559						   consumed_down);560		if (ret)561			return ret;562	}563 564	/*565	 * If there is anything reserved for PCIe bulk traffic take it566	 * into account here too.567	 */568	if (tb_tunnel_reserved_pci(port, &pci_consumed_up, &pci_consumed_down)) {569		*consumed_up += pci_consumed_up;570		*consumed_down += pci_consumed_down;571	}572 573	return 0;574}575 576/**577 * tb_consumed_dp_bandwidth() - Consumed DP bandwidth over a single link578 * @tb: Domain structure579 * @src_port: Source protocol adapter580 * @dst_port: Destination protocol adapter581 * @port: USB4 port the consumed bandwidth is calculated582 * @consumed_up: Consumed upsream bandwidth (Mb/s)583 * @consumed_down: Consumed downstream bandwidth (Mb/s)584 *585 * Calculates consumed DP bandwidth at @port between path from @src_port586 * to @dst_port. Does not take tunnel starting from @src_port and ending587 * from @src_port into account.588 *589 * If there is bandwidth reserved for any of the groups between590 * @src_port and @dst_port (but not yet used) that is also taken into591 * account in the returned consumed bandwidth.592 */593static int tb_consumed_dp_bandwidth(struct tb *tb,594				    struct tb_port *src_port,595				    struct tb_port *dst_port,596				    struct tb_port *port,597				    int *consumed_up,598				    int *consumed_down)599{600	int group_reserved[MAX_GROUPS] = {};601	struct tb_cm *tcm = tb_priv(tb);602	struct tb_tunnel *tunnel;603	bool downstream;604	int i, ret;605 606	*consumed_up = *consumed_down = 0;607 608	/*609	 * Find all DP tunnels that cross the port and reduce610	 * their consumed bandwidth from the available.611	 */612	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {613		const struct tb_bandwidth_group *group;614		int dp_consumed_up, dp_consumed_down;615 616		if (tb_tunnel_is_invalid(tunnel))617			continue;618 619		if (!tb_tunnel_is_dp(tunnel))620			continue;621 622		if (!tb_tunnel_port_on_path(tunnel, port))623			continue;624 625		/*626		 * Calculate what is reserved for groups crossing the627		 * same ports only once (as that is reserved for all the628		 * tunnels in the group).629		 */630		group = tunnel->src_port->group;631		if (group && group->reserved && !group_reserved[group->index])632			group_reserved[group->index] = group->reserved;633 634		/*635		 * Ignore the DP tunnel between src_port and dst_port636		 * because it is the same tunnel and we may be637		 * re-calculating estimated bandwidth.638		 */639		if (tunnel->src_port == src_port &&640		    tunnel->dst_port == dst_port)641			continue;642 643		ret = tb_tunnel_consumed_bandwidth(tunnel, &dp_consumed_up,644						   &dp_consumed_down);645		if (ret)646			return ret;647 648		*consumed_up += dp_consumed_up;649		*consumed_down += dp_consumed_down;650	}651 652	downstream = tb_port_path_direction_downstream(src_port, dst_port);653	for (i = 0; i < ARRAY_SIZE(group_reserved); i++) {654		if (downstream)655			*consumed_down += group_reserved[i];656		else657			*consumed_up += group_reserved[i];658	}659 660	return 0;661}662 663static bool tb_asym_supported(struct tb_port *src_port, struct tb_port *dst_port,664			      struct tb_port *port)665{666	bool downstream = tb_port_path_direction_downstream(src_port, dst_port);667	enum tb_link_width width;668 669	if (tb_is_upstream_port(port))670		width = downstream ? TB_LINK_WIDTH_ASYM_RX : TB_LINK_WIDTH_ASYM_TX;671	else672		width = downstream ? TB_LINK_WIDTH_ASYM_TX : TB_LINK_WIDTH_ASYM_RX;673 674	return tb_port_width_supported(port, width);675}676 677/**678 * tb_maximum_bandwidth() - Maximum bandwidth over a single link679 * @tb: Domain structure680 * @src_port: Source protocol adapter681 * @dst_port: Destination protocol adapter682 * @port: USB4 port the total bandwidth is calculated683 * @max_up: Maximum upstream bandwidth (Mb/s)684 * @max_down: Maximum downstream bandwidth (Mb/s)685 * @include_asym: Include bandwidth if the link is switched from686 *		  symmetric to asymmetric687 *688 * Returns maximum possible bandwidth in @max_up and @max_down over a689 * single link at @port. If @include_asym is set then includes the690 * additional banwdith if the links are transitioned into asymmetric to691 * direction from @src_port to @dst_port.692 */693static int tb_maximum_bandwidth(struct tb *tb, struct tb_port *src_port,694				struct tb_port *dst_port, struct tb_port *port,695				int *max_up, int *max_down, bool include_asym)696{697	bool downstream = tb_port_path_direction_downstream(src_port, dst_port);698	int link_speed, link_width, up_bw, down_bw;699 700	/*701	 * Can include asymmetric, only if it is actually supported by702	 * the lane adapter.703	 */704	if (!tb_asym_supported(src_port, dst_port, port))705		include_asym = false;706 707	if (tb_is_upstream_port(port)) {708		link_speed = port->sw->link_speed;709		/*710		 * sw->link_width is from upstream perspective so we use711		 * the opposite for downstream of the host router.712		 */713		if (port->sw->link_width == TB_LINK_WIDTH_ASYM_TX) {714			up_bw = link_speed * 3 * 1000;715			down_bw = link_speed * 1 * 1000;716		} else if (port->sw->link_width == TB_LINK_WIDTH_ASYM_RX) {717			up_bw = link_speed * 1 * 1000;718			down_bw = link_speed * 3 * 1000;719		} else if (include_asym) {720			/*721			 * The link is symmetric at the moment but we722			 * can switch it to asymmetric as needed. Report723			 * this bandwidth as available (even though it724			 * is not yet enabled).725			 */726			if (downstream) {727				up_bw = link_speed * 1 * 1000;728				down_bw = link_speed * 3 * 1000;729			} else {730				up_bw = link_speed * 3 * 1000;731				down_bw = link_speed * 1 * 1000;732			}733		} else {734			up_bw = link_speed * port->sw->link_width * 1000;735			down_bw = up_bw;736		}737	} else {738		link_speed = tb_port_get_link_speed(port);739		if (link_speed < 0)740			return link_speed;741 742		link_width = tb_port_get_link_width(port);743		if (link_width < 0)744			return link_width;745 746		if (link_width == TB_LINK_WIDTH_ASYM_TX) {747			up_bw = link_speed * 1 * 1000;748			down_bw = link_speed * 3 * 1000;749		} else if (link_width == TB_LINK_WIDTH_ASYM_RX) {750			up_bw = link_speed * 3 * 1000;751			down_bw = link_speed * 1 * 1000;752		} else if (include_asym) {753			/*754			 * The link is symmetric at the moment but we755			 * can switch it to asymmetric as needed. Report756			 * this bandwidth as available (even though it757			 * is not yet enabled).758			 */759			if (downstream) {760				up_bw = link_speed * 1 * 1000;761				down_bw = link_speed * 3 * 1000;762			} else {763				up_bw = link_speed * 3 * 1000;764				down_bw = link_speed * 1 * 1000;765			}766		} else {767			up_bw = link_speed * link_width * 1000;768			down_bw = up_bw;769		}770	}771 772	/* Leave 10% guard band */773	*max_up = up_bw - up_bw / 10;774	*max_down = down_bw - down_bw / 10;775 776	tb_port_dbg(port, "link maximum bandwidth %d/%d Mb/s\n", *max_up, *max_down);777	return 0;778}779 780/**781 * tb_available_bandwidth() - Available bandwidth for tunneling782 * @tb: Domain structure783 * @src_port: Source protocol adapter784 * @dst_port: Destination protocol adapter785 * @available_up: Available bandwidth upstream (Mb/s)786 * @available_down: Available bandwidth downstream (Mb/s)787 * @include_asym: Include bandwidth if the link is switched from788 *		  symmetric to asymmetric789 *790 * Calculates maximum available bandwidth for protocol tunneling between791 * @src_port and @dst_port at the moment. This is minimum of maximum792 * link bandwidth across all links reduced by currently consumed793 * bandwidth on that link.794 *795 * If @include_asym is true then includes also bandwidth that can be796 * added when the links are transitioned into asymmetric (but does not797 * transition the links).798 */799static int tb_available_bandwidth(struct tb *tb, struct tb_port *src_port,800				 struct tb_port *dst_port, int *available_up,801				 int *available_down, bool include_asym)802{803	struct tb_port *port;804	int ret;805 806	/* Maximum possible bandwidth asymmetric Gen 4 link is 120 Gb/s */807	*available_up = *available_down = 120000;808 809	/* Find the minimum available bandwidth over all links */810	tb_for_each_port_on_path(src_port, dst_port, port) {811		int max_up, max_down, consumed_up, consumed_down;812 813		if (!tb_port_is_null(port))814			continue;815 816		ret = tb_maximum_bandwidth(tb, src_port, dst_port, port,817					   &max_up, &max_down, include_asym);818		if (ret)819			return ret;820 821		ret = tb_consumed_usb3_pcie_bandwidth(tb, src_port, dst_port,822						      port, &consumed_up,823						      &consumed_down);824		if (ret)825			return ret;826		max_up -= consumed_up;827		max_down -= consumed_down;828 829		ret = tb_consumed_dp_bandwidth(tb, src_port, dst_port, port,830					       &consumed_up, &consumed_down);831		if (ret)832			return ret;833		max_up -= consumed_up;834		max_down -= consumed_down;835 836		if (max_up < *available_up)837			*available_up = max_up;838		if (max_down < *available_down)839			*available_down = max_down;840	}841 842	if (*available_up < 0)843		*available_up = 0;844	if (*available_down < 0)845		*available_down = 0;846 847	return 0;848}849 850static int tb_release_unused_usb3_bandwidth(struct tb *tb,851					    struct tb_port *src_port,852					    struct tb_port *dst_port)853{854	struct tb_tunnel *tunnel;855 856	tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);857	return tunnel ? tb_tunnel_release_unused_bandwidth(tunnel) : 0;858}859 860static void tb_reclaim_usb3_bandwidth(struct tb *tb, struct tb_port *src_port,861				      struct tb_port *dst_port)862{863	int ret, available_up, available_down;864	struct tb_tunnel *tunnel;865 866	tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);867	if (!tunnel)868		return;869 870	tb_tunnel_dbg(tunnel, "reclaiming unused bandwidth\n");871 872	/*873	 * Calculate available bandwidth for the first hop USB3 tunnel.874	 * That determines the whole USB3 bandwidth for this branch.875	 */876	ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port,877				     &available_up, &available_down, false);878	if (ret) {879		tb_tunnel_warn(tunnel, "failed to calculate available bandwidth\n");880		return;881	}882 883	tb_tunnel_dbg(tunnel, "available bandwidth %d/%d Mb/s\n", available_up,884		      available_down);885 886	tb_tunnel_reclaim_available_bandwidth(tunnel, &available_up, &available_down);887}888 889static int tb_tunnel_usb3(struct tb *tb, struct tb_switch *sw)890{891	struct tb_switch *parent = tb_switch_parent(sw);892	int ret, available_up, available_down;893	struct tb_port *up, *down, *port;894	struct tb_cm *tcm = tb_priv(tb);895	struct tb_tunnel *tunnel;896 897	if (!tb_acpi_may_tunnel_usb3()) {898		tb_dbg(tb, "USB3 tunneling disabled, not creating tunnel\n");899		return 0;900	}901 902	up = tb_switch_find_port(sw, TB_TYPE_USB3_UP);903	if (!up)904		return 0;905 906	if (!sw->link_usb4)907		return 0;908 909	/*910	 * Look up available down port. Since we are chaining it should911	 * be found right above this switch.912	 */913	port = tb_switch_downstream_port(sw);914	down = tb_find_usb3_down(parent, port);915	if (!down)916		return 0;917 918	if (tb_route(parent)) {919		struct tb_port *parent_up;920		/*921		 * Check first that the parent switch has its upstream USB3922		 * port enabled. Otherwise the chain is not complete and923		 * there is no point setting up a new tunnel.924		 */925		parent_up = tb_switch_find_port(parent, TB_TYPE_USB3_UP);926		if (!parent_up || !tb_port_is_enabled(parent_up))927			return 0;928 929		/* Make all unused bandwidth available for the new tunnel */930		ret = tb_release_unused_usb3_bandwidth(tb, down, up);931		if (ret)932			return ret;933	}934 935	ret = tb_available_bandwidth(tb, down, up, &available_up, &available_down,936				     false);937	if (ret)938		goto err_reclaim;939 940	tb_port_dbg(up, "available bandwidth for new USB3 tunnel %d/%d Mb/s\n",941		    available_up, available_down);942 943	tunnel = tb_tunnel_alloc_usb3(tb, up, down, available_up,944				      available_down);945	if (!tunnel) {946		ret = -ENOMEM;947		goto err_reclaim;948	}949 950	if (tb_tunnel_activate(tunnel)) {951		tb_port_info(up,952			     "USB3 tunnel activation failed, aborting\n");953		ret = -EIO;954		goto err_free;955	}956 957	list_add_tail(&tunnel->list, &tcm->tunnel_list);958	if (tb_route(parent))959		tb_reclaim_usb3_bandwidth(tb, down, up);960 961	return 0;962 963err_free:964	tb_tunnel_free(tunnel);965err_reclaim:966	if (tb_route(parent))967		tb_reclaim_usb3_bandwidth(tb, down, up);968 969	return ret;970}971 972static int tb_create_usb3_tunnels(struct tb_switch *sw)973{974	struct tb_port *port;975	int ret;976 977	if (!tb_acpi_may_tunnel_usb3())978		return 0;979 980	if (tb_route(sw)) {981		ret = tb_tunnel_usb3(sw->tb, sw);982		if (ret)983			return ret;984	}985 986	tb_switch_for_each_port(sw, port) {987		if (!tb_port_has_remote(port))988			continue;989		ret = tb_create_usb3_tunnels(port->remote->sw);990		if (ret)991			return ret;992	}993 994	return 0;995}996 997/**998 * tb_configure_asym() - Transition links to asymmetric if needed999 * @tb: Domain structure1000 * @src_port: Source adapter to start the transition1001 * @dst_port: Destination adapter1002 * @requested_up: Additional bandwidth (Mb/s) required upstream1003 * @requested_down: Additional bandwidth (Mb/s) required downstream1004 *1005 * Transition links between @src_port and @dst_port into asymmetric, with1006 * three lanes in the direction from @src_port towards @dst_port and one lane1007 * in the opposite direction, if the bandwidth requirements1008 * (requested + currently consumed) on that link exceed @asym_threshold.1009 *1010 * Must be called with available >= requested over all links.1011 */1012static int tb_configure_asym(struct tb *tb, struct tb_port *src_port,1013			     struct tb_port *dst_port, int requested_up,1014			     int requested_down)1015{1016	bool clx = false, clx_disabled = false, downstream;1017	struct tb_switch *sw;1018	struct tb_port *up;1019	int ret = 0;1020 1021	if (!asym_threshold)1022		return 0;1023 1024	downstream = tb_port_path_direction_downstream(src_port, dst_port);1025	/* Pick up router deepest in the hierarchy */1026	if (downstream)1027		sw = dst_port->sw;1028	else1029		sw = src_port->sw;1030 1031	tb_for_each_upstream_port_on_path(src_port, dst_port, up) {1032		struct tb_port *down = tb_switch_downstream_port(up->sw);1033		enum tb_link_width width_up, width_down;1034		int consumed_up, consumed_down;1035 1036		ret = tb_consumed_dp_bandwidth(tb, src_port, dst_port, up,1037					       &consumed_up, &consumed_down);1038		if (ret)1039			break;1040 1041		if (downstream) {1042			/*1043			 * Downstream so make sure upstream is within the 36G1044			 * (40G - guard band 10%), and the requested is above1045			 * what the threshold is.1046			 */1047			if (consumed_up + requested_up >= TB_ASYM_MIN) {1048				ret = -ENOBUFS;1049				break;1050			}1051			/* Does consumed + requested exceed the threshold */1052			if (consumed_down + requested_down < asym_threshold)1053				continue;1054 1055			width_up = TB_LINK_WIDTH_ASYM_RX;1056			width_down = TB_LINK_WIDTH_ASYM_TX;1057		} else {1058			/* Upstream, the opposite of above */1059			if (consumed_down + requested_down >= TB_ASYM_MIN) {1060				ret = -ENOBUFS;1061				break;1062			}1063			if (consumed_up + requested_up < asym_threshold)1064				continue;1065 1066			width_up = TB_LINK_WIDTH_ASYM_TX;1067			width_down = TB_LINK_WIDTH_ASYM_RX;1068		}1069 1070		if (up->sw->link_width == width_up)1071			continue;1072 1073		if (!tb_port_width_supported(up, width_up) ||1074		    !tb_port_width_supported(down, width_down))1075			continue;1076 1077		/*1078		 * Disable CL states before doing any transitions. We1079		 * delayed it until now that we know there is a real1080		 * transition taking place.1081		 */1082		if (!clx_disabled) {1083			clx = tb_disable_clx(sw);1084			clx_disabled = true;1085		}1086 1087		tb_sw_dbg(up->sw, "configuring asymmetric link\n");1088 1089		/*1090		 * Here requested + consumed > threshold so we need to1091		 * transtion the link into asymmetric now.1092		 */1093		ret = tb_switch_set_link_width(up->sw, width_up);1094		if (ret) {1095			tb_sw_warn(up->sw, "failed to set link width\n");1096			break;1097		}1098	}1099 1100	/* Re-enable CL states if they were previosly enabled */1101	if (clx)1102		tb_enable_clx(sw);1103 1104	return ret;1105}1106 1107/**1108 * tb_configure_sym() - Transition links to symmetric if possible1109 * @tb: Domain structure1110 * @src_port: Source adapter to start the transition1111 * @dst_port: Destination adapter1112 * @keep_asym: Keep asymmetric link if preferred1113 *1114 * Goes over each link from @src_port to @dst_port and tries to1115 * transition the link to symmetric if the currently consumed bandwidth1116 * allows and link asymmetric preference is ignored (if @keep_asym is %false).1117 */1118static int tb_configure_sym(struct tb *tb, struct tb_port *src_port,1119			    struct tb_port *dst_port, bool keep_asym)1120{1121	bool clx = false, clx_disabled = false, downstream;1122	struct tb_switch *sw;1123	struct tb_port *up;1124	int ret = 0;1125 1126	if (!asym_threshold)1127		return 0;1128 1129	downstream = tb_port_path_direction_downstream(src_port, dst_port);1130	/* Pick up router deepest in the hierarchy */1131	if (downstream)1132		sw = dst_port->sw;1133	else1134		sw = src_port->sw;1135 1136	tb_for_each_upstream_port_on_path(src_port, dst_port, up) {1137		int consumed_up, consumed_down;1138 1139		/* Already symmetric */1140		if (up->sw->link_width <= TB_LINK_WIDTH_DUAL)1141			continue;1142		/* Unplugged, no need to switch */1143		if (up->sw->is_unplugged)1144			continue;1145 1146		ret = tb_consumed_dp_bandwidth(tb, src_port, dst_port, up,1147					       &consumed_up, &consumed_down);1148		if (ret)1149			break;1150 1151		if (downstream) {1152			/*1153			 * Downstream so we want the consumed_down < threshold.1154			 * Upstream traffic should be less than 36G (40G1155			 * guard band 10%) as the link was configured asymmetric1156			 * already.1157			 */1158			if (consumed_down >= asym_threshold)1159				continue;1160		} else {1161			if (consumed_up >= asym_threshold)1162				continue;1163		}1164 1165		if (up->sw->link_width == TB_LINK_WIDTH_DUAL)1166			continue;1167 1168		/*1169		 * Here consumed < threshold so we can transition the1170		 * link to symmetric.1171		 *1172		 * However, if the router prefers asymmetric link we1173		 * honor that (unless @keep_asym is %false).1174		 */1175		if (keep_asym &&1176		    up->sw->preferred_link_width > TB_LINK_WIDTH_DUAL) {1177			tb_sw_dbg(up->sw, "keeping preferred asymmetric link\n");1178			continue;1179		}1180 1181		/* Disable CL states before doing any transitions */1182		if (!clx_disabled) {1183			clx = tb_disable_clx(sw);1184			clx_disabled = true;1185		}1186 1187		tb_sw_dbg(up->sw, "configuring symmetric link\n");1188 1189		ret = tb_switch_set_link_width(up->sw, TB_LINK_WIDTH_DUAL);1190		if (ret) {1191			tb_sw_warn(up->sw, "failed to set link width\n");1192			break;1193		}1194	}1195 1196	/* Re-enable CL states if they were previosly enabled */1197	if (clx)1198		tb_enable_clx(sw);1199 1200	return ret;1201}1202 1203static void tb_configure_link(struct tb_port *down, struct tb_port *up,1204			      struct tb_switch *sw)1205{1206	struct tb *tb = sw->tb;1207 1208	/* Link the routers using both links if available */1209	down->remote = up;1210	up->remote = down;1211	if (down->dual_link_port && up->dual_link_port) {1212		down->dual_link_port->remote = up->dual_link_port;1213		up->dual_link_port->remote = down->dual_link_port;1214	}1215 1216	/*1217	 * Enable lane bonding if the link is currently two single lane1218	 * links.1219	 */1220	if (sw->link_width < TB_LINK_WIDTH_DUAL)1221		tb_switch_set_link_width(sw, TB_LINK_WIDTH_DUAL);1222 1223	/*1224	 * Device router that comes up as symmetric link is1225	 * connected deeper in the hierarchy, we transition the links1226	 * above into symmetric if bandwidth allows.1227	 */1228	if (tb_switch_depth(sw) > 1 &&1229	    tb_port_get_link_generation(up) >= 4 &&1230	    up->sw->link_width == TB_LINK_WIDTH_DUAL) {1231		struct tb_port *host_port;1232 1233		host_port = tb_port_at(tb_route(sw), tb->root_switch);1234		tb_configure_sym(tb, host_port, up, false);1235	}1236 1237	/* Set the link configured */1238	tb_switch_configure_link(sw);1239}1240 1241static void tb_scan_port(struct tb_port *port);1242 1243/*1244 * tb_scan_switch() - scan for and initialize downstream switches1245 */1246static void tb_scan_switch(struct tb_switch *sw)1247{1248	struct tb_port *port;1249 1250	pm_runtime_get_sync(&sw->dev);1251 1252	tb_switch_for_each_port(sw, port)1253		tb_scan_port(port);1254 1255	pm_runtime_mark_last_busy(&sw->dev);1256	pm_runtime_put_autosuspend(&sw->dev);1257}1258 1259/*1260 * tb_scan_port() - check for and initialize switches below port1261 */1262static void tb_scan_port(struct tb_port *port)1263{1264	struct tb_cm *tcm = tb_priv(port->sw->tb);1265	struct tb_port *upstream_port;1266	bool discovery = false;1267	struct tb_switch *sw;1268 1269	if (tb_is_upstream_port(port))1270		return;1271 1272	if (tb_port_is_dpout(port) && tb_dp_port_hpd_is_active(port) == 1 &&1273	    !tb_dp_port_is_enabled(port)) {1274		tb_port_dbg(port, "DP adapter HPD set, queuing hotplug\n");1275		tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port,1276				 false);1277		return;1278	}1279 1280	if (port->config.type != TB_TYPE_PORT)1281		return;1282	if (port->dual_link_port && port->link_nr)1283		return; /*1284			 * Downstream switch is reachable through two ports.1285			 * Only scan on the primary port (link_nr == 0).1286			 */1287 1288	if (port->usb4)1289		pm_runtime_get_sync(&port->usb4->dev);1290 1291	if (tb_wait_for_port(port, false) <= 0)1292		goto out_rpm_put;1293	if (port->remote) {1294		tb_port_dbg(port, "port already has a remote\n");1295		goto out_rpm_put;1296	}1297 1298	tb_retimer_scan(port, true);1299 1300	sw = tb_switch_alloc(port->sw->tb, &port->sw->dev,1301			     tb_downstream_route(port));1302	if (IS_ERR(sw)) {1303		/*1304		 * If there is an error accessing the connected switch1305		 * it may be connected to another domain. Also we allow1306		 * the other domain to be connected to a max depth switch.1307		 */1308		if (PTR_ERR(sw) == -EIO || PTR_ERR(sw) == -EADDRNOTAVAIL)1309			tb_scan_xdomain(port);1310		goto out_rpm_put;1311	}1312 1313	if (tb_switch_configure(sw)) {1314		tb_switch_put(sw);1315		goto out_rpm_put;1316	}1317 1318	/*1319	 * If there was previously another domain connected remove it1320	 * first.1321	 */1322	if (port->xdomain) {1323		tb_xdomain_remove(port->xdomain);1324		tb_port_unconfigure_xdomain(port);1325		port->xdomain = NULL;1326	}1327 1328	/*1329	 * Do not send uevents until we have discovered all existing1330	 * tunnels and know which switches were authorized already by1331	 * the boot firmware.1332	 */1333	if (!tcm->hotplug_active) {1334		dev_set_uevent_suppress(&sw->dev, true);1335		discovery = true;1336	}1337 1338	/*1339	 * At the moment Thunderbolt 2 and beyond (devices with LC) we1340	 * can support runtime PM.1341	 */1342	sw->rpm = sw->generation > 1;1343 1344	if (tb_switch_add(sw)) {1345		tb_switch_put(sw);1346		goto out_rpm_put;1347	}1348 1349	upstream_port = tb_upstream_port(sw);1350	tb_configure_link(port, upstream_port, sw);1351 1352	/*1353	 * CL0s and CL1 are enabled and supported together.1354	 * Silently ignore CLx enabling in case CLx is not supported.1355	 */1356	if (discovery)1357		tb_sw_dbg(sw, "discovery, not touching CL states\n");1358	else if (tb_enable_clx(sw))1359		tb_sw_warn(sw, "failed to enable CL states\n");1360 1361	if (tb_enable_tmu(sw))1362		tb_sw_warn(sw, "failed to enable TMU\n");1363 1364	/*1365	 * Configuration valid needs to be set after the TMU has been1366	 * enabled for the upstream port of the router so we do it here.1367	 */1368	tb_switch_configuration_valid(sw);1369 1370	/* Scan upstream retimers */1371	tb_retimer_scan(upstream_port, true);1372 1373	/*1374	 * Create USB 3.x tunnels only when the switch is plugged to the1375	 * domain. This is because we scan the domain also during discovery1376	 * and want to discover existing USB 3.x tunnels before we create1377	 * any new.1378	 */1379	if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw))1380		tb_sw_warn(sw, "USB3 tunnel creation failed\n");1381 1382	tb_add_dp_resources(sw);1383	tb_scan_switch(sw);1384 1385out_rpm_put:1386	if (port->usb4) {1387		pm_runtime_mark_last_busy(&port->usb4->dev);1388		pm_runtime_put_autosuspend(&port->usb4->dev);1389	}1390}1391 1392static void1393tb_recalc_estimated_bandwidth_for_group(struct tb_bandwidth_group *group)1394{1395	struct tb_tunnel *first_tunnel;1396	struct tb *tb = group->tb;1397	struct tb_port *in;1398	int ret;1399 1400	tb_dbg(tb, "re-calculating bandwidth estimation for group %u\n",1401	       group->index);1402 1403	first_tunnel = NULL;1404	list_for_each_entry(in, &group->ports, group_list) {1405		int estimated_bw, estimated_up, estimated_down;1406		struct tb_tunnel *tunnel;1407		struct tb_port *out;1408 1409		if (!usb4_dp_port_bandwidth_mode_enabled(in))1410			continue;1411 1412		tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, NULL);1413		if (WARN_ON(!tunnel))1414			break;1415 1416		if (!first_tunnel) {1417			/*1418			 * Since USB3 bandwidth is shared by all DP1419			 * tunnels under the host router USB4 port, even1420			 * if they do not begin from the host router, we1421			 * can release USB3 bandwidth just once and not1422			 * for each tunnel separately.1423			 */1424			first_tunnel = tunnel;1425			ret = tb_release_unused_usb3_bandwidth(tb,1426				first_tunnel->src_port, first_tunnel->dst_port);1427			if (ret) {1428				tb_tunnel_warn(tunnel,1429					"failed to release unused bandwidth\n");1430				break;1431			}1432		}1433 1434		out = tunnel->dst_port;1435		ret = tb_available_bandwidth(tb, in, out, &estimated_up,1436					     &estimated_down, true);1437		if (ret) {1438			tb_tunnel_warn(tunnel,1439				"failed to re-calculate estimated bandwidth\n");1440			break;1441		}1442 1443		/*1444		 * Estimated bandwidth includes:1445		 *  - already allocated bandwidth for the DP tunnel1446		 *  - available bandwidth along the path1447		 *  - bandwidth allocated for USB 3.x but not used.1448		 */1449		if (tb_tunnel_direction_downstream(tunnel))1450			estimated_bw = estimated_down;1451		else1452			estimated_bw = estimated_up;1453 1454		/*1455		 * If there is reserved bandwidth for the group that is1456		 * not yet released we report that too.1457		 */1458		tb_tunnel_dbg(tunnel,1459			      "re-calculated estimated bandwidth %u (+ %u reserved) = %u Mb/s\n",1460			      estimated_bw, group->reserved,1461			      estimated_bw + group->reserved);1462 1463		if (usb4_dp_port_set_estimated_bandwidth(in,1464				estimated_bw + group->reserved))1465			tb_tunnel_warn(tunnel,1466				       "failed to update estimated bandwidth\n");1467	}1468 1469	if (first_tunnel)1470		tb_reclaim_usb3_bandwidth(tb, first_tunnel->src_port,1471					  first_tunnel->dst_port);1472 1473	tb_dbg(tb, "bandwidth estimation for group %u done\n", group->index);1474}1475 1476static void tb_recalc_estimated_bandwidth(struct tb *tb)1477{1478	struct tb_cm *tcm = tb_priv(tb);1479	int i;1480 1481	tb_dbg(tb, "bandwidth consumption changed, re-calculating estimated bandwidth\n");1482 1483	for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) {1484		struct tb_bandwidth_group *group = &tcm->groups[i];1485 1486		if (!list_empty(&group->ports))1487			tb_recalc_estimated_bandwidth_for_group(group);1488	}1489 1490	tb_dbg(tb, "bandwidth re-calculation done\n");1491}1492 1493static bool __release_group_bandwidth(struct tb_bandwidth_group *group)1494{1495	if (group->reserved) {1496		tb_dbg(group->tb, "group %d released total %d Mb/s\n", group->index,1497			group->reserved);1498		group->reserved = 0;1499		return true;1500	}1501	return false;1502}1503 1504static void __configure_group_sym(struct tb_bandwidth_group *group)1505{1506	struct tb_tunnel *tunnel;1507	struct tb_port *in;1508 1509	if (list_empty(&group->ports))1510		return;1511 1512	/*1513	 * All the tunnels in the group go through the same USB4 links1514	 * so we find the first one here and pass the IN and OUT1515	 * adapters to tb_configure_sym() which now transitions the1516	 * links back to symmetric if bandwidth requirement < asym_threshold.1517	 *1518	 * We do this here to avoid unnecessary transitions (for example1519	 * if the graphics released bandwidth for other tunnel in the1520	 * same group).1521	 */1522	in = list_first_entry(&group->ports, struct tb_port, group_list);1523	tunnel = tb_find_tunnel(group->tb, TB_TUNNEL_DP, in, NULL);1524	if (tunnel)1525		tb_configure_sym(group->tb, in, tunnel->dst_port, true);1526}1527 1528static void tb_bandwidth_group_release_work(struct work_struct *work)1529{1530	struct tb_bandwidth_group *group =1531		container_of(work, typeof(*group), release_work.work);1532	struct tb *tb = group->tb;1533 1534	mutex_lock(&tb->lock);1535	if (__release_group_bandwidth(group))1536		tb_recalc_estimated_bandwidth(tb);1537	__configure_group_sym(group);1538	mutex_unlock(&tb->lock);1539}1540 1541static void tb_init_bandwidth_groups(struct tb_cm *tcm)1542{1543	int i;1544 1545	for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) {1546		struct tb_bandwidth_group *group = &tcm->groups[i];1547 1548		group->tb = tcm_to_tb(tcm);1549		group->index = i + 1;1550		INIT_LIST_HEAD(&group->ports);1551		INIT_DELAYED_WORK(&group->release_work,1552				  tb_bandwidth_group_release_work);1553	}1554}1555 1556static void tb_bandwidth_group_attach_port(struct tb_bandwidth_group *group,1557					   struct tb_port *in)1558{1559	if (!group || WARN_ON(in->group))1560		return;1561 1562	in->group = group;1563	list_add_tail(&in->group_list, &group->ports);1564 1565	tb_port_dbg(in, "attached to bandwidth group %d\n", group->index);1566}1567 1568static struct tb_bandwidth_group *tb_find_free_bandwidth_group(struct tb_cm *tcm)1569{1570	int i;1571 1572	for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) {1573		struct tb_bandwidth_group *group = &tcm->groups[i];1574 1575		if (list_empty(&group->ports))1576			return group;1577	}1578 1579	return NULL;1580}1581 1582static struct tb_bandwidth_group *1583tb_attach_bandwidth_group(struct tb_cm *tcm, struct tb_port *in,1584			  struct tb_port *out)1585{1586	struct tb_bandwidth_group *group;1587	struct tb_tunnel *tunnel;1588 1589	/*1590	 * Find all DP tunnels that go through all the same USB4 links1591	 * as this one. Because we always setup tunnels the same way we1592	 * can just check for the routers at both ends of the tunnels1593	 * and if they are the same we have a match.1594	 */1595	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {1596		if (!tb_tunnel_is_dp(tunnel))1597			continue;1598 1599		if (tunnel->src_port->sw == in->sw &&1600		    tunnel->dst_port->sw == out->sw) {1601			group = tunnel->src_port->group;1602			if (group) {1603				tb_bandwidth_group_attach_port(group, in);1604				return group;1605			}1606		}1607	}1608 1609	/* Pick up next available group then */1610	group = tb_find_free_bandwidth_group(tcm);1611	if (group)1612		tb_bandwidth_group_attach_port(group, in);1613	else1614		tb_port_warn(in, "no available bandwidth groups\n");1615 1616	return group;1617}1618 1619static void tb_discover_bandwidth_group(struct tb_cm *tcm, struct tb_port *in,1620					struct tb_port *out)1621{1622	if (usb4_dp_port_bandwidth_mode_enabled(in)) {1623		int index, i;1624 1625		index = usb4_dp_port_group_id(in);1626		for (i = 0; i < ARRAY_SIZE(tcm->groups); i++) {1627			if (tcm->groups[i].index == index) {1628				tb_bandwidth_group_attach_port(&tcm->groups[i], in);1629				return;1630			}1631		}1632	}1633 1634	tb_attach_bandwidth_group(tcm, in, out);1635}1636 1637static void tb_detach_bandwidth_group(struct tb_port *in)1638{1639	struct tb_bandwidth_group *group = in->group;1640 1641	if (group) {1642		in->group = NULL;1643		list_del_init(&in->group_list);1644 1645		tb_port_dbg(in, "detached from bandwidth group %d\n", group->index);1646 1647		/* No more tunnels so release the reserved bandwidth if any */1648		if (list_empty(&group->ports)) {1649			cancel_delayed_work(&group->release_work);1650			__release_group_bandwidth(group);1651		}1652	}1653}1654 1655static void tb_discover_tunnels(struct tb *tb)1656{1657	struct tb_cm *tcm = tb_priv(tb);1658	struct tb_tunnel *tunnel;1659 1660	tb_switch_discover_tunnels(tb->root_switch, &tcm->tunnel_list, true);1661 1662	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {1663		if (tb_tunnel_is_pci(tunnel)) {1664			struct tb_switch *parent = tunnel->dst_port->sw;1665 1666			while (parent != tunnel->src_port->sw) {1667				parent->boot = true;1668				parent = tb_switch_parent(parent);1669			}1670		} else if (tb_tunnel_is_dp(tunnel)) {1671			struct tb_port *in = tunnel->src_port;1672			struct tb_port *out = tunnel->dst_port;1673 1674			/* Keep the domain from powering down */1675			pm_runtime_get_sync(&in->sw->dev);1676			pm_runtime_get_sync(&out->sw->dev);1677 1678			tb_discover_bandwidth_group(tcm, in, out);1679		}1680	}1681}1682 1683static void tb_deactivate_and_free_tunnel(struct tb_tunnel *tunnel)1684{1685	struct tb_port *src_port, *dst_port;1686	struct tb *tb;1687 1688	if (!tunnel)1689		return;1690 1691	tb_tunnel_deactivate(tunnel);1692	list_del(&tunnel->list);1693 1694	tb = tunnel->tb;1695	src_port = tunnel->src_port;1696	dst_port = tunnel->dst_port;1697 1698	switch (tunnel->type) {1699	case TB_TUNNEL_DP:1700		tb_detach_bandwidth_group(src_port);1701		/*1702		 * In case of DP tunnel make sure the DP IN resource is1703		 * deallocated properly.1704		 */1705		tb_switch_dealloc_dp_resource(src_port->sw, src_port);1706		/*1707		 * If bandwidth on a link is < asym_threshold1708		 * transition the link to symmetric.1709		 */1710		tb_configure_sym(tb, src_port, dst_port, true);1711		/* Now we can allow the domain to runtime suspend again */1712		pm_runtime_mark_last_busy(&dst_port->sw->dev);1713		pm_runtime_put_autosuspend(&dst_port->sw->dev);1714		pm_runtime_mark_last_busy(&src_port->sw->dev);1715		pm_runtime_put_autosuspend(&src_port->sw->dev);1716		fallthrough;1717 1718	case TB_TUNNEL_USB3:1719		tb_reclaim_usb3_bandwidth(tb, src_port, dst_port);1720		break;1721 1722	default:1723		/*1724		 * PCIe and DMA tunnels do not consume guaranteed1725		 * bandwidth.1726		 */1727		break;1728	}1729 1730	tb_tunnel_free(tunnel);1731}1732 1733/*1734 * tb_free_invalid_tunnels() - destroy tunnels of devices that have gone away1735 */1736static void tb_free_invalid_tunnels(struct tb *tb)1737{1738	struct tb_cm *tcm = tb_priv(tb);1739	struct tb_tunnel *tunnel;1740	struct tb_tunnel *n;1741 1742	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {1743		if (tb_tunnel_is_invalid(tunnel))1744			tb_deactivate_and_free_tunnel(tunnel);1745	}1746}1747 1748/*1749 * tb_free_unplugged_children() - traverse hierarchy and free unplugged switches1750 */1751static void tb_free_unplugged_children(struct tb_switch *sw)1752{1753	struct tb_port *port;1754 1755	tb_switch_for_each_port(sw, port) {1756		if (!tb_port_has_remote(port))1757			continue;1758 1759		if (port->remote->sw->is_unplugged) {1760			tb_retimer_remove_all(port);1761			tb_remove_dp_resources(port->remote->sw);1762			tb_switch_unconfigure_link(port->remote->sw);1763			tb_switch_set_link_width(port->remote->sw,1764						 TB_LINK_WIDTH_SINGLE);1765			tb_switch_remove(port->remote->sw);1766			port->remote = NULL;1767			if (port->dual_link_port)1768				port->dual_link_port->remote = NULL;1769		} else {1770			tb_free_unplugged_children(port->remote->sw);1771		}1772	}1773}1774 1775static struct tb_port *tb_find_pcie_down(struct tb_switch *sw,1776					 const struct tb_port *port)1777{1778	struct tb_port *down = NULL;1779 1780	/*1781	 * To keep plugging devices consistently in the same PCIe1782	 * hierarchy, do mapping here for switch downstream PCIe ports.1783	 */1784	if (tb_switch_is_usb4(sw)) {1785		down = usb4_switch_map_pcie_down(sw, port);1786	} else if (!tb_route(sw)) {1787		int phy_port = tb_phy_port_from_link(port->port);1788		int index;1789 1790		/*1791		 * Hard-coded Thunderbolt port to PCIe down port mapping1792		 * per controller.1793		 */1794		if (tb_switch_is_cactus_ridge(sw) ||1795		    tb_switch_is_alpine_ridge(sw))1796			index = !phy_port ? 6 : 7;1797		else if (tb_switch_is_falcon_ridge(sw))1798			index = !phy_port ? 6 : 8;1799		else if (tb_switch_is_titan_ridge(sw))1800			index = !phy_port ? 8 : 9;1801		else1802			goto out;1803 1804		/* Validate the hard-coding */1805		if (WARN_ON(index > sw->config.max_port_number))1806			goto out;1807 1808		down = &sw->ports[index];1809	}1810 1811	if (down) {1812		if (WARN_ON(!tb_port_is_pcie_down(down)))1813			goto out;1814		if (tb_pci_port_is_enabled(down))1815			goto out;1816 1817		return down;1818	}1819 1820out:1821	return tb_find_unused_port(sw, TB_TYPE_PCIE_DOWN);1822}1823 1824static struct tb_port *tb_find_dp_out(struct tb *tb, struct tb_port *in)1825{1826	struct tb_port *host_port, *port;1827	struct tb_cm *tcm = tb_priv(tb);1828 1829	host_port = tb_route(in->sw) ?1830		tb_port_at(tb_route(in->sw), tb->root_switch) : NULL;1831 1832	list_for_each_entry(port, &tcm->dp_resources, list) {1833		if (!tb_port_is_dpout(port))1834			continue;1835 1836		if (tb_port_is_enabled(port)) {1837			tb_port_dbg(port, "DP OUT in use\n");1838			continue;1839		}1840 1841		/* Needs to be on different routers */1842		if (in->sw == port->sw) {1843			tb_port_dbg(port, "skipping DP OUT on same router\n");1844			continue;1845		}1846 1847		tb_port_dbg(port, "DP OUT available\n");1848 1849		/*1850		 * Keep the DP tunnel under the topology starting from1851		 * the same host router downstream port.1852		 */1853		if (host_port && tb_route(port->sw)) {1854			struct tb_port *p;1855 1856			p = tb_port_at(tb_route(port->sw), tb->root_switch);1857			if (p != host_port)1858				continue;1859		}1860 1861		return port;1862	}1863 1864	return NULL;1865}1866 1867static bool tb_tunnel_one_dp(struct tb *tb, struct tb_port *in,1868			     struct tb_port *out)1869{1870	int available_up, available_down, ret, link_nr;1871	struct tb_cm *tcm = tb_priv(tb);1872	int consumed_up, consumed_down;1873	struct tb_tunnel *tunnel;1874 1875	/*1876	 * This is only applicable to links that are not bonded (so1877	 * when Thunderbolt 1 hardware is involved somewhere in the1878	 * topology). For these try to share the DP bandwidth between1879	 * the two lanes.1880	 */1881	link_nr = 1;1882	list_for_each_entry(tunnel, &tcm->tunnel_list, list) {1883		if (tb_tunnel_is_dp(tunnel)) {1884			link_nr = 0;1885			break;1886		}1887	}1888 1889	/*1890	 * DP stream needs the domain to be active so runtime resume1891	 * both ends of the tunnel.1892	 *1893	 * This should bring the routers in the middle active as well1894	 * and keeps the domain from runtime suspending while the DP1895	 * tunnel is active.1896	 */1897	pm_runtime_get_sync(&in->sw->dev);1898	pm_runtime_get_sync(&out->sw->dev);1899 1900	if (tb_switch_alloc_dp_resource(in->sw, in)) {1901		tb_port_dbg(in, "no resource available for DP IN, not tunneling\n");1902		goto err_rpm_put;1903	}1904 1905	if (!tb_attach_bandwidth_group(tcm, in, out))1906		goto err_dealloc_dp;1907 1908	/* Make all unused USB3 bandwidth available for the new DP tunnel */1909	ret = tb_release_unused_usb3_bandwidth(tb, in, out);1910	if (ret) {1911		tb_warn(tb, "failed to release unused bandwidth\n");1912		goto err_detach_group;1913	}1914 1915	ret = tb_available_bandwidth(tb, in, out, &available_up, &available_down,1916				     true);1917	if (ret)1918		goto err_reclaim_usb;1919 1920	tb_dbg(tb, "available bandwidth for new DP tunnel %u/%u Mb/s\n",1921	       available_up, available_down);1922 1923	tunnel = tb_tunnel_alloc_dp(tb, in, out, link_nr, available_up,1924				    available_down);1925	if (!tunnel) {1926		tb_port_dbg(out, "could not allocate DP tunnel\n");1927		goto err_reclaim_usb;1928	}1929 1930	if (tb_tunnel_activate(tunnel)) {1931		tb_port_info(out, "DP tunnel activation failed, aborting\n");1932		goto err_free;1933	}1934 1935	/* If fail reading tunnel's consumed bandwidth, tear it down */1936	ret = tb_tunnel_consumed_bandwidth(tunnel, &consumed_up, &consumed_down);1937	if (ret)1938		goto err_deactivate;1939 1940	list_add_tail(&tunnel->list, &tcm->tunnel_list);1941 1942	tb_reclaim_usb3_bandwidth(tb, in, out);1943	/*1944	 * Transition the links to asymmetric if the consumption exceeds1945	 * the threshold.1946	 */1947	tb_configure_asym(tb, in, out, consumed_up, consumed_down);1948 1949	/* Update the domain with the new bandwidth estimation */1950	tb_recalc_estimated_bandwidth(tb);1951 1952	/*1953	 * In case of DP tunnel exists, change host router's 1st children1954	 * TMU mode to HiFi for CL0s to work.1955	 */1956	tb_increase_tmu_accuracy(tunnel);1957	return true;1958 1959err_deactivate:1960	tb_tunnel_deactivate(tunnel);1961err_free:1962	tb_tunnel_free(tunnel);1963err_reclaim_usb:1964	tb_reclaim_usb3_bandwidth(tb, in, out);1965err_detach_group:1966	tb_detach_bandwidth_group(in);1967err_dealloc_dp:1968	tb_switch_dealloc_dp_resource(in->sw, in);1969err_rpm_put:1970	pm_runtime_mark_last_busy(&out->sw->dev);1971	pm_runtime_put_autosuspend(&out->sw->dev);1972	pm_runtime_mark_last_busy(&in->sw->dev);1973	pm_runtime_put_autosuspend(&in->sw->dev);1974 1975	return false;1976}1977 1978static void tb_tunnel_dp(struct tb *tb)1979{1980	struct tb_cm *tcm = tb_priv(tb);1981	struct tb_port *port, *in, *out;1982 1983	if (!tb_acpi_may_tunnel_dp()) {1984		tb_dbg(tb, "DP tunneling disabled, not creating tunnel\n");1985		return;1986	}1987 1988	/*1989	 * Find pair of inactive DP IN and DP OUT adapters and then1990	 * establish a DP tunnel between them.1991	 */1992	tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n");1993 1994	in = NULL;1995	out = NULL;1996	list_for_each_entry(port, &tcm->dp_resources, list) {1997		if (!tb_port_is_dpin(port))1998			continue;1999 2000		if (tb_port_is_enabled(port)) {2001			tb_port_dbg(port, "DP IN in use\n");2002			continue;2003		}2004 2005		in = port;2006		tb_port_dbg(in, "DP IN available\n");2007 2008		out = tb_find_dp_out(tb, port);2009		if (out)2010			tb_tunnel_one_dp(tb, in, out);2011		else2012			tb_port_dbg(in, "no suitable DP OUT adapter available, not tunneling\n");2013	}2014 2015	if (!in)2016		tb_dbg(tb, "no suitable DP IN adapter available, not tunneling\n");2017}2018 2019static void tb_enter_redrive(struct tb_port *port)2020{2021	struct tb_switch *sw = port->sw;2022 2023	if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE))2024		return;2025 2026	/*2027	 * If we get hot-unplug for the DP IN port of the host router2028	 * and the DP resource is not available anymore it means there2029	 * is a monitor connected directly to the Type-C port and we are2030	 * in "redrive" mode. For this to work we cannot enter RTD3 so2031	 * we bump up the runtime PM reference count here.2032	 */2033	if (!tb_port_is_dpin(port))2034		return;2035	if (tb_route(sw))2036		return;2037	if (!tb_switch_query_dp_resource(sw, port)) {2038		port->redrive = true;2039		pm_runtime_get(&sw->dev);2040		tb_port_dbg(port, "enter redrive mode, keeping powered\n");2041	}2042}2043 2044static void tb_exit_redrive(struct tb_port *port)2045{2046	struct tb_switch *sw = port->sw;2047 2048	if (!(sw->quirks & QUIRK_KEEP_POWER_IN_DP_REDRIVE))2049		return;2050 2051	if (!tb_port_is_dpin(port))2052		return;2053	if (tb_route(sw))2054		return;2055	if (port->redrive && tb_switch_query_dp_resource(sw, port)) {2056		port->redrive = false;2057		pm_runtime_put(&sw->dev);2058		tb_port_dbg(port, "exit redrive mode\n");2059	}2060}2061 2062static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port)2063{2064	struct tb_port *in, *out;2065	struct tb_tunnel *tunnel;2066 2067	if (tb_port_is_dpin(port)) {2068		tb_port_dbg(port, "DP IN resource unavailable\n");2069		in = port;2070		out = NULL;2071	} else {2072		tb_port_dbg(port, "DP OUT resource unavailable\n");2073		in = NULL;2074		out = port;2075	}2076 2077	tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, out);2078	if (tunnel)2079		tb_deactivate_and_free_tunnel(tunnel);2080	else2081		tb_enter_redrive(port);2082	list_del_init(&port->list);2083 2084	/*2085	 * See if there is another DP OUT port that can be used for2086	 * to create another tunnel.2087	 */2088	tb_recalc_estimated_bandwidth(tb);2089	tb_tunnel_dp(tb);2090}2091 2092static void tb_dp_resource_available(struct tb *tb, struct tb_port *port)2093{2094	struct tb_cm *tcm = tb_priv(tb);2095	struct tb_port *p;2096 2097	if (tb_port_is_enabled(port))2098		return;2099 2100	list_for_each_entry(p, &tcm->dp_resources, list) {2101		if (p == port)2102			return;2103	}2104 2105	tb_port_dbg(port, "DP %s resource available after hotplug\n",2106		    tb_port_is_dpin(port) ? "IN" : "OUT");2107	list_add_tail(&port->list, &tcm->dp_resources);2108	tb_exit_redrive(port);2109 2110	/* Look for suitable DP IN <-> DP OUT pairs now */2111	tb_tunnel_dp(tb);2112}2113 2114static void tb_disconnect_and_release_dp(struct tb *tb)2115{2116	struct tb_cm *tcm = tb_priv(tb);2117	struct tb_tunnel *tunnel, *n;2118 2119	/*2120	 * Tear down all DP tunnels and release their resources. They2121	 * will be re-established after resume based on plug events.2122	 */2123	list_for_each_entry_safe_reverse(tunnel, n, &tcm->tunnel_list, list) {2124		if (tb_tunnel_is_dp(tunnel))2125			tb_deactivate_and_free_tunnel(tunnel);2126	}2127 2128	while (!list_empty(&tcm->dp_resources)) {2129		struct tb_port *port;2130 2131		port = list_first_entry(&tcm->dp_resources,2132					struct tb_port, list);2133		list_del_init(&port->list);2134	}2135}2136 2137static int tb_disconnect_pci(struct tb *tb, struct tb_switch *sw)2138{2139	struct tb_tunnel *tunnel;2140	struct tb_port *up;2141 2142	up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);2143	if (WARN_ON(!up))2144		return -ENODEV;2145 2146	tunnel = tb_find_tunnel(tb, TB_TUNNEL_PCI, NULL, up);2147	if (WARN_ON(!tunnel))2148		return -ENODEV;2149 2150	tb_switch_xhci_disconnect(sw);2151 2152	tb_tunnel_deactivate(tunnel);2153	list_del(&tunnel->list);2154	tb_tunnel_free(tunnel);2155	return 0;2156}2157 2158static int tb_tunnel_pci(struct tb *tb, struct tb_switch *sw)2159{2160	struct tb_port *up, *down, *port;2161	struct tb_cm *tcm = tb_priv(tb);2162	struct tb_tunnel *tunnel;2163 2164	up = tb_switch_find_port(sw, TB_TYPE_PCIE_UP);2165	if (!up)2166		return 0;2167 2168	/*2169	 * Look up available down port. Since we are chaining it should2170	 * be found right above this switch.2171	 */2172	port = tb_switch_downstream_port(sw);2173	down = tb_find_pcie_down(tb_switch_parent(sw), port);2174	if (!down)2175		return 0;2176 2177	tunnel = tb_tunnel_alloc_pci(tb, up, down);2178	if (!tunnel)2179		return -ENOMEM;2180 2181	if (tb_tunnel_activate(tunnel)) {2182		tb_port_info(up,2183			     "PCIe tunnel activation failed, aborting\n");2184		tb_tunnel_free(tunnel);2185		return -EIO;2186	}2187 2188	/*2189	 * PCIe L1 is needed to enable CL0s for Titan Ridge so enable it2190	 * here.2191	 */2192	if (tb_switch_pcie_l1_enable(sw))2193		tb_sw_warn(sw, "failed to enable PCIe L1 for Titan Ridge\n");2194 2195	if (tb_switch_xhci_connect(sw))2196		tb_sw_warn(sw, "failed to connect xHCI\n");2197 2198	list_add_tail(&tunnel->list, &tcm->tunnel_list);2199	return 0;2200}2201 2202static int tb_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,2203				    int transmit_path, int transmit_ring,2204				    int receive_path, int receive_ring)2205{2206	struct tb_cm *tcm = tb_priv(tb);2207	struct tb_port *nhi_port, *dst_port;2208	struct tb_tunnel *tunnel;2209	struct tb_switch *sw;2210	int ret;2211 2212	sw = tb_to_switch(xd->dev.parent);2213	dst_port = tb_port_at(xd->route, sw);2214	nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);2215 2216	mutex_lock(&tb->lock);2217 2218	/*2219	 * When tunneling DMA paths the link should not enter CL states2220	 * so disable them now.2221	 */2222	tb_disable_clx(sw);2223 2224	tunnel = tb_tunnel_alloc_dma(tb, nhi_port, dst_port, transmit_path,2225				     transmit_ring, receive_path, receive_ring);2226	if (!tunnel) {2227		ret = -ENOMEM;2228		goto err_clx;2229	}2230 2231	if (tb_tunnel_activate(tunnel)) {2232		tb_port_info(nhi_port,2233			     "DMA tunnel activation failed, aborting\n");2234		ret = -EIO;2235		goto err_free;2236	}2237 2238	list_add_tail(&tunnel->list, &tcm->tunnel_list);2239	mutex_unlock(&tb->lock);2240	return 0;2241 2242err_free:2243	tb_tunnel_free(tunnel);2244err_clx:2245	tb_enable_clx(sw);2246	mutex_unlock(&tb->lock);2247 2248	return ret;2249}2250 2251static void __tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,2252					  int transmit_path, int transmit_ring,2253					  int receive_path, int receive_ring)2254{2255	struct tb_cm *tcm = tb_priv(tb);2256	struct tb_port *nhi_port, *dst_port;2257	struct tb_tunnel *tunnel, *n;2258	struct tb_switch *sw;2259 2260	sw = tb_to_switch(xd->dev.parent);2261	dst_port = tb_port_at(xd->route, sw);2262	nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);2263 2264	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {2265		if (!tb_tunnel_is_dma(tunnel))2266			continue;2267		if (tunnel->src_port != nhi_port || tunnel->dst_port != dst_port)2268			continue;2269 2270		if (tb_tunnel_match_dma(tunnel, transmit_path, transmit_ring,2271					receive_path, receive_ring))2272			tb_deactivate_and_free_tunnel(tunnel);2273	}2274 2275	/*2276	 * Try to re-enable CL states now, it is OK if this fails2277	 * because we may still have another DMA tunnel active through2278	 * the same host router USB4 downstream port.2279	 */2280	tb_enable_clx(sw);2281}2282 2283static int tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,2284				       int transmit_path, int transmit_ring,2285				       int receive_path, int receive_ring)2286{2287	if (!xd->is_unplugged) {2288		mutex_lock(&tb->lock);2289		__tb_disconnect_xdomain_paths(tb, xd, transmit_path,2290					      transmit_ring, receive_path,2291					      receive_ring);2292		mutex_unlock(&tb->lock);2293	}2294	return 0;2295}2296 2297/* hotplug handling */2298 2299/*2300 * tb_handle_hotplug() - handle hotplug event2301 *2302 * Executes on tb->wq.2303 */2304static void tb_handle_hotplug(struct work_struct *work)2305{2306	struct tb_hotplug_event *ev = container_of(work, typeof(*ev), work);2307	struct tb *tb = ev->tb;2308	struct tb_cm *tcm = tb_priv(tb);2309	struct tb_switch *sw;2310	struct tb_port *port;2311 2312	/* Bring the domain back from sleep if it was suspended */2313	pm_runtime_get_sync(&tb->dev);2314 2315	mutex_lock(&tb->lock);2316	if (!tcm->hotplug_active)2317		goto out; /* during init, suspend or shutdown */2318 2319	sw = tb_switch_find_by_route(tb, ev->route);2320	if (!sw) {2321		tb_warn(tb,2322			"hotplug event from non existent switch %llx:%x (unplug: %d)\n",2323			ev->route, ev->port, ev->unplug);2324		goto out;2325	}2326	if (ev->port > sw->config.max_port_number) {2327		tb_warn(tb,2328			"hotplug event from non existent port %llx:%x (unplug: %d)\n",2329			ev->route, ev->port, ev->unplug);2330		goto put_sw;2331	}2332	port = &sw->ports[ev->port];2333	if (tb_is_upstream_port(port)) {2334		tb_dbg(tb, "hotplug event for upstream port %llx:%x (unplug: %d)\n",2335		       ev->route, ev->port, ev->unplug);2336		goto put_sw;2337	}2338 2339	pm_runtime_get_sync(&sw->dev);2340 2341	if (ev->unplug) {2342		tb_retimer_remove_all(port);2343 2344		if (tb_port_has_remote(port)) {2345			tb_port_dbg(port, "switch unplugged\n");2346			tb_sw_set_unplugged(port->remote->sw);2347			tb_free_invalid_tunnels(tb);2348			tb_remove_dp_resources(port->remote->sw);2349			tb_switch_tmu_disable(port->remote->sw);2350			tb_switch_unconfigure_link(port->remote->sw);2351			tb_switch_set_link_width(port->remote->sw,2352						 TB_LINK_WIDTH_SINGLE);2353			tb_switch_remove(port->remote->sw);2354			port->remote = NULL;2355			if (port->dual_link_port)2356				port->dual_link_port->remote = NULL;2357			/* Maybe we can create another DP tunnel */2358			tb_recalc_estimated_bandwidth(tb);2359			tb_tunnel_dp(tb);2360		} else if (port->xdomain) {2361			struct tb_xdomain *xd = tb_xdomain_get(port->xdomain);2362 2363			tb_port_dbg(port, "xdomain unplugged\n");2364			/*2365			 * Service drivers are unbound during2366			 * tb_xdomain_remove() so setting XDomain as2367			 * unplugged here prevents deadlock if they call2368			 * tb_xdomain_disable_paths(). We will tear down2369			 * all the tunnels below.2370			 */2371			xd->is_unplugged = true;2372			tb_xdomain_remove(xd);2373			port->xdomain = NULL;2374			__tb_disconnect_xdomain_paths(tb, xd, -1, -1, -1, -1);2375			tb_xdomain_put(xd);2376			tb_port_unconfigure_xdomain(port);2377		} else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {2378			tb_dp_resource_unavailable(tb, port);2379		} else if (!port->port) {2380			tb_sw_dbg(sw, "xHCI disconnect request\n");2381			tb_switch_xhci_disconnect(sw);2382		} else {2383			tb_port_dbg(port,2384				   "got unplug event for disconnected port, ignoring\n");2385		}2386	} else if (port->remote) {2387		tb_port_dbg(port, "got plug event for connected port, ignoring\n");2388	} else if (!port->port && sw->authorized) {2389		tb_sw_dbg(sw, "xHCI connect request\n");2390		tb_switch_xhci_connect(sw);2391	} else {2392		if (tb_port_is_null(port)) {2393			tb_port_dbg(port, "hotplug: scanning\n");2394			tb_scan_port(port);2395			if (!port->remote)2396				tb_port_dbg(port, "hotplug: no switch found\n");2397		} else if (tb_port_is_dpout(port) || tb_port_is_dpin(port)) {2398			tb_dp_resource_available(tb, port);2399		}2400	}2401 2402	pm_runtime_mark_last_busy(&sw->dev);2403	pm_runtime_put_autosuspend(&sw->dev);2404 2405put_sw:2406	tb_switch_put(sw);2407out:2408	mutex_unlock(&tb->lock);2409 2410	pm_runtime_mark_last_busy(&tb->dev);2411	pm_runtime_put_autosuspend(&tb->dev);2412 2413	kfree(ev);2414}2415 2416static int tb_alloc_dp_bandwidth(struct tb_tunnel *tunnel, int *requested_up,2417				 int *requested_down)2418{2419	int allocated_up, allocated_down, available_up, available_down, ret;2420	int requested_up_corrected, requested_down_corrected, granularity;2421	int max_up, max_down, max_up_rounded, max_down_rounded;2422	struct tb_bandwidth_group *group;2423	struct tb *tb = tunnel->tb;2424	struct tb_port *in, *out;2425	bool downstream;2426 2427	ret = tb_tunnel_allocated_bandwidth(tunnel, &allocated_up, &allocated_down);2428	if (ret)2429		return ret;2430 2431	in = tunnel->src_port;2432	out = tunnel->dst_port;2433 2434	tb_tunnel_dbg(tunnel, "bandwidth allocated currently %d/%d Mb/s\n",2435		      allocated_up, allocated_down);2436 2437	/*2438	 * If we get rounded up request from graphics side, say HBR2 x 42439	 * that is 17500 instead of 17280 (this is because of the2440	 * granularity), we allow it too. Here the graphics has already2441	 * negotiated with the DPRX the maximum possible rates (which is2442	 * 17280 in this case).2443	 *2444	 * Since the link cannot go higher than 17280 we use that in our2445	 * calculations but the DP IN adapter Allocated BW write must be2446	 * the same value (17500) otherwise the adapter will mark it as2447	 * failed for graphics.2448	 */2449	ret = tb_tunnel_maximum_bandwidth(tunnel, &max_up, &max_down);2450	if (ret)2451		goto fail;2452 2453	ret = usb4_dp_port_granularity(in);2454	if (ret < 0)2455		goto fail;2456	granularity = ret;2457 2458	max_up_rounded = roundup(max_up, granularity);2459	max_down_rounded = roundup(max_down, granularity);2460 2461	/*2462	 * This will "fix" the request down to the maximum supported2463	 * rate * lanes if it is at the maximum rounded up level.2464	 */2465	requested_up_corrected = *requested_up;2466	if (requested_up_corrected == max_up_rounded)2467		requested_up_corrected = max_up;2468	else if (requested_up_corrected < 0)2469		requested_up_corrected = 0;2470	requested_down_corrected = *requested_down;2471	if (requested_down_corrected == max_down_rounded)2472		requested_down_corrected = max_down;2473	else if (requested_down_corrected < 0)2474		requested_down_corrected = 0;2475 2476	tb_tunnel_dbg(tunnel, "corrected bandwidth request %d/%d Mb/s\n",2477		      requested_up_corrected, requested_down_corrected);2478 2479	if ((*requested_up >= 0 && requested_up_corrected > max_up_rounded) ||2480	    (*requested_down >= 0 && requested_down_corrected > max_down_rounded)) {2481		tb_tunnel_dbg(tunnel,2482			      "bandwidth request too high (%d/%d Mb/s > %d/%d Mb/s)\n",2483			      requested_up_corrected, requested_down_corrected,2484			      max_up_rounded, max_down_rounded);2485		ret = -ENOBUFS;2486		goto fail;2487	}2488 2489	downstream = tb_tunnel_direction_downstream(tunnel);2490	group = in->group;2491 2492	if ((*requested_up >= 0 && requested_up_corrected <= allocated_up) ||2493	    (*requested_down >= 0 && requested_down_corrected <= allocated_down)) {2494		if (tunnel->bw_mode) {2495			int reserved;2496			/*2497			 * If requested bandwidth is less or equal than2498			 * what is currently allocated to that tunnel we2499			 * simply change the reservation of the tunnel2500			 * and add the released bandwidth for the group2501			 * for the next 10s. Then we release it for2502			 * others to use.2503			 */2504			if (downstream)2505				reserved = allocated_down - *requested_down;2506			else2507				reserved = allocated_up - *requested_up;2508 2509			if (reserved > 0) {2510				group->reserved += reserved;2511				tb_dbg(tb, "group %d reserved %d total %d Mb/s\n",2512				       group->index, reserved, group->reserved);2513 2514				/*2515				 * If it was not already pending,2516				 * schedule release now. If it is then2517				 * postpone it for the next 10s (unless2518				 * it is already running in which case2519				 * the 10s already expired and we should2520				 * give the reserved back to others).2521				 */2522				mod_delayed_work(system_wq, &group->release_work,2523					msecs_to_jiffies(TB_RELEASE_BW_TIMEOUT));2524			}2525		}2526 2527		return tb_tunnel_alloc_bandwidth(tunnel, requested_up,2528						 requested_down);2529	}2530 2531	/*2532	 * More bandwidth is requested. Release all the potential2533	 * bandwidth from USB3 first.2534	 */2535	ret = tb_release_unused_usb3_bandwidth(tb, in, out);2536	if (ret)2537		goto fail;2538 2539	/*2540	 * Then go over all tunnels that cross the same USB4 ports (they2541	 * are also in the same group but we use the same function here2542	 * that we use with the normal bandwidth allocation).2543	 */2544	ret = tb_available_bandwidth(tb, in, out, &available_up, &available_down,2545				     true);2546	if (ret)2547		goto reclaim;2548 2549	tb_tunnel_dbg(tunnel, "bandwidth available for allocation %d/%d (+ %u reserved) Mb/s\n",2550		      available_up, available_down, group->reserved);2551 2552	if ((*requested_up >= 0 &&2553		available_up + group->reserved >= requested_up_corrected) ||2554	    (*requested_down >= 0 &&2555		available_down + group->reserved >= requested_down_corrected)) {2556		int released = 0;2557 2558		/*2559		 * If bandwidth on a link is >= asym_threshold2560		 * transition the link to asymmetric.2561		 */2562		ret = tb_configure_asym(tb, in, out, *requested_up,2563					*requested_down);2564		if (ret) {2565			tb_configure_sym(tb, in, out, true);2566			goto fail;2567		}2568 2569		ret = tb_tunnel_alloc_bandwidth(tunnel, requested_up,2570						requested_down);2571		if (ret) {2572			tb_tunnel_warn(tunnel, "failed to allocate bandwidth\n");2573			tb_configure_sym(tb, in, out, true);2574		}2575 2576		if (downstream) {2577			if (*requested_down > available_down)2578				released = *requested_down - available_down;2579		} else {2580			if (*requested_up > available_up)2581				released = *requested_up - available_up;2582		}2583		if (released) {2584			group->reserved -= released;2585			tb_dbg(tb, "group %d released %d total %d Mb/s\n",2586			       group->index, released, group->reserved);2587		}2588	} else {2589		ret = -ENOBUFS;2590	}2591 2592reclaim:2593	tb_reclaim_usb3_bandwidth(tb, in, out);2594fail:2595	if (ret && ret != -ENODEV) {2596		/*2597		 * Write back the same allocated (so no change), this2598		 * makes the DPTX request fail on graphics side.2599		 */2600		tb_tunnel_dbg(tunnel,2601			      "failing the request by rewriting allocated %d/%d Mb/s\n",2602			      allocated_up, allocated_down);2603		tb_tunnel_alloc_bandwidth(tunnel, &allocated_up, &allocated_down);2604	}2605 2606	return ret;2607}2608 2609static void tb_handle_dp_bandwidth_request(struct work_struct *work)2610{2611	struct tb_hotplug_event *ev = container_of(work, typeof(*ev), work);2612	int requested_bw, requested_up, requested_down, ret;2613	struct tb_tunnel *tunnel;2614	struct tb *tb = ev->tb;2615	struct tb_cm *tcm = tb_priv(tb);2616	struct tb_switch *sw;2617	struct tb_port *in;2618 2619	pm_runtime_get_sync(&tb->dev);2620 2621	mutex_lock(&tb->lock);2622	if (!tcm->hotplug_active)2623		goto unlock;2624 2625	sw = tb_switch_find_by_route(tb, ev->route);2626	if (!sw) {2627		tb_warn(tb, "bandwidth request from non-existent router %llx\n",2628			ev->route);2629		goto unlock;2630	}2631 2632	in = &sw->ports[ev->port];2633	if (!tb_port_is_dpin(in)) {2634		tb_port_warn(in, "bandwidth request to non-DP IN adapter\n");2635		goto put_sw;2636	}2637 2638	tb_port_dbg(in, "handling bandwidth allocation request\n");2639 2640	tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, NULL);2641	if (!tunnel) {2642		tb_port_warn(in, "failed to find tunnel\n");2643		goto put_sw;2644	}2645 2646	if (!usb4_dp_port_bandwidth_mode_enabled(in)) {2647		if (tunnel->bw_mode) {2648			/*2649			 * Reset the tunnel back to use the legacy2650			 * allocation.2651			 */2652			tunnel->bw_mode = false;2653			tb_port_dbg(in, "DPTX disabled bandwidth allocation mode\n");2654		} else {2655			tb_port_warn(in, "bandwidth allocation mode not enabled\n");2656		}2657		goto put_sw;2658	}2659 2660	ret = usb4_dp_port_requested_bandwidth(in);2661	if (ret < 0) {2662		if (ret == -ENODATA) {2663			/*2664			 * There is no request active so this means the2665			 * BW allocation mode was enabled from graphics2666			 * side. At this point we know that the graphics2667			 * driver has read the DRPX capabilities so we2668			 * can offer an better bandwidth estimatation.2669			 */2670			tb_port_dbg(in, "DPTX enabled bandwidth allocation mode, updating estimated bandwidth\n");2671			tb_recalc_estimated_bandwidth(tb);2672		} else {2673			tb_port_warn(in, "failed to read requested bandwidth\n");2674		}2675		goto put_sw;2676	}2677	requested_bw = ret;2678 2679	tb_port_dbg(in, "requested bandwidth %d Mb/s\n", requested_bw);2680 2681	if (tb_tunnel_direction_downstream(tunnel)) {2682		requested_up = -1;2683		requested_down = requested_bw;2684	} else {2685		requested_up = requested_bw;2686		requested_down = -1;2687	}2688 2689	ret = tb_alloc_dp_bandwidth(tunnel, &requested_up, &requested_down);2690	if (ret) {2691		if (ret == -ENOBUFS)2692			tb_tunnel_warn(tunnel,2693				       "not enough bandwidth available\n");2694		else2695			tb_tunnel_warn(tunnel,2696				       "failed to change bandwidth allocation\n");2697	} else {2698		tb_tunnel_dbg(tunnel,2699			      "bandwidth allocation changed to %d/%d Mb/s\n",2700			      requested_up, requested_down);2701 2702		/* Update other clients about the allocation change */2703		tb_recalc_estimated_bandwidth(tb);2704	}2705 2706put_sw:2707	tb_switch_put(sw);2708unlock:2709	mutex_unlock(&tb->lock);2710 2711	pm_runtime_mark_last_busy(&tb->dev);2712	pm_runtime_put_autosuspend(&tb->dev);2713 2714	kfree(ev);2715}2716 2717static void tb_queue_dp_bandwidth_request(struct tb *tb, u64 route, u8 port)2718{2719	struct tb_hotplug_event *ev;2720 2721	ev = kmalloc(sizeof(*ev), GFP_KERNEL);2722	if (!ev)2723		return;2724 2725	ev->tb = tb;2726	ev->route = route;2727	ev->port = port;2728	INIT_WORK(&ev->work, tb_handle_dp_bandwidth_request);2729	queue_work(tb->wq, &ev->work);2730}2731 2732static void tb_handle_notification(struct tb *tb, u64 route,2733				   const struct cfg_error_pkg *error)2734{2735 2736	switch (error->error) {2737	case TB_CFG_ERROR_PCIE_WAKE:2738	case TB_CFG_ERROR_DP_CON_CHANGE:2739	case TB_CFG_ERROR_DPTX_DISCOVERY:2740		if (tb_cfg_ack_notification(tb->ctl, route, error))2741			tb_warn(tb, "could not ack notification on %llx\n",2742				route);2743		break;2744 2745	case TB_CFG_ERROR_DP_BW:2746		if (tb_cfg_ack_notification(tb->ctl, route, error))2747			tb_warn(tb, "could not ack notification on %llx\n",2748				route);2749		tb_queue_dp_bandwidth_request(tb, route, error->port);2750		break;2751 2752	default:2753		/* Ignore for now */2754		break;2755	}2756}2757 2758/*2759 * tb_schedule_hotplug_handler() - callback function for the control channel2760 *2761 * Delegates to tb_handle_hotplug.2762 */2763static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,2764			    const void *buf, size_t size)2765{2766	const struct cfg_event_pkg *pkg = buf;2767	u64 route = tb_cfg_get_route(&pkg->header);2768 2769	switch (type) {2770	case TB_CFG_PKG_ERROR:2771		tb_handle_notification(tb, route, (const struct cfg_error_pkg *)buf);2772		return;2773	case TB_CFG_PKG_EVENT:2774		break;2775	default:2776		tb_warn(tb, "unexpected event %#x, ignoring\n", type);2777		return;2778	}2779 2780	if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) {2781		tb_warn(tb, "could not ack plug event on %llx:%x\n", route,2782			pkg->port);2783	}2784 2785	tb_queue_hotplug(tb, route, pkg->port, pkg->unplug);2786}2787 2788static void tb_stop(struct tb *tb)2789{2790	struct tb_cm *tcm = tb_priv(tb);2791	struct tb_tunnel *tunnel;2792	struct tb_tunnel *n;2793 2794	cancel_delayed_work(&tcm->remove_work);2795	/* tunnels are only present after everything has been initialized */2796	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {2797		/*2798		 * DMA tunnels require the driver to be functional so we2799		 * tear them down. Other protocol tunnels can be left2800		 * intact.2801		 */2802		if (tb_tunnel_is_dma(tunnel))2803			tb_tunnel_deactivate(tunnel);2804		tb_tunnel_free(tunnel);2805	}2806	tb_switch_remove(tb->root_switch);2807	tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */2808}2809 2810static void tb_deinit(struct tb *tb)2811{2812	struct tb_cm *tcm = tb_priv(tb);2813	int i;2814 2815	/* Cancel all the release bandwidth workers */2816	for (i = 0; i < ARRAY_SIZE(tcm->groups); i++)2817		cancel_delayed_work_sync(&tcm->groups[i].release_work);2818}2819 2820static int tb_scan_finalize_switch(struct device *dev, void *data)2821{2822	if (tb_is_switch(dev)) {2823		struct tb_switch *sw = tb_to_switch(dev);2824 2825		/*2826		 * If we found that the switch was already setup by the2827		 * boot firmware, mark it as authorized now before we2828		 * send uevent to userspace.2829		 */2830		if (sw->boot)2831			sw->authorized = 1;2832 2833		dev_set_uevent_suppress(dev, false);2834		kobject_uevent(&dev->kobj, KOBJ_ADD);2835		device_for_each_child(dev, NULL, tb_scan_finalize_switch);2836	}2837 2838	return 0;2839}2840 2841static int tb_start(struct tb *tb, bool reset)2842{2843	struct tb_cm *tcm = tb_priv(tb);2844	bool discover = true;2845	int ret;2846 2847	tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);2848	if (IS_ERR(tb->root_switch))2849		return PTR_ERR(tb->root_switch);2850 2851	/*2852	 * ICM firmware upgrade needs running firmware and in native2853	 * mode that is not available so disable firmware upgrade of the2854	 * root switch.2855	 *2856	 * However, USB4 routers support NVM firmware upgrade if they2857	 * implement the necessary router operations.2858	 */2859	tb->root_switch->no_nvm_upgrade = !tb_switch_is_usb4(tb->root_switch);2860	/* All USB4 routers support runtime PM */2861	tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch);2862 2863	ret = tb_switch_configure(tb->root_switch);2864	if (ret) {2865		tb_switch_put(tb->root_switch);2866		return ret;2867	}2868 2869	/* Announce the switch to the world */2870	ret = tb_switch_add(tb->root_switch);2871	if (ret) {2872		tb_switch_put(tb->root_switch);2873		return ret;2874	}2875 2876	/*2877	 * To support highest CLx state, we set host router's TMU to2878	 * Normal mode.2879	 */2880	tb_switch_tmu_configure(tb->root_switch, TB_SWITCH_TMU_MODE_LOWRES);2881	/* Enable TMU if it is off */2882	tb_switch_tmu_enable(tb->root_switch);2883 2884	/*2885	 * Boot firmware might have created tunnels of its own. Since we2886	 * cannot be sure they are usable for us, tear them down and2887	 * reset the ports to handle it as new hotplug for USB4 v12888	 * routers (for USB4 v2 and beyond we already do host reset).2889	 */2890	if (reset && tb_switch_is_usb4(tb->root_switch)) {2891		discover = false;2892		if (usb4_switch_version(tb->root_switch) == 1)2893			tb_switch_reset(tb->root_switch);2894	}2895 2896	if (discover) {2897		/* Full scan to discover devices added before the driver was loaded. */2898		tb_scan_switch(tb->root_switch);2899		/* Find out tunnels created by the boot firmware */2900		tb_discover_tunnels(tb);2901		/* Add DP resources from the DP tunnels created by the boot firmware */2902		tb_discover_dp_resources(tb);2903	}2904 2905	/*2906	 * If the boot firmware did not create USB 3.x tunnels create them2907	 * now for the whole topology.2908	 */2909	tb_create_usb3_tunnels(tb->root_switch);2910	/* Add DP IN resources for the root switch */2911	tb_add_dp_resources(tb->root_switch);2912	/* Make the discovered switches available to the userspace */2913	device_for_each_child(&tb->root_switch->dev, NULL,2914			      tb_scan_finalize_switch);2915 2916	/* Allow tb_handle_hotplug to progress events */2917	tcm->hotplug_active = true;2918	return 0;2919}2920 2921static int tb_suspend_noirq(struct tb *tb)2922{2923	struct tb_cm *tcm = tb_priv(tb);2924 2925	tb_dbg(tb, "suspending...\n");2926	tb_disconnect_and_release_dp(tb);2927	tb_switch_suspend(tb->root_switch, false);2928	tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */2929	tb_dbg(tb, "suspend finished\n");2930 2931	return 0;2932}2933 2934static void tb_restore_children(struct tb_switch *sw)2935{2936	struct tb_port *port;2937 2938	/* No need to restore if the router is already unplugged */2939	if (sw->is_unplugged)2940		return;2941 2942	if (tb_enable_clx(sw))2943		tb_sw_warn(sw, "failed to re-enable CL states\n");2944 2945	if (tb_enable_tmu(sw))2946		tb_sw_warn(sw, "failed to restore TMU configuration\n");2947 2948	tb_switch_configuration_valid(sw);2949 2950	tb_switch_for_each_port(sw, port) {2951		if (!tb_port_has_remote(port) && !port->xdomain)2952			continue;2953 2954		if (port->remote) {2955			tb_switch_set_link_width(port->remote->sw,2956						 port->remote->sw->link_width);2957			tb_switch_configure_link(port->remote->sw);2958 2959			tb_restore_children(port->remote->sw);2960		} else if (port->xdomain) {2961			tb_port_configure_xdomain(port, port->xdomain);2962		}2963	}2964}2965 2966static int tb_resume_noirq(struct tb *tb)2967{2968	struct tb_cm *tcm = tb_priv(tb);2969	struct tb_tunnel *tunnel, *n;2970	unsigned int usb3_delay = 0;2971	LIST_HEAD(tunnels);2972 2973	tb_dbg(tb, "resuming...\n");2974 2975	/*2976	 * For non-USB4 hosts (Apple systems) remove any PCIe devices2977	 * the firmware might have setup.2978	 */2979	if (!tb_switch_is_usb4(tb->root_switch))2980		tb_switch_reset(tb->root_switch);2981 2982	tb_switch_resume(tb->root_switch, false);2983	tb_free_invalid_tunnels(tb);2984	tb_free_unplugged_children(tb->root_switch);2985	tb_restore_children(tb->root_switch);2986 2987	/*2988	 * If we get here from suspend to disk the boot firmware or the2989	 * restore kernel might have created tunnels of its own. Since2990	 * we cannot be sure they are usable for us we find and tear2991	 * them down.2992	 */2993	tb_switch_discover_tunnels(tb->root_switch, &tunnels, false);2994	list_for_each_entry_safe_reverse(tunnel, n, &tunnels, list) {2995		if (tb_tunnel_is_usb3(tunnel))2996			usb3_delay = 500;2997		tb_tunnel_deactivate(tunnel);2998		tb_tunnel_free(tunnel);2999	}3000 3001	/* Re-create our tunnels now */3002	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list) {3003		/* USB3 requires delay before it can be re-activated */3004		if (tb_tunnel_is_usb3(tunnel)) {3005			msleep(usb3_delay);3006			/* Only need to do it once */3007			usb3_delay = 0;3008		}3009		tb_tunnel_restart(tunnel);3010	}3011	if (!list_empty(&tcm->tunnel_list)) {3012		/*3013		 * the pcie links need some time to get going.3014		 * 100ms works for me...3015		 */3016		tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n");3017		msleep(100);3018	}3019	 /* Allow tb_handle_hotplug to progress events */3020	tcm->hotplug_active = true;3021	tb_dbg(tb, "resume finished\n");3022 3023	return 0;3024}3025 3026static int tb_free_unplugged_xdomains(struct tb_switch *sw)3027{3028	struct tb_port *port;3029	int ret = 0;3030 3031	tb_switch_for_each_port(sw, port) {3032		if (tb_is_upstream_port(port))3033			continue;3034		if (port->xdomain && port->xdomain->is_unplugged) {3035			tb_retimer_remove_all(port);3036			tb_xdomain_remove(port->xdomain);3037			tb_port_unconfigure_xdomain(port);3038			port->xdomain = NULL;3039			ret++;3040		} else if (port->remote) {3041			ret += tb_free_unplugged_xdomains(port->remote->sw);3042		}3043	}3044 3045	return ret;3046}3047 3048static int tb_freeze_noirq(struct tb *tb)3049{3050	struct tb_cm *tcm = tb_priv(tb);3051 3052	tcm->hotplug_active = false;3053	return 0;3054}3055 3056static int tb_thaw_noirq(struct tb *tb)3057{3058	struct tb_cm *tcm = tb_priv(tb);3059 3060	tcm->hotplug_active = true;3061	return 0;3062}3063 3064static void tb_complete(struct tb *tb)3065{3066	/*3067	 * Release any unplugged XDomains and if there is a case where3068	 * another domain is swapped in place of unplugged XDomain we3069	 * need to run another rescan.3070	 */3071	mutex_lock(&tb->lock);3072	if (tb_free_unplugged_xdomains(tb->root_switch))3073		tb_scan_switch(tb->root_switch);3074	mutex_unlock(&tb->lock);3075}3076 3077static int tb_runtime_suspend(struct tb *tb)3078{3079	struct tb_cm *tcm = tb_priv(tb);3080 3081	mutex_lock(&tb->lock);3082	tb_switch_suspend(tb->root_switch, true);3083	tcm->hotplug_active = false;3084	mutex_unlock(&tb->lock);3085 3086	return 0;3087}3088 3089static void tb_remove_work(struct work_struct *work)3090{3091	struct tb_cm *tcm = container_of(work, struct tb_cm, remove_work.work);3092	struct tb *tb = tcm_to_tb(tcm);3093 3094	mutex_lock(&tb->lock);3095	if (tb->root_switch) {3096		tb_free_unplugged_children(tb->root_switch);3097		tb_free_unplugged_xdomains(tb->root_switch);3098	}3099	mutex_unlock(&tb->lock);3100}3101 3102static int tb_runtime_resume(struct tb *tb)3103{3104	struct tb_cm *tcm = tb_priv(tb);3105	struct tb_tunnel *tunnel, *n;3106 3107	mutex_lock(&tb->lock);3108	tb_switch_resume(tb->root_switch, true);3109	tb_free_invalid_tunnels(tb);3110	tb_restore_children(tb->root_switch);3111	list_for_each_entry_safe(tunnel, n, &tcm->tunnel_list, list)3112		tb_tunnel_restart(tunnel);3113	tcm->hotplug_active = true;3114	mutex_unlock(&tb->lock);3115 3116	/*3117	 * Schedule cleanup of any unplugged devices. Run this in a3118	 * separate thread to avoid possible deadlock if the device3119	 * removal runtime resumes the unplugged device.3120	 */3121	queue_delayed_work(tb->wq, &tcm->remove_work, msecs_to_jiffies(50));3122	return 0;3123}3124 3125static const struct tb_cm_ops tb_cm_ops = {3126	.start = tb_start,3127	.stop = tb_stop,3128	.deinit = tb_deinit,3129	.suspend_noirq = tb_suspend_noirq,3130	.resume_noirq = tb_resume_noirq,3131	.freeze_noirq = tb_freeze_noirq,3132	.thaw_noirq = tb_thaw_noirq,3133	.complete = tb_complete,3134	.runtime_suspend = tb_runtime_suspend,3135	.runtime_resume = tb_runtime_resume,3136	.handle_event = tb_handle_event,3137	.disapprove_switch = tb_disconnect_pci,3138	.approve_switch = tb_tunnel_pci,3139	.approve_xdomain_paths = tb_approve_xdomain_paths,3140	.disconnect_xdomain_paths = tb_disconnect_xdomain_paths,3141};3142 3143/*3144 * During suspend the Thunderbolt controller is reset and all PCIe3145 * tunnels are lost. The NHI driver will try to reestablish all tunnels3146 * during resume. This adds device links between the tunneled PCIe3147 * downstream ports and the NHI so that the device core will make sure3148 * NHI is resumed first before the rest.3149 */3150static bool tb_apple_add_links(struct tb_nhi *nhi)3151{3152	struct pci_dev *upstream, *pdev;3153	bool ret;3154 3155	if (!x86_apple_machine)3156		return false;3157 3158	switch (nhi->pdev->device) {3159	case PCI_DEVICE_ID_INTEL_LIGHT_RIDGE:3160	case PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C:3161	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:3162	case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:3163		break;3164	default:3165		return false;3166	}3167 3168	upstream = pci_upstream_bridge(nhi->pdev);3169	while (upstream) {3170		if (!pci_is_pcie(upstream))3171			return false;3172		if (pci_pcie_type(upstream) == PCI_EXP_TYPE_UPSTREAM)3173			break;3174		upstream = pci_upstream_bridge(upstream);3175	}3176 3177	if (!upstream)3178		return false;3179 3180	/*3181	 * For each hotplug downstream port, create add device link3182	 * back to NHI so that PCIe tunnels can be re-established after3183	 * sleep.3184	 */3185	ret = false;3186	for_each_pci_bridge(pdev, upstream->subordinate) {3187		const struct device_link *link;3188 3189		if (!pci_is_pcie(pdev))3190			continue;3191		if (pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM ||3192		    !pdev->is_hotplug_bridge)3193			continue;3194 3195		link = device_link_add(&pdev->dev, &nhi->pdev->dev,3196				       DL_FLAG_AUTOREMOVE_SUPPLIER |3197				       DL_FLAG_PM_RUNTIME);3198		if (link) {3199			dev_dbg(&nhi->pdev->dev, "created link from %s\n",3200				dev_name(&pdev->dev));3201			ret = true;3202		} else {3203			dev_warn(&nhi->pdev->dev, "device link creation from %s failed\n",3204				 dev_name(&pdev->dev));3205		}3206	}3207 3208	return ret;3209}3210 3211struct tb *tb_probe(struct tb_nhi *nhi)3212{3213	struct tb_cm *tcm;3214	struct tb *tb;3215 3216	tb = tb_domain_alloc(nhi, TB_TIMEOUT, sizeof(*tcm));3217	if (!tb)3218		return NULL;3219 3220	if (tb_acpi_may_tunnel_pcie())3221		tb->security_level = TB_SECURITY_USER;3222	else3223		tb->security_level = TB_SECURITY_NOPCIE;3224 3225	tb->cm_ops = &tb_cm_ops;3226 3227	tcm = tb_priv(tb);3228	INIT_LIST_HEAD(&tcm->tunnel_list);3229	INIT_LIST_HEAD(&tcm->dp_resources);3230	INIT_DELAYED_WORK(&tcm->remove_work, tb_remove_work);3231	tb_init_bandwidth_groups(tcm);3232 3233	tb_dbg(tb, "using software connection manager\n");3234 3235	/*3236	 * Device links are needed to make sure we establish tunnels3237	 * before the PCIe/USB stack is resumed so complain here if we3238	 * found them missing.3239	 */3240	if (!tb_apple_add_links(nhi) && !tb_acpi_add_links(nhi))3241		tb_warn(tb, "device links to tunneled native ports are missing!\n");3242 3243	return tb;3244}3245