2557 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Internal Thunderbolt Connection Manager. This is a firmware running on4 * the Thunderbolt host controller performing most of the low-level5 * handling.6 *7 * Copyright (C) 2017, Intel Corporation8 * Authors: Michael Jamet <michael.jamet@intel.com>9 * Mika Westerberg <mika.westerberg@linux.intel.com>10 */11 12#include <linux/delay.h>13#include <linux/mutex.h>14#include <linux/moduleparam.h>15#include <linux/pci.h>16#include <linux/pm_runtime.h>17#include <linux/platform_data/x86/apple.h>18#include <linux/sizes.h>19#include <linux/slab.h>20#include <linux/workqueue.h>21 22#include "ctl.h"23#include "nhi_regs.h"24#include "tb.h"25 26#define PCIE2CIO_CMD 0x3027#define PCIE2CIO_CMD_TIMEOUT BIT(31)28#define PCIE2CIO_CMD_START BIT(30)29#define PCIE2CIO_CMD_WRITE BIT(21)30#define PCIE2CIO_CMD_CS_MASK GENMASK(20, 19)31#define PCIE2CIO_CMD_CS_SHIFT 1932#define PCIE2CIO_CMD_PORT_MASK GENMASK(18, 13)33#define PCIE2CIO_CMD_PORT_SHIFT 1334 35#define PCIE2CIO_WRDATA 0x3436#define PCIE2CIO_RDDATA 0x3837 38#define PHY_PORT_CS1 0x3739#define PHY_PORT_CS1_LINK_DISABLE BIT(14)40#define PHY_PORT_CS1_LINK_STATE_MASK GENMASK(29, 26)41#define PHY_PORT_CS1_LINK_STATE_SHIFT 2642 43#define ICM_TIMEOUT 5000 /* ms */44#define ICM_RETRIES 345#define ICM_APPROVE_TIMEOUT 10000 /* ms */46#define ICM_MAX_LINK 447 48static bool start_icm;49module_param(start_icm, bool, 0444);50MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");51 52/**53 * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status54 * @reply: Reply from ICM firmware is placed here55 * @request: Request that is sent to ICM firmware56 * @icm: Pointer to ICM private data57 */58struct usb4_switch_nvm_auth {59 struct icm_usb4_switch_op_response reply;60 struct icm_usb4_switch_op request;61 struct icm *icm;62};63 64/**65 * struct icm - Internal connection manager private data66 * @request_lock: Makes sure only one message is send to ICM at time67 * @rescan_work: Work used to rescan the surviving switches after resume68 * @upstream_port: Pointer to the PCIe upstream port this host69 * controller is connected. This is only set for systems70 * where ICM needs to be started manually71 * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides72 * (only set when @upstream_port is not %NULL)73 * @safe_mode: ICM is in safe mode74 * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)75 * @rpm: Does the controller support runtime PM (RTD3)76 * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller77 * @proto_version: Firmware protocol version78 * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)79 * @veto: Is RTD3 veto in effect80 * @is_supported: Checks if we can support ICM on this controller81 * @cio_reset: Trigger CIO reset82 * @get_mode: Read and return the ICM firmware mode (optional)83 * @get_route: Find a route string for given switch84 * @save_devices: Ask ICM to save devices to ACL when suspending (optional)85 * @driver_ready: Send driver ready message to ICM86 * @set_uuid: Set UUID for the root switch (optional)87 * @device_connected: Handle device connected ICM message88 * @device_disconnected: Handle device disconnected ICM message89 * @xdomain_connected: Handle XDomain connected ICM message90 * @xdomain_disconnected: Handle XDomain disconnected ICM message91 * @rtd3_veto: Handle RTD3 veto notification ICM message92 */93struct icm {94 struct mutex request_lock;95 struct delayed_work rescan_work;96 struct pci_dev *upstream_port;97 int vnd_cap;98 bool safe_mode;99 size_t max_boot_acl;100 bool rpm;101 bool can_upgrade_nvm;102 u8 proto_version;103 struct usb4_switch_nvm_auth *last_nvm_auth;104 bool veto;105 bool (*is_supported)(struct tb *tb);106 int (*cio_reset)(struct tb *tb);107 int (*get_mode)(struct tb *tb);108 int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);109 void (*save_devices)(struct tb *tb);110 int (*driver_ready)(struct tb *tb,111 enum tb_security_level *security_level,112 u8 *proto_version, size_t *nboot_acl, bool *rpm);113 void (*set_uuid)(struct tb *tb);114 void (*device_connected)(struct tb *tb,115 const struct icm_pkg_header *hdr);116 void (*device_disconnected)(struct tb *tb,117 const struct icm_pkg_header *hdr);118 void (*xdomain_connected)(struct tb *tb,119 const struct icm_pkg_header *hdr);120 void (*xdomain_disconnected)(struct tb *tb,121 const struct icm_pkg_header *hdr);122 void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);123};124 125struct icm_notification {126 struct work_struct work;127 struct icm_pkg_header *pkg;128 struct tb *tb;129};130 131struct ep_name_entry {132 u8 len;133 u8 type;134 u8 data[];135};136 137#define EP_NAME_INTEL_VSS 0x10138 139/* Intel Vendor specific structure */140struct intel_vss {141 u16 vendor;142 u16 model;143 u8 mc;144 u8 flags;145 u16 pci_devid;146 u32 nvm_version;147};148 149#define INTEL_VSS_FLAGS_RTD3 BIT(0)150 151static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)152{153 const void *end = ep_name + size;154 155 while (ep_name < end) {156 const struct ep_name_entry *ep = ep_name;157 158 if (!ep->len)159 break;160 if (ep_name + ep->len > end)161 break;162 163 if (ep->type == EP_NAME_INTEL_VSS)164 return (const struct intel_vss *)ep->data;165 166 ep_name += ep->len;167 }168 169 return NULL;170}171 172static bool intel_vss_is_rtd3(const void *ep_name, size_t size)173{174 const struct intel_vss *vss;175 176 vss = parse_intel_vss(ep_name, size);177 if (vss)178 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);179 180 return false;181}182 183static inline struct tb *icm_to_tb(struct icm *icm)184{185 return ((void *)icm - sizeof(struct tb));186}187 188static inline u8 phy_port_from_route(u64 route, u8 depth)189{190 u8 link;191 192 link = depth ? route >> ((depth - 1) * 8) : route;193 return tb_phy_port_from_link(link);194}195 196static inline u8 dual_link_from_link(u8 link)197{198 return link ? ((link - 1) ^ 0x01) + 1 : 0;199}200 201static inline u64 get_route(u32 route_hi, u32 route_lo)202{203 return (u64)route_hi << 32 | route_lo;204}205 206static inline u64 get_parent_route(u64 route)207{208 int depth = tb_route_length(route);209 return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;210}211 212static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)213{214 unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);215 u32 cmd;216 217 do {218 pci_read_config_dword(icm->upstream_port,219 icm->vnd_cap + PCIE2CIO_CMD, &cmd);220 if (!(cmd & PCIE2CIO_CMD_START)) {221 if (cmd & PCIE2CIO_CMD_TIMEOUT)222 break;223 return 0;224 }225 226 msleep(50);227 } while (time_before(jiffies, end));228 229 return -ETIMEDOUT;230}231 232static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,233 unsigned int port, unsigned int index, u32 *data)234{235 struct pci_dev *pdev = icm->upstream_port;236 int ret, vnd_cap = icm->vnd_cap;237 u32 cmd;238 239 cmd = index;240 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;241 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;242 cmd |= PCIE2CIO_CMD_START;243 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);244 245 ret = pci2cio_wait_completion(icm, 5000);246 if (ret)247 return ret;248 249 pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);250 return 0;251}252 253static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,254 unsigned int port, unsigned int index, u32 data)255{256 struct pci_dev *pdev = icm->upstream_port;257 int vnd_cap = icm->vnd_cap;258 u32 cmd;259 260 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);261 262 cmd = index;263 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;264 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;265 cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;266 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);267 268 return pci2cio_wait_completion(icm, 5000);269}270 271static bool icm_match(const struct tb_cfg_request *req,272 const struct ctl_pkg *pkg)273{274 const struct icm_pkg_header *res_hdr = pkg->buffer;275 const struct icm_pkg_header *req_hdr = req->request;276 277 if (pkg->frame.eof != req->response_type)278 return false;279 if (res_hdr->code != req_hdr->code)280 return false;281 282 return true;283}284 285static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)286{287 const struct icm_pkg_header *hdr = pkg->buffer;288 289 if (hdr->packet_id < req->npackets) {290 size_t offset = hdr->packet_id * req->response_size;291 292 memcpy(req->response + offset, pkg->buffer, req->response_size);293 }294 295 return hdr->packet_id == hdr->total_packets - 1;296}297 298static int icm_request(struct tb *tb, const void *request, size_t request_size,299 void *response, size_t response_size, size_t npackets,300 int retries, unsigned int timeout_msec)301{302 struct icm *icm = tb_priv(tb);303 304 do {305 struct tb_cfg_request *req;306 struct tb_cfg_result res;307 308 req = tb_cfg_request_alloc();309 if (!req)310 return -ENOMEM;311 312 req->match = icm_match;313 req->copy = icm_copy;314 req->request = request;315 req->request_size = request_size;316 req->request_type = TB_CFG_PKG_ICM_CMD;317 req->response = response;318 req->npackets = npackets;319 req->response_size = response_size;320 req->response_type = TB_CFG_PKG_ICM_RESP;321 322 mutex_lock(&icm->request_lock);323 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);324 mutex_unlock(&icm->request_lock);325 326 tb_cfg_request_put(req);327 328 if (res.err != -ETIMEDOUT)329 return res.err == 1 ? -EIO : res.err;330 331 usleep_range(20, 50);332 } while (retries--);333 334 return -ETIMEDOUT;335}336 337/*338 * If rescan is queued to run (we are resuming), postpone it to give the339 * firmware some more time to send device connected notifications for next340 * devices in the chain.341 */342static void icm_postpone_rescan(struct tb *tb)343{344 struct icm *icm = tb_priv(tb);345 346 if (delayed_work_pending(&icm->rescan_work))347 mod_delayed_work(tb->wq, &icm->rescan_work,348 msecs_to_jiffies(500));349}350 351static void icm_veto_begin(struct tb *tb)352{353 struct icm *icm = tb_priv(tb);354 355 if (!icm->veto) {356 icm->veto = true;357 /* Keep the domain powered while veto is in effect */358 pm_runtime_get(&tb->dev);359 }360}361 362static void icm_veto_end(struct tb *tb)363{364 struct icm *icm = tb_priv(tb);365 366 if (icm->veto) {367 icm->veto = false;368 /* Allow the domain suspend now */369 pm_runtime_mark_last_busy(&tb->dev);370 pm_runtime_put_autosuspend(&tb->dev);371 }372}373 374static bool icm_firmware_running(const struct tb_nhi *nhi)375{376 u32 val;377 378 val = ioread32(nhi->iobase + REG_FW_STS);379 return !!(val & REG_FW_STS_ICM_EN);380}381 382static bool icm_fr_is_supported(struct tb *tb)383{384 return !x86_apple_machine;385}386 387static inline int icm_fr_get_switch_index(u32 port)388{389 int index;390 391 if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)392 return 0;393 394 index = port >> ICM_PORT_INDEX_SHIFT;395 return index != 0xff ? index : 0;396}397 398static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)399{400 struct icm_fr_pkg_get_topology_response *switches, *sw;401 struct icm_fr_pkg_get_topology request = {402 .hdr = { .code = ICM_GET_TOPOLOGY },403 };404 size_t npackets = ICM_GET_TOPOLOGY_PACKETS;405 int ret, index;406 u8 i;407 408 switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);409 if (!switches)410 return -ENOMEM;411 412 ret = icm_request(tb, &request, sizeof(request), switches,413 sizeof(*switches), npackets, ICM_RETRIES, ICM_TIMEOUT);414 if (ret)415 goto err_free;416 417 sw = &switches[0];418 index = icm_fr_get_switch_index(sw->ports[link]);419 if (!index) {420 ret = -ENODEV;421 goto err_free;422 }423 424 sw = &switches[index];425 for (i = 1; i < depth; i++) {426 unsigned int j;427 428 if (!(sw->first_data & ICM_SWITCH_USED)) {429 ret = -ENODEV;430 goto err_free;431 }432 433 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {434 index = icm_fr_get_switch_index(sw->ports[j]);435 if (index > sw->switch_index) {436 sw = &switches[index];437 break;438 }439 }440 }441 442 *route = get_route(sw->route_hi, sw->route_lo);443 444err_free:445 kfree(switches);446 return ret;447}448 449static void icm_fr_save_devices(struct tb *tb)450{451 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);452}453 454static int455icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,456 u8 *proto_version, size_t *nboot_acl, bool *rpm)457{458 struct icm_fr_pkg_driver_ready_response reply;459 struct icm_pkg_driver_ready request = {460 .hdr.code = ICM_DRIVER_READY,461 };462 int ret;463 464 memset(&reply, 0, sizeof(reply));465 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),466 1, ICM_RETRIES, ICM_TIMEOUT);467 if (ret)468 return ret;469 470 if (security_level)471 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK;472 473 return 0;474}475 476static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)477{478 struct icm_fr_pkg_approve_device request;479 struct icm_fr_pkg_approve_device reply;480 int ret;481 482 memset(&request, 0, sizeof(request));483 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));484 request.hdr.code = ICM_APPROVE_DEVICE;485 request.connection_id = sw->connection_id;486 request.connection_key = sw->connection_key;487 488 memset(&reply, 0, sizeof(reply));489 /* Use larger timeout as establishing tunnels can take some time */490 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),491 1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);492 if (ret)493 return ret;494 495 if (reply.hdr.flags & ICM_FLAGS_ERROR) {496 tb_warn(tb, "PCIe tunnel creation failed\n");497 return -EIO;498 }499 500 return 0;501}502 503static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)504{505 struct icm_fr_pkg_add_device_key request;506 struct icm_fr_pkg_add_device_key_response reply;507 int ret;508 509 memset(&request, 0, sizeof(request));510 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));511 request.hdr.code = ICM_ADD_DEVICE_KEY;512 request.connection_id = sw->connection_id;513 request.connection_key = sw->connection_key;514 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);515 516 memset(&reply, 0, sizeof(reply));517 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),518 1, ICM_RETRIES, ICM_TIMEOUT);519 if (ret)520 return ret;521 522 if (reply.hdr.flags & ICM_FLAGS_ERROR) {523 tb_warn(tb, "Adding key to switch failed\n");524 return -EIO;525 }526 527 return 0;528}529 530static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,531 const u8 *challenge, u8 *response)532{533 struct icm_fr_pkg_challenge_device request;534 struct icm_fr_pkg_challenge_device_response reply;535 int ret;536 537 memset(&request, 0, sizeof(request));538 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));539 request.hdr.code = ICM_CHALLENGE_DEVICE;540 request.connection_id = sw->connection_id;541 request.connection_key = sw->connection_key;542 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);543 544 memset(&reply, 0, sizeof(reply));545 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),546 1, ICM_RETRIES, ICM_TIMEOUT);547 if (ret)548 return ret;549 550 if (reply.hdr.flags & ICM_FLAGS_ERROR)551 return -EKEYREJECTED;552 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)553 return -ENOKEY;554 555 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);556 557 return 0;558}559 560static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,561 int transmit_path, int transmit_ring,562 int receive_path, int receive_ring)563{564 struct icm_fr_pkg_approve_xdomain_response reply;565 struct icm_fr_pkg_approve_xdomain request;566 int ret;567 568 memset(&request, 0, sizeof(request));569 request.hdr.code = ICM_APPROVE_XDOMAIN;570 request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;571 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));572 573 request.transmit_path = transmit_path;574 request.transmit_ring = transmit_ring;575 request.receive_path = receive_path;576 request.receive_ring = receive_ring;577 578 memset(&reply, 0, sizeof(reply));579 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),580 1, ICM_RETRIES, ICM_TIMEOUT);581 if (ret)582 return ret;583 584 if (reply.hdr.flags & ICM_FLAGS_ERROR)585 return -EIO;586 587 return 0;588}589 590static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,591 int transmit_path, int transmit_ring,592 int receive_path, int receive_ring)593{594 u8 phy_port;595 u8 cmd;596 597 phy_port = tb_phy_port_from_link(xd->link);598 if (phy_port == 0)599 cmd = NHI_MAILBOX_DISCONNECT_PA;600 else601 cmd = NHI_MAILBOX_DISCONNECT_PB;602 603 nhi_mailbox_cmd(tb->nhi, cmd, 1);604 usleep_range(10, 50);605 nhi_mailbox_cmd(tb->nhi, cmd, 2);606 return 0;607}608 609static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,610 const uuid_t *uuid)611{612 struct tb *tb = parent_sw->tb;613 struct tb_switch *sw;614 615 sw = tb_switch_alloc(tb, &parent_sw->dev, route);616 if (IS_ERR(sw)) {617 tb_warn(tb, "failed to allocate switch at %llx\n", route);618 return sw;619 }620 621 sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);622 if (!sw->uuid) {623 tb_switch_put(sw);624 return ERR_PTR(-ENOMEM);625 }626 627 init_completion(&sw->rpm_complete);628 return sw;629}630 631static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)632{633 u64 route = tb_route(sw);634 int ret;635 636 /* Link the two switches now */637 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);638 tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);639 640 ret = tb_switch_add(sw);641 if (ret)642 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;643 644 return ret;645}646 647static void update_switch(struct tb_switch *sw, u64 route, u8 connection_id,648 u8 connection_key, u8 link, u8 depth, bool boot)649{650 struct tb_switch *parent_sw = tb_switch_parent(sw);651 652 /* Disconnect from parent */653 tb_switch_downstream_port(sw)->remote = NULL;654 /* Re-connect via updated port */655 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);656 657 /* Update with the new addressing information */658 sw->config.route_hi = upper_32_bits(route);659 sw->config.route_lo = lower_32_bits(route);660 sw->connection_id = connection_id;661 sw->connection_key = connection_key;662 sw->link = link;663 sw->depth = depth;664 sw->boot = boot;665 666 /* This switch still exists */667 sw->is_unplugged = false;668 669 /* Runtime resume is now complete */670 complete(&sw->rpm_complete);671}672 673static void remove_switch(struct tb_switch *sw)674{675 tb_switch_downstream_port(sw)->remote = NULL;676 tb_switch_remove(sw);677}678 679static void add_xdomain(struct tb_switch *sw, u64 route,680 const uuid_t *local_uuid, const uuid_t *remote_uuid,681 u8 link, u8 depth)682{683 struct tb_xdomain *xd;684 685 pm_runtime_get_sync(&sw->dev);686 687 xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);688 if (!xd)689 goto out;690 691 xd->link = link;692 xd->depth = depth;693 694 tb_port_at(route, sw)->xdomain = xd;695 696 tb_xdomain_add(xd);697 698out:699 pm_runtime_mark_last_busy(&sw->dev);700 pm_runtime_put_autosuspend(&sw->dev);701}702 703static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)704{705 xd->link = link;706 xd->route = route;707 xd->is_unplugged = false;708}709 710static void remove_xdomain(struct tb_xdomain *xd)711{712 struct tb_switch *sw;713 714 sw = tb_to_switch(xd->dev.parent);715 tb_port_at(xd->route, sw)->xdomain = NULL;716 tb_xdomain_remove(xd);717}718 719static void720icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)721{722 const struct icm_fr_event_device_connected *pkg =723 (const struct icm_fr_event_device_connected *)hdr;724 enum tb_security_level security_level;725 struct tb_switch *sw, *parent_sw;726 bool boot, dual_lane, speed_gen3;727 struct icm *icm = tb_priv(tb);728 bool authorized = false;729 struct tb_xdomain *xd;730 u8 link, depth;731 u64 route;732 int ret;733 734 icm_postpone_rescan(tb);735 736 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;737 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>738 ICM_LINK_INFO_DEPTH_SHIFT;739 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;740 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>741 ICM_FLAGS_SLEVEL_SHIFT;742 boot = pkg->link_info & ICM_LINK_INFO_BOOT;743 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;744 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;745 746 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {747 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",748 link, depth);749 return;750 }751 752 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);753 if (sw) {754 u8 phy_port, sw_phy_port;755 756 sw_phy_port = tb_phy_port_from_link(sw->link);757 phy_port = tb_phy_port_from_link(link);758 759 /*760 * On resume ICM will send us connected events for the761 * devices that still are present. However, that762 * information might have changed for example by the763 * fact that a switch on a dual-link connection might764 * have been enumerated using the other link now. Make765 * sure our book keeping matches that.766 */767 if (sw->depth == depth && sw_phy_port == phy_port &&768 !!sw->authorized == authorized) {769 /*770 * It was enumerated through another link so update771 * route string accordingly.772 */773 if (sw->link != link) {774 ret = icm->get_route(tb, link, depth, &route);775 if (ret) {776 tb_err(tb, "failed to update route string for switch at %u.%u\n",777 link, depth);778 tb_switch_put(sw);779 return;780 }781 } else {782 route = tb_route(sw);783 }784 785 update_switch(sw, route, pkg->connection_id,786 pkg->connection_key, link, depth, boot);787 tb_switch_put(sw);788 return;789 }790 791 /*792 * User connected the same switch to another physical793 * port or to another part of the topology. Remove the794 * existing switch now before adding the new one.795 */796 remove_switch(sw);797 tb_switch_put(sw);798 }799 800 /*801 * If the switch was not found by UUID, look for a switch on802 * same physical port (taking possible link aggregation into803 * account) and depth. If we found one it is definitely a stale804 * one so remove it first.805 */806 sw = tb_switch_find_by_link_depth(tb, link, depth);807 if (!sw) {808 u8 dual_link;809 810 dual_link = dual_link_from_link(link);811 if (dual_link)812 sw = tb_switch_find_by_link_depth(tb, dual_link, depth);813 }814 if (sw) {815 remove_switch(sw);816 tb_switch_put(sw);817 }818 819 /* Remove existing XDomain connection if found */820 xd = tb_xdomain_find_by_link_depth(tb, link, depth);821 if (xd) {822 remove_xdomain(xd);823 tb_xdomain_put(xd);824 }825 826 parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);827 if (!parent_sw) {828 tb_err(tb, "failed to find parent switch for %u.%u\n",829 link, depth);830 return;831 }832 833 ret = icm->get_route(tb, link, depth, &route);834 if (ret) {835 tb_err(tb, "failed to find route string for switch at %u.%u\n",836 link, depth);837 tb_switch_put(parent_sw);838 return;839 }840 841 pm_runtime_get_sync(&parent_sw->dev);842 843 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);844 if (!IS_ERR(sw)) {845 sw->connection_id = pkg->connection_id;846 sw->connection_key = pkg->connection_key;847 sw->link = link;848 sw->depth = depth;849 sw->authorized = authorized;850 sw->security_level = security_level;851 sw->boot = boot;852 sw->link_speed = speed_gen3 ? 20 : 10;853 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :854 TB_LINK_WIDTH_SINGLE;855 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));856 857 if (add_switch(parent_sw, sw))858 tb_switch_put(sw);859 }860 861 pm_runtime_mark_last_busy(&parent_sw->dev);862 pm_runtime_put_autosuspend(&parent_sw->dev);863 864 tb_switch_put(parent_sw);865}866 867static void868icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)869{870 const struct icm_fr_event_device_disconnected *pkg =871 (const struct icm_fr_event_device_disconnected *)hdr;872 struct tb_switch *sw;873 u8 link, depth;874 875 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;876 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>877 ICM_LINK_INFO_DEPTH_SHIFT;878 879 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {880 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);881 return;882 }883 884 sw = tb_switch_find_by_link_depth(tb, link, depth);885 if (!sw) {886 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,887 depth);888 return;889 }890 891 pm_runtime_get_sync(sw->dev.parent);892 893 remove_switch(sw);894 895 pm_runtime_mark_last_busy(sw->dev.parent);896 pm_runtime_put_autosuspend(sw->dev.parent);897 898 tb_switch_put(sw);899}900 901static void902icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)903{904 const struct icm_fr_event_xdomain_connected *pkg =905 (const struct icm_fr_event_xdomain_connected *)hdr;906 struct tb_xdomain *xd;907 struct tb_switch *sw;908 u8 link, depth;909 u64 route;910 911 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;912 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>913 ICM_LINK_INFO_DEPTH_SHIFT;914 915 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {916 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);917 return;918 }919 920 route = get_route(pkg->local_route_hi, pkg->local_route_lo);921 922 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);923 if (xd) {924 u8 xd_phy_port, phy_port;925 926 xd_phy_port = phy_port_from_route(xd->route, xd->depth);927 phy_port = phy_port_from_route(route, depth);928 929 if (xd->depth == depth && xd_phy_port == phy_port) {930 update_xdomain(xd, route, link);931 tb_xdomain_put(xd);932 return;933 }934 935 /*936 * If we find an existing XDomain connection remove it937 * now. We need to go through login handshake and938 * everything anyway to be able to re-establish the939 * connection.940 */941 remove_xdomain(xd);942 tb_xdomain_put(xd);943 }944 945 /*946 * Look if there already exists an XDomain in the same place947 * than the new one and in that case remove it because it is948 * most likely another host that got disconnected.949 */950 xd = tb_xdomain_find_by_link_depth(tb, link, depth);951 if (!xd) {952 u8 dual_link;953 954 dual_link = dual_link_from_link(link);955 if (dual_link)956 xd = tb_xdomain_find_by_link_depth(tb, dual_link,957 depth);958 }959 if (xd) {960 remove_xdomain(xd);961 tb_xdomain_put(xd);962 }963 964 /*965 * If the user disconnected a switch during suspend and966 * connected another host to the same port, remove the switch967 * first.968 */969 sw = tb_switch_find_by_route(tb, route);970 if (sw) {971 remove_switch(sw);972 tb_switch_put(sw);973 }974 975 sw = tb_switch_find_by_link_depth(tb, link, depth);976 if (!sw) {977 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,978 depth);979 return;980 }981 982 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,983 depth);984 tb_switch_put(sw);985}986 987static void988icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)989{990 const struct icm_fr_event_xdomain_disconnected *pkg =991 (const struct icm_fr_event_xdomain_disconnected *)hdr;992 struct tb_xdomain *xd;993 994 /*995 * If the connection is through one or multiple devices, the996 * XDomain device is removed along with them so it is fine if we997 * cannot find it here.998 */999 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);1000 if (xd) {1001 remove_xdomain(xd);1002 tb_xdomain_put(xd);1003 }1004}1005 1006static int icm_tr_cio_reset(struct tb *tb)1007{1008 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));1009}1010 1011static int1012icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,1013 u8 *proto_version, size_t *nboot_acl, bool *rpm)1014{1015 struct icm_tr_pkg_driver_ready_response reply;1016 struct icm_pkg_driver_ready request = {1017 .hdr.code = ICM_DRIVER_READY,1018 };1019 int ret;1020 1021 memset(&reply, 0, sizeof(reply));1022 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1023 1, 10, 250);1024 if (ret)1025 return ret;1026 1027 if (security_level)1028 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;1029 if (proto_version)1030 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>1031 ICM_TR_INFO_PROTO_VERSION_SHIFT;1032 if (nboot_acl)1033 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>1034 ICM_TR_INFO_BOOT_ACL_SHIFT;1035 if (rpm)1036 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);1037 1038 return 0;1039}1040 1041static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)1042{1043 struct icm_tr_pkg_approve_device request;1044 struct icm_tr_pkg_approve_device reply;1045 int ret;1046 1047 memset(&request, 0, sizeof(request));1048 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));1049 request.hdr.code = ICM_APPROVE_DEVICE;1050 request.route_lo = sw->config.route_lo;1051 request.route_hi = sw->config.route_hi;1052 request.connection_id = sw->connection_id;1053 1054 memset(&reply, 0, sizeof(reply));1055 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1056 1, ICM_RETRIES, ICM_APPROVE_TIMEOUT);1057 if (ret)1058 return ret;1059 1060 if (reply.hdr.flags & ICM_FLAGS_ERROR) {1061 tb_warn(tb, "PCIe tunnel creation failed\n");1062 return -EIO;1063 }1064 1065 return 0;1066}1067 1068static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)1069{1070 struct icm_tr_pkg_add_device_key_response reply;1071 struct icm_tr_pkg_add_device_key request;1072 int ret;1073 1074 memset(&request, 0, sizeof(request));1075 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));1076 request.hdr.code = ICM_ADD_DEVICE_KEY;1077 request.route_lo = sw->config.route_lo;1078 request.route_hi = sw->config.route_hi;1079 request.connection_id = sw->connection_id;1080 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);1081 1082 memset(&reply, 0, sizeof(reply));1083 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1084 1, ICM_RETRIES, ICM_TIMEOUT);1085 if (ret)1086 return ret;1087 1088 if (reply.hdr.flags & ICM_FLAGS_ERROR) {1089 tb_warn(tb, "Adding key to switch failed\n");1090 return -EIO;1091 }1092 1093 return 0;1094}1095 1096static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,1097 const u8 *challenge, u8 *response)1098{1099 struct icm_tr_pkg_challenge_device_response reply;1100 struct icm_tr_pkg_challenge_device request;1101 int ret;1102 1103 memset(&request, 0, sizeof(request));1104 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));1105 request.hdr.code = ICM_CHALLENGE_DEVICE;1106 request.route_lo = sw->config.route_lo;1107 request.route_hi = sw->config.route_hi;1108 request.connection_id = sw->connection_id;1109 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);1110 1111 memset(&reply, 0, sizeof(reply));1112 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1113 1, ICM_RETRIES, ICM_TIMEOUT);1114 if (ret)1115 return ret;1116 1117 if (reply.hdr.flags & ICM_FLAGS_ERROR)1118 return -EKEYREJECTED;1119 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)1120 return -ENOKEY;1121 1122 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);1123 1124 return 0;1125}1126 1127static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,1128 int transmit_path, int transmit_ring,1129 int receive_path, int receive_ring)1130{1131 struct icm_tr_pkg_approve_xdomain_response reply;1132 struct icm_tr_pkg_approve_xdomain request;1133 int ret;1134 1135 memset(&request, 0, sizeof(request));1136 request.hdr.code = ICM_APPROVE_XDOMAIN;1137 request.route_hi = upper_32_bits(xd->route);1138 request.route_lo = lower_32_bits(xd->route);1139 request.transmit_path = transmit_path;1140 request.transmit_ring = transmit_ring;1141 request.receive_path = receive_path;1142 request.receive_ring = receive_ring;1143 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));1144 1145 memset(&reply, 0, sizeof(reply));1146 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1147 1, ICM_RETRIES, ICM_TIMEOUT);1148 if (ret)1149 return ret;1150 1151 if (reply.hdr.flags & ICM_FLAGS_ERROR)1152 return -EIO;1153 1154 return 0;1155}1156 1157static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,1158 int stage)1159{1160 struct icm_tr_pkg_disconnect_xdomain_response reply;1161 struct icm_tr_pkg_disconnect_xdomain request;1162 int ret;1163 1164 memset(&request, 0, sizeof(request));1165 request.hdr.code = ICM_DISCONNECT_XDOMAIN;1166 request.stage = stage;1167 request.route_hi = upper_32_bits(xd->route);1168 request.route_lo = lower_32_bits(xd->route);1169 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));1170 1171 memset(&reply, 0, sizeof(reply));1172 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1173 1, ICM_RETRIES, ICM_TIMEOUT);1174 if (ret)1175 return ret;1176 1177 if (reply.hdr.flags & ICM_FLAGS_ERROR)1178 return -EIO;1179 1180 return 0;1181}1182 1183static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,1184 int transmit_path, int transmit_ring,1185 int receive_path, int receive_ring)1186{1187 int ret;1188 1189 ret = icm_tr_xdomain_tear_down(tb, xd, 1);1190 if (ret)1191 return ret;1192 1193 usleep_range(10, 50);1194 return icm_tr_xdomain_tear_down(tb, xd, 2);1195}1196 1197static void1198__icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,1199 bool force_rtd3)1200{1201 const struct icm_tr_event_device_connected *pkg =1202 (const struct icm_tr_event_device_connected *)hdr;1203 bool authorized, boot, dual_lane, speed_gen3;1204 enum tb_security_level security_level;1205 struct tb_switch *sw, *parent_sw;1206 struct tb_xdomain *xd;1207 u64 route;1208 1209 icm_postpone_rescan(tb);1210 1211 /*1212 * Currently we don't use the QoS information coming with the1213 * device connected message so simply just ignore that extra1214 * packet for now.1215 */1216 if (pkg->hdr.packet_id)1217 return;1218 1219 route = get_route(pkg->route_hi, pkg->route_lo);1220 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;1221 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>1222 ICM_FLAGS_SLEVEL_SHIFT;1223 boot = pkg->link_info & ICM_LINK_INFO_BOOT;1224 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;1225 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;1226 1227 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {1228 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",1229 route);1230 return;1231 }1232 1233 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);1234 if (sw) {1235 /* Update the switch if it is still in the same place */1236 if (tb_route(sw) == route && !!sw->authorized == authorized) {1237 update_switch(sw, route, pkg->connection_id, 0, 0, 0,1238 boot);1239 tb_switch_put(sw);1240 return;1241 }1242 1243 remove_switch(sw);1244 tb_switch_put(sw);1245 }1246 1247 /* Another switch with the same address */1248 sw = tb_switch_find_by_route(tb, route);1249 if (sw) {1250 remove_switch(sw);1251 tb_switch_put(sw);1252 }1253 1254 /* XDomain connection with the same address */1255 xd = tb_xdomain_find_by_route(tb, route);1256 if (xd) {1257 remove_xdomain(xd);1258 tb_xdomain_put(xd);1259 }1260 1261 parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));1262 if (!parent_sw) {1263 tb_err(tb, "failed to find parent switch for %llx\n", route);1264 return;1265 }1266 1267 pm_runtime_get_sync(&parent_sw->dev);1268 1269 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);1270 if (!IS_ERR(sw)) {1271 sw->connection_id = pkg->connection_id;1272 sw->authorized = authorized;1273 sw->security_level = security_level;1274 sw->boot = boot;1275 sw->link_speed = speed_gen3 ? 20 : 10;1276 sw->link_width = dual_lane ? TB_LINK_WIDTH_DUAL :1277 TB_LINK_WIDTH_SINGLE;1278 sw->rpm = force_rtd3;1279 if (!sw->rpm)1280 sw->rpm = intel_vss_is_rtd3(pkg->ep_name,1281 sizeof(pkg->ep_name));1282 1283 if (add_switch(parent_sw, sw))1284 tb_switch_put(sw);1285 }1286 1287 pm_runtime_mark_last_busy(&parent_sw->dev);1288 pm_runtime_put_autosuspend(&parent_sw->dev);1289 1290 tb_switch_put(parent_sw);1291}1292 1293static void1294icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)1295{1296 __icm_tr_device_connected(tb, hdr, false);1297}1298 1299static void1300icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)1301{1302 const struct icm_tr_event_device_disconnected *pkg =1303 (const struct icm_tr_event_device_disconnected *)hdr;1304 struct tb_switch *sw;1305 u64 route;1306 1307 route = get_route(pkg->route_hi, pkg->route_lo);1308 1309 sw = tb_switch_find_by_route(tb, route);1310 if (!sw) {1311 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);1312 return;1313 }1314 pm_runtime_get_sync(sw->dev.parent);1315 1316 remove_switch(sw);1317 1318 pm_runtime_mark_last_busy(sw->dev.parent);1319 pm_runtime_put_autosuspend(sw->dev.parent);1320 1321 tb_switch_put(sw);1322}1323 1324static void1325icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)1326{1327 const struct icm_tr_event_xdomain_connected *pkg =1328 (const struct icm_tr_event_xdomain_connected *)hdr;1329 struct tb_xdomain *xd;1330 struct tb_switch *sw;1331 u64 route;1332 1333 if (!tb->root_switch)1334 return;1335 1336 route = get_route(pkg->local_route_hi, pkg->local_route_lo);1337 1338 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);1339 if (xd) {1340 if (xd->route == route) {1341 update_xdomain(xd, route, 0);1342 tb_xdomain_put(xd);1343 return;1344 }1345 1346 remove_xdomain(xd);1347 tb_xdomain_put(xd);1348 }1349 1350 /* An existing xdomain with the same address */1351 xd = tb_xdomain_find_by_route(tb, route);1352 if (xd) {1353 remove_xdomain(xd);1354 tb_xdomain_put(xd);1355 }1356 1357 /*1358 * If the user disconnected a switch during suspend and1359 * connected another host to the same port, remove the switch1360 * first.1361 */1362 sw = tb_switch_find_by_route(tb, route);1363 if (sw) {1364 remove_switch(sw);1365 tb_switch_put(sw);1366 }1367 1368 sw = tb_switch_find_by_route(tb, get_parent_route(route));1369 if (!sw) {1370 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);1371 return;1372 }1373 1374 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);1375 tb_switch_put(sw);1376}1377 1378static void1379icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)1380{1381 const struct icm_tr_event_xdomain_disconnected *pkg =1382 (const struct icm_tr_event_xdomain_disconnected *)hdr;1383 struct tb_xdomain *xd;1384 u64 route;1385 1386 route = get_route(pkg->route_hi, pkg->route_lo);1387 1388 xd = tb_xdomain_find_by_route(tb, route);1389 if (xd) {1390 remove_xdomain(xd);1391 tb_xdomain_put(xd);1392 }1393}1394 1395static struct pci_dev *get_upstream_port(struct pci_dev *pdev)1396{1397 struct pci_dev *parent;1398 1399 parent = pci_upstream_bridge(pdev);1400 while (parent) {1401 if (!pci_is_pcie(parent))1402 return NULL;1403 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)1404 break;1405 parent = pci_upstream_bridge(parent);1406 }1407 1408 if (!parent)1409 return NULL;1410 1411 switch (parent->device) {1412 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:1413 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:1414 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:1415 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:1416 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:1417 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:1418 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:1419 return parent;1420 }1421 1422 return NULL;1423}1424 1425static bool icm_ar_is_supported(struct tb *tb)1426{1427 struct pci_dev *upstream_port;1428 struct icm *icm = tb_priv(tb);1429 1430 /*1431 * Starting from Alpine Ridge we can use ICM on Apple machines1432 * as well. We just need to reset and re-enable it first.1433 * However, only start it if explicitly asked by the user.1434 */1435 if (icm_firmware_running(tb->nhi))1436 return true;1437 if (!start_icm)1438 return false;1439 1440 /*1441 * Find the upstream PCIe port in case we need to do reset1442 * through its vendor specific registers.1443 */1444 upstream_port = get_upstream_port(tb->nhi->pdev);1445 if (upstream_port) {1446 int cap;1447 1448 cap = pci_find_ext_capability(upstream_port,1449 PCI_EXT_CAP_ID_VNDR);1450 if (cap > 0) {1451 icm->upstream_port = upstream_port;1452 icm->vnd_cap = cap;1453 1454 return true;1455 }1456 }1457 1458 return false;1459}1460 1461static int icm_ar_cio_reset(struct tb *tb)1462{1463 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));1464}1465 1466static int icm_ar_get_mode(struct tb *tb)1467{1468 struct tb_nhi *nhi = tb->nhi;1469 int retries = 60;1470 u32 val;1471 1472 do {1473 val = ioread32(nhi->iobase + REG_FW_STS);1474 if (val & REG_FW_STS_NVM_AUTH_DONE)1475 break;1476 msleep(50);1477 } while (--retries);1478 1479 if (!retries) {1480 dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");1481 return -ENODEV;1482 }1483 1484 return nhi_mailbox_mode(nhi);1485}1486 1487static int1488icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,1489 u8 *proto_version, size_t *nboot_acl, bool *rpm)1490{1491 struct icm_ar_pkg_driver_ready_response reply;1492 struct icm_pkg_driver_ready request = {1493 .hdr.code = ICM_DRIVER_READY,1494 };1495 int ret;1496 1497 memset(&reply, 0, sizeof(reply));1498 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1499 1, ICM_RETRIES, ICM_TIMEOUT);1500 if (ret)1501 return ret;1502 1503 if (security_level)1504 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;1505 if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))1506 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>1507 ICM_AR_INFO_BOOT_ACL_SHIFT;1508 if (rpm)1509 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);1510 1511 return 0;1512}1513 1514static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)1515{1516 struct icm_ar_pkg_get_route_response reply;1517 struct icm_ar_pkg_get_route request = {1518 .hdr = { .code = ICM_GET_ROUTE },1519 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,1520 };1521 int ret;1522 1523 memset(&reply, 0, sizeof(reply));1524 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1525 1, ICM_RETRIES, ICM_TIMEOUT);1526 if (ret)1527 return ret;1528 1529 if (reply.hdr.flags & ICM_FLAGS_ERROR)1530 return -EIO;1531 1532 *route = get_route(reply.route_hi, reply.route_lo);1533 return 0;1534}1535 1536static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)1537{1538 struct icm_ar_pkg_preboot_acl_response reply;1539 struct icm_ar_pkg_preboot_acl request = {1540 .hdr = { .code = ICM_PREBOOT_ACL },1541 };1542 int ret, i;1543 1544 memset(&reply, 0, sizeof(reply));1545 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1546 1, ICM_RETRIES, ICM_TIMEOUT);1547 if (ret)1548 return ret;1549 1550 if (reply.hdr.flags & ICM_FLAGS_ERROR)1551 return -EIO;1552 1553 for (i = 0; i < nuuids; i++) {1554 u32 *uuid = (u32 *)&uuids[i];1555 1556 uuid[0] = reply.acl[i].uuid_lo;1557 uuid[1] = reply.acl[i].uuid_hi;1558 1559 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {1560 /* Map empty entries to null UUID */1561 uuid[0] = 0;1562 uuid[1] = 0;1563 } else if (uuid[0] != 0 || uuid[1] != 0) {1564 /* Upper two DWs are always one's */1565 uuid[2] = 0xffffffff;1566 uuid[3] = 0xffffffff;1567 }1568 }1569 1570 return ret;1571}1572 1573static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,1574 size_t nuuids)1575{1576 struct icm_ar_pkg_preboot_acl_response reply;1577 struct icm_ar_pkg_preboot_acl request = {1578 .hdr = {1579 .code = ICM_PREBOOT_ACL,1580 .flags = ICM_FLAGS_WRITE,1581 },1582 };1583 int ret, i;1584 1585 for (i = 0; i < nuuids; i++) {1586 const u32 *uuid = (const u32 *)&uuids[i];1587 1588 if (uuid_is_null(&uuids[i])) {1589 /*1590 * Map null UUID to the empty (all one) entries1591 * for ICM.1592 */1593 request.acl[i].uuid_lo = 0xffffffff;1594 request.acl[i].uuid_hi = 0xffffffff;1595 } else {1596 /* Two high DWs need to be set to all one */1597 if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)1598 return -EINVAL;1599 1600 request.acl[i].uuid_lo = uuid[0];1601 request.acl[i].uuid_hi = uuid[1];1602 }1603 }1604 1605 memset(&reply, 0, sizeof(reply));1606 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1607 1, ICM_RETRIES, ICM_TIMEOUT);1608 if (ret)1609 return ret;1610 1611 if (reply.hdr.flags & ICM_FLAGS_ERROR)1612 return -EIO;1613 1614 return 0;1615}1616 1617static int1618icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,1619 u8 *proto_version, size_t *nboot_acl, bool *rpm)1620{1621 struct icm_tr_pkg_driver_ready_response reply;1622 struct icm_pkg_driver_ready request = {1623 .hdr.code = ICM_DRIVER_READY,1624 };1625 int ret;1626 1627 memset(&reply, 0, sizeof(reply));1628 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),1629 1, ICM_RETRIES, 20000);1630 if (ret)1631 return ret;1632 1633 if (proto_version)1634 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>1635 ICM_TR_INFO_PROTO_VERSION_SHIFT;1636 1637 /* Ice Lake always supports RTD3 */1638 if (rpm)1639 *rpm = true;1640 1641 return 0;1642}1643 1644static void icm_icl_set_uuid(struct tb *tb)1645{1646 struct tb_nhi *nhi = tb->nhi;1647 u32 uuid[4];1648 1649 pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);1650 pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);1651 uuid[2] = 0xffffffff;1652 uuid[3] = 0xffffffff;1653 1654 tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);1655}1656 1657static void1658icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)1659{1660 __icm_tr_device_connected(tb, hdr, true);1661}1662 1663static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)1664{1665 const struct icm_icl_event_rtd3_veto *pkg =1666 (const struct icm_icl_event_rtd3_veto *)hdr;1667 1668 tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);1669 1670 if (pkg->veto_reason)1671 icm_veto_begin(tb);1672 else1673 icm_veto_end(tb);1674}1675 1676static bool icm_tgl_is_supported(struct tb *tb)1677{1678 unsigned long end = jiffies + msecs_to_jiffies(10);1679 1680 do {1681 u32 val;1682 1683 val = ioread32(tb->nhi->iobase + REG_FW_STS);1684 if (val & REG_FW_STS_NVM_AUTH_DONE)1685 return true;1686 usleep_range(100, 500);1687 } while (time_before(jiffies, end));1688 1689 return false;1690}1691 1692static void icm_handle_notification(struct work_struct *work)1693{1694 struct icm_notification *n = container_of(work, typeof(*n), work);1695 struct tb *tb = n->tb;1696 struct icm *icm = tb_priv(tb);1697 1698 mutex_lock(&tb->lock);1699 1700 /*1701 * When the domain is stopped we flush its workqueue but before1702 * that the root switch is removed. In that case we should treat1703 * the queued events as being canceled.1704 */1705 if (tb->root_switch) {1706 switch (n->pkg->code) {1707 case ICM_EVENT_DEVICE_CONNECTED:1708 icm->device_connected(tb, n->pkg);1709 break;1710 case ICM_EVENT_DEVICE_DISCONNECTED:1711 icm->device_disconnected(tb, n->pkg);1712 break;1713 case ICM_EVENT_XDOMAIN_CONNECTED:1714 if (tb_is_xdomain_enabled())1715 icm->xdomain_connected(tb, n->pkg);1716 break;1717 case ICM_EVENT_XDOMAIN_DISCONNECTED:1718 if (tb_is_xdomain_enabled())1719 icm->xdomain_disconnected(tb, n->pkg);1720 break;1721 case ICM_EVENT_RTD3_VETO:1722 icm->rtd3_veto(tb, n->pkg);1723 break;1724 }1725 }1726 1727 mutex_unlock(&tb->lock);1728 1729 kfree(n->pkg);1730 kfree(n);1731}1732 1733static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,1734 const void *buf, size_t size)1735{1736 struct icm_notification *n;1737 1738 n = kmalloc(sizeof(*n), GFP_KERNEL);1739 if (!n)1740 return;1741 1742 n->pkg = kmemdup(buf, size, GFP_KERNEL);1743 if (!n->pkg) {1744 kfree(n);1745 return;1746 }1747 1748 INIT_WORK(&n->work, icm_handle_notification);1749 n->tb = tb;1750 1751 queue_work(tb->wq, &n->work);1752}1753 1754static int1755__icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,1756 u8 *proto_version, size_t *nboot_acl, bool *rpm)1757{1758 struct icm *icm = tb_priv(tb);1759 unsigned int retries = 50;1760 int ret;1761 1762 ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,1763 rpm);1764 if (ret) {1765 tb_err(tb, "failed to send driver ready to ICM\n");1766 return ret;1767 }1768 1769 /*1770 * Hold on here until the switch config space is accessible so1771 * that we can read root switch config successfully.1772 */1773 do {1774 struct tb_cfg_result res;1775 u32 tmp;1776 1777 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,1778 0, 1, 100);1779 if (!res.err)1780 return 0;1781 1782 msleep(50);1783 } while (--retries);1784 1785 tb_err(tb, "failed to read root switch config space, giving up\n");1786 return -ETIMEDOUT;1787}1788 1789static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)1790{1791 struct icm *icm = tb_priv(tb);1792 u32 val;1793 1794 if (!icm->upstream_port)1795 return -ENODEV;1796 1797 /* Put ARC to wait for CIO reset event to happen */1798 val = ioread32(nhi->iobase + REG_FW_STS);1799 val |= REG_FW_STS_CIO_RESET_REQ;1800 iowrite32(val, nhi->iobase + REG_FW_STS);1801 1802 /* Re-start ARC */1803 val = ioread32(nhi->iobase + REG_FW_STS);1804 val |= REG_FW_STS_ICM_EN_INVERT;1805 val |= REG_FW_STS_ICM_EN_CPU;1806 iowrite32(val, nhi->iobase + REG_FW_STS);1807 1808 /* Trigger CIO reset now */1809 return icm->cio_reset(tb);1810}1811 1812static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)1813{1814 unsigned int retries = 10;1815 int ret;1816 u32 val;1817 1818 /* Check if the ICM firmware is already running */1819 if (icm_firmware_running(nhi))1820 return 0;1821 1822 dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");1823 1824 ret = icm_firmware_reset(tb, nhi);1825 if (ret)1826 return ret;1827 1828 /* Wait until the ICM firmware tells us it is up and running */1829 do {1830 /* Check that the ICM firmware is running */1831 val = ioread32(nhi->iobase + REG_FW_STS);1832 if (val & REG_FW_STS_NVM_AUTH_DONE)1833 return 0;1834 1835 msleep(300);1836 } while (--retries);1837 1838 return -ETIMEDOUT;1839}1840 1841static int icm_reset_phy_port(struct tb *tb, int phy_port)1842{1843 struct icm *icm = tb_priv(tb);1844 u32 state0, state1;1845 int port0, port1;1846 u32 val0, val1;1847 int ret;1848 1849 if (!icm->upstream_port)1850 return 0;1851 1852 if (phy_port) {1853 port0 = 3;1854 port1 = 4;1855 } else {1856 port0 = 1;1857 port1 = 2;1858 }1859 1860 /*1861 * Read link status of both null ports belonging to a single1862 * physical port.1863 */1864 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);1865 if (ret)1866 return ret;1867 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);1868 if (ret)1869 return ret;1870 1871 state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;1872 state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;1873 state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;1874 state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;1875 1876 /* If they are both up we need to reset them now */1877 if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)1878 return 0;1879 1880 val0 |= PHY_PORT_CS1_LINK_DISABLE;1881 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);1882 if (ret)1883 return ret;1884 1885 val1 |= PHY_PORT_CS1_LINK_DISABLE;1886 ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);1887 if (ret)1888 return ret;1889 1890 /* Wait a bit and then re-enable both ports */1891 usleep_range(10, 100);1892 1893 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);1894 if (ret)1895 return ret;1896 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);1897 if (ret)1898 return ret;1899 1900 val0 &= ~PHY_PORT_CS1_LINK_DISABLE;1901 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);1902 if (ret)1903 return ret;1904 1905 val1 &= ~PHY_PORT_CS1_LINK_DISABLE;1906 return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);1907}1908 1909static int icm_firmware_init(struct tb *tb)1910{1911 struct icm *icm = tb_priv(tb);1912 struct tb_nhi *nhi = tb->nhi;1913 int ret;1914 1915 ret = icm_firmware_start(tb, nhi);1916 if (ret) {1917 dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");1918 return ret;1919 }1920 1921 if (icm->get_mode) {1922 ret = icm->get_mode(tb);1923 1924 switch (ret) {1925 case NHI_FW_SAFE_MODE:1926 icm->safe_mode = true;1927 break;1928 1929 case NHI_FW_CM_MODE:1930 /* Ask ICM to accept all Thunderbolt devices */1931 nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);1932 break;1933 1934 default:1935 if (ret < 0)1936 return ret;1937 1938 tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);1939 return -ENODEV;1940 }1941 }1942 1943 /*1944 * Reset both physical ports if there is anything connected to1945 * them already.1946 */1947 ret = icm_reset_phy_port(tb, 0);1948 if (ret)1949 dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");1950 ret = icm_reset_phy_port(tb, 1);1951 if (ret)1952 dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");1953 1954 return 0;1955}1956 1957static int icm_driver_ready(struct tb *tb)1958{1959 struct icm *icm = tb_priv(tb);1960 int ret;1961 1962 ret = icm_firmware_init(tb);1963 if (ret)1964 return ret;1965 1966 if (icm->safe_mode) {1967 tb_info(tb, "Thunderbolt host controller is in safe mode.\n");1968 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");1969 tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n");1970 return 0;1971 }1972 1973 ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,1974 &tb->nboot_acl, &icm->rpm);1975 if (ret)1976 return ret;1977 1978 /*1979 * Make sure the number of supported preboot ACL matches what we1980 * expect or disable the whole feature.1981 */1982 if (tb->nboot_acl > icm->max_boot_acl)1983 tb->nboot_acl = 0;1984 1985 if (icm->proto_version >= 3)1986 tb_dbg(tb, "USB4 proxy operations supported\n");1987 1988 return 0;1989}1990 1991static int icm_suspend(struct tb *tb)1992{1993 struct icm *icm = tb_priv(tb);1994 1995 if (icm->save_devices)1996 icm->save_devices(tb);1997 1998 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);1999 return 0;2000}2001 2002/*2003 * Mark all switches (except root switch) below this one unplugged. ICM2004 * firmware will send us an updated list of switches after we have send2005 * it driver ready command. If a switch is not in that list it will be2006 * removed when we perform rescan.2007 */2008static void icm_unplug_children(struct tb_switch *sw)2009{2010 struct tb_port *port;2011 2012 if (tb_route(sw))2013 sw->is_unplugged = true;2014 2015 tb_switch_for_each_port(sw, port) {2016 if (port->xdomain)2017 port->xdomain->is_unplugged = true;2018 else if (tb_port_has_remote(port))2019 icm_unplug_children(port->remote->sw);2020 }2021}2022 2023static int complete_rpm(struct device *dev, void *data)2024{2025 struct tb_switch *sw = tb_to_switch(dev);2026 2027 if (sw)2028 complete(&sw->rpm_complete);2029 return 0;2030}2031 2032static void remove_unplugged_switch(struct tb_switch *sw)2033{2034 struct device *parent = get_device(sw->dev.parent);2035 2036 pm_runtime_get_sync(parent);2037 2038 /*2039 * Signal this and switches below for rpm_complete because2040 * tb_switch_remove() calls pm_runtime_get_sync() that then waits2041 * for it.2042 */2043 complete_rpm(&sw->dev, NULL);2044 bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);2045 tb_switch_remove(sw);2046 2047 pm_runtime_mark_last_busy(parent);2048 pm_runtime_put_autosuspend(parent);2049 2050 put_device(parent);2051}2052 2053static void icm_free_unplugged_children(struct tb_switch *sw)2054{2055 struct tb_port *port;2056 2057 tb_switch_for_each_port(sw, port) {2058 if (port->xdomain && port->xdomain->is_unplugged) {2059 tb_xdomain_remove(port->xdomain);2060 port->xdomain = NULL;2061 } else if (tb_port_has_remote(port)) {2062 if (port->remote->sw->is_unplugged) {2063 remove_unplugged_switch(port->remote->sw);2064 port->remote = NULL;2065 } else {2066 icm_free_unplugged_children(port->remote->sw);2067 }2068 }2069 }2070}2071 2072static void icm_rescan_work(struct work_struct *work)2073{2074 struct icm *icm = container_of(work, struct icm, rescan_work.work);2075 struct tb *tb = icm_to_tb(icm);2076 2077 mutex_lock(&tb->lock);2078 if (tb->root_switch)2079 icm_free_unplugged_children(tb->root_switch);2080 mutex_unlock(&tb->lock);2081}2082 2083static void icm_complete(struct tb *tb)2084{2085 struct icm *icm = tb_priv(tb);2086 2087 if (tb->nhi->going_away)2088 return;2089 2090 /*2091 * If RTD3 was vetoed before we entered system suspend allow it2092 * again now before driver ready is sent. Firmware sends a new RTD32093 * veto if it is still the case after we have sent it driver ready2094 * command.2095 */2096 icm_veto_end(tb);2097 icm_unplug_children(tb->root_switch);2098 2099 /*2100 * Now all existing children should be resumed, start events2101 * from ICM to get updated status.2102 */2103 __icm_driver_ready(tb, NULL, NULL, NULL, NULL);2104 2105 /*2106 * We do not get notifications of devices that have been2107 * unplugged during suspend so schedule rescan to clean them up2108 * if any.2109 */2110 queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));2111}2112 2113static int icm_runtime_suspend(struct tb *tb)2114{2115 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);2116 return 0;2117}2118 2119static int icm_runtime_suspend_switch(struct tb_switch *sw)2120{2121 if (tb_route(sw))2122 reinit_completion(&sw->rpm_complete);2123 return 0;2124}2125 2126static int icm_runtime_resume_switch(struct tb_switch *sw)2127{2128 if (tb_route(sw)) {2129 if (!wait_for_completion_timeout(&sw->rpm_complete,2130 msecs_to_jiffies(500))) {2131 dev_dbg(&sw->dev, "runtime resuming timed out\n");2132 }2133 }2134 return 0;2135}2136 2137static int icm_runtime_resume(struct tb *tb)2138{2139 /*2140 * We can reuse the same resume functionality than with system2141 * suspend.2142 */2143 icm_complete(tb);2144 return 0;2145}2146 2147static int icm_start(struct tb *tb, bool not_used)2148{2149 struct icm *icm = tb_priv(tb);2150 int ret;2151 2152 if (icm->safe_mode)2153 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);2154 else2155 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);2156 if (IS_ERR(tb->root_switch))2157 return PTR_ERR(tb->root_switch);2158 2159 tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;2160 tb->root_switch->rpm = icm->rpm;2161 2162 if (icm->set_uuid)2163 icm->set_uuid(tb);2164 2165 ret = tb_switch_add(tb->root_switch);2166 if (ret) {2167 tb_switch_put(tb->root_switch);2168 tb->root_switch = NULL;2169 }2170 2171 return ret;2172}2173 2174static void icm_stop(struct tb *tb)2175{2176 struct icm *icm = tb_priv(tb);2177 2178 cancel_delayed_work(&icm->rescan_work);2179 tb_switch_remove(tb->root_switch);2180 tb->root_switch = NULL;2181 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);2182 kfree(icm->last_nvm_auth);2183 icm->last_nvm_auth = NULL;2184}2185 2186static int icm_disconnect_pcie_paths(struct tb *tb)2187{2188 return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);2189}2190 2191static void icm_usb4_switch_nvm_auth_complete(void *data)2192{2193 struct usb4_switch_nvm_auth *auth = data;2194 struct icm *icm = auth->icm;2195 struct tb *tb = icm_to_tb(icm);2196 2197 tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",2198 get_route(auth->reply.route_hi, auth->reply.route_lo),2199 auth->reply.hdr.flags, auth->reply.status);2200 2201 mutex_lock(&tb->lock);2202 if (WARN_ON(icm->last_nvm_auth))2203 kfree(icm->last_nvm_auth);2204 icm->last_nvm_auth = auth;2205 mutex_unlock(&tb->lock);2206}2207 2208static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)2209{2210 struct usb4_switch_nvm_auth *auth;2211 struct icm *icm = tb_priv(tb);2212 struct tb_cfg_request *req;2213 int ret;2214 2215 auth = kzalloc(sizeof(*auth), GFP_KERNEL);2216 if (!auth)2217 return -ENOMEM;2218 2219 auth->icm = icm;2220 auth->request.hdr.code = ICM_USB4_SWITCH_OP;2221 auth->request.route_hi = upper_32_bits(route);2222 auth->request.route_lo = lower_32_bits(route);2223 auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;2224 2225 req = tb_cfg_request_alloc();2226 if (!req) {2227 ret = -ENOMEM;2228 goto err_free_auth;2229 }2230 2231 req->match = icm_match;2232 req->copy = icm_copy;2233 req->request = &auth->request;2234 req->request_size = sizeof(auth->request);2235 req->request_type = TB_CFG_PKG_ICM_CMD;2236 req->response = &auth->reply;2237 req->npackets = 1;2238 req->response_size = sizeof(auth->reply);2239 req->response_type = TB_CFG_PKG_ICM_RESP;2240 2241 tb_dbg(tb, "NVM_AUTH request for %llx\n", route);2242 2243 mutex_lock(&icm->request_lock);2244 ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,2245 auth);2246 mutex_unlock(&icm->request_lock);2247 2248 tb_cfg_request_put(req);2249 if (ret)2250 goto err_free_auth;2251 return 0;2252 2253err_free_auth:2254 kfree(auth);2255 return ret;2256}2257 2258static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,2259 u8 *status, const void *tx_data, size_t tx_data_len,2260 void *rx_data, size_t rx_data_len)2261{2262 struct icm_usb4_switch_op_response reply;2263 struct icm_usb4_switch_op request;2264 struct tb *tb = sw->tb;2265 struct icm *icm = tb_priv(tb);2266 u64 route = tb_route(sw);2267 int ret;2268 2269 /*2270 * USB4 router operation proxy is supported in firmware if the2271 * protocol version is 3 or higher.2272 */2273 if (icm->proto_version < 3)2274 return -EOPNOTSUPP;2275 2276 /*2277 * NVM_AUTH is a special USB4 proxy operation that does not2278 * return immediately so handle it separately.2279 */2280 if (opcode == USB4_SWITCH_OP_NVM_AUTH)2281 return icm_usb4_switch_nvm_authenticate(tb, route);2282 2283 memset(&request, 0, sizeof(request));2284 request.hdr.code = ICM_USB4_SWITCH_OP;2285 request.route_hi = upper_32_bits(route);2286 request.route_lo = lower_32_bits(route);2287 request.opcode = opcode;2288 if (metadata)2289 request.metadata = *metadata;2290 2291 if (tx_data_len) {2292 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;2293 if (tx_data_len < ARRAY_SIZE(request.data))2294 request.data_len_valid =2295 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;2296 memcpy(request.data, tx_data, tx_data_len * sizeof(u32));2297 }2298 2299 memset(&reply, 0, sizeof(reply));2300 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),2301 1, ICM_RETRIES, ICM_TIMEOUT);2302 if (ret)2303 return ret;2304 2305 if (reply.hdr.flags & ICM_FLAGS_ERROR)2306 return -EIO;2307 2308 if (status)2309 *status = reply.status;2310 2311 if (metadata)2312 *metadata = reply.metadata;2313 2314 if (rx_data_len)2315 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));2316 2317 return 0;2318}2319 2320static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,2321 u32 *status)2322{2323 struct usb4_switch_nvm_auth *auth;2324 struct tb *tb = sw->tb;2325 struct icm *icm = tb_priv(tb);2326 int ret = 0;2327 2328 if (icm->proto_version < 3)2329 return -EOPNOTSUPP;2330 2331 auth = icm->last_nvm_auth;2332 icm->last_nvm_auth = NULL;2333 2334 if (auth && auth->reply.route_hi == sw->config.route_hi &&2335 auth->reply.route_lo == sw->config.route_lo) {2336 tb_dbg(tb, "NVM_AUTH found for %llx flags %#x status %#x\n",2337 tb_route(sw), auth->reply.hdr.flags, auth->reply.status);2338 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)2339 ret = -EIO;2340 else2341 *status = auth->reply.status;2342 } else {2343 *status = 0;2344 }2345 2346 kfree(auth);2347 return ret;2348}2349 2350/* Falcon Ridge */2351static const struct tb_cm_ops icm_fr_ops = {2352 .driver_ready = icm_driver_ready,2353 .start = icm_start,2354 .stop = icm_stop,2355 .suspend = icm_suspend,2356 .complete = icm_complete,2357 .handle_event = icm_handle_event,2358 .approve_switch = icm_fr_approve_switch,2359 .add_switch_key = icm_fr_add_switch_key,2360 .challenge_switch_key = icm_fr_challenge_switch_key,2361 .disconnect_pcie_paths = icm_disconnect_pcie_paths,2362 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,2363 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,2364};2365 2366/* Alpine Ridge */2367static const struct tb_cm_ops icm_ar_ops = {2368 .driver_ready = icm_driver_ready,2369 .start = icm_start,2370 .stop = icm_stop,2371 .suspend = icm_suspend,2372 .complete = icm_complete,2373 .runtime_suspend = icm_runtime_suspend,2374 .runtime_resume = icm_runtime_resume,2375 .runtime_suspend_switch = icm_runtime_suspend_switch,2376 .runtime_resume_switch = icm_runtime_resume_switch,2377 .handle_event = icm_handle_event,2378 .get_boot_acl = icm_ar_get_boot_acl,2379 .set_boot_acl = icm_ar_set_boot_acl,2380 .approve_switch = icm_fr_approve_switch,2381 .add_switch_key = icm_fr_add_switch_key,2382 .challenge_switch_key = icm_fr_challenge_switch_key,2383 .disconnect_pcie_paths = icm_disconnect_pcie_paths,2384 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,2385 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,2386};2387 2388/* Titan Ridge */2389static const struct tb_cm_ops icm_tr_ops = {2390 .driver_ready = icm_driver_ready,2391 .start = icm_start,2392 .stop = icm_stop,2393 .suspend = icm_suspend,2394 .complete = icm_complete,2395 .runtime_suspend = icm_runtime_suspend,2396 .runtime_resume = icm_runtime_resume,2397 .runtime_suspend_switch = icm_runtime_suspend_switch,2398 .runtime_resume_switch = icm_runtime_resume_switch,2399 .handle_event = icm_handle_event,2400 .get_boot_acl = icm_ar_get_boot_acl,2401 .set_boot_acl = icm_ar_set_boot_acl,2402 .approve_switch = icm_tr_approve_switch,2403 .add_switch_key = icm_tr_add_switch_key,2404 .challenge_switch_key = icm_tr_challenge_switch_key,2405 .disconnect_pcie_paths = icm_disconnect_pcie_paths,2406 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,2407 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,2408 .usb4_switch_op = icm_usb4_switch_op,2409 .usb4_switch_nvm_authenticate_status =2410 icm_usb4_switch_nvm_authenticate_status,2411};2412 2413/* Ice Lake */2414static const struct tb_cm_ops icm_icl_ops = {2415 .driver_ready = icm_driver_ready,2416 .start = icm_start,2417 .stop = icm_stop,2418 .complete = icm_complete,2419 .runtime_suspend = icm_runtime_suspend,2420 .runtime_resume = icm_runtime_resume,2421 .handle_event = icm_handle_event,2422 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,2423 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,2424 .usb4_switch_op = icm_usb4_switch_op,2425 .usb4_switch_nvm_authenticate_status =2426 icm_usb4_switch_nvm_authenticate_status,2427};2428 2429struct tb *icm_probe(struct tb_nhi *nhi)2430{2431 struct icm *icm;2432 struct tb *tb;2433 2434 tb = tb_domain_alloc(nhi, ICM_TIMEOUT, sizeof(struct icm));2435 if (!tb)2436 return NULL;2437 2438 icm = tb_priv(tb);2439 INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);2440 mutex_init(&icm->request_lock);2441 2442 switch (nhi->pdev->device) {2443 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:2444 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:2445 icm->can_upgrade_nvm = true;2446 icm->is_supported = icm_fr_is_supported;2447 icm->get_route = icm_fr_get_route;2448 icm->save_devices = icm_fr_save_devices;2449 icm->driver_ready = icm_fr_driver_ready;2450 icm->device_connected = icm_fr_device_connected;2451 icm->device_disconnected = icm_fr_device_disconnected;2452 icm->xdomain_connected = icm_fr_xdomain_connected;2453 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;2454 tb->cm_ops = &icm_fr_ops;2455 break;2456 2457 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:2458 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:2459 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:2460 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:2461 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:2462 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;2463 /*2464 * NVM upgrade has not been tested on Apple systems and2465 * they don't provide images publicly either. To be on2466 * the safe side prevent root switch NVM upgrade on Macs2467 * for now.2468 */2469 icm->can_upgrade_nvm = !x86_apple_machine;2470 icm->is_supported = icm_ar_is_supported;2471 icm->cio_reset = icm_ar_cio_reset;2472 icm->get_mode = icm_ar_get_mode;2473 icm->get_route = icm_ar_get_route;2474 icm->save_devices = icm_fr_save_devices;2475 icm->driver_ready = icm_ar_driver_ready;2476 icm->device_connected = icm_fr_device_connected;2477 icm->device_disconnected = icm_fr_device_disconnected;2478 icm->xdomain_connected = icm_fr_xdomain_connected;2479 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;2480 tb->cm_ops = &icm_ar_ops;2481 break;2482 2483 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:2484 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:2485 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;2486 icm->can_upgrade_nvm = !x86_apple_machine;2487 icm->is_supported = icm_ar_is_supported;2488 icm->cio_reset = icm_tr_cio_reset;2489 icm->get_mode = icm_ar_get_mode;2490 icm->driver_ready = icm_tr_driver_ready;2491 icm->device_connected = icm_tr_device_connected;2492 icm->device_disconnected = icm_tr_device_disconnected;2493 icm->xdomain_connected = icm_tr_xdomain_connected;2494 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;2495 tb->cm_ops = &icm_tr_ops;2496 break;2497 2498 case PCI_DEVICE_ID_INTEL_ICL_NHI0:2499 case PCI_DEVICE_ID_INTEL_ICL_NHI1:2500 icm->is_supported = icm_fr_is_supported;2501 icm->driver_ready = icm_icl_driver_ready;2502 icm->set_uuid = icm_icl_set_uuid;2503 icm->device_connected = icm_icl_device_connected;2504 icm->device_disconnected = icm_tr_device_disconnected;2505 icm->xdomain_connected = icm_tr_xdomain_connected;2506 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;2507 icm->rtd3_veto = icm_icl_rtd3_veto;2508 tb->cm_ops = &icm_icl_ops;2509 break;2510 2511 case PCI_DEVICE_ID_INTEL_TGL_NHI0:2512 case PCI_DEVICE_ID_INTEL_TGL_NHI1:2513 case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:2514 case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:2515 case PCI_DEVICE_ID_INTEL_ADL_NHI0:2516 case PCI_DEVICE_ID_INTEL_ADL_NHI1:2517 case PCI_DEVICE_ID_INTEL_RPL_NHI0:2518 case PCI_DEVICE_ID_INTEL_RPL_NHI1:2519 case PCI_DEVICE_ID_INTEL_MTL_M_NHI0:2520 case PCI_DEVICE_ID_INTEL_MTL_P_NHI0:2521 case PCI_DEVICE_ID_INTEL_MTL_P_NHI1:2522 icm->is_supported = icm_tgl_is_supported;2523 icm->driver_ready = icm_icl_driver_ready;2524 icm->set_uuid = icm_icl_set_uuid;2525 icm->device_connected = icm_icl_device_connected;2526 icm->device_disconnected = icm_tr_device_disconnected;2527 icm->xdomain_connected = icm_tr_xdomain_connected;2528 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;2529 icm->rtd3_veto = icm_icl_rtd3_veto;2530 tb->cm_ops = &icm_icl_ops;2531 break;2532 2533 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_2C_NHI:2534 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:2535 icm->can_upgrade_nvm = true;2536 icm->is_supported = icm_tgl_is_supported;2537 icm->get_mode = icm_ar_get_mode;2538 icm->driver_ready = icm_tr_driver_ready;2539 icm->device_connected = icm_tr_device_connected;2540 icm->device_disconnected = icm_tr_device_disconnected;2541 icm->xdomain_connected = icm_tr_xdomain_connected;2542 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;2543 tb->cm_ops = &icm_tr_ops;2544 break;2545 }2546 2547 if (!icm->is_supported || !icm->is_supported(tb)) {2548 dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");2549 tb_domain_put(tb);2550 return NULL;2551 }2552 2553 tb_dbg(tb, "using firmware connection manager\n");2554 2555 return tb;2556}2557