542 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * PCI Express I/O Virtualization (IOV) support4 * Address Translation Service 1.05 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com>6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com>7 *8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com>9 * Copyright (C) 2011 Advanced Micro Devices,10 */11 12#include <linux/bitfield.h>13#include <linux/export.h>14#include <linux/pci-ats.h>15#include <linux/pci.h>16#include <linux/slab.h>17 18#include "pci.h"19 20void pci_ats_init(struct pci_dev *dev)21{22 int pos;23 24 if (pci_ats_disabled())25 return;26 27 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS);28 if (!pos)29 return;30 31 dev->ats_cap = pos;32}33 34/**35 * pci_ats_supported - check if the device can use ATS36 * @dev: the PCI device37 *38 * Returns true if the device supports ATS and is allowed to use it, false39 * otherwise.40 */41bool pci_ats_supported(struct pci_dev *dev)42{43 if (!dev->ats_cap)44 return false;45 46 return (dev->untrusted == 0);47}48EXPORT_SYMBOL_GPL(pci_ats_supported);49 50/**51 * pci_prepare_ats - Setup the PS for ATS52 * @dev: the PCI device53 * @ps: the IOMMU page shift54 *55 * This must be done by the IOMMU driver on the PF before any VFs are created to56 * ensure that the VF can have ATS enabled.57 *58 * Returns 0 on success, or negative on failure.59 */60int pci_prepare_ats(struct pci_dev *dev, int ps)61{62 u16 ctrl;63 64 if (!pci_ats_supported(dev))65 return -EINVAL;66 67 if (WARN_ON(dev->ats_enabled))68 return -EBUSY;69 70 if (ps < PCI_ATS_MIN_STU)71 return -EINVAL;72 73 if (dev->is_virtfn)74 return 0;75 76 dev->ats_stu = ps;77 ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);78 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);79 return 0;80}81EXPORT_SYMBOL_GPL(pci_prepare_ats);82 83/**84 * pci_enable_ats - enable the ATS capability85 * @dev: the PCI device86 * @ps: the IOMMU page shift87 *88 * Returns 0 on success, or negative on failure.89 */90int pci_enable_ats(struct pci_dev *dev, int ps)91{92 u16 ctrl;93 struct pci_dev *pdev;94 95 if (!pci_ats_supported(dev))96 return -EINVAL;97 98 if (WARN_ON(dev->ats_enabled))99 return -EBUSY;100 101 if (ps < PCI_ATS_MIN_STU)102 return -EINVAL;103 104 /*105 * Note that enabling ATS on a VF fails unless it's already enabled106 * with the same STU on the PF.107 */108 ctrl = PCI_ATS_CTRL_ENABLE;109 if (dev->is_virtfn) {110 pdev = pci_physfn(dev);111 if (pdev->ats_stu != ps)112 return -EINVAL;113 } else {114 dev->ats_stu = ps;115 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);116 }117 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);118 119 dev->ats_enabled = 1;120 return 0;121}122EXPORT_SYMBOL_GPL(pci_enable_ats);123 124/**125 * pci_disable_ats - disable the ATS capability126 * @dev: the PCI device127 */128void pci_disable_ats(struct pci_dev *dev)129{130 u16 ctrl;131 132 if (WARN_ON(!dev->ats_enabled))133 return;134 135 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl);136 ctrl &= ~PCI_ATS_CTRL_ENABLE;137 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);138 139 dev->ats_enabled = 0;140}141EXPORT_SYMBOL_GPL(pci_disable_ats);142 143void pci_restore_ats_state(struct pci_dev *dev)144{145 u16 ctrl;146 147 if (!dev->ats_enabled)148 return;149 150 ctrl = PCI_ATS_CTRL_ENABLE;151 if (!dev->is_virtfn)152 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU);153 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl);154}155 156/**157 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth158 * @dev: the PCI device159 *160 * Returns the queue depth on success, or negative on failure.161 *162 * The ATS spec uses 0 in the Invalidate Queue Depth field to163 * indicate that the function can accept 32 Invalidate Request.164 * But here we use the `real' values (i.e. 1~32) for the Queue165 * Depth; and 0 indicates the function shares the Queue with166 * other functions (doesn't exclusively own a Queue).167 */168int pci_ats_queue_depth(struct pci_dev *dev)169{170 u16 cap;171 172 if (!dev->ats_cap)173 return -EINVAL;174 175 if (dev->is_virtfn)176 return 0;177 178 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap);179 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP;180}181 182/**183 * pci_ats_page_aligned - Return Page Aligned Request bit status.184 * @pdev: the PCI device185 *186 * Returns 1, if the Untranslated Addresses generated by the device187 * are always aligned or 0 otherwise.188 *189 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit190 * is set, it indicates the Untranslated Addresses generated by the191 * device are always aligned to a 4096 byte boundary.192 */193int pci_ats_page_aligned(struct pci_dev *pdev)194{195 u16 cap;196 197 if (!pdev->ats_cap)198 return 0;199 200 pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap);201 202 if (cap & PCI_ATS_CAP_PAGE_ALIGNED)203 return 1;204 205 return 0;206}207 208#ifdef CONFIG_PCI_PRI209void pci_pri_init(struct pci_dev *pdev)210{211 u16 status;212 213 pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI);214 215 if (!pdev->pri_cap)216 return;217 218 pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status);219 if (status & PCI_PRI_STATUS_PASID)220 pdev->pasid_required = 1;221}222 223/**224 * pci_enable_pri - Enable PRI capability225 * @pdev: PCI device structure226 * @reqs: outstanding requests227 *228 * Returns 0 on success, negative value on error229 */230int pci_enable_pri(struct pci_dev *pdev, u32 reqs)231{232 u16 control, status;233 u32 max_requests;234 int pri = pdev->pri_cap;235 236 /*237 * VFs must not implement the PRI Capability. If their PF238 * implements PRI, it is shared by the VFs, so if the PF PRI is239 * enabled, it is also enabled for the VF.240 */241 if (pdev->is_virtfn) {242 if (pci_physfn(pdev)->pri_enabled)243 return 0;244 return -EINVAL;245 }246 247 if (WARN_ON(pdev->pri_enabled))248 return -EBUSY;249 250 if (!pri)251 return -EINVAL;252 253 pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status);254 if (!(status & PCI_PRI_STATUS_STOPPED))255 return -EBUSY;256 257 pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests);258 reqs = min(max_requests, reqs);259 pdev->pri_reqs_alloc = reqs;260 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);261 262 control = PCI_PRI_CTRL_ENABLE;263 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);264 265 pdev->pri_enabled = 1;266 267 return 0;268}269 270/**271 * pci_disable_pri - Disable PRI capability272 * @pdev: PCI device structure273 *274 * Only clears the enabled-bit, regardless of its former value275 */276void pci_disable_pri(struct pci_dev *pdev)277{278 u16 control;279 int pri = pdev->pri_cap;280 281 /* VFs share the PF PRI */282 if (pdev->is_virtfn)283 return;284 285 if (WARN_ON(!pdev->pri_enabled))286 return;287 288 if (!pri)289 return;290 291 pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control);292 control &= ~PCI_PRI_CTRL_ENABLE;293 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);294 295 pdev->pri_enabled = 0;296}297EXPORT_SYMBOL_GPL(pci_disable_pri);298 299/**300 * pci_restore_pri_state - Restore PRI301 * @pdev: PCI device structure302 */303void pci_restore_pri_state(struct pci_dev *pdev)304{305 u16 control = PCI_PRI_CTRL_ENABLE;306 u32 reqs = pdev->pri_reqs_alloc;307 int pri = pdev->pri_cap;308 309 if (pdev->is_virtfn)310 return;311 312 if (!pdev->pri_enabled)313 return;314 315 if (!pri)316 return;317 318 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs);319 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);320}321 322/**323 * pci_reset_pri - Resets device's PRI state324 * @pdev: PCI device structure325 *326 * The PRI capability must be disabled before this function is called.327 * Returns 0 on success, negative value on error.328 */329int pci_reset_pri(struct pci_dev *pdev)330{331 u16 control;332 int pri = pdev->pri_cap;333 334 if (pdev->is_virtfn)335 return 0;336 337 if (WARN_ON(pdev->pri_enabled))338 return -EBUSY;339 340 if (!pri)341 return -EINVAL;342 343 control = PCI_PRI_CTRL_RESET;344 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control);345 346 return 0;347}348 349/**350 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit351 * status.352 * @pdev: PCI device structure353 *354 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise.355 */356int pci_prg_resp_pasid_required(struct pci_dev *pdev)357{358 if (pdev->is_virtfn)359 pdev = pci_physfn(pdev);360 361 return pdev->pasid_required;362}363 364/**365 * pci_pri_supported - Check if PRI is supported.366 * @pdev: PCI device structure367 *368 * Returns true if PRI capability is present, false otherwise.369 */370bool pci_pri_supported(struct pci_dev *pdev)371{372 /* VFs share the PF PRI */373 if (pci_physfn(pdev)->pri_cap)374 return true;375 return false;376}377EXPORT_SYMBOL_GPL(pci_pri_supported);378#endif /* CONFIG_PCI_PRI */379 380#ifdef CONFIG_PCI_PASID381void pci_pasid_init(struct pci_dev *pdev)382{383 pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID);384}385 386/**387 * pci_enable_pasid - Enable the PASID capability388 * @pdev: PCI device structure389 * @features: Features to enable390 *391 * Returns 0 on success, negative value on error. This function checks392 * whether the features are actually supported by the device and returns393 * an error if not.394 */395int pci_enable_pasid(struct pci_dev *pdev, int features)396{397 u16 control, supported;398 int pasid = pdev->pasid_cap;399 400 /*401 * VFs must not implement the PASID Capability, but if a PF402 * supports PASID, its VFs share the PF PASID configuration.403 */404 if (pdev->is_virtfn) {405 if (pci_physfn(pdev)->pasid_enabled)406 return 0;407 return -EINVAL;408 }409 410 if (WARN_ON(pdev->pasid_enabled))411 return -EBUSY;412 413 if (!pdev->eetlp_prefix_path && !pdev->pasid_no_tlp)414 return -EINVAL;415 416 if (!pasid)417 return -EINVAL;418 419 if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR | PCI_ACS_UF))420 return -EINVAL;421 422 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);423 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;424 425 /* User wants to enable anything unsupported? */426 if ((supported & features) != features)427 return -EINVAL;428 429 control = PCI_PASID_CTRL_ENABLE | features;430 pdev->pasid_features = features;431 432 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);433 434 pdev->pasid_enabled = 1;435 436 return 0;437}438EXPORT_SYMBOL_GPL(pci_enable_pasid);439 440/**441 * pci_disable_pasid - Disable the PASID capability442 * @pdev: PCI device structure443 */444void pci_disable_pasid(struct pci_dev *pdev)445{446 u16 control = 0;447 int pasid = pdev->pasid_cap;448 449 /* VFs share the PF PASID configuration */450 if (pdev->is_virtfn)451 return;452 453 if (WARN_ON(!pdev->pasid_enabled))454 return;455 456 if (!pasid)457 return;458 459 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);460 461 pdev->pasid_enabled = 0;462}463EXPORT_SYMBOL_GPL(pci_disable_pasid);464 465/**466 * pci_restore_pasid_state - Restore PASID capabilities467 * @pdev: PCI device structure468 */469void pci_restore_pasid_state(struct pci_dev *pdev)470{471 u16 control;472 int pasid = pdev->pasid_cap;473 474 if (pdev->is_virtfn)475 return;476 477 if (!pdev->pasid_enabled)478 return;479 480 if (!pasid)481 return;482 483 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features;484 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control);485}486 487/**488 * pci_pasid_features - Check which PASID features are supported489 * @pdev: PCI device structure490 *491 * Return a negative value when no PASID capability is present.492 * Otherwise return a bitmask with supported features. Current493 * features reported are:494 * PCI_PASID_CAP_EXEC - Execute permission supported495 * PCI_PASID_CAP_PRIV - Privileged mode supported496 */497int pci_pasid_features(struct pci_dev *pdev)498{499 u16 supported;500 int pasid;501 502 if (pdev->is_virtfn)503 pdev = pci_physfn(pdev);504 505 pasid = pdev->pasid_cap;506 if (!pasid)507 return -EINVAL;508 509 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);510 511 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV;512 513 return supported;514}515EXPORT_SYMBOL_GPL(pci_pasid_features);516 517/**518 * pci_max_pasids - Get maximum number of PASIDs supported by device519 * @pdev: PCI device structure520 *521 * Returns negative value when PASID capability is not present.522 * Otherwise it returns the number of supported PASIDs.523 */524int pci_max_pasids(struct pci_dev *pdev)525{526 u16 supported;527 int pasid;528 529 if (pdev->is_virtfn)530 pdev = pci_physfn(pdev);531 532 pasid = pdev->pasid_cap;533 if (!pasid)534 return -EINVAL;535 536 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported);537 538 return (1 << FIELD_GET(PCI_PASID_CAP_WIDTH, supported));539}540EXPORT_SYMBOL_GPL(pci_max_pasids);541#endif /* CONFIG_PCI_PASID */542