419 lines · c
1// SPDX-License-Identifier: GPL-2.0-only2/*3 * Xen grant DMA-mapping layer - contains special DMA-mapping routines4 * for providing grant references as DMA addresses to be used by frontends5 * (e.g. virtio) in Xen guests6 *7 * Copyright (c) 2021, Juergen Gross <jgross@suse.com>8 */9 10#include <linux/module.h>11#include <linux/dma-map-ops.h>12#include <linux/of.h>13#include <linux/pci.h>14#include <linux/pfn.h>15#include <linux/xarray.h>16#include <linux/virtio_anchor.h>17#include <linux/virtio.h>18#include <xen/xen.h>19#include <xen/xen-ops.h>20#include <xen/grant_table.h>21 22struct xen_grant_dma_data {23 /* The ID of backend domain */24 domid_t backend_domid;25 /* Is device behaving sane? */26 bool broken;27};28 29static DEFINE_XARRAY_FLAGS(xen_grant_dma_devices, XA_FLAGS_LOCK_IRQ);30 31#define XEN_GRANT_DMA_ADDR_OFF (1ULL << 63)32 33static inline dma_addr_t grant_to_dma(grant_ref_t grant)34{35 return XEN_GRANT_DMA_ADDR_OFF | ((dma_addr_t)grant << XEN_PAGE_SHIFT);36}37 38static inline grant_ref_t dma_to_grant(dma_addr_t dma)39{40 return (grant_ref_t)((dma & ~XEN_GRANT_DMA_ADDR_OFF) >> XEN_PAGE_SHIFT);41}42 43static struct xen_grant_dma_data *find_xen_grant_dma_data(struct device *dev)44{45 struct xen_grant_dma_data *data;46 unsigned long flags;47 48 xa_lock_irqsave(&xen_grant_dma_devices, flags);49 data = xa_load(&xen_grant_dma_devices, (unsigned long)dev);50 xa_unlock_irqrestore(&xen_grant_dma_devices, flags);51 52 return data;53}54 55static int store_xen_grant_dma_data(struct device *dev,56 struct xen_grant_dma_data *data)57{58 unsigned long flags;59 int ret;60 61 xa_lock_irqsave(&xen_grant_dma_devices, flags);62 ret = xa_err(__xa_store(&xen_grant_dma_devices, (unsigned long)dev, data,63 GFP_ATOMIC));64 xa_unlock_irqrestore(&xen_grant_dma_devices, flags);65 66 return ret;67}68 69/*70 * DMA ops for Xen frontends (e.g. virtio).71 *72 * Used to act as a kind of software IOMMU for Xen guests by using grants as73 * DMA addresses.74 * Such a DMA address is formed by using the grant reference as a frame75 * number and setting the highest address bit (this bit is for the backend76 * to be able to distinguish it from e.g. a mmio address).77 */78static void *xen_grant_dma_alloc(struct device *dev, size_t size,79 dma_addr_t *dma_handle, gfp_t gfp,80 unsigned long attrs)81{82 struct xen_grant_dma_data *data;83 unsigned int i, n_pages = XEN_PFN_UP(size);84 unsigned long pfn;85 grant_ref_t grant;86 void *ret;87 88 data = find_xen_grant_dma_data(dev);89 if (!data)90 return NULL;91 92 if (unlikely(data->broken))93 return NULL;94 95 ret = alloc_pages_exact(n_pages * XEN_PAGE_SIZE, gfp);96 if (!ret)97 return NULL;98 99 pfn = virt_to_pfn(ret);100 101 if (gnttab_alloc_grant_reference_seq(n_pages, &grant)) {102 free_pages_exact(ret, n_pages * XEN_PAGE_SIZE);103 return NULL;104 }105 106 for (i = 0; i < n_pages; i++) {107 gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,108 pfn_to_gfn(pfn + i), 0);109 }110 111 *dma_handle = grant_to_dma(grant);112 113 return ret;114}115 116static void xen_grant_dma_free(struct device *dev, size_t size, void *vaddr,117 dma_addr_t dma_handle, unsigned long attrs)118{119 struct xen_grant_dma_data *data;120 unsigned int i, n_pages = XEN_PFN_UP(size);121 grant_ref_t grant;122 123 data = find_xen_grant_dma_data(dev);124 if (!data)125 return;126 127 if (unlikely(data->broken))128 return;129 130 grant = dma_to_grant(dma_handle);131 132 for (i = 0; i < n_pages; i++) {133 if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {134 dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");135 data->broken = true;136 return;137 }138 }139 140 gnttab_free_grant_reference_seq(grant, n_pages);141 142 free_pages_exact(vaddr, n_pages * XEN_PAGE_SIZE);143}144 145static struct page *xen_grant_dma_alloc_pages(struct device *dev, size_t size,146 dma_addr_t *dma_handle,147 enum dma_data_direction dir,148 gfp_t gfp)149{150 void *vaddr;151 152 vaddr = xen_grant_dma_alloc(dev, size, dma_handle, gfp, 0);153 if (!vaddr)154 return NULL;155 156 return virt_to_page(vaddr);157}158 159static void xen_grant_dma_free_pages(struct device *dev, size_t size,160 struct page *vaddr, dma_addr_t dma_handle,161 enum dma_data_direction dir)162{163 xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, 0);164}165 166static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page,167 unsigned long offset, size_t size,168 enum dma_data_direction dir,169 unsigned long attrs)170{171 struct xen_grant_dma_data *data;172 unsigned long dma_offset = xen_offset_in_page(offset),173 pfn_offset = XEN_PFN_DOWN(offset);174 unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);175 grant_ref_t grant;176 dma_addr_t dma_handle;177 178 if (WARN_ON(dir == DMA_NONE))179 return DMA_MAPPING_ERROR;180 181 data = find_xen_grant_dma_data(dev);182 if (!data)183 return DMA_MAPPING_ERROR;184 185 if (unlikely(data->broken))186 return DMA_MAPPING_ERROR;187 188 if (gnttab_alloc_grant_reference_seq(n_pages, &grant))189 return DMA_MAPPING_ERROR;190 191 for (i = 0; i < n_pages; i++) {192 gnttab_grant_foreign_access_ref(grant + i, data->backend_domid,193 pfn_to_gfn(page_to_xen_pfn(page) + i + pfn_offset),194 dir == DMA_TO_DEVICE);195 }196 197 dma_handle = grant_to_dma(grant) + dma_offset;198 199 return dma_handle;200}201 202static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,203 size_t size, enum dma_data_direction dir,204 unsigned long attrs)205{206 struct xen_grant_dma_data *data;207 unsigned long dma_offset = xen_offset_in_page(dma_handle);208 unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size);209 grant_ref_t grant;210 211 if (WARN_ON(dir == DMA_NONE))212 return;213 214 data = find_xen_grant_dma_data(dev);215 if (!data)216 return;217 218 if (unlikely(data->broken))219 return;220 221 grant = dma_to_grant(dma_handle);222 223 for (i = 0; i < n_pages; i++) {224 if (unlikely(!gnttab_end_foreign_access_ref(grant + i))) {225 dev_alert(dev, "Grant still in use by backend domain, disabled for further use\n");226 data->broken = true;227 return;228 }229 }230 231 gnttab_free_grant_reference_seq(grant, n_pages);232}233 234static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg,235 int nents, enum dma_data_direction dir,236 unsigned long attrs)237{238 struct scatterlist *s;239 unsigned int i;240 241 if (WARN_ON(dir == DMA_NONE))242 return;243 244 for_each_sg(sg, s, nents, i)245 xen_grant_dma_unmap_page(dev, s->dma_address, sg_dma_len(s), dir,246 attrs);247}248 249static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg,250 int nents, enum dma_data_direction dir,251 unsigned long attrs)252{253 struct scatterlist *s;254 unsigned int i;255 256 if (WARN_ON(dir == DMA_NONE))257 return -EINVAL;258 259 for_each_sg(sg, s, nents, i) {260 s->dma_address = xen_grant_dma_map_page(dev, sg_page(s), s->offset,261 s->length, dir, attrs);262 if (s->dma_address == DMA_MAPPING_ERROR)263 goto out;264 265 sg_dma_len(s) = s->length;266 }267 268 return nents;269 270out:271 xen_grant_dma_unmap_sg(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);272 sg_dma_len(sg) = 0;273 274 return -EIO;275}276 277static int xen_grant_dma_supported(struct device *dev, u64 mask)278{279 return mask == DMA_BIT_MASK(64);280}281 282static const struct dma_map_ops xen_grant_dma_ops = {283 .alloc = xen_grant_dma_alloc,284 .free = xen_grant_dma_free,285 .alloc_pages_op = xen_grant_dma_alloc_pages,286 .free_pages = xen_grant_dma_free_pages,287 .mmap = dma_common_mmap,288 .get_sgtable = dma_common_get_sgtable,289 .map_page = xen_grant_dma_map_page,290 .unmap_page = xen_grant_dma_unmap_page,291 .map_sg = xen_grant_dma_map_sg,292 .unmap_sg = xen_grant_dma_unmap_sg,293 .dma_supported = xen_grant_dma_supported,294};295 296static struct device_node *xen_dt_get_node(struct device *dev)297{298 if (dev_is_pci(dev)) {299 struct pci_dev *pdev = to_pci_dev(dev);300 struct pci_bus *bus = pdev->bus;301 302 /* Walk up to the root bus to look for PCI Host controller */303 while (!pci_is_root_bus(bus))304 bus = bus->parent;305 306 if (!bus->bridge->parent)307 return NULL;308 return of_node_get(bus->bridge->parent->of_node);309 }310 311 return of_node_get(dev->of_node);312}313 314static int xen_dt_grant_init_backend_domid(struct device *dev,315 struct device_node *np,316 domid_t *backend_domid)317{318 struct of_phandle_args iommu_spec = { .args_count = 1 };319 320 if (dev_is_pci(dev)) {321 struct pci_dev *pdev = to_pci_dev(dev);322 u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);323 324 if (of_map_id(np, rid, "iommu-map", "iommu-map-mask", &iommu_spec.np,325 iommu_spec.args)) {326 dev_dbg(dev, "Cannot translate ID\n");327 return -ESRCH;328 }329 } else {330 if (of_parse_phandle_with_args(np, "iommus", "#iommu-cells",331 0, &iommu_spec)) {332 dev_dbg(dev, "Cannot parse iommus property\n");333 return -ESRCH;334 }335 }336 337 if (!of_device_is_compatible(iommu_spec.np, "xen,grant-dma") ||338 iommu_spec.args_count != 1) {339 dev_dbg(dev, "Incompatible IOMMU node\n");340 of_node_put(iommu_spec.np);341 return -ESRCH;342 }343 344 of_node_put(iommu_spec.np);345 346 /*347 * The endpoint ID here means the ID of the domain where the348 * corresponding backend is running349 */350 *backend_domid = iommu_spec.args[0];351 352 return 0;353}354 355static int xen_grant_init_backend_domid(struct device *dev,356 domid_t *backend_domid)357{358 struct device_node *np;359 int ret = -ENODEV;360 361 np = xen_dt_get_node(dev);362 if (np) {363 ret = xen_dt_grant_init_backend_domid(dev, np, backend_domid);364 of_node_put(np);365 } else if (IS_ENABLED(CONFIG_XEN_VIRTIO_FORCE_GRANT) || xen_pv_domain()) {366 dev_info(dev, "Using dom0 as backend\n");367 *backend_domid = 0;368 ret = 0;369 }370 371 return ret;372}373 374static void xen_grant_setup_dma_ops(struct device *dev, domid_t backend_domid)375{376 struct xen_grant_dma_data *data;377 378 data = find_xen_grant_dma_data(dev);379 if (data) {380 dev_err(dev, "Xen grant DMA data is already created\n");381 return;382 }383 384 data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);385 if (!data)386 goto err;387 388 data->backend_domid = backend_domid;389 390 if (store_xen_grant_dma_data(dev, data)) {391 dev_err(dev, "Cannot store Xen grant DMA data\n");392 goto err;393 }394 395 dev->dma_ops = &xen_grant_dma_ops;396 397 return;398 399err:400 devm_kfree(dev, data);401 dev_err(dev, "Cannot set up Xen grant DMA ops, retain platform DMA ops\n");402}403 404bool xen_virtio_restricted_mem_acc(struct virtio_device *dev)405{406 domid_t backend_domid;407 408 if (!xen_grant_init_backend_domid(dev->dev.parent, &backend_domid)) {409 xen_grant_setup_dma_ops(dev->dev.parent, backend_domid);410 return true;411 }412 413 return false;414}415 416MODULE_DESCRIPTION("Xen grant DMA-mapping layer");417MODULE_AUTHOR("Juergen Gross <jgross@suse.com>");418MODULE_LICENSE("GPL");419