552 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR MIT2 3/*4 * Xen frontend/backend page directory based shared buffer5 * helper module.6 *7 * Copyright (C) 2018 EPAM Systems Inc.8 *9 * Author: Oleksandr Andrushchenko <oleksandr_andrushchenko@epam.com>10 */11 12#include <linux/module.h>13#include <linux/errno.h>14#include <linux/mm.h>15 16#include <asm/xen/hypervisor.h>17#include <xen/balloon.h>18#include <xen/xen.h>19#include <xen/xenbus.h>20#include <xen/interface/io/ring.h>21 22#include <xen/xen-front-pgdir-shbuf.h>23 24/*25 * This structure represents the structure of a shared page26 * that contains grant references to the pages of the shared27 * buffer. This structure is common to many Xen para-virtualized28 * protocols at include/xen/interface/io/29 */30struct xen_page_directory {31 grant_ref_t gref_dir_next_page;32#define XEN_GREF_LIST_END 033 grant_ref_t gref[]; /* Variable length */34};35 36/*37 * Shared buffer ops which are differently implemented38 * depending on the allocation mode, e.g. if the buffer39 * is allocated by the corresponding backend or frontend.40 * Some of the operations.41 */42struct xen_front_pgdir_shbuf_ops {43 /*44 * Calculate number of grefs required to handle this buffer,45 * e.g. if grefs are required for page directory only or the buffer46 * pages as well.47 */48 void (*calc_num_grefs)(struct xen_front_pgdir_shbuf *buf);49 50 /* Fill page directory according to para-virtual display protocol. */51 void (*fill_page_dir)(struct xen_front_pgdir_shbuf *buf);52 53 /* Claim grant references for the pages of the buffer. */54 int (*grant_refs_for_buffer)(struct xen_front_pgdir_shbuf *buf,55 grant_ref_t *priv_gref_head, int gref_idx);56 57 /* Map grant references of the buffer. */58 int (*map)(struct xen_front_pgdir_shbuf *buf);59 60 /* Unmap grant references of the buffer. */61 int (*unmap)(struct xen_front_pgdir_shbuf *buf);62};63 64/*65 * Get granted reference to the very first page of the66 * page directory. Usually this is passed to the backend,67 * so it can find/fill the grant references to the buffer's68 * pages.69 *70 * \param buf shared buffer which page directory is of interest.71 * \return granted reference to the very first page of the72 * page directory.73 */74grant_ref_t75xen_front_pgdir_shbuf_get_dir_start(struct xen_front_pgdir_shbuf *buf)76{77 if (!buf->grefs)78 return INVALID_GRANT_REF;79 80 return buf->grefs[0];81}82EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_get_dir_start);83 84/*85 * Map granted references of the shared buffer.86 *87 * Depending on the shared buffer mode of allocation88 * (be_alloc flag) this can either do nothing (for buffers89 * shared by the frontend itself) or map the provided granted90 * references onto the backing storage (buf->pages).91 *92 * \param buf shared buffer which grants to be mapped.93 * \return zero on success or a negative number on failure.94 */95int xen_front_pgdir_shbuf_map(struct xen_front_pgdir_shbuf *buf)96{97 if (buf->ops && buf->ops->map)98 return buf->ops->map(buf);99 100 /* No need to map own grant references. */101 return 0;102}103EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_map);104 105/*106 * Unmap granted references of the shared buffer.107 *108 * Depending on the shared buffer mode of allocation109 * (be_alloc flag) this can either do nothing (for buffers110 * shared by the frontend itself) or unmap the provided granted111 * references.112 *113 * \param buf shared buffer which grants to be unmapped.114 * \return zero on success or a negative number on failure.115 */116int xen_front_pgdir_shbuf_unmap(struct xen_front_pgdir_shbuf *buf)117{118 if (buf->ops && buf->ops->unmap)119 return buf->ops->unmap(buf);120 121 /* No need to unmap own grant references. */122 return 0;123}124EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_unmap);125 126/*127 * Free all the resources of the shared buffer.128 *129 * \param buf shared buffer which resources to be freed.130 */131void xen_front_pgdir_shbuf_free(struct xen_front_pgdir_shbuf *buf)132{133 if (buf->grefs) {134 int i;135 136 for (i = 0; i < buf->num_grefs; i++)137 if (buf->grefs[i] != INVALID_GRANT_REF)138 gnttab_end_foreign_access(buf->grefs[i], NULL);139 }140 kfree(buf->grefs);141 kfree(buf->directory);142}143EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_free);144 145/*146 * Number of grefs a page can hold with respect to the147 * struct xen_page_directory header.148 */149#define XEN_NUM_GREFS_PER_PAGE ((PAGE_SIZE - \150 offsetof(struct xen_page_directory, \151 gref)) / sizeof(grant_ref_t))152 153/*154 * Get the number of pages the page directory consumes itself.155 *156 * \param buf shared buffer.157 */158static int get_num_pages_dir(struct xen_front_pgdir_shbuf *buf)159{160 return DIV_ROUND_UP(buf->num_pages, XEN_NUM_GREFS_PER_PAGE);161}162 163/*164 * Calculate the number of grant references needed to share the buffer165 * and its pages when backend allocates the buffer.166 *167 * \param buf shared buffer.168 */169static void backend_calc_num_grefs(struct xen_front_pgdir_shbuf *buf)170{171 /* Only for pages the page directory consumes itself. */172 buf->num_grefs = get_num_pages_dir(buf);173}174 175/*176 * Calculate the number of grant references needed to share the buffer177 * and its pages when frontend allocates the buffer.178 *179 * \param buf shared buffer.180 */181static void guest_calc_num_grefs(struct xen_front_pgdir_shbuf *buf)182{183 /*184 * Number of pages the page directory consumes itself185 * plus grefs for the buffer pages.186 */187 buf->num_grefs = get_num_pages_dir(buf) + buf->num_pages;188}189 190#define xen_page_to_vaddr(page) \191 ((uintptr_t)pfn_to_kaddr(page_to_xen_pfn(page)))192 193/*194 * Unmap the buffer previously mapped with grant references195 * provided by the backend.196 *197 * \param buf shared buffer.198 * \return zero on success or a negative number on failure.199 */200static int backend_unmap(struct xen_front_pgdir_shbuf *buf)201{202 struct gnttab_unmap_grant_ref *unmap_ops;203 int i, ret;204 205 if (!buf->pages || !buf->backend_map_handles || !buf->grefs)206 return 0;207 208 unmap_ops = kcalloc(buf->num_pages, sizeof(*unmap_ops),209 GFP_KERNEL);210 if (!unmap_ops)211 return -ENOMEM;212 213 for (i = 0; i < buf->num_pages; i++) {214 phys_addr_t addr;215 216 addr = xen_page_to_vaddr(buf->pages[i]);217 gnttab_set_unmap_op(&unmap_ops[i], addr, GNTMAP_host_map,218 buf->backend_map_handles[i]);219 }220 221 ret = gnttab_unmap_refs(unmap_ops, NULL, buf->pages,222 buf->num_pages);223 224 for (i = 0; i < buf->num_pages; i++) {225 if (unlikely(unmap_ops[i].status != GNTST_okay))226 dev_err(&buf->xb_dev->dev,227 "Failed to unmap page %d: %d\n",228 i, unmap_ops[i].status);229 }230 231 if (ret)232 dev_err(&buf->xb_dev->dev,233 "Failed to unmap grant references, ret %d", ret);234 235 kfree(unmap_ops);236 kfree(buf->backend_map_handles);237 buf->backend_map_handles = NULL;238 return ret;239}240 241/*242 * Map the buffer with grant references provided by the backend.243 *244 * \param buf shared buffer.245 * \return zero on success or a negative number on failure.246 */247static int backend_map(struct xen_front_pgdir_shbuf *buf)248{249 struct gnttab_map_grant_ref *map_ops = NULL;250 unsigned char *ptr;251 int ret, cur_gref, cur_dir_page, cur_page, grefs_left;252 253 map_ops = kcalloc(buf->num_pages, sizeof(*map_ops), GFP_KERNEL);254 if (!map_ops)255 return -ENOMEM;256 257 buf->backend_map_handles = kcalloc(buf->num_pages,258 sizeof(*buf->backend_map_handles),259 GFP_KERNEL);260 if (!buf->backend_map_handles) {261 kfree(map_ops);262 return -ENOMEM;263 }264 265 /*266 * Read page directory to get grefs from the backend: for external267 * buffer we only allocate buf->grefs for the page directory,268 * so buf->num_grefs has number of pages in the page directory itself.269 */270 ptr = buf->directory;271 grefs_left = buf->num_pages;272 cur_page = 0;273 for (cur_dir_page = 0; cur_dir_page < buf->num_grefs; cur_dir_page++) {274 struct xen_page_directory *page_dir =275 (struct xen_page_directory *)ptr;276 int to_copy = XEN_NUM_GREFS_PER_PAGE;277 278 if (to_copy > grefs_left)279 to_copy = grefs_left;280 281 for (cur_gref = 0; cur_gref < to_copy; cur_gref++) {282 phys_addr_t addr;283 284 addr = xen_page_to_vaddr(buf->pages[cur_page]);285 gnttab_set_map_op(&map_ops[cur_page], addr,286 GNTMAP_host_map,287 page_dir->gref[cur_gref],288 buf->xb_dev->otherend_id);289 cur_page++;290 }291 292 grefs_left -= to_copy;293 ptr += PAGE_SIZE;294 }295 ret = gnttab_map_refs(map_ops, NULL, buf->pages, buf->num_pages);296 297 /* Save handles even if error, so we can unmap. */298 for (cur_page = 0; cur_page < buf->num_pages; cur_page++) {299 if (likely(map_ops[cur_page].status == GNTST_okay)) {300 buf->backend_map_handles[cur_page] =301 map_ops[cur_page].handle;302 } else {303 buf->backend_map_handles[cur_page] =304 INVALID_GRANT_HANDLE;305 if (!ret)306 ret = -ENXIO;307 dev_err(&buf->xb_dev->dev,308 "Failed to map page %d: %d\n",309 cur_page, map_ops[cur_page].status);310 }311 }312 313 if (ret) {314 dev_err(&buf->xb_dev->dev,315 "Failed to map grant references, ret %d", ret);316 backend_unmap(buf);317 }318 319 kfree(map_ops);320 return ret;321}322 323/*324 * Fill page directory with grant references to the pages of the325 * page directory itself.326 *327 * The grant references to the buffer pages are provided by the328 * backend in this case.329 *330 * \param buf shared buffer.331 */332static void backend_fill_page_dir(struct xen_front_pgdir_shbuf *buf)333{334 struct xen_page_directory *page_dir;335 unsigned char *ptr;336 int i, num_pages_dir;337 338 ptr = buf->directory;339 num_pages_dir = get_num_pages_dir(buf);340 341 /* Fill only grefs for the page directory itself. */342 for (i = 0; i < num_pages_dir - 1; i++) {343 page_dir = (struct xen_page_directory *)ptr;344 345 page_dir->gref_dir_next_page = buf->grefs[i + 1];346 ptr += PAGE_SIZE;347 }348 /* Last page must say there is no more pages. */349 page_dir = (struct xen_page_directory *)ptr;350 page_dir->gref_dir_next_page = XEN_GREF_LIST_END;351}352 353/*354 * Fill page directory with grant references to the pages of the355 * page directory and the buffer we share with the backend.356 *357 * \param buf shared buffer.358 */359static void guest_fill_page_dir(struct xen_front_pgdir_shbuf *buf)360{361 unsigned char *ptr;362 int cur_gref, grefs_left, to_copy, i, num_pages_dir;363 364 ptr = buf->directory;365 num_pages_dir = get_num_pages_dir(buf);366 367 /*368 * While copying, skip grefs at start, they are for pages369 * granted for the page directory itself.370 */371 cur_gref = num_pages_dir;372 grefs_left = buf->num_pages;373 for (i = 0; i < num_pages_dir; i++) {374 struct xen_page_directory *page_dir =375 (struct xen_page_directory *)ptr;376 377 if (grefs_left <= XEN_NUM_GREFS_PER_PAGE) {378 to_copy = grefs_left;379 page_dir->gref_dir_next_page = XEN_GREF_LIST_END;380 } else {381 to_copy = XEN_NUM_GREFS_PER_PAGE;382 page_dir->gref_dir_next_page = buf->grefs[i + 1];383 }384 memcpy(&page_dir->gref, &buf->grefs[cur_gref],385 to_copy * sizeof(grant_ref_t));386 ptr += PAGE_SIZE;387 grefs_left -= to_copy;388 cur_gref += to_copy;389 }390}391 392/*393 * Grant references to the frontend's buffer pages.394 *395 * These will be shared with the backend, so it can396 * access the buffer's data.397 *398 * \param buf shared buffer.399 * \return zero on success or a negative number on failure.400 */401static int guest_grant_refs_for_buffer(struct xen_front_pgdir_shbuf *buf,402 grant_ref_t *priv_gref_head,403 int gref_idx)404{405 int i, cur_ref, otherend_id;406 407 otherend_id = buf->xb_dev->otherend_id;408 for (i = 0; i < buf->num_pages; i++) {409 cur_ref = gnttab_claim_grant_reference(priv_gref_head);410 if (cur_ref < 0)411 return cur_ref;412 413 gnttab_grant_foreign_access_ref(cur_ref, otherend_id,414 xen_page_to_gfn(buf->pages[i]),415 0);416 buf->grefs[gref_idx++] = cur_ref;417 }418 return 0;419}420 421/*422 * Grant all the references needed to share the buffer.423 *424 * Grant references to the page directory pages and, if425 * needed, also to the pages of the shared buffer data.426 *427 * \param buf shared buffer.428 * \return zero on success or a negative number on failure.429 */430static int grant_references(struct xen_front_pgdir_shbuf *buf)431{432 grant_ref_t priv_gref_head;433 int ret, i, j, cur_ref;434 int otherend_id, num_pages_dir;435 436 ret = gnttab_alloc_grant_references(buf->num_grefs, &priv_gref_head);437 if (ret < 0) {438 dev_err(&buf->xb_dev->dev,439 "Cannot allocate grant references\n");440 return ret;441 }442 443 otherend_id = buf->xb_dev->otherend_id;444 j = 0;445 num_pages_dir = get_num_pages_dir(buf);446 for (i = 0; i < num_pages_dir; i++) {447 unsigned long frame;448 449 cur_ref = gnttab_claim_grant_reference(&priv_gref_head);450 if (cur_ref < 0)451 return cur_ref;452 453 frame = xen_page_to_gfn(virt_to_page(buf->directory +454 PAGE_SIZE * i));455 gnttab_grant_foreign_access_ref(cur_ref, otherend_id, frame, 0);456 buf->grefs[j++] = cur_ref;457 }458 459 if (buf->ops->grant_refs_for_buffer) {460 ret = buf->ops->grant_refs_for_buffer(buf, &priv_gref_head, j);461 if (ret)462 return ret;463 }464 465 gnttab_free_grant_references(priv_gref_head);466 return 0;467}468 469/*470 * Allocate all required structures to mange shared buffer.471 *472 * \param buf shared buffer.473 * \return zero on success or a negative number on failure.474 */475static int alloc_storage(struct xen_front_pgdir_shbuf *buf)476{477 buf->grefs = kcalloc(buf->num_grefs, sizeof(*buf->grefs), GFP_KERNEL);478 if (!buf->grefs)479 return -ENOMEM;480 481 buf->directory = kcalloc(get_num_pages_dir(buf), PAGE_SIZE, GFP_KERNEL);482 if (!buf->directory)483 return -ENOMEM;484 485 return 0;486}487 488/*489 * For backend allocated buffers we don't need grant_refs_for_buffer490 * as those grant references are allocated at backend side.491 */492static const struct xen_front_pgdir_shbuf_ops backend_ops = {493 .calc_num_grefs = backend_calc_num_grefs,494 .fill_page_dir = backend_fill_page_dir,495 .map = backend_map,496 .unmap = backend_unmap497};498 499/*500 * For locally granted references we do not need to map/unmap501 * the references.502 */503static const struct xen_front_pgdir_shbuf_ops local_ops = {504 .calc_num_grefs = guest_calc_num_grefs,505 .fill_page_dir = guest_fill_page_dir,506 .grant_refs_for_buffer = guest_grant_refs_for_buffer,507};508 509/*510 * Allocate a new instance of a shared buffer.511 *512 * \param cfg configuration to be used while allocating a new shared buffer.513 * \return zero on success or a negative number on failure.514 */515int xen_front_pgdir_shbuf_alloc(struct xen_front_pgdir_shbuf_cfg *cfg)516{517 struct xen_front_pgdir_shbuf *buf = cfg->pgdir;518 int ret;519 520 if (cfg->be_alloc)521 buf->ops = &backend_ops;522 else523 buf->ops = &local_ops;524 buf->xb_dev = cfg->xb_dev;525 buf->num_pages = cfg->num_pages;526 buf->pages = cfg->pages;527 528 buf->ops->calc_num_grefs(buf);529 530 ret = alloc_storage(buf);531 if (ret)532 goto fail;533 534 ret = grant_references(buf);535 if (ret)536 goto fail;537 538 buf->ops->fill_page_dir(buf);539 540 return 0;541 542fail:543 xen_front_pgdir_shbuf_free(buf);544 return ret;545}546EXPORT_SYMBOL_GPL(xen_front_pgdir_shbuf_alloc);547 548MODULE_DESCRIPTION("Xen frontend/backend page directory based "549 "shared buffer handling");550MODULE_AUTHOR("Oleksandr Andrushchenko");551MODULE_LICENSE("GPL");552