1784 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright 2020-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.4 */5 6/**7 * DOC: Enclave lifetime management driver for Nitro Enclaves (NE).8 * Nitro is a hypervisor that has been developed by Amazon.9 */10 11#include <linux/anon_inodes.h>12#include <linux/capability.h>13#include <linux/cpu.h>14#include <linux/device.h>15#include <linux/file.h>16#include <linux/hugetlb.h>17#include <linux/limits.h>18#include <linux/list.h>19#include <linux/miscdevice.h>20#include <linux/mm.h>21#include <linux/mman.h>22#include <linux/module.h>23#include <linux/mutex.h>24#include <linux/nitro_enclaves.h>25#include <linux/pci.h>26#include <linux/poll.h>27#include <linux/range.h>28#include <linux/slab.h>29#include <linux/types.h>30#include <uapi/linux/vm_sockets.h>31 32#include "ne_misc_dev.h"33#include "ne_pci_dev.h"34 35/**36 * NE_CPUS_SIZE - Size for max 128 CPUs, for now, in a cpu-list string, comma37 * separated. The NE CPU pool includes CPUs from a single NUMA38 * node.39 */40#define NE_CPUS_SIZE (512)41 42/**43 * NE_EIF_LOAD_OFFSET - The offset where to copy the Enclave Image Format (EIF)44 * image in enclave memory.45 */46#define NE_EIF_LOAD_OFFSET (8 * 1024UL * 1024UL)47 48/**49 * NE_MIN_ENCLAVE_MEM_SIZE - The minimum memory size an enclave can be launched50 * with.51 */52#define NE_MIN_ENCLAVE_MEM_SIZE (64 * 1024UL * 1024UL)53 54/**55 * NE_MIN_MEM_REGION_SIZE - The minimum size of an enclave memory region.56 */57#define NE_MIN_MEM_REGION_SIZE (2 * 1024UL * 1024UL)58 59/**60 * NE_PARENT_VM_CID - The CID for the vsock device of the primary / parent VM.61 */62#define NE_PARENT_VM_CID (3)63 64static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg);65 66static const struct file_operations ne_fops = {67 .owner = THIS_MODULE,68 .llseek = noop_llseek,69 .unlocked_ioctl = ne_ioctl,70};71 72static struct miscdevice ne_misc_dev = {73 .minor = MISC_DYNAMIC_MINOR,74 .name = "nitro_enclaves",75 .fops = &ne_fops,76 .mode = 0660,77};78 79struct ne_devs ne_devs = {80 .ne_misc_dev = &ne_misc_dev,81};82 83/*84 * TODO: Update logic to create new sysfs entries instead of using85 * a kernel parameter e.g. if multiple sysfs files needed.86 */87static int ne_set_kernel_param(const char *val, const struct kernel_param *kp);88 89static const struct kernel_param_ops ne_cpu_pool_ops = {90 .get = param_get_string,91 .set = ne_set_kernel_param,92};93 94static char ne_cpus[NE_CPUS_SIZE];95static struct kparam_string ne_cpus_arg = {96 .maxlen = sizeof(ne_cpus),97 .string = ne_cpus,98};99 100module_param_cb(ne_cpus, &ne_cpu_pool_ops, &ne_cpus_arg, 0644);101/* https://www.kernel.org/doc/html/latest/admin-guide/kernel-parameters.html#cpu-lists */102MODULE_PARM_DESC(ne_cpus, "<cpu-list> - CPU pool used for Nitro Enclaves");103 104/**105 * struct ne_cpu_pool - CPU pool used for Nitro Enclaves.106 * @avail_threads_per_core: Available full CPU cores to be dedicated to107 * enclave(s). The cpumasks from the array, indexed108 * by core id, contain all the threads from the109 * available cores, that are not set for created110 * enclave(s). The full CPU cores are part of the111 * NE CPU pool.112 * @mutex: Mutex for the access to the NE CPU pool.113 * @nr_parent_vm_cores : The size of the available threads per core array.114 * The total number of CPU cores available on the115 * primary / parent VM.116 * @nr_threads_per_core: The number of threads that a full CPU core has.117 * @numa_node: NUMA node of the CPUs in the pool.118 */119struct ne_cpu_pool {120 cpumask_var_t *avail_threads_per_core;121 struct mutex mutex;122 unsigned int nr_parent_vm_cores;123 unsigned int nr_threads_per_core;124 int numa_node;125};126 127static struct ne_cpu_pool ne_cpu_pool;128 129/**130 * struct ne_phys_contig_mem_regions - Contiguous physical memory regions.131 * @num: The number of regions that currently has.132 * @regions: The array of physical memory regions.133 */134struct ne_phys_contig_mem_regions {135 unsigned long num;136 struct range *regions;137};138 139/**140 * ne_check_enclaves_created() - Verify if at least one enclave has been created.141 * @void: No parameters provided.142 *143 * Context: Process context.144 * Return:145 * * True if at least one enclave is created.146 * * False otherwise.147 */148static bool ne_check_enclaves_created(void)149{150 struct ne_pci_dev *ne_pci_dev = ne_devs.ne_pci_dev;151 bool ret = false;152 153 if (!ne_pci_dev)154 return ret;155 156 mutex_lock(&ne_pci_dev->enclaves_list_mutex);157 158 if (!list_empty(&ne_pci_dev->enclaves_list))159 ret = true;160 161 mutex_unlock(&ne_pci_dev->enclaves_list_mutex);162 163 return ret;164}165 166/**167 * ne_setup_cpu_pool() - Set the NE CPU pool after handling sanity checks such168 * as not sharing CPU cores with the primary / parent VM169 * or not using CPU 0, which should remain available for170 * the primary / parent VM. Offline the CPUs from the171 * pool after the checks passed.172 * @ne_cpu_list: The CPU list used for setting NE CPU pool.173 *174 * Context: Process context.175 * Return:176 * * 0 on success.177 * * Negative return value on failure.178 */179static int ne_setup_cpu_pool(const char *ne_cpu_list)180{181 int core_id = -1;182 unsigned int cpu = 0;183 cpumask_var_t cpu_pool;184 unsigned int cpu_sibling = 0;185 unsigned int i = 0;186 int numa_node = -1;187 int rc = -EINVAL;188 189 if (!zalloc_cpumask_var(&cpu_pool, GFP_KERNEL))190 return -ENOMEM;191 192 mutex_lock(&ne_cpu_pool.mutex);193 194 rc = cpulist_parse(ne_cpu_list, cpu_pool);195 if (rc < 0) {196 pr_err("%s: Error in cpulist parse [rc=%d]\n", ne_misc_dev.name, rc);197 198 goto free_pool_cpumask;199 }200 201 cpu = cpumask_any(cpu_pool);202 if (cpu >= nr_cpu_ids) {203 pr_err("%s: No CPUs available in CPU pool\n", ne_misc_dev.name);204 205 rc = -EINVAL;206 207 goto free_pool_cpumask;208 }209 210 /*211 * Check if the CPUs are online, to further get info about them212 * e.g. numa node, core id, siblings.213 */214 for_each_cpu(cpu, cpu_pool)215 if (cpu_is_offline(cpu)) {216 pr_err("%s: CPU %d is offline, has to be online to get its metadata\n",217 ne_misc_dev.name, cpu);218 219 rc = -EINVAL;220 221 goto free_pool_cpumask;222 }223 224 /*225 * Check if the CPUs from the NE CPU pool are from the same NUMA node.226 */227 for_each_cpu(cpu, cpu_pool)228 if (numa_node < 0) {229 numa_node = cpu_to_node(cpu);230 if (numa_node < 0) {231 pr_err("%s: Invalid NUMA node %d\n",232 ne_misc_dev.name, numa_node);233 234 rc = -EINVAL;235 236 goto free_pool_cpumask;237 }238 } else {239 if (numa_node != cpu_to_node(cpu)) {240 pr_err("%s: CPUs with different NUMA nodes\n",241 ne_misc_dev.name);242 243 rc = -EINVAL;244 245 goto free_pool_cpumask;246 }247 }248 249 /*250 * Check if CPU 0 and its siblings are included in the provided CPU pool251 * They should remain available for the primary / parent VM.252 */253 if (cpumask_test_cpu(0, cpu_pool)) {254 pr_err("%s: CPU 0 has to remain available\n", ne_misc_dev.name);255 256 rc = -EINVAL;257 258 goto free_pool_cpumask;259 }260 261 for_each_cpu(cpu_sibling, topology_sibling_cpumask(0)) {262 if (cpumask_test_cpu(cpu_sibling, cpu_pool)) {263 pr_err("%s: CPU sibling %d for CPU 0 is in CPU pool\n",264 ne_misc_dev.name, cpu_sibling);265 266 rc = -EINVAL;267 268 goto free_pool_cpumask;269 }270 }271 272 /*273 * Check if CPU siblings are included in the provided CPU pool. The274 * expectation is that full CPU cores are made available in the CPU pool275 * for enclaves.276 */277 for_each_cpu(cpu, cpu_pool) {278 for_each_cpu(cpu_sibling, topology_sibling_cpumask(cpu)) {279 if (!cpumask_test_cpu(cpu_sibling, cpu_pool)) {280 pr_err("%s: CPU %d is not in CPU pool\n",281 ne_misc_dev.name, cpu_sibling);282 283 rc = -EINVAL;284 285 goto free_pool_cpumask;286 }287 }288 }289 290 /* Calculate the number of threads from a full CPU core. */291 cpu = cpumask_any(cpu_pool);292 for_each_cpu(cpu_sibling, topology_sibling_cpumask(cpu))293 ne_cpu_pool.nr_threads_per_core++;294 295 ne_cpu_pool.nr_parent_vm_cores = nr_cpu_ids / ne_cpu_pool.nr_threads_per_core;296 297 ne_cpu_pool.avail_threads_per_core = kcalloc(ne_cpu_pool.nr_parent_vm_cores,298 sizeof(*ne_cpu_pool.avail_threads_per_core),299 GFP_KERNEL);300 if (!ne_cpu_pool.avail_threads_per_core) {301 rc = -ENOMEM;302 303 goto free_pool_cpumask;304 }305 306 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++)307 if (!zalloc_cpumask_var(&ne_cpu_pool.avail_threads_per_core[i], GFP_KERNEL)) {308 rc = -ENOMEM;309 310 goto free_cores_cpumask;311 }312 313 /*314 * Split the NE CPU pool in threads per core to keep the CPU topology315 * after offlining the CPUs.316 */317 for_each_cpu(cpu, cpu_pool) {318 core_id = topology_core_id(cpu);319 if (core_id < 0 || core_id >= ne_cpu_pool.nr_parent_vm_cores) {320 pr_err("%s: Invalid core id %d for CPU %d\n",321 ne_misc_dev.name, core_id, cpu);322 323 rc = -EINVAL;324 325 goto clear_cpumask;326 }327 328 cpumask_set_cpu(cpu, ne_cpu_pool.avail_threads_per_core[core_id]);329 }330 331 /*332 * CPUs that are given to enclave(s) should not be considered online333 * by Linux anymore, as the hypervisor will degrade them to floating.334 * The physical CPUs (full cores) are carved out of the primary / parent335 * VM and given to the enclave VM. The same number of vCPUs would run336 * on less pCPUs for the primary / parent VM.337 *338 * We offline them here, to not degrade performance and expose correct339 * topology to Linux and user space.340 */341 for_each_cpu(cpu, cpu_pool) {342 rc = remove_cpu(cpu);343 if (rc != 0) {344 pr_err("%s: CPU %d is not offlined [rc=%d]\n",345 ne_misc_dev.name, cpu, rc);346 347 goto online_cpus;348 }349 }350 351 free_cpumask_var(cpu_pool);352 353 ne_cpu_pool.numa_node = numa_node;354 355 mutex_unlock(&ne_cpu_pool.mutex);356 357 return 0;358 359online_cpus:360 for_each_cpu(cpu, cpu_pool)361 add_cpu(cpu);362clear_cpumask:363 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++)364 cpumask_clear(ne_cpu_pool.avail_threads_per_core[i]);365free_cores_cpumask:366 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++)367 free_cpumask_var(ne_cpu_pool.avail_threads_per_core[i]);368 kfree(ne_cpu_pool.avail_threads_per_core);369free_pool_cpumask:370 free_cpumask_var(cpu_pool);371 ne_cpu_pool.nr_parent_vm_cores = 0;372 ne_cpu_pool.nr_threads_per_core = 0;373 ne_cpu_pool.numa_node = -1;374 mutex_unlock(&ne_cpu_pool.mutex);375 376 return rc;377}378 379/**380 * ne_teardown_cpu_pool() - Online the CPUs from the NE CPU pool and cleanup the381 * CPU pool.382 * @void: No parameters provided.383 *384 * Context: Process context.385 */386static void ne_teardown_cpu_pool(void)387{388 unsigned int cpu = 0;389 unsigned int i = 0;390 int rc = -EINVAL;391 392 mutex_lock(&ne_cpu_pool.mutex);393 394 if (!ne_cpu_pool.nr_parent_vm_cores) {395 mutex_unlock(&ne_cpu_pool.mutex);396 397 return;398 }399 400 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++) {401 for_each_cpu(cpu, ne_cpu_pool.avail_threads_per_core[i]) {402 rc = add_cpu(cpu);403 if (rc != 0)404 pr_err("%s: CPU %d is not onlined [rc=%d]\n",405 ne_misc_dev.name, cpu, rc);406 }407 408 cpumask_clear(ne_cpu_pool.avail_threads_per_core[i]);409 410 free_cpumask_var(ne_cpu_pool.avail_threads_per_core[i]);411 }412 413 kfree(ne_cpu_pool.avail_threads_per_core);414 ne_cpu_pool.nr_parent_vm_cores = 0;415 ne_cpu_pool.nr_threads_per_core = 0;416 ne_cpu_pool.numa_node = -1;417 418 mutex_unlock(&ne_cpu_pool.mutex);419}420 421/**422 * ne_set_kernel_param() - Set the NE CPU pool value via the NE kernel parameter.423 * @val: NE CPU pool string value.424 * @kp : NE kernel parameter associated with the NE CPU pool.425 *426 * Context: Process context.427 * Return:428 * * 0 on success.429 * * Negative return value on failure.430 */431static int ne_set_kernel_param(const char *val, const struct kernel_param *kp)432{433 char error_val[] = "";434 int rc = -EINVAL;435 436 if (!capable(CAP_SYS_ADMIN))437 return -EPERM;438 439 if (ne_check_enclaves_created()) {440 pr_err("%s: The CPU pool is used by enclave(s)\n", ne_misc_dev.name);441 442 return -EPERM;443 }444 445 ne_teardown_cpu_pool();446 447 rc = ne_setup_cpu_pool(val);448 if (rc < 0) {449 pr_err("%s: Error in setup CPU pool [rc=%d]\n", ne_misc_dev.name, rc);450 451 param_set_copystring(error_val, kp);452 453 return rc;454 }455 456 rc = param_set_copystring(val, kp);457 if (rc < 0) {458 pr_err("%s: Error in param set copystring [rc=%d]\n", ne_misc_dev.name, rc);459 460 ne_teardown_cpu_pool();461 462 param_set_copystring(error_val, kp);463 464 return rc;465 }466 467 return 0;468}469 470/**471 * ne_donated_cpu() - Check if the provided CPU is already used by the enclave.472 * @ne_enclave : Private data associated with the current enclave.473 * @cpu: CPU to check if already used.474 *475 * Context: Process context. This function is called with the ne_enclave mutex held.476 * Return:477 * * True if the provided CPU is already used by the enclave.478 * * False otherwise.479 */480static bool ne_donated_cpu(struct ne_enclave *ne_enclave, unsigned int cpu)481{482 if (cpumask_test_cpu(cpu, ne_enclave->vcpu_ids))483 return true;484 485 return false;486}487 488/**489 * ne_get_unused_core_from_cpu_pool() - Get the id of a full core from the490 * NE CPU pool.491 * @void: No parameters provided.492 *493 * Context: Process context. This function is called with the ne_enclave and494 * ne_cpu_pool mutexes held.495 * Return:496 * * Core id.497 * * -1 if no CPU core available in the pool.498 */499static int ne_get_unused_core_from_cpu_pool(void)500{501 int core_id = -1;502 unsigned int i = 0;503 504 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++)505 if (!cpumask_empty(ne_cpu_pool.avail_threads_per_core[i])) {506 core_id = i;507 508 break;509 }510 511 return core_id;512}513 514/**515 * ne_set_enclave_threads_per_core() - Set the threads of the provided core in516 * the enclave data structure.517 * @ne_enclave : Private data associated with the current enclave.518 * @core_id: Core id to get its threads from the NE CPU pool.519 * @vcpu_id: vCPU id part of the provided core.520 *521 * Context: Process context. This function is called with the ne_enclave and522 * ne_cpu_pool mutexes held.523 * Return:524 * * 0 on success.525 * * Negative return value on failure.526 */527static int ne_set_enclave_threads_per_core(struct ne_enclave *ne_enclave,528 int core_id, u32 vcpu_id)529{530 unsigned int cpu = 0;531 532 if (core_id < 0 && vcpu_id == 0) {533 dev_err_ratelimited(ne_misc_dev.this_device,534 "No CPUs available in NE CPU pool\n");535 536 return -NE_ERR_NO_CPUS_AVAIL_IN_POOL;537 }538 539 if (core_id < 0) {540 dev_err_ratelimited(ne_misc_dev.this_device,541 "CPU %d is not in NE CPU pool\n", vcpu_id);542 543 return -NE_ERR_VCPU_NOT_IN_CPU_POOL;544 }545 546 if (core_id >= ne_enclave->nr_parent_vm_cores) {547 dev_err_ratelimited(ne_misc_dev.this_device,548 "Invalid core id %d - ne_enclave\n", core_id);549 550 return -NE_ERR_VCPU_INVALID_CPU_CORE;551 }552 553 for_each_cpu(cpu, ne_cpu_pool.avail_threads_per_core[core_id])554 cpumask_set_cpu(cpu, ne_enclave->threads_per_core[core_id]);555 556 cpumask_clear(ne_cpu_pool.avail_threads_per_core[core_id]);557 558 return 0;559}560 561/**562 * ne_get_cpu_from_cpu_pool() - Get a CPU from the NE CPU pool, either from the563 * remaining sibling(s) of a CPU core or the first564 * sibling of a new CPU core.565 * @ne_enclave : Private data associated with the current enclave.566 * @vcpu_id: vCPU to get from the NE CPU pool.567 *568 * Context: Process context. This function is called with the ne_enclave mutex held.569 * Return:570 * * 0 on success.571 * * Negative return value on failure.572 */573static int ne_get_cpu_from_cpu_pool(struct ne_enclave *ne_enclave, u32 *vcpu_id)574{575 int core_id = -1;576 unsigned int cpu = 0;577 unsigned int i = 0;578 int rc = -EINVAL;579 580 /*581 * If previously allocated a thread of a core to this enclave, first582 * check remaining sibling(s) for new CPU allocations, so that full583 * CPU cores are used for the enclave.584 */585 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++)586 for_each_cpu(cpu, ne_enclave->threads_per_core[i])587 if (!ne_donated_cpu(ne_enclave, cpu)) {588 *vcpu_id = cpu;589 590 return 0;591 }592 593 mutex_lock(&ne_cpu_pool.mutex);594 595 /*596 * If no remaining siblings, get a core from the NE CPU pool and keep597 * track of all the threads in the enclave threads per core data structure.598 */599 core_id = ne_get_unused_core_from_cpu_pool();600 601 rc = ne_set_enclave_threads_per_core(ne_enclave, core_id, *vcpu_id);602 if (rc < 0)603 goto unlock_mutex;604 605 *vcpu_id = cpumask_any(ne_enclave->threads_per_core[core_id]);606 607 rc = 0;608 609unlock_mutex:610 mutex_unlock(&ne_cpu_pool.mutex);611 612 return rc;613}614 615/**616 * ne_get_vcpu_core_from_cpu_pool() - Get from the NE CPU pool the id of the617 * core associated with the provided vCPU.618 * @vcpu_id: Provided vCPU id to get its associated core id.619 *620 * Context: Process context. This function is called with the ne_enclave and621 * ne_cpu_pool mutexes held.622 * Return:623 * * Core id.624 * * -1 if the provided vCPU is not in the pool.625 */626static int ne_get_vcpu_core_from_cpu_pool(u32 vcpu_id)627{628 int core_id = -1;629 unsigned int i = 0;630 631 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++)632 if (cpumask_test_cpu(vcpu_id, ne_cpu_pool.avail_threads_per_core[i])) {633 core_id = i;634 635 break;636 }637 638 return core_id;639}640 641/**642 * ne_check_cpu_in_cpu_pool() - Check if the given vCPU is in the available CPUs643 * from the pool.644 * @ne_enclave : Private data associated with the current enclave.645 * @vcpu_id: ID of the vCPU to check if available in the NE CPU pool.646 *647 * Context: Process context. This function is called with the ne_enclave mutex held.648 * Return:649 * * 0 on success.650 * * Negative return value on failure.651 */652static int ne_check_cpu_in_cpu_pool(struct ne_enclave *ne_enclave, u32 vcpu_id)653{654 int core_id = -1;655 unsigned int i = 0;656 int rc = -EINVAL;657 658 if (ne_donated_cpu(ne_enclave, vcpu_id)) {659 dev_err_ratelimited(ne_misc_dev.this_device,660 "CPU %d already used\n", vcpu_id);661 662 return -NE_ERR_VCPU_ALREADY_USED;663 }664 665 /*666 * If previously allocated a thread of a core to this enclave, but not667 * the full core, first check remaining sibling(s).668 */669 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++)670 if (cpumask_test_cpu(vcpu_id, ne_enclave->threads_per_core[i]))671 return 0;672 673 mutex_lock(&ne_cpu_pool.mutex);674 675 /*676 * If no remaining siblings, get from the NE CPU pool the core677 * associated with the vCPU and keep track of all the threads in the678 * enclave threads per core data structure.679 */680 core_id = ne_get_vcpu_core_from_cpu_pool(vcpu_id);681 682 rc = ne_set_enclave_threads_per_core(ne_enclave, core_id, vcpu_id);683 if (rc < 0)684 goto unlock_mutex;685 686 rc = 0;687 688unlock_mutex:689 mutex_unlock(&ne_cpu_pool.mutex);690 691 return rc;692}693 694/**695 * ne_add_vcpu_ioctl() - Add a vCPU to the slot associated with the current696 * enclave.697 * @ne_enclave : Private data associated with the current enclave.698 * @vcpu_id: ID of the CPU to be associated with the given slot,699 * apic id on x86.700 *701 * Context: Process context. This function is called with the ne_enclave mutex held.702 * Return:703 * * 0 on success.704 * * Negative return value on failure.705 */706static int ne_add_vcpu_ioctl(struct ne_enclave *ne_enclave, u32 vcpu_id)707{708 struct ne_pci_dev_cmd_reply cmd_reply = {};709 struct pci_dev *pdev = ne_devs.ne_pci_dev->pdev;710 int rc = -EINVAL;711 struct slot_add_vcpu_req slot_add_vcpu_req = {};712 713 if (ne_enclave->mm != current->mm)714 return -EIO;715 716 slot_add_vcpu_req.slot_uid = ne_enclave->slot_uid;717 slot_add_vcpu_req.vcpu_id = vcpu_id;718 719 rc = ne_do_request(pdev, SLOT_ADD_VCPU,720 &slot_add_vcpu_req, sizeof(slot_add_vcpu_req),721 &cmd_reply, sizeof(cmd_reply));722 if (rc < 0) {723 dev_err_ratelimited(ne_misc_dev.this_device,724 "Error in slot add vCPU [rc=%d]\n", rc);725 726 return rc;727 }728 729 cpumask_set_cpu(vcpu_id, ne_enclave->vcpu_ids);730 731 ne_enclave->nr_vcpus++;732 733 return 0;734}735 736/**737 * ne_sanity_check_user_mem_region() - Sanity check the user space memory738 * region received during the set user739 * memory region ioctl call.740 * @ne_enclave : Private data associated with the current enclave.741 * @mem_region : User space memory region to be sanity checked.742 *743 * Context: Process context. This function is called with the ne_enclave mutex held.744 * Return:745 * * 0 on success.746 * * Negative return value on failure.747 */748static int ne_sanity_check_user_mem_region(struct ne_enclave *ne_enclave,749 struct ne_user_memory_region mem_region)750{751 struct ne_mem_region *ne_mem_region = NULL;752 753 if (ne_enclave->mm != current->mm)754 return -EIO;755 756 if (mem_region.memory_size & (NE_MIN_MEM_REGION_SIZE - 1)) {757 dev_err_ratelimited(ne_misc_dev.this_device,758 "User space memory size is not multiple of 2 MiB\n");759 760 return -NE_ERR_INVALID_MEM_REGION_SIZE;761 }762 763 if (!IS_ALIGNED(mem_region.userspace_addr, NE_MIN_MEM_REGION_SIZE)) {764 dev_err_ratelimited(ne_misc_dev.this_device,765 "User space address is not 2 MiB aligned\n");766 767 return -NE_ERR_UNALIGNED_MEM_REGION_ADDR;768 }769 770 if ((mem_region.userspace_addr & (NE_MIN_MEM_REGION_SIZE - 1)) ||771 !access_ok((void __user *)(unsigned long)mem_region.userspace_addr,772 mem_region.memory_size)) {773 dev_err_ratelimited(ne_misc_dev.this_device,774 "Invalid user space address range\n");775 776 return -NE_ERR_INVALID_MEM_REGION_ADDR;777 }778 779 list_for_each_entry(ne_mem_region, &ne_enclave->mem_regions_list,780 mem_region_list_entry) {781 u64 memory_size = ne_mem_region->memory_size;782 u64 userspace_addr = ne_mem_region->userspace_addr;783 784 if ((userspace_addr <= mem_region.userspace_addr &&785 mem_region.userspace_addr < (userspace_addr + memory_size)) ||786 (mem_region.userspace_addr <= userspace_addr &&787 (mem_region.userspace_addr + mem_region.memory_size) > userspace_addr)) {788 dev_err_ratelimited(ne_misc_dev.this_device,789 "User space memory region already used\n");790 791 return -NE_ERR_MEM_REGION_ALREADY_USED;792 }793 }794 795 return 0;796}797 798/**799 * ne_sanity_check_user_mem_region_page() - Sanity check a page from the user space800 * memory region received during the set801 * user memory region ioctl call.802 * @ne_enclave : Private data associated with the current enclave.803 * @mem_region_page: Page from the user space memory region to be sanity checked.804 *805 * Context: Process context. This function is called with the ne_enclave mutex held.806 * Return:807 * * 0 on success.808 * * Negative return value on failure.809 */810static int ne_sanity_check_user_mem_region_page(struct ne_enclave *ne_enclave,811 struct page *mem_region_page)812{813 if (!PageHuge(mem_region_page)) {814 dev_err_ratelimited(ne_misc_dev.this_device,815 "Not a hugetlbfs page\n");816 817 return -NE_ERR_MEM_NOT_HUGE_PAGE;818 }819 820 if (page_size(mem_region_page) & (NE_MIN_MEM_REGION_SIZE - 1)) {821 dev_err_ratelimited(ne_misc_dev.this_device,822 "Page size not multiple of 2 MiB\n");823 824 return -NE_ERR_INVALID_PAGE_SIZE;825 }826 827 if (ne_enclave->numa_node != page_to_nid(mem_region_page)) {828 dev_err_ratelimited(ne_misc_dev.this_device,829 "Page is not from NUMA node %d\n",830 ne_enclave->numa_node);831 832 return -NE_ERR_MEM_DIFFERENT_NUMA_NODE;833 }834 835 return 0;836}837 838/**839 * ne_sanity_check_phys_mem_region() - Sanity check the start address and the size840 * of a physical memory region.841 * @phys_mem_region_paddr : Physical start address of the region to be sanity checked.842 * @phys_mem_region_size : Length of the region to be sanity checked.843 *844 * Context: Process context. This function is called with the ne_enclave mutex held.845 * Return:846 * * 0 on success.847 * * Negative return value on failure.848 */849static int ne_sanity_check_phys_mem_region(u64 phys_mem_region_paddr,850 u64 phys_mem_region_size)851{852 if (phys_mem_region_size & (NE_MIN_MEM_REGION_SIZE - 1)) {853 dev_err_ratelimited(ne_misc_dev.this_device,854 "Physical mem region size is not multiple of 2 MiB\n");855 856 return -EINVAL;857 }858 859 if (!IS_ALIGNED(phys_mem_region_paddr, NE_MIN_MEM_REGION_SIZE)) {860 dev_err_ratelimited(ne_misc_dev.this_device,861 "Physical mem region address is not 2 MiB aligned\n");862 863 return -EINVAL;864 }865 866 return 0;867}868 869/**870 * ne_merge_phys_contig_memory_regions() - Add a memory region and merge the adjacent871 * regions if they are physically contiguous.872 * @phys_contig_regions : Private data associated with the contiguous physical memory regions.873 * @page_paddr : Physical start address of the region to be added.874 * @page_size : Length of the region to be added.875 *876 * Context: Process context. This function is called with the ne_enclave mutex held.877 * Return:878 * * 0 on success.879 * * Negative return value on failure.880 */881static int882ne_merge_phys_contig_memory_regions(struct ne_phys_contig_mem_regions *phys_contig_regions,883 u64 page_paddr, u64 page_size)884{885 unsigned long num = phys_contig_regions->num;886 int rc = 0;887 888 rc = ne_sanity_check_phys_mem_region(page_paddr, page_size);889 if (rc < 0)890 return rc;891 892 /* Physically contiguous, just merge */893 if (num && (phys_contig_regions->regions[num - 1].end + 1) == page_paddr) {894 phys_contig_regions->regions[num - 1].end += page_size;895 } else {896 phys_contig_regions->regions[num].start = page_paddr;897 phys_contig_regions->regions[num].end = page_paddr + page_size - 1;898 phys_contig_regions->num++;899 }900 901 return 0;902}903 904/**905 * ne_set_user_memory_region_ioctl() - Add user space memory region to the slot906 * associated with the current enclave.907 * @ne_enclave : Private data associated with the current enclave.908 * @mem_region : User space memory region to be associated with the given slot.909 *910 * Context: Process context. This function is called with the ne_enclave mutex held.911 * Return:912 * * 0 on success.913 * * Negative return value on failure.914 */915static int ne_set_user_memory_region_ioctl(struct ne_enclave *ne_enclave,916 struct ne_user_memory_region mem_region)917{918 long gup_rc = 0;919 unsigned long i = 0;920 unsigned long max_nr_pages = 0;921 unsigned long memory_size = 0;922 struct ne_mem_region *ne_mem_region = NULL;923 struct pci_dev *pdev = ne_devs.ne_pci_dev->pdev;924 struct ne_phys_contig_mem_regions phys_contig_mem_regions = {};925 int rc = -EINVAL;926 927 rc = ne_sanity_check_user_mem_region(ne_enclave, mem_region);928 if (rc < 0)929 return rc;930 931 ne_mem_region = kzalloc(sizeof(*ne_mem_region), GFP_KERNEL);932 if (!ne_mem_region)933 return -ENOMEM;934 935 max_nr_pages = mem_region.memory_size / NE_MIN_MEM_REGION_SIZE;936 937 ne_mem_region->pages = kcalloc(max_nr_pages, sizeof(*ne_mem_region->pages),938 GFP_KERNEL);939 if (!ne_mem_region->pages) {940 rc = -ENOMEM;941 942 goto free_mem_region;943 }944 945 phys_contig_mem_regions.regions = kcalloc(max_nr_pages,946 sizeof(*phys_contig_mem_regions.regions),947 GFP_KERNEL);948 if (!phys_contig_mem_regions.regions) {949 rc = -ENOMEM;950 951 goto free_mem_region;952 }953 954 do {955 i = ne_mem_region->nr_pages;956 957 if (i == max_nr_pages) {958 dev_err_ratelimited(ne_misc_dev.this_device,959 "Reached max nr of pages in the pages data struct\n");960 961 rc = -ENOMEM;962 963 goto put_pages;964 }965 966 gup_rc = get_user_pages_unlocked(mem_region.userspace_addr + memory_size, 1,967 ne_mem_region->pages + i, FOLL_GET);968 969 if (gup_rc < 0) {970 rc = gup_rc;971 972 dev_err_ratelimited(ne_misc_dev.this_device,973 "Error in get user pages [rc=%d]\n", rc);974 975 goto put_pages;976 }977 978 rc = ne_sanity_check_user_mem_region_page(ne_enclave, ne_mem_region->pages[i]);979 if (rc < 0)980 goto put_pages;981 982 rc = ne_merge_phys_contig_memory_regions(&phys_contig_mem_regions,983 page_to_phys(ne_mem_region->pages[i]),984 page_size(ne_mem_region->pages[i]));985 if (rc < 0)986 goto put_pages;987 988 memory_size += page_size(ne_mem_region->pages[i]);989 990 ne_mem_region->nr_pages++;991 } while (memory_size < mem_region.memory_size);992 993 if ((ne_enclave->nr_mem_regions + phys_contig_mem_regions.num) >994 ne_enclave->max_mem_regions) {995 dev_err_ratelimited(ne_misc_dev.this_device,996 "Reached max memory regions %lld\n",997 ne_enclave->max_mem_regions);998 999 rc = -NE_ERR_MEM_MAX_REGIONS;1000 1001 goto put_pages;1002 }1003 1004 for (i = 0; i < phys_contig_mem_regions.num; i++) {1005 u64 phys_region_addr = phys_contig_mem_regions.regions[i].start;1006 u64 phys_region_size = range_len(&phys_contig_mem_regions.regions[i]);1007 1008 rc = ne_sanity_check_phys_mem_region(phys_region_addr, phys_region_size);1009 if (rc < 0)1010 goto put_pages;1011 }1012 1013 ne_mem_region->memory_size = mem_region.memory_size;1014 ne_mem_region->userspace_addr = mem_region.userspace_addr;1015 1016 list_add(&ne_mem_region->mem_region_list_entry, &ne_enclave->mem_regions_list);1017 1018 for (i = 0; i < phys_contig_mem_regions.num; i++) {1019 struct ne_pci_dev_cmd_reply cmd_reply = {};1020 struct slot_add_mem_req slot_add_mem_req = {};1021 1022 slot_add_mem_req.slot_uid = ne_enclave->slot_uid;1023 slot_add_mem_req.paddr = phys_contig_mem_regions.regions[i].start;1024 slot_add_mem_req.size = range_len(&phys_contig_mem_regions.regions[i]);1025 1026 rc = ne_do_request(pdev, SLOT_ADD_MEM,1027 &slot_add_mem_req, sizeof(slot_add_mem_req),1028 &cmd_reply, sizeof(cmd_reply));1029 if (rc < 0) {1030 dev_err_ratelimited(ne_misc_dev.this_device,1031 "Error in slot add mem [rc=%d]\n", rc);1032 1033 kfree(phys_contig_mem_regions.regions);1034 1035 /*1036 * Exit here without put pages as memory regions may1037 * already been added.1038 */1039 return rc;1040 }1041 1042 ne_enclave->mem_size += slot_add_mem_req.size;1043 ne_enclave->nr_mem_regions++;1044 }1045 1046 kfree(phys_contig_mem_regions.regions);1047 1048 return 0;1049 1050put_pages:1051 for (i = 0; i < ne_mem_region->nr_pages; i++)1052 put_page(ne_mem_region->pages[i]);1053free_mem_region:1054 kfree(phys_contig_mem_regions.regions);1055 kfree(ne_mem_region->pages);1056 kfree(ne_mem_region);1057 1058 return rc;1059}1060 1061/**1062 * ne_start_enclave_ioctl() - Trigger enclave start after the enclave resources,1063 * such as memory and CPU, have been set.1064 * @ne_enclave : Private data associated with the current enclave.1065 * @enclave_start_info : Enclave info that includes enclave cid and flags.1066 *1067 * Context: Process context. This function is called with the ne_enclave mutex held.1068 * Return:1069 * * 0 on success.1070 * * Negative return value on failure.1071 */1072static int ne_start_enclave_ioctl(struct ne_enclave *ne_enclave,1073 struct ne_enclave_start_info *enclave_start_info)1074{1075 struct ne_pci_dev_cmd_reply cmd_reply = {};1076 unsigned int cpu = 0;1077 struct enclave_start_req enclave_start_req = {};1078 unsigned int i = 0;1079 struct pci_dev *pdev = ne_devs.ne_pci_dev->pdev;1080 int rc = -EINVAL;1081 1082 if (!ne_enclave->nr_mem_regions) {1083 dev_err_ratelimited(ne_misc_dev.this_device,1084 "Enclave has no mem regions\n");1085 1086 return -NE_ERR_NO_MEM_REGIONS_ADDED;1087 }1088 1089 if (ne_enclave->mem_size < NE_MIN_ENCLAVE_MEM_SIZE) {1090 dev_err_ratelimited(ne_misc_dev.this_device,1091 "Enclave memory is less than %ld\n",1092 NE_MIN_ENCLAVE_MEM_SIZE);1093 1094 return -NE_ERR_ENCLAVE_MEM_MIN_SIZE;1095 }1096 1097 if (!ne_enclave->nr_vcpus) {1098 dev_err_ratelimited(ne_misc_dev.this_device,1099 "Enclave has no vCPUs\n");1100 1101 return -NE_ERR_NO_VCPUS_ADDED;1102 }1103 1104 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++)1105 for_each_cpu(cpu, ne_enclave->threads_per_core[i])1106 if (!cpumask_test_cpu(cpu, ne_enclave->vcpu_ids)) {1107 dev_err_ratelimited(ne_misc_dev.this_device,1108 "Full CPU cores not used\n");1109 1110 return -NE_ERR_FULL_CORES_NOT_USED;1111 }1112 1113 enclave_start_req.enclave_cid = enclave_start_info->enclave_cid;1114 enclave_start_req.flags = enclave_start_info->flags;1115 enclave_start_req.slot_uid = ne_enclave->slot_uid;1116 1117 rc = ne_do_request(pdev, ENCLAVE_START,1118 &enclave_start_req, sizeof(enclave_start_req),1119 &cmd_reply, sizeof(cmd_reply));1120 if (rc < 0) {1121 dev_err_ratelimited(ne_misc_dev.this_device,1122 "Error in enclave start [rc=%d]\n", rc);1123 1124 return rc;1125 }1126 1127 ne_enclave->state = NE_STATE_RUNNING;1128 1129 enclave_start_info->enclave_cid = cmd_reply.enclave_cid;1130 1131 return 0;1132}1133 1134/**1135 * ne_enclave_ioctl() - Ioctl function provided by the enclave file.1136 * @file: File associated with this ioctl function.1137 * @cmd: The command that is set for the ioctl call.1138 * @arg: The argument that is provided for the ioctl call.1139 *1140 * Context: Process context.1141 * Return:1142 * * 0 on success.1143 * * Negative return value on failure.1144 */1145static long ne_enclave_ioctl(struct file *file, unsigned int cmd, unsigned long arg)1146{1147 struct ne_enclave *ne_enclave = file->private_data;1148 1149 switch (cmd) {1150 case NE_ADD_VCPU: {1151 int rc = -EINVAL;1152 u32 vcpu_id = 0;1153 1154 if (copy_from_user(&vcpu_id, (void __user *)arg, sizeof(vcpu_id)))1155 return -EFAULT;1156 1157 mutex_lock(&ne_enclave->enclave_info_mutex);1158 1159 if (ne_enclave->state != NE_STATE_INIT) {1160 dev_err_ratelimited(ne_misc_dev.this_device,1161 "Enclave is not in init state\n");1162 1163 mutex_unlock(&ne_enclave->enclave_info_mutex);1164 1165 return -NE_ERR_NOT_IN_INIT_STATE;1166 }1167 1168 if (vcpu_id >= (ne_enclave->nr_parent_vm_cores *1169 ne_enclave->nr_threads_per_core)) {1170 dev_err_ratelimited(ne_misc_dev.this_device,1171 "vCPU id higher than max CPU id\n");1172 1173 mutex_unlock(&ne_enclave->enclave_info_mutex);1174 1175 return -NE_ERR_INVALID_VCPU;1176 }1177 1178 if (!vcpu_id) {1179 /* Use the CPU pool for choosing a CPU for the enclave. */1180 rc = ne_get_cpu_from_cpu_pool(ne_enclave, &vcpu_id);1181 if (rc < 0) {1182 dev_err_ratelimited(ne_misc_dev.this_device,1183 "Error in get CPU from pool [rc=%d]\n",1184 rc);1185 1186 mutex_unlock(&ne_enclave->enclave_info_mutex);1187 1188 return rc;1189 }1190 } else {1191 /* Check if the provided vCPU is available in the NE CPU pool. */1192 rc = ne_check_cpu_in_cpu_pool(ne_enclave, vcpu_id);1193 if (rc < 0) {1194 dev_err_ratelimited(ne_misc_dev.this_device,1195 "Error in check CPU %d in pool [rc=%d]\n",1196 vcpu_id, rc);1197 1198 mutex_unlock(&ne_enclave->enclave_info_mutex);1199 1200 return rc;1201 }1202 }1203 1204 rc = ne_add_vcpu_ioctl(ne_enclave, vcpu_id);1205 if (rc < 0) {1206 mutex_unlock(&ne_enclave->enclave_info_mutex);1207 1208 return rc;1209 }1210 1211 mutex_unlock(&ne_enclave->enclave_info_mutex);1212 1213 if (copy_to_user((void __user *)arg, &vcpu_id, sizeof(vcpu_id)))1214 return -EFAULT;1215 1216 return 0;1217 }1218 1219 case NE_GET_IMAGE_LOAD_INFO: {1220 struct ne_image_load_info image_load_info = {};1221 1222 if (copy_from_user(&image_load_info, (void __user *)arg, sizeof(image_load_info)))1223 return -EFAULT;1224 1225 mutex_lock(&ne_enclave->enclave_info_mutex);1226 1227 if (ne_enclave->state != NE_STATE_INIT) {1228 dev_err_ratelimited(ne_misc_dev.this_device,1229 "Enclave is not in init state\n");1230 1231 mutex_unlock(&ne_enclave->enclave_info_mutex);1232 1233 return -NE_ERR_NOT_IN_INIT_STATE;1234 }1235 1236 mutex_unlock(&ne_enclave->enclave_info_mutex);1237 1238 if (!image_load_info.flags ||1239 image_load_info.flags >= NE_IMAGE_LOAD_MAX_FLAG_VAL) {1240 dev_err_ratelimited(ne_misc_dev.this_device,1241 "Incorrect flag in enclave image load info\n");1242 1243 return -NE_ERR_INVALID_FLAG_VALUE;1244 }1245 1246 if (image_load_info.flags == NE_EIF_IMAGE)1247 image_load_info.memory_offset = NE_EIF_LOAD_OFFSET;1248 1249 if (copy_to_user((void __user *)arg, &image_load_info, sizeof(image_load_info)))1250 return -EFAULT;1251 1252 return 0;1253 }1254 1255 case NE_SET_USER_MEMORY_REGION: {1256 struct ne_user_memory_region mem_region = {};1257 int rc = -EINVAL;1258 1259 if (copy_from_user(&mem_region, (void __user *)arg, sizeof(mem_region)))1260 return -EFAULT;1261 1262 if (mem_region.flags >= NE_MEMORY_REGION_MAX_FLAG_VAL) {1263 dev_err_ratelimited(ne_misc_dev.this_device,1264 "Incorrect flag for user memory region\n");1265 1266 return -NE_ERR_INVALID_FLAG_VALUE;1267 }1268 1269 mutex_lock(&ne_enclave->enclave_info_mutex);1270 1271 if (ne_enclave->state != NE_STATE_INIT) {1272 dev_err_ratelimited(ne_misc_dev.this_device,1273 "Enclave is not in init state\n");1274 1275 mutex_unlock(&ne_enclave->enclave_info_mutex);1276 1277 return -NE_ERR_NOT_IN_INIT_STATE;1278 }1279 1280 rc = ne_set_user_memory_region_ioctl(ne_enclave, mem_region);1281 if (rc < 0) {1282 mutex_unlock(&ne_enclave->enclave_info_mutex);1283 1284 return rc;1285 }1286 1287 mutex_unlock(&ne_enclave->enclave_info_mutex);1288 1289 return 0;1290 }1291 1292 case NE_START_ENCLAVE: {1293 struct ne_enclave_start_info enclave_start_info = {};1294 int rc = -EINVAL;1295 1296 if (copy_from_user(&enclave_start_info, (void __user *)arg,1297 sizeof(enclave_start_info)))1298 return -EFAULT;1299 1300 if (enclave_start_info.flags >= NE_ENCLAVE_START_MAX_FLAG_VAL) {1301 dev_err_ratelimited(ne_misc_dev.this_device,1302 "Incorrect flag in enclave start info\n");1303 1304 return -NE_ERR_INVALID_FLAG_VALUE;1305 }1306 1307 /*1308 * Do not use well-known CIDs - 0, 1, 2 - for enclaves.1309 * VMADDR_CID_ANY = -1U1310 * VMADDR_CID_HYPERVISOR = 01311 * VMADDR_CID_LOCAL = 11312 * VMADDR_CID_HOST = 21313 * Note: 0 is used as a placeholder to auto-generate an enclave CID.1314 * http://man7.org/linux/man-pages/man7/vsock.7.html1315 */1316 if (enclave_start_info.enclave_cid > 0 &&1317 enclave_start_info.enclave_cid <= VMADDR_CID_HOST) {1318 dev_err_ratelimited(ne_misc_dev.this_device,1319 "Well-known CID value, not to be used for enclaves\n");1320 1321 return -NE_ERR_INVALID_ENCLAVE_CID;1322 }1323 1324 if (enclave_start_info.enclave_cid == U32_MAX) {1325 dev_err_ratelimited(ne_misc_dev.this_device,1326 "Well-known CID value, not to be used for enclaves\n");1327 1328 return -NE_ERR_INVALID_ENCLAVE_CID;1329 }1330 1331 /*1332 * Do not use the CID of the primary / parent VM for enclaves.1333 */1334 if (enclave_start_info.enclave_cid == NE_PARENT_VM_CID) {1335 dev_err_ratelimited(ne_misc_dev.this_device,1336 "CID of the parent VM, not to be used for enclaves\n");1337 1338 return -NE_ERR_INVALID_ENCLAVE_CID;1339 }1340 1341 /* 64-bit CIDs are not yet supported for the vsock device. */1342 if (enclave_start_info.enclave_cid > U32_MAX) {1343 dev_err_ratelimited(ne_misc_dev.this_device,1344 "64-bit CIDs not yet supported for the vsock device\n");1345 1346 return -NE_ERR_INVALID_ENCLAVE_CID;1347 }1348 1349 mutex_lock(&ne_enclave->enclave_info_mutex);1350 1351 if (ne_enclave->state != NE_STATE_INIT) {1352 dev_err_ratelimited(ne_misc_dev.this_device,1353 "Enclave is not in init state\n");1354 1355 mutex_unlock(&ne_enclave->enclave_info_mutex);1356 1357 return -NE_ERR_NOT_IN_INIT_STATE;1358 }1359 1360 rc = ne_start_enclave_ioctl(ne_enclave, &enclave_start_info);1361 if (rc < 0) {1362 mutex_unlock(&ne_enclave->enclave_info_mutex);1363 1364 return rc;1365 }1366 1367 mutex_unlock(&ne_enclave->enclave_info_mutex);1368 1369 if (copy_to_user((void __user *)arg, &enclave_start_info,1370 sizeof(enclave_start_info)))1371 return -EFAULT;1372 1373 return 0;1374 }1375 1376 default:1377 return -ENOTTY;1378 }1379 1380 return 0;1381}1382 1383/**1384 * ne_enclave_remove_all_mem_region_entries() - Remove all memory region entries1385 * from the enclave data structure.1386 * @ne_enclave : Private data associated with the current enclave.1387 *1388 * Context: Process context. This function is called with the ne_enclave mutex held.1389 */1390static void ne_enclave_remove_all_mem_region_entries(struct ne_enclave *ne_enclave)1391{1392 unsigned long i = 0;1393 struct ne_mem_region *ne_mem_region = NULL;1394 struct ne_mem_region *ne_mem_region_tmp = NULL;1395 1396 list_for_each_entry_safe(ne_mem_region, ne_mem_region_tmp,1397 &ne_enclave->mem_regions_list,1398 mem_region_list_entry) {1399 list_del(&ne_mem_region->mem_region_list_entry);1400 1401 for (i = 0; i < ne_mem_region->nr_pages; i++)1402 put_page(ne_mem_region->pages[i]);1403 1404 kfree(ne_mem_region->pages);1405 1406 kfree(ne_mem_region);1407 }1408}1409 1410/**1411 * ne_enclave_remove_all_vcpu_id_entries() - Remove all vCPU id entries from1412 * the enclave data structure.1413 * @ne_enclave : Private data associated with the current enclave.1414 *1415 * Context: Process context. This function is called with the ne_enclave mutex held.1416 */1417static void ne_enclave_remove_all_vcpu_id_entries(struct ne_enclave *ne_enclave)1418{1419 unsigned int cpu = 0;1420 unsigned int i = 0;1421 1422 mutex_lock(&ne_cpu_pool.mutex);1423 1424 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++) {1425 for_each_cpu(cpu, ne_enclave->threads_per_core[i])1426 /* Update the available NE CPU pool. */1427 cpumask_set_cpu(cpu, ne_cpu_pool.avail_threads_per_core[i]);1428 1429 free_cpumask_var(ne_enclave->threads_per_core[i]);1430 }1431 1432 mutex_unlock(&ne_cpu_pool.mutex);1433 1434 kfree(ne_enclave->threads_per_core);1435 1436 free_cpumask_var(ne_enclave->vcpu_ids);1437}1438 1439/**1440 * ne_pci_dev_remove_enclave_entry() - Remove the enclave entry from the data1441 * structure that is part of the NE PCI1442 * device private data.1443 * @ne_enclave : Private data associated with the current enclave.1444 * @ne_pci_dev : Private data associated with the PCI device.1445 *1446 * Context: Process context. This function is called with the ne_pci_dev enclave1447 * mutex held.1448 */1449static void ne_pci_dev_remove_enclave_entry(struct ne_enclave *ne_enclave,1450 struct ne_pci_dev *ne_pci_dev)1451{1452 struct ne_enclave *ne_enclave_entry = NULL;1453 struct ne_enclave *ne_enclave_entry_tmp = NULL;1454 1455 list_for_each_entry_safe(ne_enclave_entry, ne_enclave_entry_tmp,1456 &ne_pci_dev->enclaves_list, enclave_list_entry) {1457 if (ne_enclave_entry->slot_uid == ne_enclave->slot_uid) {1458 list_del(&ne_enclave_entry->enclave_list_entry);1459 1460 break;1461 }1462 }1463}1464 1465/**1466 * ne_enclave_release() - Release function provided by the enclave file.1467 * @inode: Inode associated with this file release function.1468 * @file: File associated with this release function.1469 *1470 * Context: Process context.1471 * Return:1472 * * 0 on success.1473 * * Negative return value on failure.1474 */1475static int ne_enclave_release(struct inode *inode, struct file *file)1476{1477 struct ne_pci_dev_cmd_reply cmd_reply = {};1478 struct enclave_stop_req enclave_stop_request = {};1479 struct ne_enclave *ne_enclave = file->private_data;1480 struct ne_pci_dev *ne_pci_dev = ne_devs.ne_pci_dev;1481 struct pci_dev *pdev = ne_pci_dev->pdev;1482 int rc = -EINVAL;1483 struct slot_free_req slot_free_req = {};1484 1485 if (!ne_enclave)1486 return 0;1487 1488 /*1489 * Early exit in case there is an error in the enclave creation logic1490 * and fput() is called on the cleanup path.1491 */1492 if (!ne_enclave->slot_uid)1493 return 0;1494 1495 /*1496 * Acquire the enclave list mutex before the enclave mutex1497 * in order to avoid deadlocks with @ref ne_event_work_handler.1498 */1499 mutex_lock(&ne_pci_dev->enclaves_list_mutex);1500 mutex_lock(&ne_enclave->enclave_info_mutex);1501 1502 if (ne_enclave->state != NE_STATE_INIT && ne_enclave->state != NE_STATE_STOPPED) {1503 enclave_stop_request.slot_uid = ne_enclave->slot_uid;1504 1505 rc = ne_do_request(pdev, ENCLAVE_STOP,1506 &enclave_stop_request, sizeof(enclave_stop_request),1507 &cmd_reply, sizeof(cmd_reply));1508 if (rc < 0) {1509 dev_err_ratelimited(ne_misc_dev.this_device,1510 "Error in enclave stop [rc=%d]\n", rc);1511 1512 goto unlock_mutex;1513 }1514 1515 memset(&cmd_reply, 0, sizeof(cmd_reply));1516 }1517 1518 slot_free_req.slot_uid = ne_enclave->slot_uid;1519 1520 rc = ne_do_request(pdev, SLOT_FREE,1521 &slot_free_req, sizeof(slot_free_req),1522 &cmd_reply, sizeof(cmd_reply));1523 if (rc < 0) {1524 dev_err_ratelimited(ne_misc_dev.this_device,1525 "Error in slot free [rc=%d]\n", rc);1526 1527 goto unlock_mutex;1528 }1529 1530 ne_pci_dev_remove_enclave_entry(ne_enclave, ne_pci_dev);1531 ne_enclave_remove_all_mem_region_entries(ne_enclave);1532 ne_enclave_remove_all_vcpu_id_entries(ne_enclave);1533 1534 mutex_unlock(&ne_enclave->enclave_info_mutex);1535 mutex_unlock(&ne_pci_dev->enclaves_list_mutex);1536 1537 kfree(ne_enclave);1538 1539 return 0;1540 1541unlock_mutex:1542 mutex_unlock(&ne_enclave->enclave_info_mutex);1543 mutex_unlock(&ne_pci_dev->enclaves_list_mutex);1544 1545 return rc;1546}1547 1548/**1549 * ne_enclave_poll() - Poll functionality used for enclave out-of-band events.1550 * @file: File associated with this poll function.1551 * @wait: Poll table data structure.1552 *1553 * Context: Process context.1554 * Return:1555 * * Poll mask.1556 */1557static __poll_t ne_enclave_poll(struct file *file, poll_table *wait)1558{1559 __poll_t mask = 0;1560 struct ne_enclave *ne_enclave = file->private_data;1561 1562 poll_wait(file, &ne_enclave->eventq, wait);1563 1564 if (ne_enclave->has_event)1565 mask |= EPOLLHUP;1566 1567 return mask;1568}1569 1570static const struct file_operations ne_enclave_fops = {1571 .owner = THIS_MODULE,1572 .llseek = noop_llseek,1573 .poll = ne_enclave_poll,1574 .unlocked_ioctl = ne_enclave_ioctl,1575 .release = ne_enclave_release,1576};1577 1578/**1579 * ne_create_vm_ioctl() - Alloc slot to be associated with an enclave. Create1580 * enclave file descriptor to be further used for enclave1581 * resources handling e.g. memory regions and CPUs.1582 * @ne_pci_dev : Private data associated with the PCI device.1583 * @slot_uid: User pointer to store the generated unique slot id1584 * associated with an enclave to.1585 *1586 * Context: Process context. This function is called with the ne_pci_dev enclave1587 * mutex held.1588 * Return:1589 * * Enclave fd on success.1590 * * Negative return value on failure.1591 */1592static int ne_create_vm_ioctl(struct ne_pci_dev *ne_pci_dev, u64 __user *slot_uid)1593{1594 struct ne_pci_dev_cmd_reply cmd_reply = {};1595 int enclave_fd = -1;1596 struct file *enclave_file = NULL;1597 unsigned int i = 0;1598 struct ne_enclave *ne_enclave = NULL;1599 struct pci_dev *pdev = ne_pci_dev->pdev;1600 int rc = -EINVAL;1601 struct slot_alloc_req slot_alloc_req = {};1602 1603 mutex_lock(&ne_cpu_pool.mutex);1604 1605 for (i = 0; i < ne_cpu_pool.nr_parent_vm_cores; i++)1606 if (!cpumask_empty(ne_cpu_pool.avail_threads_per_core[i]))1607 break;1608 1609 if (i == ne_cpu_pool.nr_parent_vm_cores) {1610 dev_err_ratelimited(ne_misc_dev.this_device,1611 "No CPUs available in CPU pool\n");1612 1613 mutex_unlock(&ne_cpu_pool.mutex);1614 1615 return -NE_ERR_NO_CPUS_AVAIL_IN_POOL;1616 }1617 1618 mutex_unlock(&ne_cpu_pool.mutex);1619 1620 ne_enclave = kzalloc(sizeof(*ne_enclave), GFP_KERNEL);1621 if (!ne_enclave)1622 return -ENOMEM;1623 1624 mutex_lock(&ne_cpu_pool.mutex);1625 1626 ne_enclave->nr_parent_vm_cores = ne_cpu_pool.nr_parent_vm_cores;1627 ne_enclave->nr_threads_per_core = ne_cpu_pool.nr_threads_per_core;1628 ne_enclave->numa_node = ne_cpu_pool.numa_node;1629 1630 mutex_unlock(&ne_cpu_pool.mutex);1631 1632 ne_enclave->threads_per_core = kcalloc(ne_enclave->nr_parent_vm_cores,1633 sizeof(*ne_enclave->threads_per_core),1634 GFP_KERNEL);1635 if (!ne_enclave->threads_per_core) {1636 rc = -ENOMEM;1637 1638 goto free_ne_enclave;1639 }1640 1641 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++)1642 if (!zalloc_cpumask_var(&ne_enclave->threads_per_core[i], GFP_KERNEL)) {1643 rc = -ENOMEM;1644 1645 goto free_cpumask;1646 }1647 1648 if (!zalloc_cpumask_var(&ne_enclave->vcpu_ids, GFP_KERNEL)) {1649 rc = -ENOMEM;1650 1651 goto free_cpumask;1652 }1653 1654 enclave_fd = get_unused_fd_flags(O_CLOEXEC);1655 if (enclave_fd < 0) {1656 rc = enclave_fd;1657 1658 dev_err_ratelimited(ne_misc_dev.this_device,1659 "Error in getting unused fd [rc=%d]\n", rc);1660 1661 goto free_cpumask;1662 }1663 1664 enclave_file = anon_inode_getfile("ne-vm", &ne_enclave_fops, ne_enclave, O_RDWR);1665 if (IS_ERR(enclave_file)) {1666 rc = PTR_ERR(enclave_file);1667 1668 dev_err_ratelimited(ne_misc_dev.this_device,1669 "Error in anon inode get file [rc=%d]\n", rc);1670 1671 goto put_fd;1672 }1673 1674 rc = ne_do_request(pdev, SLOT_ALLOC,1675 &slot_alloc_req, sizeof(slot_alloc_req),1676 &cmd_reply, sizeof(cmd_reply));1677 if (rc < 0) {1678 dev_err_ratelimited(ne_misc_dev.this_device,1679 "Error in slot alloc [rc=%d]\n", rc);1680 1681 goto put_file;1682 }1683 1684 init_waitqueue_head(&ne_enclave->eventq);1685 ne_enclave->has_event = false;1686 mutex_init(&ne_enclave->enclave_info_mutex);1687 ne_enclave->max_mem_regions = cmd_reply.mem_regions;1688 INIT_LIST_HEAD(&ne_enclave->mem_regions_list);1689 ne_enclave->mm = current->mm;1690 ne_enclave->slot_uid = cmd_reply.slot_uid;1691 ne_enclave->state = NE_STATE_INIT;1692 1693 list_add(&ne_enclave->enclave_list_entry, &ne_pci_dev->enclaves_list);1694 1695 if (copy_to_user(slot_uid, &ne_enclave->slot_uid, sizeof(ne_enclave->slot_uid))) {1696 /*1697 * As we're holding the only reference to 'enclave_file', fput()1698 * will call ne_enclave_release() which will do a proper cleanup1699 * of all so far allocated resources, leaving only the unused fd1700 * for us to free.1701 */1702 fput(enclave_file);1703 put_unused_fd(enclave_fd);1704 1705 return -EFAULT;1706 }1707 1708 fd_install(enclave_fd, enclave_file);1709 1710 return enclave_fd;1711 1712put_file:1713 fput(enclave_file);1714put_fd:1715 put_unused_fd(enclave_fd);1716free_cpumask:1717 free_cpumask_var(ne_enclave->vcpu_ids);1718 for (i = 0; i < ne_enclave->nr_parent_vm_cores; i++)1719 free_cpumask_var(ne_enclave->threads_per_core[i]);1720 kfree(ne_enclave->threads_per_core);1721free_ne_enclave:1722 kfree(ne_enclave);1723 1724 return rc;1725}1726 1727/**1728 * ne_ioctl() - Ioctl function provided by the NE misc device.1729 * @file: File associated with this ioctl function.1730 * @cmd: The command that is set for the ioctl call.1731 * @arg: The argument that is provided for the ioctl call.1732 *1733 * Context: Process context.1734 * Return:1735 * * Ioctl result (e.g. enclave file descriptor) on success.1736 * * Negative return value on failure.1737 */1738static long ne_ioctl(struct file *file, unsigned int cmd, unsigned long arg)1739{1740 switch (cmd) {1741 case NE_CREATE_VM: {1742 int enclave_fd = -1;1743 struct ne_pci_dev *ne_pci_dev = ne_devs.ne_pci_dev;1744 u64 __user *slot_uid = (void __user *)arg;1745 1746 mutex_lock(&ne_pci_dev->enclaves_list_mutex);1747 enclave_fd = ne_create_vm_ioctl(ne_pci_dev, slot_uid);1748 mutex_unlock(&ne_pci_dev->enclaves_list_mutex);1749 1750 return enclave_fd;1751 }1752 1753 default:1754 return -ENOTTY;1755 }1756 1757 return 0;1758}1759 1760#if defined(CONFIG_NITRO_ENCLAVES_MISC_DEV_TEST)1761#include "ne_misc_dev_test.c"1762#endif1763 1764static int __init ne_init(void)1765{1766 mutex_init(&ne_cpu_pool.mutex);1767 1768 return pci_register_driver(&ne_pci_driver);1769}1770 1771static void __exit ne_exit(void)1772{1773 pci_unregister_driver(&ne_pci_driver);1774 1775 ne_teardown_cpu_pool();1776}1777 1778module_init(ne_init);1779module_exit(ne_exit);1780 1781MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");1782MODULE_DESCRIPTION("Nitro Enclaves Driver");1783MODULE_LICENSE("GPL v2");1784