1193 lines · c
1// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause2/*3 * Copyright(c) 2015 - 2020 Intel Corporation.4 */5 6#include <linux/topology.h>7#include <linux/cpumask.h>8#include <linux/interrupt.h>9#include <linux/numa.h>10 11#include "hfi.h"12#include "affinity.h"13#include "sdma.h"14#include "trace.h"15 16struct hfi1_affinity_node_list node_affinity = {17 .list = LIST_HEAD_INIT(node_affinity.list),18 .lock = __MUTEX_INITIALIZER(node_affinity.lock)19};20 21/* Name of IRQ types, indexed by enum irq_type */22static const char * const irq_type_names[] = {23 "SDMA",24 "RCVCTXT",25 "NETDEVCTXT",26 "GENERAL",27 "OTHER",28};29 30/* Per NUMA node count of HFI devices */31static unsigned int *hfi1_per_node_cntr;32 33static inline void init_cpu_mask_set(struct cpu_mask_set *set)34{35 cpumask_clear(&set->mask);36 cpumask_clear(&set->used);37 set->gen = 0;38}39 40/* Increment generation of CPU set if needed */41static void _cpu_mask_set_gen_inc(struct cpu_mask_set *set)42{43 if (cpumask_equal(&set->mask, &set->used)) {44 /*45 * We've used up all the CPUs, bump up the generation46 * and reset the 'used' map47 */48 set->gen++;49 cpumask_clear(&set->used);50 }51}52 53static void _cpu_mask_set_gen_dec(struct cpu_mask_set *set)54{55 if (cpumask_empty(&set->used) && set->gen) {56 set->gen--;57 cpumask_copy(&set->used, &set->mask);58 }59}60 61/* Get the first CPU from the list of unused CPUs in a CPU set data structure */62static int cpu_mask_set_get_first(struct cpu_mask_set *set, cpumask_var_t diff)63{64 int cpu;65 66 if (!diff || !set)67 return -EINVAL;68 69 _cpu_mask_set_gen_inc(set);70 71 /* Find out CPUs left in CPU mask */72 cpumask_andnot(diff, &set->mask, &set->used);73 74 cpu = cpumask_first(diff);75 if (cpu >= nr_cpu_ids) /* empty */76 cpu = -EINVAL;77 else78 cpumask_set_cpu(cpu, &set->used);79 80 return cpu;81}82 83static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu)84{85 if (!set)86 return;87 88 cpumask_clear_cpu(cpu, &set->used);89 _cpu_mask_set_gen_dec(set);90}91 92/* Initialize non-HT cpu cores mask */93void init_real_cpu_mask(void)94{95 int possible, curr_cpu, i, ht;96 97 cpumask_clear(&node_affinity.real_cpu_mask);98 99 /* Start with cpu online mask as the real cpu mask */100 cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);101 102 /*103 * Remove HT cores from the real cpu mask. Do this in two steps below.104 */105 possible = cpumask_weight(&node_affinity.real_cpu_mask);106 ht = cpumask_weight(topology_sibling_cpumask(107 cpumask_first(&node_affinity.real_cpu_mask)));108 /*109 * Step 1. Skip over the first N HT siblings and use them as the110 * "real" cores. Assumes that HT cores are not enumerated in111 * succession (except in the single core case).112 */113 curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);114 for (i = 0; i < possible / ht; i++)115 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);116 /*117 * Step 2. Remove the remaining HT siblings. Use cpumask_next() to118 * skip any gaps.119 */120 for (; i < possible; i++) {121 cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);122 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);123 }124}125 126int node_affinity_init(void)127{128 int node;129 struct pci_dev *dev = NULL;130 const struct pci_device_id *ids = hfi1_pci_tbl;131 132 cpumask_clear(&node_affinity.proc.used);133 cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);134 135 node_affinity.proc.gen = 0;136 node_affinity.num_core_siblings =137 cpumask_weight(topology_sibling_cpumask(138 cpumask_first(&node_affinity.proc.mask)139 ));140 node_affinity.num_possible_nodes = num_possible_nodes();141 node_affinity.num_online_nodes = num_online_nodes();142 node_affinity.num_online_cpus = num_online_cpus();143 144 /*145 * The real cpu mask is part of the affinity struct but it has to be146 * initialized early. It is needed to calculate the number of user147 * contexts in set_up_context_variables().148 */149 init_real_cpu_mask();150 151 hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes,152 sizeof(*hfi1_per_node_cntr), GFP_KERNEL);153 if (!hfi1_per_node_cntr)154 return -ENOMEM;155 156 while (ids->vendor) {157 dev = NULL;158 while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {159 node = pcibus_to_node(dev->bus);160 if (node < 0)161 goto out;162 163 hfi1_per_node_cntr[node]++;164 }165 ids++;166 }167 168 return 0;169 170out:171 /*172 * Invalid PCI NUMA node information found, note it, and populate173 * our database 1:1.174 */175 pr_err("HFI: Invalid PCI NUMA node. Performance may be affected\n");176 pr_err("HFI: System BIOS may need to be upgraded\n");177 for (node = 0; node < node_affinity.num_possible_nodes; node++)178 hfi1_per_node_cntr[node] = 1;179 180 pci_dev_put(dev);181 182 return 0;183}184 185static void node_affinity_destroy(struct hfi1_affinity_node *entry)186{187 free_percpu(entry->comp_vect_affinity);188 kfree(entry);189}190 191void node_affinity_destroy_all(void)192{193 struct list_head *pos, *q;194 struct hfi1_affinity_node *entry;195 196 mutex_lock(&node_affinity.lock);197 list_for_each_safe(pos, q, &node_affinity.list) {198 entry = list_entry(pos, struct hfi1_affinity_node,199 list);200 list_del(pos);201 node_affinity_destroy(entry);202 }203 mutex_unlock(&node_affinity.lock);204 kfree(hfi1_per_node_cntr);205}206 207static struct hfi1_affinity_node *node_affinity_allocate(int node)208{209 struct hfi1_affinity_node *entry;210 211 entry = kzalloc(sizeof(*entry), GFP_KERNEL);212 if (!entry)213 return NULL;214 entry->node = node;215 entry->comp_vect_affinity = alloc_percpu(u16);216 INIT_LIST_HEAD(&entry->list);217 218 return entry;219}220 221/*222 * It appends an entry to the list.223 * It *must* be called with node_affinity.lock held.224 */225static void node_affinity_add_tail(struct hfi1_affinity_node *entry)226{227 list_add_tail(&entry->list, &node_affinity.list);228}229 230/* It must be called with node_affinity.lock held */231static struct hfi1_affinity_node *node_affinity_lookup(int node)232{233 struct hfi1_affinity_node *entry;234 235 list_for_each_entry(entry, &node_affinity.list, list) {236 if (entry->node == node)237 return entry;238 }239 240 return NULL;241}242 243static int per_cpu_affinity_get(cpumask_var_t possible_cpumask,244 u16 __percpu *comp_vect_affinity)245{246 int curr_cpu;247 u16 cntr;248 u16 prev_cntr;249 int ret_cpu;250 251 if (!possible_cpumask) {252 ret_cpu = -EINVAL;253 goto fail;254 }255 256 if (!comp_vect_affinity) {257 ret_cpu = -EINVAL;258 goto fail;259 }260 261 ret_cpu = cpumask_first(possible_cpumask);262 if (ret_cpu >= nr_cpu_ids) {263 ret_cpu = -EINVAL;264 goto fail;265 }266 267 prev_cntr = *per_cpu_ptr(comp_vect_affinity, ret_cpu);268 for_each_cpu(curr_cpu, possible_cpumask) {269 cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);270 271 if (cntr < prev_cntr) {272 ret_cpu = curr_cpu;273 prev_cntr = cntr;274 }275 }276 277 *per_cpu_ptr(comp_vect_affinity, ret_cpu) += 1;278 279fail:280 return ret_cpu;281}282 283static int per_cpu_affinity_put_max(cpumask_var_t possible_cpumask,284 u16 __percpu *comp_vect_affinity)285{286 int curr_cpu;287 int max_cpu;288 u16 cntr;289 u16 prev_cntr;290 291 if (!possible_cpumask)292 return -EINVAL;293 294 if (!comp_vect_affinity)295 return -EINVAL;296 297 max_cpu = cpumask_first(possible_cpumask);298 if (max_cpu >= nr_cpu_ids)299 return -EINVAL;300 301 prev_cntr = *per_cpu_ptr(comp_vect_affinity, max_cpu);302 for_each_cpu(curr_cpu, possible_cpumask) {303 cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);304 305 if (cntr > prev_cntr) {306 max_cpu = curr_cpu;307 prev_cntr = cntr;308 }309 }310 311 *per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1;312 313 return max_cpu;314}315 316/*317 * Non-interrupt CPUs are used first, then interrupt CPUs.318 * Two already allocated cpu masks must be passed.319 */320static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd,321 struct hfi1_affinity_node *entry,322 cpumask_var_t non_intr_cpus,323 cpumask_var_t available_cpus)324 __must_hold(&node_affinity.lock)325{326 int cpu;327 struct cpu_mask_set *set = dd->comp_vect;328 329 lockdep_assert_held(&node_affinity.lock);330 if (!non_intr_cpus) {331 cpu = -1;332 goto fail;333 }334 335 if (!available_cpus) {336 cpu = -1;337 goto fail;338 }339 340 /* Available CPUs for pinning completion vectors */341 _cpu_mask_set_gen_inc(set);342 cpumask_andnot(available_cpus, &set->mask, &set->used);343 344 /* Available CPUs without SDMA engine interrupts */345 cpumask_andnot(non_intr_cpus, available_cpus,346 &entry->def_intr.used);347 348 /* If there are non-interrupt CPUs available, use them first */349 if (!cpumask_empty(non_intr_cpus))350 cpu = cpumask_first(non_intr_cpus);351 else /* Otherwise, use interrupt CPUs */352 cpu = cpumask_first(available_cpus);353 354 if (cpu >= nr_cpu_ids) { /* empty */355 cpu = -1;356 goto fail;357 }358 cpumask_set_cpu(cpu, &set->used);359 360fail:361 return cpu;362}363 364static void _dev_comp_vect_cpu_put(struct hfi1_devdata *dd, int cpu)365{366 struct cpu_mask_set *set = dd->comp_vect;367 368 if (cpu < 0)369 return;370 371 cpu_mask_set_put(set, cpu);372}373 374/* _dev_comp_vect_mappings_destroy() is reentrant */375static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)376{377 int i, cpu;378 379 if (!dd->comp_vect_mappings)380 return;381 382 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {383 cpu = dd->comp_vect_mappings[i];384 _dev_comp_vect_cpu_put(dd, cpu);385 dd->comp_vect_mappings[i] = -1;386 hfi1_cdbg(AFFINITY,387 "[%s] Release CPU %d from completion vector %d",388 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i);389 }390 391 kfree(dd->comp_vect_mappings);392 dd->comp_vect_mappings = NULL;393}394 395/*396 * This function creates the table for looking up CPUs for completion vectors.397 * num_comp_vectors needs to have been initilized before calling this function.398 */399static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,400 struct hfi1_affinity_node *entry)401 __must_hold(&node_affinity.lock)402{403 int i, cpu, ret;404 cpumask_var_t non_intr_cpus;405 cpumask_var_t available_cpus;406 407 lockdep_assert_held(&node_affinity.lock);408 409 if (!zalloc_cpumask_var(&non_intr_cpus, GFP_KERNEL))410 return -ENOMEM;411 412 if (!zalloc_cpumask_var(&available_cpus, GFP_KERNEL)) {413 free_cpumask_var(non_intr_cpus);414 return -ENOMEM;415 }416 417 dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus,418 sizeof(*dd->comp_vect_mappings),419 GFP_KERNEL);420 if (!dd->comp_vect_mappings) {421 ret = -ENOMEM;422 goto fail;423 }424 for (i = 0; i < dd->comp_vect_possible_cpus; i++)425 dd->comp_vect_mappings[i] = -1;426 427 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {428 cpu = _dev_comp_vect_cpu_get(dd, entry, non_intr_cpus,429 available_cpus);430 if (cpu < 0) {431 ret = -EINVAL;432 goto fail;433 }434 435 dd->comp_vect_mappings[i] = cpu;436 hfi1_cdbg(AFFINITY,437 "[%s] Completion Vector %d -> CPU %d",438 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu);439 }440 441 free_cpumask_var(available_cpus);442 free_cpumask_var(non_intr_cpus);443 return 0;444 445fail:446 free_cpumask_var(available_cpus);447 free_cpumask_var(non_intr_cpus);448 _dev_comp_vect_mappings_destroy(dd);449 450 return ret;451}452 453int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd)454{455 int ret;456 struct hfi1_affinity_node *entry;457 458 mutex_lock(&node_affinity.lock);459 entry = node_affinity_lookup(dd->node);460 if (!entry) {461 ret = -EINVAL;462 goto unlock;463 }464 ret = _dev_comp_vect_mappings_create(dd, entry);465unlock:466 mutex_unlock(&node_affinity.lock);467 468 return ret;469}470 471void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd)472{473 _dev_comp_vect_mappings_destroy(dd);474}475 476int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect)477{478 struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);479 struct hfi1_devdata *dd = dd_from_dev(verbs_dev);480 481 if (!dd->comp_vect_mappings)482 return -EINVAL;483 if (comp_vect >= dd->comp_vect_possible_cpus)484 return -EINVAL;485 486 return dd->comp_vect_mappings[comp_vect];487}488 489/*490 * It assumes dd->comp_vect_possible_cpus is available.491 */492static int _dev_comp_vect_cpu_mask_init(struct hfi1_devdata *dd,493 struct hfi1_affinity_node *entry,494 bool first_dev_init)495 __must_hold(&node_affinity.lock)496{497 int i, j, curr_cpu;498 int possible_cpus_comp_vect = 0;499 struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask;500 501 lockdep_assert_held(&node_affinity.lock);502 /*503 * If there's only one CPU available for completion vectors, then504 * there will only be one completion vector available. Othewise,505 * the number of completion vector available will be the number of506 * available CPUs divide it by the number of devices in the507 * local NUMA node.508 */509 if (cpumask_weight(&entry->comp_vect_mask) == 1) {510 possible_cpus_comp_vect = 1;511 dd_dev_warn(dd,512 "Number of kernel receive queues is too large for completion vector affinity to be effective\n");513 } else {514 possible_cpus_comp_vect +=515 cpumask_weight(&entry->comp_vect_mask) /516 hfi1_per_node_cntr[dd->node];517 518 /*519 * If the completion vector CPUs available doesn't divide520 * evenly among devices, then the first device device to be521 * initialized gets an extra CPU.522 */523 if (first_dev_init &&524 cpumask_weight(&entry->comp_vect_mask) %525 hfi1_per_node_cntr[dd->node] != 0)526 possible_cpus_comp_vect++;527 }528 529 dd->comp_vect_possible_cpus = possible_cpus_comp_vect;530 531 /* Reserving CPUs for device completion vector */532 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {533 curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask,534 entry->comp_vect_affinity);535 if (curr_cpu < 0)536 goto fail;537 538 cpumask_set_cpu(curr_cpu, dev_comp_vect_mask);539 }540 541 hfi1_cdbg(AFFINITY,542 "[%s] Completion vector affinity CPU set(s) %*pbl",543 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi),544 cpumask_pr_args(dev_comp_vect_mask));545 546 return 0;547 548fail:549 for (j = 0; j < i; j++)550 per_cpu_affinity_put_max(&entry->comp_vect_mask,551 entry->comp_vect_affinity);552 553 return curr_cpu;554}555 556/*557 * It assumes dd->comp_vect_possible_cpus is available.558 */559static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd,560 struct hfi1_affinity_node *entry)561 __must_hold(&node_affinity.lock)562{563 int i, cpu;564 565 lockdep_assert_held(&node_affinity.lock);566 if (!dd->comp_vect_possible_cpus)567 return;568 569 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {570 cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask,571 entry->comp_vect_affinity);572 /* Clearing CPU in device completion vector cpu mask */573 if (cpu >= 0)574 cpumask_clear_cpu(cpu, &dd->comp_vect->mask);575 }576 577 dd->comp_vect_possible_cpus = 0;578}579 580/*581 * Interrupt affinity.582 *583 * non-rcv avail gets a default mask that584 * starts as possible cpus with threads reset585 * and each rcv avail reset.586 *587 * rcv avail gets node relative 1 wrapping back588 * to the node relative 1 as necessary.589 *590 */591int hfi1_dev_affinity_init(struct hfi1_devdata *dd)592{593 struct hfi1_affinity_node *entry;594 const struct cpumask *local_mask;595 int curr_cpu, possible, i, ret;596 bool new_entry = false;597 598 local_mask = cpumask_of_node(dd->node);599 if (cpumask_first(local_mask) >= nr_cpu_ids)600 local_mask = topology_core_cpumask(0);601 602 mutex_lock(&node_affinity.lock);603 entry = node_affinity_lookup(dd->node);604 605 /*606 * If this is the first time this NUMA node's affinity is used,607 * create an entry in the global affinity structure and initialize it.608 */609 if (!entry) {610 entry = node_affinity_allocate(dd->node);611 if (!entry) {612 dd_dev_err(dd,613 "Unable to allocate global affinity node\n");614 ret = -ENOMEM;615 goto fail;616 }617 new_entry = true;618 619 init_cpu_mask_set(&entry->def_intr);620 init_cpu_mask_set(&entry->rcv_intr);621 cpumask_clear(&entry->comp_vect_mask);622 cpumask_clear(&entry->general_intr_mask);623 /* Use the "real" cpu mask of this node as the default */624 cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,625 local_mask);626 627 /* fill in the receive list */628 possible = cpumask_weight(&entry->def_intr.mask);629 curr_cpu = cpumask_first(&entry->def_intr.mask);630 631 if (possible == 1) {632 /* only one CPU, everyone will use it */633 cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);634 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);635 } else {636 /*637 * The general/control context will be the first CPU in638 * the default list, so it is removed from the default639 * list and added to the general interrupt list.640 */641 cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);642 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);643 curr_cpu = cpumask_next(curr_cpu,644 &entry->def_intr.mask);645 646 /*647 * Remove the remaining kernel receive queues from648 * the default list and add them to the receive list.649 */650 for (i = 0;651 i < (dd->n_krcv_queues - 1) *652 hfi1_per_node_cntr[dd->node];653 i++) {654 cpumask_clear_cpu(curr_cpu,655 &entry->def_intr.mask);656 cpumask_set_cpu(curr_cpu,657 &entry->rcv_intr.mask);658 curr_cpu = cpumask_next(curr_cpu,659 &entry->def_intr.mask);660 if (curr_cpu >= nr_cpu_ids)661 break;662 }663 664 /*665 * If there ends up being 0 CPU cores leftover for SDMA666 * engines, use the same CPU cores as general/control667 * context.668 */669 if (cpumask_empty(&entry->def_intr.mask))670 cpumask_copy(&entry->def_intr.mask,671 &entry->general_intr_mask);672 }673 674 /* Determine completion vector CPUs for the entire node */675 cpumask_and(&entry->comp_vect_mask,676 &node_affinity.real_cpu_mask, local_mask);677 cpumask_andnot(&entry->comp_vect_mask,678 &entry->comp_vect_mask,679 &entry->rcv_intr.mask);680 cpumask_andnot(&entry->comp_vect_mask,681 &entry->comp_vect_mask,682 &entry->general_intr_mask);683 684 /*685 * If there ends up being 0 CPU cores leftover for completion686 * vectors, use the same CPU core as the general/control687 * context.688 */689 if (cpumask_empty(&entry->comp_vect_mask))690 cpumask_copy(&entry->comp_vect_mask,691 &entry->general_intr_mask);692 }693 694 ret = _dev_comp_vect_cpu_mask_init(dd, entry, new_entry);695 if (ret < 0)696 goto fail;697 698 if (new_entry)699 node_affinity_add_tail(entry);700 701 dd->affinity_entry = entry;702 mutex_unlock(&node_affinity.lock);703 704 return 0;705 706fail:707 if (new_entry)708 node_affinity_destroy(entry);709 mutex_unlock(&node_affinity.lock);710 return ret;711}712 713void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd)714{715 struct hfi1_affinity_node *entry;716 717 mutex_lock(&node_affinity.lock);718 if (!dd->affinity_entry)719 goto unlock;720 entry = node_affinity_lookup(dd->node);721 if (!entry)722 goto unlock;723 724 /*725 * Free device completion vector CPUs to be used by future726 * completion vectors727 */728 _dev_comp_vect_cpu_mask_clean_up(dd, entry);729unlock:730 dd->affinity_entry = NULL;731 mutex_unlock(&node_affinity.lock);732}733 734/*735 * Function updates the irq affinity hint for msix after it has been changed736 * by the user using the /proc/irq interface. This function only accepts737 * one cpu in the mask.738 */739static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu)740{741 struct sdma_engine *sde = msix->arg;742 struct hfi1_devdata *dd = sde->dd;743 struct hfi1_affinity_node *entry;744 struct cpu_mask_set *set;745 int i, old_cpu;746 747 if (cpu > num_online_cpus() || cpu == sde->cpu)748 return;749 750 mutex_lock(&node_affinity.lock);751 entry = node_affinity_lookup(dd->node);752 if (!entry)753 goto unlock;754 755 old_cpu = sde->cpu;756 sde->cpu = cpu;757 cpumask_clear(&msix->mask);758 cpumask_set_cpu(cpu, &msix->mask);759 dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n",760 msix->irq, irq_type_names[msix->type],761 sde->this_idx, cpu);762 irq_set_affinity_hint(msix->irq, &msix->mask);763 764 /*765 * Set the new cpu in the hfi1_affinity_node and clean766 * the old cpu if it is not used by any other IRQ767 */768 set = &entry->def_intr;769 cpumask_set_cpu(cpu, &set->mask);770 cpumask_set_cpu(cpu, &set->used);771 for (i = 0; i < dd->msix_info.max_requested; i++) {772 struct hfi1_msix_entry *other_msix;773 774 other_msix = &dd->msix_info.msix_entries[i];775 if (other_msix->type != IRQ_SDMA || other_msix == msix)776 continue;777 778 if (cpumask_test_cpu(old_cpu, &other_msix->mask))779 goto unlock;780 }781 cpumask_clear_cpu(old_cpu, &set->mask);782 cpumask_clear_cpu(old_cpu, &set->used);783unlock:784 mutex_unlock(&node_affinity.lock);785}786 787static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify,788 const cpumask_t *mask)789{790 int cpu = cpumask_first(mask);791 struct hfi1_msix_entry *msix = container_of(notify,792 struct hfi1_msix_entry,793 notify);794 795 /* Only one CPU configuration supported currently */796 hfi1_update_sdma_affinity(msix, cpu);797}798 799static void hfi1_irq_notifier_release(struct kref *ref)800{801 /*802 * This is required by affinity notifier. We don't have anything to803 * free here.804 */805}806 807static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix)808{809 struct irq_affinity_notify *notify = &msix->notify;810 811 notify->irq = msix->irq;812 notify->notify = hfi1_irq_notifier_notify;813 notify->release = hfi1_irq_notifier_release;814 815 if (irq_set_affinity_notifier(notify->irq, notify))816 pr_err("Failed to register sdma irq affinity notifier for irq %d\n",817 notify->irq);818}819 820static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix)821{822 struct irq_affinity_notify *notify = &msix->notify;823 824 if (irq_set_affinity_notifier(notify->irq, NULL))825 pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n",826 notify->irq);827}828 829/*830 * Function sets the irq affinity for msix.831 * It *must* be called with node_affinity.lock held.832 */833static int get_irq_affinity(struct hfi1_devdata *dd,834 struct hfi1_msix_entry *msix)835{836 cpumask_var_t diff;837 struct hfi1_affinity_node *entry;838 struct cpu_mask_set *set = NULL;839 struct sdma_engine *sde = NULL;840 struct hfi1_ctxtdata *rcd = NULL;841 char extra[64];842 int cpu = -1;843 844 extra[0] = '\0';845 cpumask_clear(&msix->mask);846 847 entry = node_affinity_lookup(dd->node);848 849 switch (msix->type) {850 case IRQ_SDMA:851 sde = (struct sdma_engine *)msix->arg;852 scnprintf(extra, 64, "engine %u", sde->this_idx);853 set = &entry->def_intr;854 break;855 case IRQ_GENERAL:856 cpu = cpumask_first(&entry->general_intr_mask);857 break;858 case IRQ_RCVCTXT:859 rcd = (struct hfi1_ctxtdata *)msix->arg;860 if (rcd->ctxt == HFI1_CTRL_CTXT)861 cpu = cpumask_first(&entry->general_intr_mask);862 else863 set = &entry->rcv_intr;864 scnprintf(extra, 64, "ctxt %u", rcd->ctxt);865 break;866 case IRQ_NETDEVCTXT:867 rcd = (struct hfi1_ctxtdata *)msix->arg;868 set = &entry->def_intr;869 scnprintf(extra, 64, "ctxt %u", rcd->ctxt);870 break;871 default:872 dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type);873 return -EINVAL;874 }875 876 /*877 * The general and control contexts are placed on a particular878 * CPU, which is set above. Skip accounting for it. Everything else879 * finds its CPU here.880 */881 if (cpu == -1 && set) {882 if (!zalloc_cpumask_var(&diff, GFP_KERNEL))883 return -ENOMEM;884 885 cpu = cpu_mask_set_get_first(set, diff);886 if (cpu < 0) {887 free_cpumask_var(diff);888 dd_dev_err(dd, "Failure to obtain CPU for IRQ\n");889 return cpu;890 }891 892 free_cpumask_var(diff);893 }894 895 cpumask_set_cpu(cpu, &msix->mask);896 dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n",897 msix->irq, irq_type_names[msix->type],898 extra, cpu);899 irq_set_affinity_hint(msix->irq, &msix->mask);900 901 if (msix->type == IRQ_SDMA) {902 sde->cpu = cpu;903 hfi1_setup_sdma_notifier(msix);904 }905 906 return 0;907}908 909int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)910{911 int ret;912 913 mutex_lock(&node_affinity.lock);914 ret = get_irq_affinity(dd, msix);915 mutex_unlock(&node_affinity.lock);916 return ret;917}918 919void hfi1_put_irq_affinity(struct hfi1_devdata *dd,920 struct hfi1_msix_entry *msix)921{922 struct cpu_mask_set *set = NULL;923 struct hfi1_affinity_node *entry;924 925 mutex_lock(&node_affinity.lock);926 entry = node_affinity_lookup(dd->node);927 928 switch (msix->type) {929 case IRQ_SDMA:930 set = &entry->def_intr;931 hfi1_cleanup_sdma_notifier(msix);932 break;933 case IRQ_GENERAL:934 /* Don't do accounting for general contexts */935 break;936 case IRQ_RCVCTXT: {937 struct hfi1_ctxtdata *rcd = msix->arg;938 939 /* Don't do accounting for control contexts */940 if (rcd->ctxt != HFI1_CTRL_CTXT)941 set = &entry->rcv_intr;942 break;943 }944 case IRQ_NETDEVCTXT:945 set = &entry->def_intr;946 break;947 default:948 mutex_unlock(&node_affinity.lock);949 return;950 }951 952 if (set) {953 cpumask_andnot(&set->used, &set->used, &msix->mask);954 _cpu_mask_set_gen_dec(set);955 }956 957 irq_set_affinity_hint(msix->irq, NULL);958 cpumask_clear(&msix->mask);959 mutex_unlock(&node_affinity.lock);960}961 962/* This should be called with node_affinity.lock held */963static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,964 struct hfi1_affinity_node_list *affinity)965{966 int possible, curr_cpu, i;967 uint num_cores_per_socket = node_affinity.num_online_cpus /968 affinity->num_core_siblings /969 node_affinity.num_online_nodes;970 971 cpumask_copy(hw_thread_mask, &affinity->proc.mask);972 if (affinity->num_core_siblings > 0) {973 /* Removing other siblings not needed for now */974 possible = cpumask_weight(hw_thread_mask);975 curr_cpu = cpumask_first(hw_thread_mask);976 for (i = 0;977 i < num_cores_per_socket * node_affinity.num_online_nodes;978 i++)979 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);980 981 for (; i < possible; i++) {982 cpumask_clear_cpu(curr_cpu, hw_thread_mask);983 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);984 }985 986 /* Identifying correct HW threads within physical cores */987 cpumask_shift_left(hw_thread_mask, hw_thread_mask,988 num_cores_per_socket *989 node_affinity.num_online_nodes *990 hw_thread_no);991 }992}993 994int hfi1_get_proc_affinity(int node)995{996 int cpu = -1, ret, i;997 struct hfi1_affinity_node *entry;998 cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;999 const struct cpumask *node_mask,1000 *proc_mask = current->cpus_ptr;1001 struct hfi1_affinity_node_list *affinity = &node_affinity;1002 struct cpu_mask_set *set = &affinity->proc;1003 1004 /*1005 * check whether process/context affinity has already1006 * been set1007 */1008 if (current->nr_cpus_allowed == 1) {1009 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl",1010 current->pid, current->comm,1011 cpumask_pr_args(proc_mask));1012 /*1013 * Mark the pre-set CPU as used. This is atomic so we don't1014 * need the lock1015 */1016 cpu = cpumask_first(proc_mask);1017 cpumask_set_cpu(cpu, &set->used);1018 goto done;1019 } else if (current->nr_cpus_allowed < cpumask_weight(&set->mask)) {1020 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl",1021 current->pid, current->comm,1022 cpumask_pr_args(proc_mask));1023 goto done;1024 }1025 1026 /*1027 * The process does not have a preset CPU affinity so find one to1028 * recommend using the following algorithm:1029 *1030 * For each user process that is opening a context on HFI Y:1031 * a) If all cores are filled, reinitialize the bitmask1032 * b) Fill real cores first, then HT cores (First set of HT1033 * cores on all physical cores, then second set of HT core,1034 * and, so on) in the following order:1035 *1036 * 1. Same NUMA node as HFI Y and not running an IRQ1037 * handler1038 * 2. Same NUMA node as HFI Y and running an IRQ handler1039 * 3. Different NUMA node to HFI Y and not running an IRQ1040 * handler1041 * 4. Different NUMA node to HFI Y and running an IRQ1042 * handler1043 * c) Mark core as filled in the bitmask. As user processes are1044 * done, clear cores from the bitmask.1045 */1046 1047 ret = zalloc_cpumask_var(&diff, GFP_KERNEL);1048 if (!ret)1049 goto done;1050 ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);1051 if (!ret)1052 goto free_diff;1053 ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);1054 if (!ret)1055 goto free_hw_thread_mask;1056 ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);1057 if (!ret)1058 goto free_available_mask;1059 1060 mutex_lock(&affinity->lock);1061 /*1062 * If we've used all available HW threads, clear the mask and start1063 * overloading.1064 */1065 _cpu_mask_set_gen_inc(set);1066 1067 /*1068 * If NUMA node has CPUs used by interrupt handlers, include them in the1069 * interrupt handler mask.1070 */1071 entry = node_affinity_lookup(node);1072 if (entry) {1073 cpumask_copy(intrs_mask, (entry->def_intr.gen ?1074 &entry->def_intr.mask :1075 &entry->def_intr.used));1076 cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?1077 &entry->rcv_intr.mask :1078 &entry->rcv_intr.used));1079 cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);1080 }1081 hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",1082 cpumask_pr_args(intrs_mask));1083 1084 cpumask_copy(hw_thread_mask, &set->mask);1085 1086 /*1087 * If HT cores are enabled, identify which HW threads within the1088 * physical cores should be used.1089 */1090 if (affinity->num_core_siblings > 0) {1091 for (i = 0; i < affinity->num_core_siblings; i++) {1092 find_hw_thread_mask(i, hw_thread_mask, affinity);1093 1094 /*1095 * If there's at least one available core for this HW1096 * thread number, stop looking for a core.1097 *1098 * diff will always be not empty at least once in this1099 * loop as the used mask gets reset when1100 * (set->mask == set->used) before this loop.1101 */1102 cpumask_andnot(diff, hw_thread_mask, &set->used);1103 if (!cpumask_empty(diff))1104 break;1105 }1106 }1107 hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",1108 cpumask_pr_args(hw_thread_mask));1109 1110 node_mask = cpumask_of_node(node);1111 hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,1112 cpumask_pr_args(node_mask));1113 1114 /* Get cpumask of available CPUs on preferred NUMA */1115 cpumask_and(available_mask, hw_thread_mask, node_mask);1116 cpumask_andnot(available_mask, available_mask, &set->used);1117 hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,1118 cpumask_pr_args(available_mask));1119 1120 /*1121 * At first, we don't want to place processes on the same1122 * CPUs as interrupt handlers. Then, CPUs running interrupt1123 * handlers are used.1124 *1125 * 1) If diff is not empty, then there are CPUs not running1126 * non-interrupt handlers available, so diff gets copied1127 * over to available_mask.1128 * 2) If diff is empty, then all CPUs not running interrupt1129 * handlers are taken, so available_mask contains all1130 * available CPUs running interrupt handlers.1131 * 3) If available_mask is empty, then all CPUs on the1132 * preferred NUMA node are taken, so other NUMA nodes are1133 * used for process assignments using the same method as1134 * the preferred NUMA node.1135 */1136 cpumask_andnot(diff, available_mask, intrs_mask);1137 if (!cpumask_empty(diff))1138 cpumask_copy(available_mask, diff);1139 1140 /* If we don't have CPUs on the preferred node, use other NUMA nodes */1141 if (cpumask_empty(available_mask)) {1142 cpumask_andnot(available_mask, hw_thread_mask, &set->used);1143 /* Excluding preferred NUMA cores */1144 cpumask_andnot(available_mask, available_mask, node_mask);1145 hfi1_cdbg(PROC,1146 "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",1147 cpumask_pr_args(available_mask));1148 1149 /*1150 * At first, we don't want to place processes on the same1151 * CPUs as interrupt handlers.1152 */1153 cpumask_andnot(diff, available_mask, intrs_mask);1154 if (!cpumask_empty(diff))1155 cpumask_copy(available_mask, diff);1156 }1157 hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",1158 cpumask_pr_args(available_mask));1159 1160 cpu = cpumask_first(available_mask);1161 if (cpu >= nr_cpu_ids) /* empty */1162 cpu = -1;1163 else1164 cpumask_set_cpu(cpu, &set->used);1165 1166 mutex_unlock(&affinity->lock);1167 hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);1168 1169 free_cpumask_var(intrs_mask);1170free_available_mask:1171 free_cpumask_var(available_mask);1172free_hw_thread_mask:1173 free_cpumask_var(hw_thread_mask);1174free_diff:1175 free_cpumask_var(diff);1176done:1177 return cpu;1178}1179 1180void hfi1_put_proc_affinity(int cpu)1181{1182 struct hfi1_affinity_node_list *affinity = &node_affinity;1183 struct cpu_mask_set *set = &affinity->proc;1184 1185 if (cpu < 0)1186 return;1187 1188 mutex_lock(&affinity->lock);1189 cpu_mask_set_put(set, cpu);1190 hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);1191 mutex_unlock(&affinity->lock);1192}1193