785 lines · c
1// SPDX-License-Identifier: GPL-2.0-or-later2/*3 * coupled.c - helper functions to enter the same idle state on multiple cpus4 *5 * Copyright (c) 2011 Google, Inc.6 *7 * Author: Colin Cross <ccross@android.com>8 */9 10#include <linux/kernel.h>11#include <linux/cpu.h>12#include <linux/cpuidle.h>13#include <linux/mutex.h>14#include <linux/sched.h>15#include <linux/slab.h>16#include <linux/spinlock.h>17 18#include "cpuidle.h"19 20/**21 * DOC: Coupled cpuidle states22 *23 * On some ARM SMP SoCs (OMAP4460, Tegra 2, and probably more), the24 * cpus cannot be independently powered down, either due to25 * sequencing restrictions (on Tegra 2, cpu 0 must be the last to26 * power down), or due to HW bugs (on OMAP4460, a cpu powering up27 * will corrupt the gic state unless the other cpu runs a work28 * around). Each cpu has a power state that it can enter without29 * coordinating with the other cpu (usually Wait For Interrupt, or30 * WFI), and one or more "coupled" power states that affect blocks31 * shared between the cpus (L2 cache, interrupt controller, and32 * sometimes the whole SoC). Entering a coupled power state must33 * be tightly controlled on both cpus.34 *35 * This file implements a solution, where each cpu will wait in the36 * WFI state until all cpus are ready to enter a coupled state, at37 * which point the coupled state function will be called on all38 * cpus at approximately the same time.39 *40 * Once all cpus are ready to enter idle, they are woken by an smp41 * cross call. At this point, there is a chance that one of the42 * cpus will find work to do, and choose not to enter idle. A43 * final pass is needed to guarantee that all cpus will call the44 * power state enter function at the same time. During this pass,45 * each cpu will increment the ready counter, and continue once the46 * ready counter matches the number of online coupled cpus. If any47 * cpu exits idle, the other cpus will decrement their counter and48 * retry.49 *50 * requested_state stores the deepest coupled idle state each cpu51 * is ready for. It is assumed that the states are indexed from52 * shallowest (highest power, lowest exit latency) to deepest53 * (lowest power, highest exit latency). The requested_state54 * variable is not locked. It is only written from the cpu that55 * it stores (or by the on/offlining cpu if that cpu is offline),56 * and only read after all the cpus are ready for the coupled idle57 * state are no longer updating it.58 *59 * Three atomic counters are used. alive_count tracks the number60 * of cpus in the coupled set that are currently or soon will be61 * online. waiting_count tracks the number of cpus that are in62 * the waiting loop, in the ready loop, or in the coupled idle state.63 * ready_count tracks the number of cpus that are in the ready loop64 * or in the coupled idle state.65 *66 * To use coupled cpuidle states, a cpuidle driver must:67 *68 * Set struct cpuidle_device.coupled_cpus to the mask of all69 * coupled cpus, usually the same as cpu_possible_mask if all cpus70 * are part of the same cluster. The coupled_cpus mask must be71 * set in the struct cpuidle_device for each cpu.72 *73 * Set struct cpuidle_device.safe_state to a state that is not a74 * coupled state. This is usually WFI.75 *76 * Set CPUIDLE_FLAG_COUPLED in struct cpuidle_state.flags for each77 * state that affects multiple cpus.78 *79 * Provide a struct cpuidle_state.enter function for each state80 * that affects multiple cpus. This function is guaranteed to be81 * called on all cpus at approximately the same time. The driver82 * should ensure that the cpus all abort together if any cpu tries83 * to abort once the function is called. The function should return84 * with interrupts still disabled.85 */86 87/**88 * struct cpuidle_coupled - data for set of cpus that share a coupled idle state89 * @coupled_cpus: mask of cpus that are part of the coupled set90 * @requested_state: array of requested states for cpus in the coupled set91 * @ready_waiting_counts: combined count of cpus in ready or waiting loops92 * @abort_barrier: synchronisation point for abort cases93 * @online_count: count of cpus that are online94 * @refcnt: reference count of cpuidle devices that are using this struct95 * @prevent: flag to prevent coupled idle while a cpu is hotplugging96 */97struct cpuidle_coupled {98 cpumask_t coupled_cpus;99 int requested_state[NR_CPUS];100 atomic_t ready_waiting_counts;101 atomic_t abort_barrier;102 int online_count;103 int refcnt;104 int prevent;105};106 107#define WAITING_BITS 16108#define MAX_WAITING_CPUS (1 << WAITING_BITS)109#define WAITING_MASK (MAX_WAITING_CPUS - 1)110#define READY_MASK (~WAITING_MASK)111 112#define CPUIDLE_COUPLED_NOT_IDLE (-1)113 114static DEFINE_PER_CPU(call_single_data_t, cpuidle_coupled_poke_cb);115 116/*117 * The cpuidle_coupled_poke_pending mask is used to avoid calling118 * __smp_call_function_single with the per cpu call_single_data_t struct already119 * in use. This prevents a deadlock where two cpus are waiting for each others120 * call_single_data_t struct to be available121 */122static cpumask_t cpuidle_coupled_poke_pending;123 124/*125 * The cpuidle_coupled_poked mask is used to ensure that each cpu has been poked126 * once to minimize entering the ready loop with a poke pending, which would127 * require aborting and retrying.128 */129static cpumask_t cpuidle_coupled_poked;130 131/**132 * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus133 * @dev: cpuidle_device of the calling cpu134 * @a: atomic variable to hold the barrier135 *136 * No caller to this function will return from this function until all online137 * cpus in the same coupled group have called this function. Once any caller138 * has returned from this function, the barrier is immediately available for139 * reuse.140 *141 * The atomic variable must be initialized to 0 before any cpu calls142 * this function, will be reset to 0 before any cpu returns from this function.143 *144 * Must only be called from within a coupled idle state handler145 * (state.enter when state.flags has CPUIDLE_FLAG_COUPLED set).146 *147 * Provides full smp barrier semantics before and after calling.148 */149void cpuidle_coupled_parallel_barrier(struct cpuidle_device *dev, atomic_t *a)150{151 int n = dev->coupled->online_count;152 153 smp_mb__before_atomic();154 atomic_inc(a);155 156 while (atomic_read(a) < n)157 cpu_relax();158 159 if (atomic_inc_return(a) == n * 2) {160 atomic_set(a, 0);161 return;162 }163 164 while (atomic_read(a) > n)165 cpu_relax();166}167 168/**169 * cpuidle_state_is_coupled - check if a state is part of a coupled set170 * @drv: struct cpuidle_driver for the platform171 * @state: index of the target state in drv->states172 *173 * Returns true if the target state is coupled with cpus besides this one174 */175bool cpuidle_state_is_coupled(struct cpuidle_driver *drv, int state)176{177 return drv->states[state].flags & CPUIDLE_FLAG_COUPLED;178}179 180/**181 * cpuidle_coupled_state_verify - check if the coupled states are correctly set.182 * @drv: struct cpuidle_driver for the platform183 *184 * Returns 0 for valid state values, a negative error code otherwise:185 * * -EINVAL if any coupled state(safe_state_index) is wrongly set.186 */187int cpuidle_coupled_state_verify(struct cpuidle_driver *drv)188{189 int i;190 191 for (i = drv->state_count - 1; i >= 0; i--) {192 if (cpuidle_state_is_coupled(drv, i) &&193 (drv->safe_state_index == i ||194 drv->safe_state_index < 0 ||195 drv->safe_state_index >= drv->state_count))196 return -EINVAL;197 }198 199 return 0;200}201 202/**203 * cpuidle_coupled_set_ready - mark a cpu as ready204 * @coupled: the struct coupled that contains the current cpu205 */206static inline void cpuidle_coupled_set_ready(struct cpuidle_coupled *coupled)207{208 atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);209}210 211/**212 * cpuidle_coupled_set_not_ready - mark a cpu as not ready213 * @coupled: the struct coupled that contains the current cpu214 *215 * Decrements the ready counter, unless the ready (and thus the waiting) counter216 * is equal to the number of online cpus. Prevents a race where one cpu217 * decrements the waiting counter and then re-increments it just before another218 * cpu has decremented its ready counter, leading to the ready counter going219 * down from the number of online cpus without going through the coupled idle220 * state.221 *222 * Returns 0 if the counter was decremented successfully, -EINVAL if the ready223 * counter was equal to the number of online cpus.224 */225static226inline int cpuidle_coupled_set_not_ready(struct cpuidle_coupled *coupled)227{228 int all;229 int ret;230 231 all = coupled->online_count | (coupled->online_count << WAITING_BITS);232 ret = atomic_add_unless(&coupled->ready_waiting_counts,233 -MAX_WAITING_CPUS, all);234 235 return ret ? 0 : -EINVAL;236}237 238/**239 * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready240 * @coupled: the struct coupled that contains the current cpu241 *242 * Returns true if all of the cpus in a coupled set are out of the ready loop.243 */244static inline int cpuidle_coupled_no_cpus_ready(struct cpuidle_coupled *coupled)245{246 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;247 return r == 0;248}249 250/**251 * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready252 * @coupled: the struct coupled that contains the current cpu253 *254 * Returns true if all cpus coupled to this target state are in the ready loop255 */256static inline bool cpuidle_coupled_cpus_ready(struct cpuidle_coupled *coupled)257{258 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS;259 return r == coupled->online_count;260}261 262/**263 * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting264 * @coupled: the struct coupled that contains the current cpu265 *266 * Returns true if all cpus coupled to this target state are in the wait loop267 */268static inline bool cpuidle_coupled_cpus_waiting(struct cpuidle_coupled *coupled)269{270 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;271 return w == coupled->online_count;272}273 274/**275 * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting276 * @coupled: the struct coupled that contains the current cpu277 *278 * Returns true if all of the cpus in a coupled set are out of the waiting loop.279 */280static inline int cpuidle_coupled_no_cpus_waiting(struct cpuidle_coupled *coupled)281{282 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK;283 return w == 0;284}285 286/**287 * cpuidle_coupled_get_state - determine the deepest idle state288 * @dev: struct cpuidle_device for this cpu289 * @coupled: the struct coupled that contains the current cpu290 *291 * Returns the deepest idle state that all coupled cpus can enter292 */293static inline int cpuidle_coupled_get_state(struct cpuidle_device *dev,294 struct cpuidle_coupled *coupled)295{296 int i;297 int state = INT_MAX;298 299 /*300 * Read barrier ensures that read of requested_state is ordered after301 * reads of ready_count. Matches the write barriers302 * cpuidle_set_state_waiting.303 */304 smp_rmb();305 306 for_each_cpu(i, &coupled->coupled_cpus)307 if (cpu_online(i) && coupled->requested_state[i] < state)308 state = coupled->requested_state[i];309 310 return state;311}312 313static void cpuidle_coupled_handle_poke(void *info)314{315 int cpu = (unsigned long)info;316 cpumask_set_cpu(cpu, &cpuidle_coupled_poked);317 cpumask_clear_cpu(cpu, &cpuidle_coupled_poke_pending);318}319 320/**321 * cpuidle_coupled_poke - wake up a cpu that may be waiting322 * @cpu: target cpu323 *324 * Ensures that the target cpu exits it's waiting idle state (if it is in it)325 * and will see updates to waiting_count before it re-enters it's waiting idle326 * state.327 *328 * If cpuidle_coupled_poked_mask is already set for the target cpu, that cpu329 * either has or will soon have a pending IPI that will wake it out of idle,330 * or it is currently processing the IPI and is not in idle.331 */332static void cpuidle_coupled_poke(int cpu)333{334 call_single_data_t *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);335 336 if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending))337 smp_call_function_single_async(cpu, csd);338}339 340/**341 * cpuidle_coupled_poke_others - wake up all other cpus that may be waiting342 * @this_cpu: target cpu343 * @coupled: the struct coupled that contains the current cpu344 *345 * Calls cpuidle_coupled_poke on all other online cpus.346 */347static void cpuidle_coupled_poke_others(int this_cpu,348 struct cpuidle_coupled *coupled)349{350 int cpu;351 352 for_each_cpu(cpu, &coupled->coupled_cpus)353 if (cpu != this_cpu && cpu_online(cpu))354 cpuidle_coupled_poke(cpu);355}356 357/**358 * cpuidle_coupled_set_waiting - mark this cpu as in the wait loop359 * @cpu: target cpu360 * @coupled: the struct coupled that contains the current cpu361 * @next_state: the index in drv->states of the requested state for this cpu362 *363 * Updates the requested idle state for the specified cpuidle device.364 * Returns the number of waiting cpus.365 */366static int cpuidle_coupled_set_waiting(int cpu,367 struct cpuidle_coupled *coupled, int next_state)368{369 coupled->requested_state[cpu] = next_state;370 371 /*372 * The atomic_inc_return provides a write barrier to order the write373 * to requested_state with the later write that increments ready_count.374 */375 return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK;376}377 378/**379 * cpuidle_coupled_set_not_waiting - mark this cpu as leaving the wait loop380 * @cpu: target cpu381 * @coupled: the struct coupled that contains the current cpu382 *383 * Removes the requested idle state for the specified cpuidle device.384 */385static void cpuidle_coupled_set_not_waiting(int cpu,386 struct cpuidle_coupled *coupled)387{388 /*389 * Decrementing waiting count can race with incrementing it in390 * cpuidle_coupled_set_waiting, but that's OK. Worst case, some391 * cpus will increment ready_count and then spin until they392 * notice that this cpu has cleared it's requested_state.393 */394 atomic_dec(&coupled->ready_waiting_counts);395 396 coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE;397}398 399/**400 * cpuidle_coupled_set_done - mark this cpu as leaving the ready loop401 * @cpu: the current cpu402 * @coupled: the struct coupled that contains the current cpu403 *404 * Marks this cpu as no longer in the ready and waiting loops. Decrements405 * the waiting count first to prevent another cpu looping back in and seeing406 * this cpu as waiting just before it exits idle.407 */408static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled)409{410 cpuidle_coupled_set_not_waiting(cpu, coupled);411 atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts);412}413 414/**415 * cpuidle_coupled_clear_pokes - spin until the poke interrupt is processed416 * @cpu: this cpu417 *418 * Turns on interrupts and spins until any outstanding poke interrupts have419 * been processed and the poke bit has been cleared.420 *421 * Other interrupts may also be processed while interrupts are enabled, so422 * need_resched() must be tested after this function returns to make sure423 * the interrupt didn't schedule work that should take the cpu out of idle.424 *425 * Returns 0 if no poke was pending, 1 if a poke was cleared.426 */427static int cpuidle_coupled_clear_pokes(int cpu)428{429 if (!cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))430 return 0;431 432 local_irq_enable();433 while (cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending))434 cpu_relax();435 local_irq_disable();436 437 return 1;438}439 440static bool cpuidle_coupled_any_pokes_pending(struct cpuidle_coupled *coupled)441{442 return cpumask_first_and_and(cpu_online_mask, &coupled->coupled_cpus,443 &cpuidle_coupled_poke_pending) < nr_cpu_ids;444}445 446/**447 * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus448 * @dev: struct cpuidle_device for the current cpu449 * @drv: struct cpuidle_driver for the platform450 * @next_state: index of the requested state in drv->states451 *452 * Coordinate with coupled cpus to enter the target state. This is a two453 * stage process. In the first stage, the cpus are operating independently,454 * and may call into cpuidle_enter_state_coupled at completely different times.455 * To save as much power as possible, the first cpus to call this function will456 * go to an intermediate state (the cpuidle_device's safe state), and wait for457 * all the other cpus to call this function. Once all coupled cpus are idle,458 * the second stage will start. Each coupled cpu will spin until all cpus have459 * guaranteed that they will call the target_state.460 *461 * This function must be called with interrupts disabled. It may enable462 * interrupts while preparing for idle, and it will always return with463 * interrupts enabled.464 */465int cpuidle_enter_state_coupled(struct cpuidle_device *dev,466 struct cpuidle_driver *drv, int next_state)467{468 int entered_state = -1;469 struct cpuidle_coupled *coupled = dev->coupled;470 int w;471 472 if (!coupled)473 return -EINVAL;474 475 while (coupled->prevent) {476 cpuidle_coupled_clear_pokes(dev->cpu);477 if (need_resched()) {478 local_irq_enable();479 return entered_state;480 }481 entered_state = cpuidle_enter_state(dev, drv,482 drv->safe_state_index);483 local_irq_disable();484 }485 486 /* Read barrier ensures online_count is read after prevent is cleared */487 smp_rmb();488 489reset:490 cpumask_clear_cpu(dev->cpu, &cpuidle_coupled_poked);491 492 w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state);493 /*494 * If this is the last cpu to enter the waiting state, poke495 * all the other cpus out of their waiting state so they can496 * enter a deeper state. This can race with one of the cpus497 * exiting the waiting state due to an interrupt and498 * decrementing waiting_count, see comment below.499 */500 if (w == coupled->online_count) {501 cpumask_set_cpu(dev->cpu, &cpuidle_coupled_poked);502 cpuidle_coupled_poke_others(dev->cpu, coupled);503 }504 505retry:506 /*507 * Wait for all coupled cpus to be idle, using the deepest state508 * allowed for a single cpu. If this was not the poking cpu, wait509 * for at least one poke before leaving to avoid a race where510 * two cpus could arrive at the waiting loop at the same time,511 * but the first of the two to arrive could skip the loop without512 * processing the pokes from the last to arrive.513 */514 while (!cpuidle_coupled_cpus_waiting(coupled) ||515 !cpumask_test_cpu(dev->cpu, &cpuidle_coupled_poked)) {516 if (cpuidle_coupled_clear_pokes(dev->cpu))517 continue;518 519 if (need_resched()) {520 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);521 goto out;522 }523 524 if (coupled->prevent) {525 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);526 goto out;527 }528 529 entered_state = cpuidle_enter_state(dev, drv,530 drv->safe_state_index);531 local_irq_disable();532 }533 534 cpuidle_coupled_clear_pokes(dev->cpu);535 if (need_resched()) {536 cpuidle_coupled_set_not_waiting(dev->cpu, coupled);537 goto out;538 }539 540 /*541 * Make sure final poke status for this cpu is visible before setting542 * cpu as ready.543 */544 smp_wmb();545 546 /*547 * All coupled cpus are probably idle. There is a small chance that548 * one of the other cpus just became active. Increment the ready count,549 * and spin until all coupled cpus have incremented the counter. Once a550 * cpu has incremented the ready counter, it cannot abort idle and must551 * spin until either all cpus have incremented the ready counter, or552 * another cpu leaves idle and decrements the waiting counter.553 */554 555 cpuidle_coupled_set_ready(coupled);556 while (!cpuidle_coupled_cpus_ready(coupled)) {557 /* Check if any other cpus bailed out of idle. */558 if (!cpuidle_coupled_cpus_waiting(coupled))559 if (!cpuidle_coupled_set_not_ready(coupled))560 goto retry;561 562 cpu_relax();563 }564 565 /*566 * Make sure read of all cpus ready is done before reading pending pokes567 */568 smp_rmb();569 570 /*571 * There is a small chance that a cpu left and reentered idle after this572 * cpu saw that all cpus were waiting. The cpu that reentered idle will573 * have sent this cpu a poke, which will still be pending after the574 * ready loop. The pending interrupt may be lost by the interrupt575 * controller when entering the deep idle state. It's not possible to576 * clear a pending interrupt without turning interrupts on and handling577 * it, and it's too late to turn on interrupts here, so reset the578 * coupled idle state of all cpus and retry.579 */580 if (cpuidle_coupled_any_pokes_pending(coupled)) {581 cpuidle_coupled_set_done(dev->cpu, coupled);582 /* Wait for all cpus to see the pending pokes */583 cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier);584 goto reset;585 }586 587 /* all cpus have acked the coupled state */588 next_state = cpuidle_coupled_get_state(dev, coupled);589 590 entered_state = cpuidle_enter_state(dev, drv, next_state);591 592 cpuidle_coupled_set_done(dev->cpu, coupled);593 594out:595 /*596 * Normal cpuidle states are expected to return with irqs enabled.597 * That leads to an inefficiency where a cpu receiving an interrupt598 * that brings it out of idle will process that interrupt before599 * exiting the idle enter function and decrementing ready_count. All600 * other cpus will need to spin waiting for the cpu that is processing601 * the interrupt. If the driver returns with interrupts disabled,602 * all other cpus will loop back into the safe idle state instead of603 * spinning, saving power.604 *605 * Calling local_irq_enable here allows coupled states to return with606 * interrupts disabled, but won't cause problems for drivers that607 * exit with interrupts enabled.608 */609 local_irq_enable();610 611 /*612 * Wait until all coupled cpus have exited idle. There is no risk that613 * a cpu exits and re-enters the ready state because this cpu has614 * already decremented its waiting_count.615 */616 while (!cpuidle_coupled_no_cpus_ready(coupled))617 cpu_relax();618 619 return entered_state;620}621 622static void cpuidle_coupled_update_online_cpus(struct cpuidle_coupled *coupled)623{624 coupled->online_count = cpumask_weight_and(cpu_online_mask, &coupled->coupled_cpus);625}626 627/**628 * cpuidle_coupled_register_device - register a coupled cpuidle device629 * @dev: struct cpuidle_device for the current cpu630 *631 * Called from cpuidle_register_device to handle coupled idle init. Finds the632 * cpuidle_coupled struct for this set of coupled cpus, or creates one if none633 * exists yet.634 */635int cpuidle_coupled_register_device(struct cpuidle_device *dev)636{637 int cpu;638 struct cpuidle_device *other_dev;639 call_single_data_t *csd;640 struct cpuidle_coupled *coupled;641 642 if (cpumask_empty(&dev->coupled_cpus))643 return 0;644 645 for_each_cpu(cpu, &dev->coupled_cpus) {646 other_dev = per_cpu(cpuidle_devices, cpu);647 if (other_dev && other_dev->coupled) {648 coupled = other_dev->coupled;649 goto have_coupled;650 }651 }652 653 /* No existing coupled info found, create a new one */654 coupled = kzalloc(sizeof(struct cpuidle_coupled), GFP_KERNEL);655 if (!coupled)656 return -ENOMEM;657 658 coupled->coupled_cpus = dev->coupled_cpus;659 660have_coupled:661 dev->coupled = coupled;662 if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus)))663 coupled->prevent++;664 665 cpuidle_coupled_update_online_cpus(coupled);666 667 coupled->refcnt++;668 669 csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu);670 INIT_CSD(csd, cpuidle_coupled_handle_poke, (void *)(unsigned long)dev->cpu);671 672 return 0;673}674 675/**676 * cpuidle_coupled_unregister_device - unregister a coupled cpuidle device677 * @dev: struct cpuidle_device for the current cpu678 *679 * Called from cpuidle_unregister_device to tear down coupled idle. Removes the680 * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if681 * this was the last cpu in the set.682 */683void cpuidle_coupled_unregister_device(struct cpuidle_device *dev)684{685 struct cpuidle_coupled *coupled = dev->coupled;686 687 if (cpumask_empty(&dev->coupled_cpus))688 return;689 690 if (--coupled->refcnt)691 kfree(coupled);692 dev->coupled = NULL;693}694 695/**696 * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state697 * @coupled: the struct coupled that contains the cpu that is changing state698 *699 * Disables coupled cpuidle on a coupled set of cpus. Used to ensure that700 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.701 */702static void cpuidle_coupled_prevent_idle(struct cpuidle_coupled *coupled)703{704 int cpu = get_cpu();705 706 /* Force all cpus out of the waiting loop. */707 coupled->prevent++;708 cpuidle_coupled_poke_others(cpu, coupled);709 put_cpu();710 while (!cpuidle_coupled_no_cpus_waiting(coupled))711 cpu_relax();712}713 714/**715 * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state716 * @coupled: the struct coupled that contains the cpu that is changing state717 *718 * Enables coupled cpuidle on a coupled set of cpus. Used to ensure that719 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.720 */721static void cpuidle_coupled_allow_idle(struct cpuidle_coupled *coupled)722{723 int cpu = get_cpu();724 725 /*726 * Write barrier ensures readers see the new online_count when they727 * see prevent == 0.728 */729 smp_wmb();730 coupled->prevent--;731 /* Force cpus out of the prevent loop. */732 cpuidle_coupled_poke_others(cpu, coupled);733 put_cpu();734}735 736static int coupled_cpu_online(unsigned int cpu)737{738 struct cpuidle_device *dev;739 740 mutex_lock(&cpuidle_lock);741 742 dev = per_cpu(cpuidle_devices, cpu);743 if (dev && dev->coupled) {744 cpuidle_coupled_update_online_cpus(dev->coupled);745 cpuidle_coupled_allow_idle(dev->coupled);746 }747 748 mutex_unlock(&cpuidle_lock);749 return 0;750}751 752static int coupled_cpu_up_prepare(unsigned int cpu)753{754 struct cpuidle_device *dev;755 756 mutex_lock(&cpuidle_lock);757 758 dev = per_cpu(cpuidle_devices, cpu);759 if (dev && dev->coupled)760 cpuidle_coupled_prevent_idle(dev->coupled);761 762 mutex_unlock(&cpuidle_lock);763 return 0;764}765 766static int __init cpuidle_coupled_init(void)767{768 int ret;769 770 ret = cpuhp_setup_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE,771 "cpuidle/coupled:prepare",772 coupled_cpu_up_prepare,773 coupled_cpu_online);774 if (ret)775 return ret;776 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,777 "cpuidle/coupled:online",778 coupled_cpu_online,779 coupled_cpu_up_prepare);780 if (ret < 0)781 cpuhp_remove_state_nocalls(CPUHP_CPUIDLE_COUPLED_PREPARE);782 return ret;783}784core_initcall(cpuidle_coupled_init);785