423 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Copyright 2018 Linaro Limited4 *5 * Author: Daniel Lezcano <daniel.lezcano@linaro.org>6 *7 * The idle injection framework provides a way to force CPUs to enter idle8 * states for a specified fraction of time over a specified period.9 *10 * It relies on the smpboot kthreads feature providing common code for CPU11 * hotplug and thread [un]parking.12 *13 * All of the kthreads used for idle injection are created at init time.14 *15 * Next, the users of the idle injection framework provide a cpumask via16 * its register function. The kthreads will be synchronized with respect to17 * this cpumask.18 *19 * The idle + run duration is specified via separate helpers and that allows20 * idle injection to be started.21 *22 * The idle injection kthreads will call play_idle_precise() with the idle23 * duration and max allowed latency specified as per the above.24 *25 * After all of them have been woken up, a timer is set to start the next idle26 * injection cycle.27 *28 * The timer interrupt handler will wake up the idle injection kthreads for29 * all of the CPUs in the cpumask provided by the user.30 *31 * Idle injection is stopped synchronously and no leftover idle injection32 * kthread activity after its completion is guaranteed.33 *34 * It is up to the user of this framework to provide a lock for higher-level35 * synchronization to prevent race conditions like starting idle injection36 * while unregistering from the framework.37 */38#define pr_fmt(fmt) "ii_dev: " fmt39 40#include <linux/cpu.h>41#include <linux/hrtimer.h>42#include <linux/kthread.h>43#include <linux/sched.h>44#include <linux/slab.h>45#include <linux/smpboot.h>46#include <linux/idle_inject.h>47 48#include <uapi/linux/sched/types.h>49 50/**51 * struct idle_inject_thread - task on/off switch structure52 * @tsk: task injecting the idle cycles53 * @should_run: whether or not to run the task (for the smpboot kthread API)54 */55struct idle_inject_thread {56 struct task_struct *tsk;57 int should_run;58};59 60/**61 * struct idle_inject_device - idle injection data62 * @timer: idle injection period timer63 * @idle_duration_us: duration of CPU idle time to inject64 * @run_duration_us: duration of CPU run time to allow65 * @latency_us: max allowed latency66 * @update: Optional callback deciding whether or not to skip idle67 * injection in the given cycle.68 * @cpumask: mask of CPUs affected by idle injection69 *70 * This structure is used to define per instance idle inject device data. Each71 * instance has an idle duration, a run duration and mask of CPUs to inject72 * idle.73 *74 * Actual CPU idle time is injected by calling kernel scheduler interface75 * play_idle_precise(). There is one optional callback that can be registered76 * by calling idle_inject_register_full():77 *78 * update() - This callback is invoked just before waking up CPUs to inject79 * idle. If it returns false, CPUs are not woken up to inject idle in the given80 * cycle. It also allows the caller to readjust the idle and run duration by81 * calling idle_inject_set_duration() for the next cycle.82 */83struct idle_inject_device {84 struct hrtimer timer;85 unsigned int idle_duration_us;86 unsigned int run_duration_us;87 unsigned int latency_us;88 bool (*update)(void);89 unsigned long cpumask[];90};91 92static DEFINE_PER_CPU(struct idle_inject_thread, idle_inject_thread);93static DEFINE_PER_CPU(struct idle_inject_device *, idle_inject_device);94 95/**96 * idle_inject_wakeup - Wake up idle injection threads97 * @ii_dev: target idle injection device98 *99 * Every idle injection task associated with the given idle injection device100 * and running on an online CPU will be woken up.101 */102static void idle_inject_wakeup(struct idle_inject_device *ii_dev)103{104 struct idle_inject_thread *iit;105 unsigned int cpu;106 107 for_each_cpu_and(cpu, to_cpumask(ii_dev->cpumask), cpu_online_mask) {108 iit = per_cpu_ptr(&idle_inject_thread, cpu);109 iit->should_run = 1;110 wake_up_process(iit->tsk);111 }112}113 114/**115 * idle_inject_timer_fn - idle injection timer function116 * @timer: idle injection hrtimer117 *118 * This function is called when the idle injection timer expires. It wakes up119 * idle injection tasks associated with the timer and they, in turn, invoke120 * play_idle_precise() to inject a specified amount of CPU idle time.121 *122 * Return: HRTIMER_RESTART.123 */124static enum hrtimer_restart idle_inject_timer_fn(struct hrtimer *timer)125{126 unsigned int duration_us;127 struct idle_inject_device *ii_dev =128 container_of(timer, struct idle_inject_device, timer);129 130 if (!ii_dev->update || ii_dev->update())131 idle_inject_wakeup(ii_dev);132 133 duration_us = READ_ONCE(ii_dev->run_duration_us);134 duration_us += READ_ONCE(ii_dev->idle_duration_us);135 136 hrtimer_forward_now(timer, ns_to_ktime(duration_us * NSEC_PER_USEC));137 138 return HRTIMER_RESTART;139}140 141/**142 * idle_inject_fn - idle injection work function143 * @cpu: the CPU owning the task144 *145 * This function calls play_idle_precise() to inject a specified amount of CPU146 * idle time.147 */148static void idle_inject_fn(unsigned int cpu)149{150 struct idle_inject_device *ii_dev;151 struct idle_inject_thread *iit;152 153 ii_dev = per_cpu(idle_inject_device, cpu);154 iit = per_cpu_ptr(&idle_inject_thread, cpu);155 156 /*157 * Let the smpboot main loop know that the task should not run again.158 */159 iit->should_run = 0;160 161 play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC,162 READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC);163}164 165/**166 * idle_inject_set_duration - idle and run duration update helper167 * @ii_dev: idle injection control device structure168 * @run_duration_us: CPU run time to allow in microseconds169 * @idle_duration_us: CPU idle time to inject in microseconds170 */171void idle_inject_set_duration(struct idle_inject_device *ii_dev,172 unsigned int run_duration_us,173 unsigned int idle_duration_us)174{175 if (run_duration_us + idle_duration_us) {176 WRITE_ONCE(ii_dev->run_duration_us, run_duration_us);177 WRITE_ONCE(ii_dev->idle_duration_us, idle_duration_us);178 }179 if (!run_duration_us)180 pr_debug("CPU is forced to 100 percent idle\n");181}182EXPORT_SYMBOL_NS_GPL(idle_inject_set_duration, IDLE_INJECT);183 184/**185 * idle_inject_get_duration - idle and run duration retrieval helper186 * @ii_dev: idle injection control device structure187 * @run_duration_us: memory location to store the current CPU run time188 * @idle_duration_us: memory location to store the current CPU idle time189 */190void idle_inject_get_duration(struct idle_inject_device *ii_dev,191 unsigned int *run_duration_us,192 unsigned int *idle_duration_us)193{194 *run_duration_us = READ_ONCE(ii_dev->run_duration_us);195 *idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);196}197EXPORT_SYMBOL_NS_GPL(idle_inject_get_duration, IDLE_INJECT);198 199/**200 * idle_inject_set_latency - set the maximum latency allowed201 * @ii_dev: idle injection control device structure202 * @latency_us: set the latency requirement for the idle state203 */204void idle_inject_set_latency(struct idle_inject_device *ii_dev,205 unsigned int latency_us)206{207 WRITE_ONCE(ii_dev->latency_us, latency_us);208}209EXPORT_SYMBOL_NS_GPL(idle_inject_set_latency, IDLE_INJECT);210 211/**212 * idle_inject_start - start idle injections213 * @ii_dev: idle injection control device structure214 *215 * The function starts idle injection by first waking up all of the idle216 * injection kthreads associated with @ii_dev to let them inject CPU idle time217 * sets up a timer to start the next idle injection period.218 *219 * Return: -EINVAL if the CPU idle or CPU run time is not set or 0 on success.220 */221int idle_inject_start(struct idle_inject_device *ii_dev)222{223 unsigned int idle_duration_us = READ_ONCE(ii_dev->idle_duration_us);224 unsigned int run_duration_us = READ_ONCE(ii_dev->run_duration_us);225 226 if (!(idle_duration_us + run_duration_us))227 return -EINVAL;228 229 pr_debug("Starting injecting idle cycles on CPUs '%*pbl'\n",230 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));231 232 idle_inject_wakeup(ii_dev);233 234 hrtimer_start(&ii_dev->timer,235 ns_to_ktime((idle_duration_us + run_duration_us) *236 NSEC_PER_USEC),237 HRTIMER_MODE_REL);238 239 return 0;240}241EXPORT_SYMBOL_NS_GPL(idle_inject_start, IDLE_INJECT);242 243/**244 * idle_inject_stop - stops idle injections245 * @ii_dev: idle injection control device structure246 *247 * The function stops idle injection and waits for the threads to finish work.248 * If CPU idle time is being injected when this function runs, then it will249 * wait until the end of the cycle.250 *251 * When it returns, there is no more idle injection kthread activity. The252 * kthreads are scheduled out and the periodic timer is off.253 */254void idle_inject_stop(struct idle_inject_device *ii_dev)255{256 struct idle_inject_thread *iit;257 unsigned int cpu;258 259 pr_debug("Stopping idle injection on CPUs '%*pbl'\n",260 cpumask_pr_args(to_cpumask(ii_dev->cpumask)));261 262 hrtimer_cancel(&ii_dev->timer);263 264 /*265 * Stopping idle injection requires all of the idle injection kthreads266 * associated with the given cpumask to be parked and stay that way, so267 * prevent CPUs from going online at this point. Any CPUs going online268 * after the loop below will be covered by clearing the should_run flag269 * that will cause the smpboot main loop to schedule them out.270 */271 cpu_hotplug_disable();272 273 /*274 * Iterate over all (online + offline) CPUs here in case one of them275 * goes offline with the should_run flag set so as to prevent its idle276 * injection kthread from running when the CPU goes online again after277 * the ii_dev has been freed.278 */279 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {280 iit = per_cpu_ptr(&idle_inject_thread, cpu);281 iit->should_run = 0;282 283 wait_task_inactive(iit->tsk, TASK_ANY);284 }285 286 cpu_hotplug_enable();287}288EXPORT_SYMBOL_NS_GPL(idle_inject_stop, IDLE_INJECT);289 290/**291 * idle_inject_setup - prepare the current task for idle injection292 * @cpu: not used293 *294 * Called once, this function is in charge of setting the current task's295 * scheduler parameters to make it an RT task.296 */297static void idle_inject_setup(unsigned int cpu)298{299 sched_set_fifo(current);300}301 302/**303 * idle_inject_should_run - function helper for the smpboot API304 * @cpu: CPU the kthread is running on305 *306 * Return: whether or not the thread can run.307 */308static int idle_inject_should_run(unsigned int cpu)309{310 struct idle_inject_thread *iit =311 per_cpu_ptr(&idle_inject_thread, cpu);312 313 return iit->should_run;314}315 316/**317 * idle_inject_register_full - initialize idle injection on a set of CPUs318 * @cpumask: CPUs to be affected by idle injection319 * @update: This callback is called just before waking up CPUs to inject320 * idle321 *322 * This function creates an idle injection control device structure for the323 * given set of CPUs and initializes the timer associated with it. This324 * function also allows to register update()callback.325 * It does not start any injection cycles.326 *327 * Return: NULL if memory allocation fails, idle injection control device328 * pointer on success.329 */330 331struct idle_inject_device *idle_inject_register_full(struct cpumask *cpumask,332 bool (*update)(void))333{334 struct idle_inject_device *ii_dev;335 int cpu, cpu_rb;336 337 ii_dev = kzalloc(sizeof(*ii_dev) + cpumask_size(), GFP_KERNEL);338 if (!ii_dev)339 return NULL;340 341 cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask);342 hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);343 ii_dev->timer.function = idle_inject_timer_fn;344 ii_dev->latency_us = UINT_MAX;345 ii_dev->update = update;346 347 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) {348 349 if (per_cpu(idle_inject_device, cpu)) {350 pr_err("cpu%d is already registered\n", cpu);351 goto out_rollback;352 }353 354 per_cpu(idle_inject_device, cpu) = ii_dev;355 }356 357 return ii_dev;358 359out_rollback:360 for_each_cpu(cpu_rb, to_cpumask(ii_dev->cpumask)) {361 if (cpu == cpu_rb)362 break;363 per_cpu(idle_inject_device, cpu_rb) = NULL;364 }365 366 kfree(ii_dev);367 368 return NULL;369}370EXPORT_SYMBOL_NS_GPL(idle_inject_register_full, IDLE_INJECT);371 372/**373 * idle_inject_register - initialize idle injection on a set of CPUs374 * @cpumask: CPUs to be affected by idle injection375 *376 * This function creates an idle injection control device structure for the377 * given set of CPUs and initializes the timer associated with it. It does not378 * start any injection cycles.379 *380 * Return: NULL if memory allocation fails, idle injection control device381 * pointer on success.382 */383struct idle_inject_device *idle_inject_register(struct cpumask *cpumask)384{385 return idle_inject_register_full(cpumask, NULL);386}387EXPORT_SYMBOL_NS_GPL(idle_inject_register, IDLE_INJECT);388 389/**390 * idle_inject_unregister - unregister idle injection control device391 * @ii_dev: idle injection control device to unregister392 *393 * The function stops idle injection for the given control device,394 * unregisters its kthreads and frees memory allocated when that device was395 * created.396 */397void idle_inject_unregister(struct idle_inject_device *ii_dev)398{399 unsigned int cpu;400 401 idle_inject_stop(ii_dev);402 403 for_each_cpu(cpu, to_cpumask(ii_dev->cpumask))404 per_cpu(idle_inject_device, cpu) = NULL;405 406 kfree(ii_dev);407}408EXPORT_SYMBOL_NS_GPL(idle_inject_unregister, IDLE_INJECT);409 410static struct smp_hotplug_thread idle_inject_threads = {411 .store = &idle_inject_thread.tsk,412 .setup = idle_inject_setup,413 .thread_fn = idle_inject_fn,414 .thread_comm = "idle_inject/%u",415 .thread_should_run = idle_inject_should_run,416};417 418static int __init idle_inject_init(void)419{420 return smpboot_register_percpu_thread(&idle_inject_threads);421}422early_initcall(idle_inject_init);423