227 lines · c
1/*2 * SPDX-License-Identifier: MIT3 *4 * Copyright © 2019 Intel Corporation5 */6 7#include <linux/wait_bit.h>8 9#include "intel_runtime_pm.h"10#include "intel_wakeref.h"11#include "i915_drv.h"12 13int __intel_wakeref_get_first(struct intel_wakeref *wf)14{15 intel_wakeref_t wakeref;16 int ret = 0;17 18 wakeref = intel_runtime_pm_get(&wf->i915->runtime_pm);19 /*20 * Treat get/put as different subclasses, as we may need to run21 * the put callback from under the shrinker and do not want to22 * cross-contanimate that callback with any extra work performed23 * upon acquiring the wakeref.24 */25 mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);26 27 if (!atomic_read(&wf->count)) {28 INTEL_WAKEREF_BUG_ON(wf->wakeref);29 wf->wakeref = wakeref;30 wakeref = 0;31 32 ret = wf->ops->get(wf);33 if (ret) {34 wakeref = xchg(&wf->wakeref, 0);35 wake_up_var(&wf->wakeref);36 goto unlock;37 }38 39 smp_mb__before_atomic(); /* release wf->count */40 }41 42 atomic_inc(&wf->count);43 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);44 45unlock:46 mutex_unlock(&wf->mutex);47 if (unlikely(wakeref))48 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);49 50 return ret;51}52 53static void ____intel_wakeref_put_last(struct intel_wakeref *wf)54{55 intel_wakeref_t wakeref = 0;56 57 INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);58 if (unlikely(!atomic_dec_and_test(&wf->count)))59 goto unlock;60 61 /* ops->put() must reschedule its own release on error/deferral */62 if (likely(!wf->ops->put(wf))) {63 INTEL_WAKEREF_BUG_ON(!wf->wakeref);64 wakeref = xchg(&wf->wakeref, 0);65 wake_up_var(&wf->wakeref);66 }67 68unlock:69 mutex_unlock(&wf->mutex);70 if (wakeref)71 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);72}73 74void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)75{76 INTEL_WAKEREF_BUG_ON(delayed_work_pending(&wf->work));77 78 /* Assume we are not in process context and so cannot sleep. */79 if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {80 mod_delayed_work(wf->i915->unordered_wq, &wf->work,81 FIELD_GET(INTEL_WAKEREF_PUT_DELAY, flags));82 return;83 }84 85 ____intel_wakeref_put_last(wf);86}87 88static void __intel_wakeref_put_work(struct work_struct *wrk)89{90 struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work.work);91 92 if (atomic_add_unless(&wf->count, -1, 1))93 return;94 95 mutex_lock(&wf->mutex);96 ____intel_wakeref_put_last(wf);97}98 99void __intel_wakeref_init(struct intel_wakeref *wf,100 struct drm_i915_private *i915,101 const struct intel_wakeref_ops *ops,102 struct intel_wakeref_lockclass *key,103 const char *name)104{105 wf->i915 = i915;106 wf->ops = ops;107 108 __mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex);109 atomic_set(&wf->count, 0);110 wf->wakeref = 0;111 112 INIT_DELAYED_WORK(&wf->work, __intel_wakeref_put_work);113 lockdep_init_map(&wf->work.work.lockdep_map,114 "wakeref.work", &key->work, 0);115 116#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_WAKEREF)117 ref_tracker_dir_init(&wf->debug, INTEL_REFTRACK_DEAD_COUNT, name);118#endif119}120 121int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)122{123 int err;124 125 might_sleep();126 127 err = wait_var_event_killable(&wf->wakeref,128 !intel_wakeref_is_active(wf));129 if (err)130 return err;131 132 intel_wakeref_unlock_wait(wf);133 return 0;134}135 136static void wakeref_auto_timeout(struct timer_list *t)137{138 struct intel_wakeref_auto *wf = from_timer(wf, t, timer);139 intel_wakeref_t wakeref;140 unsigned long flags;141 142 if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags))143 return;144 145 wakeref = fetch_and_zero(&wf->wakeref);146 spin_unlock_irqrestore(&wf->lock, flags);147 148 intel_runtime_pm_put(&wf->i915->runtime_pm, wakeref);149}150 151void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,152 struct drm_i915_private *i915)153{154 spin_lock_init(&wf->lock);155 timer_setup(&wf->timer, wakeref_auto_timeout, 0);156 refcount_set(&wf->count, 0);157 wf->wakeref = 0;158 wf->i915 = i915;159}160 161void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)162{163 unsigned long flags;164 165 if (!timeout) {166 if (del_timer_sync(&wf->timer))167 wakeref_auto_timeout(&wf->timer);168 return;169 }170 171 /* Our mission is that we only extend an already active wakeref */172 assert_rpm_wakelock_held(&wf->i915->runtime_pm);173 174 if (!refcount_inc_not_zero(&wf->count)) {175 spin_lock_irqsave(&wf->lock, flags);176 if (!refcount_inc_not_zero(&wf->count)) {177 INTEL_WAKEREF_BUG_ON(wf->wakeref);178 wf->wakeref =179 intel_runtime_pm_get_if_in_use(&wf->i915->runtime_pm);180 refcount_set(&wf->count, 1);181 }182 spin_unlock_irqrestore(&wf->lock, flags);183 }184 185 /*186 * If we extend a pending timer, we will only get a single timer187 * callback and so need to cancel the local inc by running the188 * elided callback to keep the wf->count balanced.189 */190 if (mod_timer(&wf->timer, jiffies + timeout))191 wakeref_auto_timeout(&wf->timer);192}193 194void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)195{196 intel_wakeref_auto(wf, 0);197 INTEL_WAKEREF_BUG_ON(wf->wakeref);198}199 200void intel_ref_tracker_show(struct ref_tracker_dir *dir,201 struct drm_printer *p)202{203 const size_t buf_size = PAGE_SIZE;204 char *buf, *sb, *se;205 size_t count;206 207 buf = kmalloc(buf_size, GFP_NOWAIT);208 if (!buf)209 return;210 211 count = ref_tracker_dir_snprint(dir, buf, buf_size);212 if (!count)213 goto free;214 /* printk does not like big buffers, so we split it */215 for (sb = buf; *sb; sb = se + 1) {216 se = strchrnul(sb, '\n');217 drm_printf(p, "%.*s", (int)(se - sb + 1), sb);218 if (!*se)219 break;220 }221 if (count >= buf_size)222 drm_printf(p, "\n...dropped %zd extra bytes of leak report.\n",223 count + 1 - buf_size);224free:225 kfree(buf);226}227