2164 lines · c
1/*2 * drm_irq.c IRQ and vblank support3 *4 * \author Rickard E. (Rik) Faith <faith@valinux.com>5 * \author Gareth Hughes <gareth@valinux.com>6 *7 * Permission is hereby granted, free of charge, to any person obtaining a8 * copy of this software and associated documentation files (the "Software"),9 * to deal in the Software without restriction, including without limitation10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,11 * and/or sell copies of the Software, and to permit persons to whom the12 * Software is furnished to do so, subject to the following conditions:13 *14 * The above copyright notice and this permission notice (including the next15 * paragraph) shall be included in all copies or substantial portions of the16 * Software.17 *18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR24 * OTHER DEALINGS IN THE SOFTWARE.25 */26 27#include <linux/export.h>28#include <linux/kthread.h>29#include <linux/moduleparam.h>30 31#include <drm/drm_crtc.h>32#include <drm/drm_drv.h>33#include <drm/drm_framebuffer.h>34#include <drm/drm_managed.h>35#include <drm/drm_modeset_helper_vtables.h>36#include <drm/drm_print.h>37#include <drm/drm_vblank.h>38 39#include "drm_internal.h"40#include "drm_trace.h"41 42/**43 * DOC: vblank handling44 *45 * From the computer's perspective, every time the monitor displays46 * a new frame the scanout engine has "scanned out" the display image47 * from top to bottom, one row of pixels at a time. The current row48 * of pixels is referred to as the current scanline.49 *50 * In addition to the display's visible area, there's usually a couple of51 * extra scanlines which aren't actually displayed on the screen.52 * These extra scanlines don't contain image data and are occasionally used53 * for features like audio and infoframes. The region made up of these54 * scanlines is referred to as the vertical blanking region, or vblank for55 * short.56 *57 * For historical reference, the vertical blanking period was designed to58 * give the electron gun (on CRTs) enough time to move back to the top of59 * the screen to start scanning out the next frame. Similar for horizontal60 * blanking periods. They were designed to give the electron gun enough61 * time to move back to the other side of the screen to start scanning the62 * next scanline.63 *64 * ::65 *66 *67 * physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽68 * top of | |69 * display | |70 * | New frame |71 * | |72 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|73 * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,74 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the75 * | | frame as it76 * | | travels down77 * | | ("scan out")78 * | Old frame |79 * | |80 * | |81 * | |82 * | | physical83 * | | bottom of84 * vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display85 * blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆86 * region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆87 * ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆88 * start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽89 * new frame90 *91 * "Physical top of display" is the reference point for the high-precision/92 * corrected timestamp.93 *94 * On a lot of display hardware, programming needs to take effect during the95 * vertical blanking period so that settings like gamma, the image buffer96 * buffer to be scanned out, etc. can safely be changed without showing97 * any visual artifacts on the screen. In some unforgiving hardware, some of98 * this programming has to both start and end in the same vblank. To help99 * with the timing of the hardware programming, an interrupt is usually100 * available to notify the driver when it can start the updating of registers.101 * The interrupt is in this context named the vblank interrupt.102 *103 * The vblank interrupt may be fired at different points depending on the104 * hardware. Some hardware implementations will fire the interrupt when the105 * new frame start, other implementations will fire the interrupt at different106 * points in time.107 *108 * Vertical blanking plays a major role in graphics rendering. To achieve109 * tear-free display, users must synchronize page flips and/or rendering to110 * vertical blanking. The DRM API offers ioctls to perform page flips111 * synchronized to vertical blanking and wait for vertical blanking.112 *113 * The DRM core handles most of the vertical blanking management logic, which114 * involves filtering out spurious interrupts, keeping race-free blanking115 * counters, coping with counter wrap-around and resets and keeping use counts.116 * It relies on the driver to generate vertical blanking interrupts and117 * optionally provide a hardware vertical blanking counter.118 *119 * Drivers must initialize the vertical blanking handling core with a call to120 * drm_vblank_init(). Minimally, a driver needs to implement121 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call122 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank123 * support.124 *125 * Vertical blanking interrupts can be enabled by the DRM core or by drivers126 * themselves (for instance to handle page flipping operations). The DRM core127 * maintains a vertical blanking use count to ensure that the interrupts are not128 * disabled while a user still needs them. To increment the use count, drivers129 * call drm_crtc_vblank_get() and release the vblank reference again with130 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are131 * guaranteed to be enabled.132 *133 * On many hardware disabling the vblank interrupt cannot be done in a race-free134 * manner, see &drm_vblank_crtc_config.disable_immediate and135 * &drm_driver.max_vblank_count. In that case the vblank core only disables the136 * vblanks after a timer has expired, which can be configured through the137 * ``vblankoffdelay`` module parameter.138 *139 * Drivers for hardware without support for vertical-blanking interrupts140 * must not call drm_vblank_init(). For such drivers, atomic helpers will141 * automatically generate fake vblank events as part of the display update.142 * This functionality also can be controlled by the driver by enabling and143 * disabling struct drm_crtc_state.no_vblank.144 */145 146/* Retry timestamp calculation up to 3 times to satisfy147 * drm_timestamp_precision before giving up.148 */149#define DRM_TIMESTAMP_MAXRETRIES 3150 151/* Threshold in nanoseconds for detection of redundant152 * vblank irq in drm_handle_vblank(). 1 msec should be ok.153 */154#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000155 156static bool157drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,158 ktime_t *tvblank, bool in_vblank_irq);159 160static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */161 162static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */163 164module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);165module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);166MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");167MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");168 169static struct drm_vblank_crtc *170drm_vblank_crtc(struct drm_device *dev, unsigned int pipe)171{172 return &dev->vblank[pipe];173}174 175struct drm_vblank_crtc *176drm_crtc_vblank_crtc(struct drm_crtc *crtc)177{178 return drm_vblank_crtc(crtc->dev, drm_crtc_index(crtc));179}180EXPORT_SYMBOL(drm_crtc_vblank_crtc);181 182static void store_vblank(struct drm_device *dev, unsigned int pipe,183 u32 vblank_count_inc,184 ktime_t t_vblank, u32 last)185{186 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);187 188 assert_spin_locked(&dev->vblank_time_lock);189 190 vblank->last = last;191 192 write_seqlock(&vblank->seqlock);193 vblank->time = t_vblank;194 atomic64_add(vblank_count_inc, &vblank->count);195 write_sequnlock(&vblank->seqlock);196}197 198static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)199{200 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);201 202 return vblank->max_vblank_count ?: dev->max_vblank_count;203}204 205/*206 * "No hw counter" fallback implementation of .get_vblank_counter() hook,207 * if there is no usable hardware frame counter available.208 */209static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)210{211 drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);212 return 0;213}214 215static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)216{217 if (drm_core_check_feature(dev, DRIVER_MODESET)) {218 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);219 220 if (drm_WARN_ON(dev, !crtc))221 return 0;222 223 if (crtc->funcs->get_vblank_counter)224 return crtc->funcs->get_vblank_counter(crtc);225 }226 227 return drm_vblank_no_hw_counter(dev, pipe);228}229 230/*231 * Reset the stored timestamp for the current vblank count to correspond232 * to the last vblank occurred.233 *234 * Only to be called from drm_crtc_vblank_on().235 *236 * Note: caller must hold &drm_device.vbl_lock since this reads & writes237 * device vblank fields.238 */239static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)240{241 u32 cur_vblank;242 bool rc;243 ktime_t t_vblank;244 int count = DRM_TIMESTAMP_MAXRETRIES;245 246 spin_lock(&dev->vblank_time_lock);247 248 /*249 * sample the current counter to avoid random jumps250 * when drm_vblank_enable() applies the diff251 */252 do {253 cur_vblank = __get_vblank_counter(dev, pipe);254 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);255 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);256 257 /*258 * Only reinitialize corresponding vblank timestamp if high-precision query259 * available and didn't fail. Otherwise reinitialize delayed at next vblank260 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.261 */262 if (!rc)263 t_vblank = 0;264 265 /*266 * +1 to make sure user will never see the same267 * vblank counter value before and after a modeset268 */269 store_vblank(dev, pipe, 1, t_vblank, cur_vblank);270 271 spin_unlock(&dev->vblank_time_lock);272}273 274/*275 * Call back into the driver to update the appropriate vblank counter276 * (specified by @pipe). Deal with wraparound, if it occurred, and277 * update the last read value so we can deal with wraparound on the next278 * call if necessary.279 *280 * Only necessary when going from off->on, to account for frames we281 * didn't get an interrupt for.282 *283 * Note: caller must hold &drm_device.vbl_lock since this reads & writes284 * device vblank fields.285 */286static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,287 bool in_vblank_irq)288{289 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);290 u32 cur_vblank, diff;291 bool rc;292 ktime_t t_vblank;293 int count = DRM_TIMESTAMP_MAXRETRIES;294 int framedur_ns = vblank->framedur_ns;295 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);296 297 /*298 * Interrupts were disabled prior to this call, so deal with counter299 * wrap if needed.300 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events301 * here if the register is small or we had vblank interrupts off for302 * a long time.303 *304 * We repeat the hardware vblank counter & timestamp query until305 * we get consistent results. This to prevent races between gpu306 * updating its hardware counter while we are retrieving the307 * corresponding vblank timestamp.308 */309 do {310 cur_vblank = __get_vblank_counter(dev, pipe);311 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);312 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);313 314 if (max_vblank_count) {315 /* trust the hw counter when it's around */316 diff = (cur_vblank - vblank->last) & max_vblank_count;317 } else if (rc && framedur_ns) {318 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));319 320 /*321 * Figure out how many vblanks we've missed based322 * on the difference in the timestamps and the323 * frame/field duration.324 */325 326 drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."327 " diff_ns = %lld, framedur_ns = %d)\n",328 pipe, (long long)diff_ns, framedur_ns);329 330 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);331 332 if (diff == 0 && in_vblank_irq)333 drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",334 pipe);335 } else {336 /* some kind of default for drivers w/o accurate vbl timestamping */337 diff = in_vblank_irq ? 1 : 0;338 }339 340 /*341 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset342 * interval? If so then vblank irqs keep running and it will likely343 * happen that the hardware vblank counter is not trustworthy as it344 * might reset at some point in that interval and vblank timestamps345 * are not trustworthy either in that interval. Iow. this can result346 * in a bogus diff >> 1 which must be avoided as it would cause347 * random large forward jumps of the software vblank counter.348 */349 if (diff > 1 && (vblank->inmodeset & 0x2)) {350 drm_dbg_vbl(dev,351 "clamping vblank bump to 1 on crtc %u: diffr=%u"352 " due to pre-modeset.\n", pipe, diff);353 diff = 1;354 }355 356 drm_dbg_vbl(dev, "updating vblank count on crtc %u:"357 " current=%llu, diff=%u, hw=%u hw_last=%u\n",358 pipe, (unsigned long long)atomic64_read(&vblank->count),359 diff, cur_vblank, vblank->last);360 361 if (diff == 0) {362 drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);363 return;364 }365 366 /*367 * Only reinitialize corresponding vblank timestamp if high-precision query368 * available and didn't fail, or we were called from the vblank interrupt.369 * Otherwise reinitialize delayed at next vblank interrupt and assign 0370 * for now, to mark the vblanktimestamp as invalid.371 */372 if (!rc && !in_vblank_irq)373 t_vblank = 0;374 375 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);376}377 378u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)379{380 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);381 u64 count;382 383 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))384 return 0;385 386 count = atomic64_read(&vblank->count);387 388 /*389 * This read barrier corresponds to the implicit write barrier of the390 * write seqlock in store_vblank(). Note that this is the only place391 * where we need an explicit barrier, since all other access goes392 * through drm_vblank_count_and_time(), which already has the required393 * read barrier curtesy of the read seqlock.394 */395 smp_rmb();396 397 return count;398}399 400/**401 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter402 * @crtc: which counter to retrieve403 *404 * This function is similar to drm_crtc_vblank_count() but this function405 * interpolates to handle a race with vblank interrupts using the high precision406 * timestamping support.407 *408 * This is mostly useful for hardware that can obtain the scanout position, but409 * doesn't have a hardware frame counter.410 */411u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)412{413 struct drm_device *dev = crtc->dev;414 unsigned int pipe = drm_crtc_index(crtc);415 u64 vblank;416 unsigned long flags;417 418 drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&419 !crtc->funcs->get_vblank_timestamp,420 "This function requires support for accurate vblank timestamps.");421 422 spin_lock_irqsave(&dev->vblank_time_lock, flags);423 424 drm_update_vblank_count(dev, pipe, false);425 vblank = drm_vblank_count(dev, pipe);426 427 spin_unlock_irqrestore(&dev->vblank_time_lock, flags);428 429 return vblank;430}431EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);432 433static void __disable_vblank(struct drm_device *dev, unsigned int pipe)434{435 if (drm_core_check_feature(dev, DRIVER_MODESET)) {436 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);437 438 if (drm_WARN_ON(dev, !crtc))439 return;440 441 if (crtc->funcs->disable_vblank)442 crtc->funcs->disable_vblank(crtc);443 }444}445 446/*447 * Disable vblank irq's on crtc, make sure that last vblank count448 * of hardware and corresponding consistent software vblank counter449 * are preserved, even if there are any spurious vblank irq's after450 * disable.451 */452void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)453{454 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);455 unsigned long irqflags;456 457 assert_spin_locked(&dev->vbl_lock);458 459 /* Prevent vblank irq processing while disabling vblank irqs,460 * so no updates of timestamps or count can happen after we've461 * disabled. Needed to prevent races in case of delayed irq's.462 */463 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);464 465 /*466 * Update vblank count and disable vblank interrupts only if the467 * interrupts were enabled. This avoids calling the ->disable_vblank()468 * operation in atomic context with the hardware potentially runtime469 * suspended.470 */471 if (!vblank->enabled)472 goto out;473 474 /*475 * Update the count and timestamp to maintain the476 * appearance that the counter has been ticking all along until477 * this time. This makes the count account for the entire time478 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().479 */480 drm_update_vblank_count(dev, pipe, false);481 __disable_vblank(dev, pipe);482 vblank->enabled = false;483 484out:485 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);486}487 488static void vblank_disable_fn(struct timer_list *t)489{490 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);491 struct drm_device *dev = vblank->dev;492 unsigned int pipe = vblank->pipe;493 unsigned long irqflags;494 495 spin_lock_irqsave(&dev->vbl_lock, irqflags);496 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {497 drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);498 drm_vblank_disable_and_save(dev, pipe);499 }500 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);501}502 503static void drm_vblank_init_release(struct drm_device *dev, void *ptr)504{505 struct drm_vblank_crtc *vblank = ptr;506 507 drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&508 drm_core_check_feature(dev, DRIVER_MODESET));509 510 drm_vblank_destroy_worker(vblank);511 del_timer_sync(&vblank->disable_timer);512}513 514/**515 * drm_vblank_init - initialize vblank support516 * @dev: DRM device517 * @num_crtcs: number of CRTCs supported by @dev518 *519 * This function initializes vblank support for @num_crtcs display pipelines.520 * Cleanup is handled automatically through a cleanup function added with521 * drmm_add_action_or_reset().522 *523 * Returns:524 * Zero on success or a negative error code on failure.525 */526int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)527{528 int ret;529 unsigned int i;530 531 spin_lock_init(&dev->vbl_lock);532 spin_lock_init(&dev->vblank_time_lock);533 534 dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);535 if (!dev->vblank)536 return -ENOMEM;537 538 dev->num_crtcs = num_crtcs;539 540 for (i = 0; i < num_crtcs; i++) {541 struct drm_vblank_crtc *vblank = &dev->vblank[i];542 543 vblank->dev = dev;544 vblank->pipe = i;545 init_waitqueue_head(&vblank->queue);546 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);547 seqlock_init(&vblank->seqlock);548 549 ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,550 vblank);551 if (ret)552 return ret;553 554 ret = drm_vblank_worker_init(vblank);555 if (ret)556 return ret;557 }558 559 return 0;560}561EXPORT_SYMBOL(drm_vblank_init);562 563/**564 * drm_dev_has_vblank - test if vblanking has been initialized for565 * a device566 * @dev: the device567 *568 * Drivers may call this function to test if vblank support is569 * initialized for a device. For most hardware this means that vblanking570 * can also be enabled.571 *572 * Atomic helpers use this function to initialize573 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().574 *575 * Returns:576 * True if vblanking has been initialized for the given device, false577 * otherwise.578 */579bool drm_dev_has_vblank(const struct drm_device *dev)580{581 return dev->num_crtcs != 0;582}583EXPORT_SYMBOL(drm_dev_has_vblank);584 585/**586 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC587 * @crtc: which CRTC's vblank waitqueue to retrieve588 *589 * This function returns a pointer to the vblank waitqueue for the CRTC.590 * Drivers can use this to implement vblank waits using wait_event() and related591 * functions.592 */593wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)594{595 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;596}597EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);598 599 600/**601 * drm_calc_timestamping_constants - calculate vblank timestamp constants602 * @crtc: drm_crtc whose timestamp constants should be updated.603 * @mode: display mode containing the scanout timings604 *605 * Calculate and store various constants which are later needed by vblank and606 * swap-completion timestamping, e.g, by607 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from608 * CRTC's true scanout timing, so they take things like panel scaling or609 * other adjustments into account.610 */611void drm_calc_timestamping_constants(struct drm_crtc *crtc,612 const struct drm_display_mode *mode)613{614 struct drm_device *dev = crtc->dev;615 unsigned int pipe = drm_crtc_index(crtc);616 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);617 int linedur_ns = 0, framedur_ns = 0;618 int dotclock = mode->crtc_clock;619 620 if (!drm_dev_has_vblank(dev))621 return;622 623 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))624 return;625 626 /* Valid dotclock? */627 if (dotclock > 0) {628 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;629 630 /*631 * Convert scanline length in pixels and video632 * dot clock to line duration and frame duration633 * in nanoseconds:634 */635 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);636 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);637 638 /*639 * Fields of interlaced scanout modes are only half a frame duration.640 */641 if (mode->flags & DRM_MODE_FLAG_INTERLACE)642 framedur_ns /= 2;643 } else {644 drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",645 crtc->base.id);646 }647 648 vblank->linedur_ns = linedur_ns;649 vblank->framedur_ns = framedur_ns;650 drm_mode_copy(&vblank->hwmode, mode);651 652 drm_dbg_core(dev,653 "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",654 crtc->base.id, mode->crtc_htotal,655 mode->crtc_vtotal, mode->crtc_vdisplay);656 drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",657 crtc->base.id, dotclock, framedur_ns, linedur_ns);658}659EXPORT_SYMBOL(drm_calc_timestamping_constants);660 661/**662 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank663 * timestamp helper664 * @crtc: CRTC whose vblank timestamp to retrieve665 * @max_error: Desired maximum allowable error in timestamps (nanosecs)666 * On return contains true maximum error of timestamp667 * @vblank_time: Pointer to time which should receive the timestamp668 * @in_vblank_irq:669 * True when called from drm_crtc_handle_vblank(). Some drivers670 * need to apply some workarounds for gpu-specific vblank irq quirks671 * if flag is set.672 * @get_scanout_position:673 * Callback function to retrieve the scanout position. See674 * @struct drm_crtc_helper_funcs.get_scanout_position.675 *676 * Implements calculation of exact vblank timestamps from given drm_display_mode677 * timings and current video scanout position of a CRTC.678 *679 * The current implementation only handles standard video modes. For double scan680 * and interlaced modes the driver is supposed to adjust the hardware mode681 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to682 * match the scanout position reported.683 *684 * Note that atomic drivers must call drm_calc_timestamping_constants() before685 * enabling a CRTC. The atomic helpers already take care of that in686 * drm_atomic_helper_calc_timestamping_constants().687 *688 * Returns:689 * Returns true on success, and false on failure, i.e. when no accurate690 * timestamp could be acquired.691 */692bool693drm_crtc_vblank_helper_get_vblank_timestamp_internal(694 struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,695 bool in_vblank_irq,696 drm_vblank_get_scanout_position_func get_scanout_position)697{698 struct drm_device *dev = crtc->dev;699 unsigned int pipe = crtc->index;700 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];701 struct timespec64 ts_etime, ts_vblank_time;702 ktime_t stime, etime;703 bool vbl_status;704 const struct drm_display_mode *mode;705 int vpos, hpos, i;706 int delta_ns, duration_ns;707 708 if (pipe >= dev->num_crtcs) {709 drm_err(dev, "Invalid crtc %u\n", pipe);710 return false;711 }712 713 /* Scanout position query not supported? Should not happen. */714 if (!get_scanout_position) {715 drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");716 return false;717 }718 719 if (drm_drv_uses_atomic_modeset(dev))720 mode = &vblank->hwmode;721 else722 mode = &crtc->hwmode;723 724 /* If mode timing undefined, just return as no-op:725 * Happens during initial modesetting of a crtc.726 */727 if (mode->crtc_clock == 0) {728 drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",729 pipe);730 drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));731 return false;732 }733 734 /* Get current scanout position with system timestamp.735 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times736 * if single query takes longer than max_error nanoseconds.737 *738 * This guarantees a tight bound on maximum error if739 * code gets preempted or delayed for some reason.740 */741 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {742 /*743 * Get vertical and horizontal scanout position vpos, hpos,744 * and bounding timestamps stime, etime, pre/post query.745 */746 vbl_status = get_scanout_position(crtc, in_vblank_irq,747 &vpos, &hpos,748 &stime, &etime,749 mode);750 751 /* Return as no-op if scanout query unsupported or failed. */752 if (!vbl_status) {753 drm_dbg_core(dev,754 "crtc %u : scanoutpos query failed.\n",755 pipe);756 return false;757 }758 759 /* Compute uncertainty in timestamp of scanout position query. */760 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);761 762 /* Accept result with < max_error nsecs timing uncertainty. */763 if (duration_ns <= *max_error)764 break;765 }766 767 /* Noisy system timing? */768 if (i == DRM_TIMESTAMP_MAXRETRIES) {769 drm_dbg_core(dev,770 "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",771 pipe, duration_ns / 1000, *max_error / 1000, i);772 }773 774 /* Return upper bound of timestamp precision error. */775 *max_error = duration_ns;776 777 /* Convert scanout position into elapsed time at raw_time query778 * since start of scanout at first display scanline. delta_ns779 * can be negative if start of scanout hasn't happened yet.780 */781 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),782 mode->crtc_clock);783 784 /* Subtract time delta from raw timestamp to get final785 * vblank_time timestamp for end of vblank.786 */787 *vblank_time = ktime_sub_ns(etime, delta_ns);788 789 if (!drm_debug_enabled(DRM_UT_VBL))790 return true;791 792 ts_etime = ktime_to_timespec64(etime);793 ts_vblank_time = ktime_to_timespec64(*vblank_time);794 795 drm_dbg_vbl(dev,796 "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",797 pipe, hpos, vpos,798 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,799 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,800 duration_ns / 1000, i);801 802 return true;803}804EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);805 806/**807 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp808 * helper809 * @crtc: CRTC whose vblank timestamp to retrieve810 * @max_error: Desired maximum allowable error in timestamps (nanosecs)811 * On return contains true maximum error of timestamp812 * @vblank_time: Pointer to time which should receive the timestamp813 * @in_vblank_irq:814 * True when called from drm_crtc_handle_vblank(). Some drivers815 * need to apply some workarounds for gpu-specific vblank irq quirks816 * if flag is set.817 *818 * Implements calculation of exact vblank timestamps from given drm_display_mode819 * timings and current video scanout position of a CRTC. This can be directly820 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms821 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.822 *823 * The current implementation only handles standard video modes. For double scan824 * and interlaced modes the driver is supposed to adjust the hardware mode825 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to826 * match the scanout position reported.827 *828 * Note that atomic drivers must call drm_calc_timestamping_constants() before829 * enabling a CRTC. The atomic helpers already take care of that in830 * drm_atomic_helper_calc_timestamping_constants().831 *832 * Returns:833 * Returns true on success, and false on failure, i.e. when no accurate834 * timestamp could be acquired.835 */836bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,837 int *max_error,838 ktime_t *vblank_time,839 bool in_vblank_irq)840{841 return drm_crtc_vblank_helper_get_vblank_timestamp_internal(842 crtc, max_error, vblank_time, in_vblank_irq,843 crtc->helper_private->get_scanout_position);844}845EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);846 847/**848 * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most849 * recent vblank interval850 * @crtc: CRTC whose vblank timestamp to retrieve851 * @tvblank: Pointer to target time which should receive the timestamp852 * @in_vblank_irq:853 * True when called from drm_crtc_handle_vblank(). Some drivers854 * need to apply some workarounds for gpu-specific vblank irq quirks855 * if flag is set.856 *857 * Fetches the system timestamp corresponding to the time of the most recent858 * vblank interval on specified CRTC. May call into kms-driver to859 * compute the timestamp with a high-precision GPU specific method.860 *861 * Returns zero if timestamp originates from uncorrected do_gettimeofday()862 * call, i.e., it isn't very precisely locked to the true vblank.863 *864 * Returns:865 * True if timestamp is considered to be very precise, false otherwise.866 */867static bool868drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank,869 bool in_vblank_irq)870{871 bool ret = false;872 873 /* Define requested maximum error on timestamps (nanoseconds). */874 int max_error = (int) drm_timestamp_precision * 1000;875 876 /* Query driver if possible and precision timestamping enabled. */877 if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {878 ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,879 tvblank, in_vblank_irq);880 }881 882 /* GPU high precision timestamp query unsupported or failed.883 * Return current monotonic/gettimeofday timestamp as best estimate.884 */885 if (!ret)886 *tvblank = ktime_get();887 888 return ret;889}890 891static bool892drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,893 ktime_t *tvblank, bool in_vblank_irq)894{895 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);896 897 return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq);898}899 900/**901 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value902 * @crtc: which counter to retrieve903 *904 * Fetches the "cooked" vblank count value that represents the number of905 * vblank events since the system was booted, including lost events due to906 * modesetting activity. Note that this timer isn't correct against a racing907 * vblank interrupt (since it only reports the software vblank counter), see908 * drm_crtc_accurate_vblank_count() for such use-cases.909 *910 * Note that for a given vblank counter value drm_crtc_handle_vblank()911 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()912 * provide a barrier: Any writes done before calling913 * drm_crtc_handle_vblank() will be visible to callers of the later914 * functions, if the vblank count is the same or a later one.915 *916 * See also &drm_vblank_crtc.count.917 *918 * Returns:919 * The software vblank counter.920 */921u64 drm_crtc_vblank_count(struct drm_crtc *crtc)922{923 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));924}925EXPORT_SYMBOL(drm_crtc_vblank_count);926 927/**928 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the929 * system timestamp corresponding to that vblank counter value.930 * @dev: DRM device931 * @pipe: index of CRTC whose counter to retrieve932 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.933 *934 * Fetches the "cooked" vblank count value that represents the number of935 * vblank events since the system was booted, including lost events due to936 * modesetting activity. Returns corresponding system timestamp of the time937 * of the vblank interval that corresponds to the current vblank counter value.938 *939 * This is the legacy version of drm_crtc_vblank_count_and_time().940 */941static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,942 ktime_t *vblanktime)943{944 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);945 u64 vblank_count;946 unsigned int seq;947 948 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {949 *vblanktime = 0;950 return 0;951 }952 953 do {954 seq = read_seqbegin(&vblank->seqlock);955 vblank_count = atomic64_read(&vblank->count);956 *vblanktime = vblank->time;957 } while (read_seqretry(&vblank->seqlock, seq));958 959 return vblank_count;960}961 962/**963 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value964 * and the system timestamp corresponding to that vblank counter value965 * @crtc: which counter to retrieve966 * @vblanktime: Pointer to time to receive the vblank timestamp.967 *968 * Fetches the "cooked" vblank count value that represents the number of969 * vblank events since the system was booted, including lost events due to970 * modesetting activity. Returns corresponding system timestamp of the time971 * of the vblank interval that corresponds to the current vblank counter value.972 *973 * Note that for a given vblank counter value drm_crtc_handle_vblank()974 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()975 * provide a barrier: Any writes done before calling976 * drm_crtc_handle_vblank() will be visible to callers of the later977 * functions, if the vblank count is the same or a later one.978 *979 * See also &drm_vblank_crtc.count.980 */981u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,982 ktime_t *vblanktime)983{984 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),985 vblanktime);986}987EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);988 989/**990 * drm_crtc_next_vblank_start - calculate the time of the next vblank991 * @crtc: the crtc for which to calculate next vblank time992 * @vblanktime: pointer to time to receive the next vblank timestamp.993 *994 * Calculate the expected time of the start of the next vblank period,995 * based on time of previous vblank and frame duration996 */997int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)998{999 struct drm_vblank_crtc *vblank;1000 struct drm_display_mode *mode;1001 u64 vblank_start;1002 1003 if (!drm_dev_has_vblank(crtc->dev))1004 return -EINVAL;1005 1006 vblank = drm_crtc_vblank_crtc(crtc);1007 mode = &vblank->hwmode;1008 1009 if (!vblank->framedur_ns || !vblank->linedur_ns)1010 return -EINVAL;1011 1012 if (!drm_crtc_get_last_vbltimestamp(crtc, vblanktime, false))1013 return -EINVAL;1014 1015 vblank_start = DIV_ROUND_DOWN_ULL(1016 (u64)vblank->framedur_ns * mode->crtc_vblank_start,1017 mode->crtc_vtotal);1018 *vblanktime = ktime_add(*vblanktime, ns_to_ktime(vblank_start));1019 1020 return 0;1021}1022EXPORT_SYMBOL(drm_crtc_next_vblank_start);1023 1024static void send_vblank_event(struct drm_device *dev,1025 struct drm_pending_vblank_event *e,1026 u64 seq, ktime_t now)1027{1028 struct timespec64 tv;1029 1030 switch (e->event.base.type) {1031 case DRM_EVENT_VBLANK:1032 case DRM_EVENT_FLIP_COMPLETE:1033 tv = ktime_to_timespec64(now);1034 e->event.vbl.sequence = seq;1035 /*1036 * e->event is a user space structure, with hardcoded unsigned1037 * 32-bit seconds/microseconds. This is safe as we always use1038 * monotonic timestamps since linux-4.151039 */1040 e->event.vbl.tv_sec = tv.tv_sec;1041 e->event.vbl.tv_usec = tv.tv_nsec / 1000;1042 break;1043 case DRM_EVENT_CRTC_SEQUENCE:1044 if (seq)1045 e->event.seq.sequence = seq;1046 e->event.seq.time_ns = ktime_to_ns(now);1047 break;1048 }1049 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);1050 /*1051 * Use the same timestamp for any associated fence signal to avoid1052 * mismatch in timestamps for vsync & fence events triggered by the1053 * same HW event. Frameworks like SurfaceFlinger in Android expects the1054 * retire-fence timestamp to match exactly with HW vsync as it uses it1055 * for its software vsync modeling.1056 */1057 drm_send_event_timestamp_locked(dev, &e->base, now);1058}1059 1060/**1061 * drm_crtc_arm_vblank_event - arm vblank event after pageflip1062 * @crtc: the source CRTC of the vblank event1063 * @e: the event to send1064 *1065 * A lot of drivers need to generate vblank events for the very next vblank1066 * interrupt. For example when the page flip interrupt happens when the page1067 * flip gets armed, but not when it actually executes within the next vblank1068 * period. This helper function implements exactly the required vblank arming1069 * behaviour.1070 *1071 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an1072 * atomic commit must ensure that the next vblank happens at exactly the same1073 * time as the atomic commit is committed to the hardware. This function itself1074 * does **not** protect against the next vblank interrupt racing with either this1075 * function call or the atomic commit operation. A possible sequence could be:1076 *1077 * 1. Driver commits new hardware state into vblank-synchronized registers.1078 * 2. A vblank happens, committing the hardware state. Also the corresponding1079 * vblank interrupt is fired off and fully processed by the interrupt1080 * handler.1081 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().1082 * 4. The event is only send out for the next vblank, which is wrong.1083 *1084 * An equivalent race can happen when the driver calls1085 * drm_crtc_arm_vblank_event() before writing out the new hardware state.1086 *1087 * The only way to make this work safely is to prevent the vblank from firing1088 * (and the hardware from committing anything else) until the entire atomic1089 * commit sequence has run to completion. If the hardware does not have such a1090 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.1091 * Instead drivers need to manually send out the event from their interrupt1092 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no1093 * possible race with the hardware committing the atomic update.1094 *1095 * Caller must hold a vblank reference for the event @e acquired by a1096 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.1097 */1098void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,1099 struct drm_pending_vblank_event *e)1100{1101 struct drm_device *dev = crtc->dev;1102 unsigned int pipe = drm_crtc_index(crtc);1103 1104 assert_spin_locked(&dev->event_lock);1105 1106 e->pipe = pipe;1107 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;1108 list_add_tail(&e->base.link, &dev->vblank_event_list);1109}1110EXPORT_SYMBOL(drm_crtc_arm_vblank_event);1111 1112/**1113 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip1114 * @crtc: the source CRTC of the vblank event1115 * @e: the event to send1116 *1117 * Updates sequence # and timestamp on event for the most recently processed1118 * vblank, and sends it to userspace. Caller must hold event lock.1119 *1120 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain1121 * situation, especially to send out events for atomic commit operations.1122 */1123void drm_crtc_send_vblank_event(struct drm_crtc *crtc,1124 struct drm_pending_vblank_event *e)1125{1126 struct drm_device *dev = crtc->dev;1127 u64 seq;1128 unsigned int pipe = drm_crtc_index(crtc);1129 ktime_t now;1130 1131 if (drm_dev_has_vblank(dev)) {1132 seq = drm_vblank_count_and_time(dev, pipe, &now);1133 } else {1134 seq = 0;1135 1136 now = ktime_get();1137 }1138 e->pipe = pipe;1139 send_vblank_event(dev, e, seq, now);1140}1141EXPORT_SYMBOL(drm_crtc_send_vblank_event);1142 1143static int __enable_vblank(struct drm_device *dev, unsigned int pipe)1144{1145 if (drm_core_check_feature(dev, DRIVER_MODESET)) {1146 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);1147 1148 if (drm_WARN_ON(dev, !crtc))1149 return 0;1150 1151 if (crtc->funcs->enable_vblank)1152 return crtc->funcs->enable_vblank(crtc);1153 }1154 1155 return -EINVAL;1156}1157 1158static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)1159{1160 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);1161 int ret = 0;1162 1163 assert_spin_locked(&dev->vbl_lock);1164 1165 spin_lock(&dev->vblank_time_lock);1166 1167 if (!vblank->enabled) {1168 /*1169 * Enable vblank irqs under vblank_time_lock protection.1170 * All vblank count & timestamp updates are held off1171 * until we are done reinitializing master counter and1172 * timestamps. Filtercode in drm_handle_vblank() will1173 * prevent double-accounting of same vblank interval.1174 */1175 ret = __enable_vblank(dev, pipe);1176 drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",1177 pipe, ret);1178 if (ret) {1179 atomic_dec(&vblank->refcount);1180 } else {1181 drm_update_vblank_count(dev, pipe, 0);1182 /* drm_update_vblank_count() includes a wmb so we just1183 * need to ensure that the compiler emits the write1184 * to mark the vblank as enabled after the call1185 * to drm_update_vblank_count().1186 */1187 WRITE_ONCE(vblank->enabled, true);1188 }1189 }1190 1191 spin_unlock(&dev->vblank_time_lock);1192 1193 return ret;1194}1195 1196int drm_vblank_get(struct drm_device *dev, unsigned int pipe)1197{1198 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);1199 unsigned long irqflags;1200 int ret = 0;1201 1202 if (!drm_dev_has_vblank(dev))1203 return -EINVAL;1204 1205 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))1206 return -EINVAL;1207 1208 spin_lock_irqsave(&dev->vbl_lock, irqflags);1209 /* Going from 0->1 means we have to enable interrupts again */1210 if (atomic_add_return(1, &vblank->refcount) == 1) {1211 ret = drm_vblank_enable(dev, pipe);1212 } else {1213 if (!vblank->enabled) {1214 atomic_dec(&vblank->refcount);1215 ret = -EINVAL;1216 }1217 }1218 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);1219 1220 return ret;1221}1222 1223/**1224 * drm_crtc_vblank_get - get a reference count on vblank events1225 * @crtc: which CRTC to own1226 *1227 * Acquire a reference count on vblank events to avoid having them disabled1228 * while in use.1229 *1230 * Returns:1231 * Zero on success or a negative error code on failure.1232 */1233int drm_crtc_vblank_get(struct drm_crtc *crtc)1234{1235 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));1236}1237EXPORT_SYMBOL(drm_crtc_vblank_get);1238 1239void drm_vblank_put(struct drm_device *dev, unsigned int pipe)1240{1241 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);1242 int vblank_offdelay = vblank->config.offdelay_ms;1243 1244 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))1245 return;1246 1247 if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))1248 return;1249 1250 /* Last user schedules interrupt disable */1251 if (atomic_dec_and_test(&vblank->refcount)) {1252 if (!vblank_offdelay)1253 return;1254 else if (vblank_offdelay < 0)1255 vblank_disable_fn(&vblank->disable_timer);1256 else if (!vblank->config.disable_immediate)1257 mod_timer(&vblank->disable_timer,1258 jiffies + ((vblank_offdelay * HZ) / 1000));1259 }1260}1261 1262/**1263 * drm_crtc_vblank_put - give up ownership of vblank events1264 * @crtc: which counter to give up1265 *1266 * Release ownership of a given vblank counter, turning off interrupts1267 * if possible. Disable interrupts after &drm_vblank_crtc_config.offdelay_ms1268 * milliseconds.1269 */1270void drm_crtc_vblank_put(struct drm_crtc *crtc)1271{1272 drm_vblank_put(crtc->dev, drm_crtc_index(crtc));1273}1274EXPORT_SYMBOL(drm_crtc_vblank_put);1275 1276/**1277 * drm_wait_one_vblank - wait for one vblank1278 * @dev: DRM device1279 * @pipe: CRTC index1280 *1281 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.1282 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.1283 * due to lack of driver support or because the crtc is off.1284 *1285 * This is the legacy version of drm_crtc_wait_one_vblank().1286 */1287void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)1288{1289 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);1290 int ret;1291 u64 last;1292 1293 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))1294 return;1295 1296 ret = drm_vblank_get(dev, pipe);1297 if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",1298 pipe, ret))1299 return;1300 1301 last = drm_vblank_count(dev, pipe);1302 1303 ret = wait_event_timeout(vblank->queue,1304 last != drm_vblank_count(dev, pipe),1305 msecs_to_jiffies(100));1306 1307 drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);1308 1309 drm_vblank_put(dev, pipe);1310}1311EXPORT_SYMBOL(drm_wait_one_vblank);1312 1313/**1314 * drm_crtc_wait_one_vblank - wait for one vblank1315 * @crtc: DRM crtc1316 *1317 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.1318 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.1319 * due to lack of driver support or because the crtc is off.1320 */1321void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)1322{1323 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));1324}1325EXPORT_SYMBOL(drm_crtc_wait_one_vblank);1326 1327/**1328 * drm_crtc_vblank_off - disable vblank events on a CRTC1329 * @crtc: CRTC in question1330 *1331 * Drivers can use this function to shut down the vblank interrupt handling when1332 * disabling a crtc. This function ensures that the latest vblank frame count is1333 * stored so that drm_vblank_on can restore it again.1334 *1335 * Drivers must use this function when the hardware vblank counter can get1336 * reset, e.g. when suspending or disabling the @crtc in general.1337 */1338void drm_crtc_vblank_off(struct drm_crtc *crtc)1339{1340 struct drm_device *dev = crtc->dev;1341 unsigned int pipe = drm_crtc_index(crtc);1342 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);1343 struct drm_pending_vblank_event *e, *t;1344 ktime_t now;1345 u64 seq;1346 1347 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))1348 return;1349 1350 /*1351 * Grab event_lock early to prevent vblank work from being scheduled1352 * while we're in the middle of shutting down vblank interrupts1353 */1354 spin_lock_irq(&dev->event_lock);1355 1356 spin_lock(&dev->vbl_lock);1357 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",1358 pipe, vblank->enabled, vblank->inmodeset);1359 1360 /* Avoid redundant vblank disables without previous1361 * drm_crtc_vblank_on(). */1362 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)1363 drm_vblank_disable_and_save(dev, pipe);1364 1365 wake_up(&vblank->queue);1366 1367 /*1368 * Prevent subsequent drm_vblank_get() from re-enabling1369 * the vblank interrupt by bumping the refcount.1370 */1371 if (!vblank->inmodeset) {1372 atomic_inc(&vblank->refcount);1373 vblank->inmodeset = 1;1374 }1375 spin_unlock(&dev->vbl_lock);1376 1377 /* Send any queued vblank events, lest the natives grow disquiet */1378 seq = drm_vblank_count_and_time(dev, pipe, &now);1379 1380 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {1381 if (e->pipe != pipe)1382 continue;1383 drm_dbg_core(dev, "Sending premature vblank event on disable: "1384 "wanted %llu, current %llu\n",1385 e->sequence, seq);1386 list_del(&e->base.link);1387 drm_vblank_put(dev, pipe);1388 send_vblank_event(dev, e, seq, now);1389 }1390 1391 /* Cancel any leftover pending vblank work */1392 drm_vblank_cancel_pending_works(vblank);1393 1394 spin_unlock_irq(&dev->event_lock);1395 1396 /* Will be reset by the modeset helpers when re-enabling the crtc by1397 * calling drm_calc_timestamping_constants(). */1398 vblank->hwmode.crtc_clock = 0;1399 1400 /* Wait for any vblank work that's still executing to finish */1401 drm_vblank_flush_worker(vblank);1402}1403EXPORT_SYMBOL(drm_crtc_vblank_off);1404 1405/**1406 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC1407 * @crtc: CRTC in question1408 *1409 * Drivers can use this function to reset the vblank state to off at load time.1410 * Drivers should use this together with the drm_crtc_vblank_off() and1411 * drm_crtc_vblank_on() functions. The difference compared to1412 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter1413 * and hence doesn't need to call any driver hooks.1414 *1415 * This is useful for recovering driver state e.g. on driver load, or on resume.1416 */1417void drm_crtc_vblank_reset(struct drm_crtc *crtc)1418{1419 struct drm_device *dev = crtc->dev;1420 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);1421 1422 spin_lock_irq(&dev->vbl_lock);1423 /*1424 * Prevent subsequent drm_vblank_get() from enabling the vblank1425 * interrupt by bumping the refcount.1426 */1427 if (!vblank->inmodeset) {1428 atomic_inc(&vblank->refcount);1429 vblank->inmodeset = 1;1430 }1431 spin_unlock_irq(&dev->vbl_lock);1432 1433 drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));1434 drm_WARN_ON(dev, !list_empty(&vblank->pending_work));1435}1436EXPORT_SYMBOL(drm_crtc_vblank_reset);1437 1438/**1439 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value1440 * @crtc: CRTC in question1441 * @max_vblank_count: max hardware vblank counter value1442 *1443 * Update the maximum hardware vblank counter value for @crtc1444 * at runtime. Useful for hardware where the operation of the1445 * hardware vblank counter depends on the currently active1446 * display configuration.1447 *1448 * For example, if the hardware vblank counter does not work1449 * when a specific connector is active the maximum can be set1450 * to zero. And when that specific connector isn't active the1451 * maximum can again be set to the appropriate non-zero value.1452 *1453 * If used, must be called before drm_vblank_on().1454 */1455void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,1456 u32 max_vblank_count)1457{1458 struct drm_device *dev = crtc->dev;1459 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);1460 1461 drm_WARN_ON(dev, dev->max_vblank_count);1462 drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));1463 1464 vblank->max_vblank_count = max_vblank_count;1465}1466EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);1467 1468/**1469 * drm_crtc_vblank_on_config - enable vblank events on a CRTC with custom1470 * configuration options1471 * @crtc: CRTC in question1472 * @config: Vblank configuration value1473 *1474 * See drm_crtc_vblank_on(). In addition, this function allows you to provide a1475 * custom vblank configuration for a given CRTC.1476 *1477 * Note that @config is copied, the pointer does not need to stay valid beyond1478 * this function call. For details of the parameters see1479 * struct drm_vblank_crtc_config.1480 */1481void drm_crtc_vblank_on_config(struct drm_crtc *crtc,1482 const struct drm_vblank_crtc_config *config)1483{1484 struct drm_device *dev = crtc->dev;1485 unsigned int pipe = drm_crtc_index(crtc);1486 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);1487 1488 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))1489 return;1490 1491 spin_lock_irq(&dev->vbl_lock);1492 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",1493 pipe, vblank->enabled, vblank->inmodeset);1494 1495 vblank->config = *config;1496 1497 /* Drop our private "prevent drm_vblank_get" refcount */1498 if (vblank->inmodeset) {1499 atomic_dec(&vblank->refcount);1500 vblank->inmodeset = 0;1501 }1502 1503 drm_reset_vblank_timestamp(dev, pipe);1504 1505 /*1506 * re-enable interrupts if there are users left, or the1507 * user wishes vblank interrupts to be enabled all the time.1508 */1509 if (atomic_read(&vblank->refcount) != 0 || !vblank->config.offdelay_ms)1510 drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));1511 spin_unlock_irq(&dev->vbl_lock);1512}1513EXPORT_SYMBOL(drm_crtc_vblank_on_config);1514 1515/**1516 * drm_crtc_vblank_on - enable vblank events on a CRTC1517 * @crtc: CRTC in question1518 *1519 * This functions restores the vblank interrupt state captured with1520 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note1521 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be1522 * unbalanced and so can also be unconditionally called in driver load code to1523 * reflect the current hardware state of the crtc.1524 *1525 * Note that unlike in drm_crtc_vblank_on_config(), default values are used.1526 */1527void drm_crtc_vblank_on(struct drm_crtc *crtc)1528{1529 const struct drm_vblank_crtc_config config = {1530 .offdelay_ms = drm_vblank_offdelay,1531 .disable_immediate = crtc->dev->vblank_disable_immediate1532 };1533 1534 drm_crtc_vblank_on_config(crtc, &config);1535}1536EXPORT_SYMBOL(drm_crtc_vblank_on);1537 1538static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)1539{1540 ktime_t t_vblank;1541 struct drm_vblank_crtc *vblank;1542 int framedur_ns;1543 u64 diff_ns;1544 u32 cur_vblank, diff = 1;1545 int count = DRM_TIMESTAMP_MAXRETRIES;1546 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);1547 1548 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))1549 return;1550 1551 assert_spin_locked(&dev->vbl_lock);1552 assert_spin_locked(&dev->vblank_time_lock);1553 1554 vblank = drm_vblank_crtc(dev, pipe);1555 drm_WARN_ONCE(dev,1556 drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,1557 "Cannot compute missed vblanks without frame duration\n");1558 framedur_ns = vblank->framedur_ns;1559 1560 do {1561 cur_vblank = __get_vblank_counter(dev, pipe);1562 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);1563 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);1564 1565 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));1566 if (framedur_ns)1567 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);1568 1569 1570 drm_dbg_vbl(dev,1571 "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",1572 diff, diff_ns, framedur_ns, cur_vblank - vblank->last);1573 vblank->last = (cur_vblank - diff) & max_vblank_count;1574}1575 1576/**1577 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.1578 * @crtc: CRTC in question1579 *1580 * Power manamement features can cause frame counter resets between vblank1581 * disable and enable. Drivers can use this function in their1582 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since1583 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the1584 * vblank counter.1585 *1586 * Note that drivers must have race-free high-precision timestamping support,1587 * i.e. &drm_crtc_funcs.get_vblank_timestamp must be hooked up and1588 * &drm_vblank_crtc_config.disable_immediate must be set to indicate the1589 * time-stamping functions are race-free against vblank hardware counter1590 * increments.1591 */1592void drm_crtc_vblank_restore(struct drm_crtc *crtc)1593{1594 struct drm_device *dev = crtc->dev;1595 unsigned int pipe = drm_crtc_index(crtc);1596 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);1597 1598 drm_WARN_ON_ONCE(dev, !crtc->funcs->get_vblank_timestamp);1599 drm_WARN_ON_ONCE(dev, vblank->inmodeset);1600 drm_WARN_ON_ONCE(dev, !vblank->config.disable_immediate);1601 1602 drm_vblank_restore(dev, pipe);1603}1604EXPORT_SYMBOL(drm_crtc_vblank_restore);1605 1606static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,1607 u64 req_seq,1608 union drm_wait_vblank *vblwait,1609 struct drm_file *file_priv)1610{1611 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);1612 struct drm_pending_vblank_event *e;1613 ktime_t now;1614 u64 seq;1615 int ret;1616 1617 e = kzalloc(sizeof(*e), GFP_KERNEL);1618 if (e == NULL) {1619 ret = -ENOMEM;1620 goto err_put;1621 }1622 1623 e->pipe = pipe;1624 e->event.base.type = DRM_EVENT_VBLANK;1625 e->event.base.length = sizeof(e->event.vbl);1626 e->event.vbl.user_data = vblwait->request.signal;1627 e->event.vbl.crtc_id = 0;1628 if (drm_core_check_feature(dev, DRIVER_MODESET)) {1629 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);1630 1631 if (crtc)1632 e->event.vbl.crtc_id = crtc->base.id;1633 }1634 1635 spin_lock_irq(&dev->event_lock);1636 1637 /*1638 * drm_crtc_vblank_off() might have been called after we called1639 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the1640 * vblank disable, so no need for further locking. The reference from1641 * drm_vblank_get() protects against vblank disable from another source.1642 */1643 if (!READ_ONCE(vblank->enabled)) {1644 ret = -EINVAL;1645 goto err_unlock;1646 }1647 1648 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,1649 &e->event.base);1650 1651 if (ret)1652 goto err_unlock;1653 1654 seq = drm_vblank_count_and_time(dev, pipe, &now);1655 1656 drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",1657 req_seq, seq, pipe);1658 1659 trace_drm_vblank_event_queued(file_priv, pipe, req_seq);1660 1661 e->sequence = req_seq;1662 if (drm_vblank_passed(seq, req_seq)) {1663 drm_vblank_put(dev, pipe);1664 send_vblank_event(dev, e, seq, now);1665 vblwait->reply.sequence = seq;1666 } else {1667 /* drm_handle_vblank_events will call drm_vblank_put */1668 list_add_tail(&e->base.link, &dev->vblank_event_list);1669 vblwait->reply.sequence = req_seq;1670 }1671 1672 spin_unlock_irq(&dev->event_lock);1673 1674 return 0;1675 1676err_unlock:1677 spin_unlock_irq(&dev->event_lock);1678 kfree(e);1679err_put:1680 drm_vblank_put(dev, pipe);1681 return ret;1682}1683 1684static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)1685{1686 if (vblwait->request.sequence)1687 return false;1688 1689 return _DRM_VBLANK_RELATIVE ==1690 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |1691 _DRM_VBLANK_EVENT |1692 _DRM_VBLANK_NEXTONMISS));1693}1694 1695/*1696 * Widen a 32-bit param to 64-bits.1697 *1698 * \param narrow 32-bit value (missing upper 32 bits)1699 * \param near 64-bit value that should be 'close' to near1700 *1701 * This function returns a 64-bit value using the lower 32-bits from1702 * 'narrow' and constructing the upper 32-bits so that the result is1703 * as close as possible to 'near'.1704 */1705 1706static u64 widen_32_to_64(u32 narrow, u64 near)1707{1708 return near + (s32) (narrow - near);1709}1710 1711static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,1712 struct drm_wait_vblank_reply *reply)1713{1714 ktime_t now;1715 struct timespec64 ts;1716 1717 /*1718 * drm_wait_vblank_reply is a UAPI structure that uses 'long'1719 * to store the seconds. This is safe as we always use monotonic1720 * timestamps since linux-4.15.1721 */1722 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);1723 ts = ktime_to_timespec64(now);1724 reply->tval_sec = (u32)ts.tv_sec;1725 reply->tval_usec = ts.tv_nsec / 1000;1726}1727 1728static bool drm_wait_vblank_supported(struct drm_device *dev)1729{1730 return drm_dev_has_vblank(dev);1731}1732 1733int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,1734 struct drm_file *file_priv)1735{1736 struct drm_crtc *crtc;1737 struct drm_vblank_crtc *vblank;1738 union drm_wait_vblank *vblwait = data;1739 int ret;1740 u64 req_seq, seq;1741 unsigned int pipe_index;1742 unsigned int flags, pipe, high_pipe;1743 1744 if (!drm_wait_vblank_supported(dev))1745 return -EOPNOTSUPP;1746 1747 if (vblwait->request.type & _DRM_VBLANK_SIGNAL)1748 return -EINVAL;1749 1750 if (vblwait->request.type &1751 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |1752 _DRM_VBLANK_HIGH_CRTC_MASK)) {1753 drm_dbg_core(dev,1754 "Unsupported type value 0x%x, supported mask 0x%x\n",1755 vblwait->request.type,1756 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |1757 _DRM_VBLANK_HIGH_CRTC_MASK));1758 return -EINVAL;1759 }1760 1761 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;1762 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);1763 if (high_pipe)1764 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;1765 else1766 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;1767 1768 /* Convert lease-relative crtc index into global crtc index */1769 if (drm_core_check_feature(dev, DRIVER_MODESET)) {1770 pipe = 0;1771 drm_for_each_crtc(crtc, dev) {1772 if (drm_lease_held(file_priv, crtc->base.id)) {1773 if (pipe_index == 0)1774 break;1775 pipe_index--;1776 }1777 pipe++;1778 }1779 } else {1780 pipe = pipe_index;1781 }1782 1783 if (pipe >= dev->num_crtcs)1784 return -EINVAL;1785 1786 vblank = &dev->vblank[pipe];1787 1788 /* If the counter is currently enabled and accurate, short-circuit1789 * queries to return the cached timestamp of the last vblank.1790 */1791 if (vblank->config.disable_immediate &&1792 drm_wait_vblank_is_query(vblwait) &&1793 READ_ONCE(vblank->enabled)) {1794 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);1795 return 0;1796 }1797 1798 ret = drm_vblank_get(dev, pipe);1799 if (ret) {1800 drm_dbg_core(dev,1801 "crtc %d failed to acquire vblank counter, %d\n",1802 pipe, ret);1803 return ret;1804 }1805 seq = drm_vblank_count(dev, pipe);1806 1807 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {1808 case _DRM_VBLANK_RELATIVE:1809 req_seq = seq + vblwait->request.sequence;1810 vblwait->request.sequence = req_seq;1811 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;1812 break;1813 case _DRM_VBLANK_ABSOLUTE:1814 req_seq = widen_32_to_64(vblwait->request.sequence, seq);1815 break;1816 default:1817 ret = -EINVAL;1818 goto done;1819 }1820 1821 if ((flags & _DRM_VBLANK_NEXTONMISS) &&1822 drm_vblank_passed(seq, req_seq)) {1823 req_seq = seq + 1;1824 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;1825 vblwait->request.sequence = req_seq;1826 }1827 1828 if (flags & _DRM_VBLANK_EVENT) {1829 /* must hold on to the vblank ref until the event fires1830 * drm_vblank_put will be called asynchronously1831 */1832 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);1833 }1834 1835 if (req_seq != seq) {1836 int wait;1837 1838 drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",1839 req_seq, pipe);1840 wait = wait_event_interruptible_timeout(vblank->queue,1841 drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||1842 !READ_ONCE(vblank->enabled),1843 msecs_to_jiffies(3000));1844 1845 switch (wait) {1846 case 0:1847 /* timeout */1848 ret = -EBUSY;1849 break;1850 case -ERESTARTSYS:1851 /* interrupted by signal */1852 ret = -EINTR;1853 break;1854 default:1855 ret = 0;1856 break;1857 }1858 }1859 1860 if (ret != -EINTR) {1861 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);1862 1863 drm_dbg_core(dev, "crtc %d returning %u to client\n",1864 pipe, vblwait->reply.sequence);1865 } else {1866 drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",1867 pipe);1868 }1869 1870done:1871 drm_vblank_put(dev, pipe);1872 return ret;1873}1874 1875static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)1876{1877 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);1878 bool high_prec = false;1879 struct drm_pending_vblank_event *e, *t;1880 ktime_t now;1881 u64 seq;1882 1883 assert_spin_locked(&dev->event_lock);1884 1885 seq = drm_vblank_count_and_time(dev, pipe, &now);1886 1887 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {1888 if (e->pipe != pipe)1889 continue;1890 if (!drm_vblank_passed(seq, e->sequence))1891 continue;1892 1893 drm_dbg_core(dev, "vblank event on %llu, current %llu\n",1894 e->sequence, seq);1895 1896 list_del(&e->base.link);1897 drm_vblank_put(dev, pipe);1898 send_vblank_event(dev, e, seq, now);1899 }1900 1901 if (crtc && crtc->funcs->get_vblank_timestamp)1902 high_prec = true;1903 1904 trace_drm_vblank_event(pipe, seq, now, high_prec);1905}1906 1907/**1908 * drm_handle_vblank - handle a vblank event1909 * @dev: DRM device1910 * @pipe: index of CRTC where this event occurred1911 *1912 * Drivers should call this routine in their vblank interrupt handlers to1913 * update the vblank counter and send any signals that may be pending.1914 *1915 * This is the legacy version of drm_crtc_handle_vblank().1916 */1917bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)1918{1919 struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);1920 unsigned long irqflags;1921 bool disable_irq;1922 1923 if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))1924 return false;1925 1926 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))1927 return false;1928 1929 spin_lock_irqsave(&dev->event_lock, irqflags);1930 1931 /* Need timestamp lock to prevent concurrent execution with1932 * vblank enable/disable, as this would cause inconsistent1933 * or corrupted timestamps and vblank counts.1934 */1935 spin_lock(&dev->vblank_time_lock);1936 1937 /* Vblank irq handling disabled. Nothing to do. */1938 if (!vblank->enabled) {1939 spin_unlock(&dev->vblank_time_lock);1940 spin_unlock_irqrestore(&dev->event_lock, irqflags);1941 return false;1942 }1943 1944 drm_update_vblank_count(dev, pipe, true);1945 1946 spin_unlock(&dev->vblank_time_lock);1947 1948 wake_up(&vblank->queue);1949 1950 /* With instant-off, we defer disabling the interrupt until after1951 * we finish processing the following vblank after all events have1952 * been signaled. The disable has to be last (after1953 * drm_handle_vblank_events) so that the timestamp is always accurate.1954 */1955 disable_irq = (vblank->config.disable_immediate &&1956 vblank->config.offdelay_ms > 0 &&1957 !atomic_read(&vblank->refcount));1958 1959 drm_handle_vblank_events(dev, pipe);1960 drm_handle_vblank_works(vblank);1961 1962 spin_unlock_irqrestore(&dev->event_lock, irqflags);1963 1964 if (disable_irq)1965 vblank_disable_fn(&vblank->disable_timer);1966 1967 return true;1968}1969EXPORT_SYMBOL(drm_handle_vblank);1970 1971/**1972 * drm_crtc_handle_vblank - handle a vblank event1973 * @crtc: where this event occurred1974 *1975 * Drivers should call this routine in their vblank interrupt handlers to1976 * update the vblank counter and send any signals that may be pending.1977 *1978 * This is the native KMS version of drm_handle_vblank().1979 *1980 * Note that for a given vblank counter value drm_crtc_handle_vblank()1981 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()1982 * provide a barrier: Any writes done before calling1983 * drm_crtc_handle_vblank() will be visible to callers of the later1984 * functions, if the vblank count is the same or a later one.1985 *1986 * See also &drm_vblank_crtc.count.1987 *1988 * Returns:1989 * True if the event was successfully handled, false on failure.1990 */1991bool drm_crtc_handle_vblank(struct drm_crtc *crtc)1992{1993 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));1994}1995EXPORT_SYMBOL(drm_crtc_handle_vblank);1996 1997/*1998 * Get crtc VBLANK count.1999 *2000 * \param dev DRM device2001 * \param data user argument, pointing to a drm_crtc_get_sequence structure.2002 * \param file_priv drm file private for the user's open file descriptor2003 */2004 2005int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,2006 struct drm_file *file_priv)2007{2008 struct drm_crtc *crtc;2009 struct drm_vblank_crtc *vblank;2010 int pipe;2011 struct drm_crtc_get_sequence *get_seq = data;2012 ktime_t now;2013 bool vblank_enabled;2014 int ret;2015 2016 if (!drm_core_check_feature(dev, DRIVER_MODESET))2017 return -EOPNOTSUPP;2018 2019 if (!drm_dev_has_vblank(dev))2020 return -EOPNOTSUPP;2021 2022 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);2023 if (!crtc)2024 return -ENOENT;2025 2026 pipe = drm_crtc_index(crtc);2027 2028 vblank = drm_crtc_vblank_crtc(crtc);2029 vblank_enabled = READ_ONCE(vblank->config.disable_immediate) &&2030 READ_ONCE(vblank->enabled);2031 2032 if (!vblank_enabled) {2033 ret = drm_crtc_vblank_get(crtc);2034 if (ret) {2035 drm_dbg_core(dev,2036 "crtc %d failed to acquire vblank counter, %d\n",2037 pipe, ret);2038 return ret;2039 }2040 }2041 drm_modeset_lock(&crtc->mutex, NULL);2042 if (crtc->state)2043 get_seq->active = crtc->state->enable;2044 else2045 get_seq->active = crtc->enabled;2046 drm_modeset_unlock(&crtc->mutex);2047 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);2048 get_seq->sequence_ns = ktime_to_ns(now);2049 if (!vblank_enabled)2050 drm_crtc_vblank_put(crtc);2051 return 0;2052}2053 2054/*2055 * Queue a event for VBLANK sequence2056 *2057 * \param dev DRM device2058 * \param data user argument, pointing to a drm_crtc_queue_sequence structure.2059 * \param file_priv drm file private for the user's open file descriptor2060 */2061 2062int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,2063 struct drm_file *file_priv)2064{2065 struct drm_crtc *crtc;2066 struct drm_vblank_crtc *vblank;2067 int pipe;2068 struct drm_crtc_queue_sequence *queue_seq = data;2069 ktime_t now;2070 struct drm_pending_vblank_event *e;2071 u32 flags;2072 u64 seq;2073 u64 req_seq;2074 int ret;2075 2076 if (!drm_core_check_feature(dev, DRIVER_MODESET))2077 return -EOPNOTSUPP;2078 2079 if (!drm_dev_has_vblank(dev))2080 return -EOPNOTSUPP;2081 2082 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);2083 if (!crtc)2084 return -ENOENT;2085 2086 flags = queue_seq->flags;2087 /* Check valid flag bits */2088 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|2089 DRM_CRTC_SEQUENCE_NEXT_ON_MISS))2090 return -EINVAL;2091 2092 pipe = drm_crtc_index(crtc);2093 2094 vblank = drm_crtc_vblank_crtc(crtc);2095 2096 e = kzalloc(sizeof(*e), GFP_KERNEL);2097 if (e == NULL)2098 return -ENOMEM;2099 2100 ret = drm_crtc_vblank_get(crtc);2101 if (ret) {2102 drm_dbg_core(dev,2103 "crtc %d failed to acquire vblank counter, %d\n",2104 pipe, ret);2105 goto err_free;2106 }2107 2108 seq = drm_vblank_count_and_time(dev, pipe, &now);2109 req_seq = queue_seq->sequence;2110 2111 if (flags & DRM_CRTC_SEQUENCE_RELATIVE)2112 req_seq += seq;2113 2114 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq))2115 req_seq = seq + 1;2116 2117 e->pipe = pipe;2118 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;2119 e->event.base.length = sizeof(e->event.seq);2120 e->event.seq.user_data = queue_seq->user_data;2121 2122 spin_lock_irq(&dev->event_lock);2123 2124 /*2125 * drm_crtc_vblank_off() might have been called after we called2126 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the2127 * vblank disable, so no need for further locking. The reference from2128 * drm_crtc_vblank_get() protects against vblank disable from another source.2129 */2130 if (!READ_ONCE(vblank->enabled)) {2131 ret = -EINVAL;2132 goto err_unlock;2133 }2134 2135 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,2136 &e->event.base);2137 2138 if (ret)2139 goto err_unlock;2140 2141 e->sequence = req_seq;2142 2143 if (drm_vblank_passed(seq, req_seq)) {2144 drm_crtc_vblank_put(crtc);2145 send_vblank_event(dev, e, seq, now);2146 queue_seq->sequence = seq;2147 } else {2148 /* drm_handle_vblank_events will call drm_vblank_put */2149 list_add_tail(&e->base.link, &dev->vblank_event_list);2150 queue_seq->sequence = req_seq;2151 }2152 2153 spin_unlock_irq(&dev->event_lock);2154 return 0;2155 2156err_unlock:2157 spin_unlock_irq(&dev->event_lock);2158 drm_crtc_vblank_put(crtc);2159err_free:2160 kfree(e);2161 return ret;2162}2163 2164