459 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Generic userspace implementations of gettimeofday() and similar.4 */5#include <vdso/datapage.h>6#include <vdso/helpers.h>7 8#ifndef vdso_calc_ns9 10#ifdef VDSO_DELTA_NOMASK11# define VDSO_DELTA_MASK(vd) ULLONG_MAX12#else13# define VDSO_DELTA_MASK(vd) (vd->mask)14#endif15 16#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT17static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)18{19 return delta < vd->max_cycles;20}21#else22static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)23{24 return true;25}26#endif27 28#ifndef vdso_shift_ns29static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)30{31 return ns >> shift;32}33#endif34 35/*36 * Default implementation which works for all sane clocksources. That37 * obviously excludes x86/TSC.38 */39static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)40{41 u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd);42 43 if (likely(vdso_delta_ok(vd, delta)))44 return vdso_shift_ns((delta * vd->mult) + base, vd->shift);45 46 return mul_u64_u32_add_u64_shr(delta, vd->mult, base, vd->shift);47}48#endif /* vdso_calc_ns */49 50#ifndef __arch_vdso_hres_capable51static inline bool __arch_vdso_hres_capable(void)52{53 return true;54}55#endif56 57#ifndef vdso_clocksource_ok58static inline bool vdso_clocksource_ok(const struct vdso_data *vd)59{60 return vd->clock_mode != VDSO_CLOCKMODE_NONE;61}62#endif63 64#ifndef vdso_cycles_ok65static inline bool vdso_cycles_ok(u64 cycles)66{67 return true;68}69#endif70 71#ifdef CONFIG_TIME_NS72static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,73 struct __kernel_timespec *ts)74{75 const struct timens_offset *offs = &vdns->offset[clk];76 const struct vdso_timestamp *vdso_ts;77 const struct vdso_data *vd;78 u64 cycles, ns;79 u32 seq;80 s64 sec;81 82 vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);83 vd = __arch_get_timens_vdso_data(vd);84 if (clk != CLOCK_MONOTONIC_RAW)85 vd = &vd[CS_HRES_COARSE];86 else87 vd = &vd[CS_RAW];88 vdso_ts = &vd->basetime[clk];89 90 do {91 seq = vdso_read_begin(vd);92 93 if (unlikely(!vdso_clocksource_ok(vd)))94 return -1;95 96 cycles = __arch_get_hw_counter(vd->clock_mode, vd);97 if (unlikely(!vdso_cycles_ok(cycles)))98 return -1;99 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);100 sec = vdso_ts->sec;101 } while (unlikely(vdso_read_retry(vd, seq)));102 103 /* Add the namespace offset */104 sec += offs->sec;105 ns += offs->nsec;106 107 /*108 * Do this outside the loop: a race inside the loop could result109 * in __iter_div_u64_rem() being extremely slow.110 */111 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);112 ts->tv_nsec = ns;113 114 return 0;115}116#else117static __always_inline118const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)119{120 return NULL;121}122 123static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,124 struct __kernel_timespec *ts)125{126 return -EINVAL;127}128#endif129 130static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,131 struct __kernel_timespec *ts)132{133 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];134 u64 cycles, sec, ns;135 u32 seq;136 137 /* Allows to compile the high resolution parts out */138 if (!__arch_vdso_hres_capable())139 return -1;140 141 do {142 /*143 * Open coded function vdso_read_begin() to handle144 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a145 * special VVAR page installed which has vd->seq set to 1 and146 * vd->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time147 * namespace affected tasks this does not affect performance148 * because if vd->seq is odd, i.e. a concurrent update is in149 * progress the extra check for vd->clock_mode is just a few150 * extra instructions while spin waiting for vd->seq to become151 * even again.152 */153 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {154 if (IS_ENABLED(CONFIG_TIME_NS) &&155 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)156 return do_hres_timens(vd, clk, ts);157 cpu_relax();158 }159 smp_rmb();160 161 if (unlikely(!vdso_clocksource_ok(vd)))162 return -1;163 164 cycles = __arch_get_hw_counter(vd->clock_mode, vd);165 if (unlikely(!vdso_cycles_ok(cycles)))166 return -1;167 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);168 sec = vdso_ts->sec;169 } while (unlikely(vdso_read_retry(vd, seq)));170 171 /*172 * Do this outside the loop: a race inside the loop could result173 * in __iter_div_u64_rem() being extremely slow.174 */175 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);176 ts->tv_nsec = ns;177 178 return 0;179}180 181#ifdef CONFIG_TIME_NS182static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,183 struct __kernel_timespec *ts)184{185 const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns);186 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];187 const struct timens_offset *offs = &vdns->offset[clk];188 u64 nsec;189 s64 sec;190 s32 seq;191 192 do {193 seq = vdso_read_begin(vd);194 sec = vdso_ts->sec;195 nsec = vdso_ts->nsec;196 } while (unlikely(vdso_read_retry(vd, seq)));197 198 /* Add the namespace offset */199 sec += offs->sec;200 nsec += offs->nsec;201 202 /*203 * Do this outside the loop: a race inside the loop could result204 * in __iter_div_u64_rem() being extremely slow.205 */206 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);207 ts->tv_nsec = nsec;208 return 0;209}210#else211static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,212 struct __kernel_timespec *ts)213{214 return -1;215}216#endif217 218static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,219 struct __kernel_timespec *ts)220{221 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];222 u32 seq;223 224 do {225 /*226 * Open coded function vdso_read_begin() to handle227 * VDSO_CLOCK_TIMENS. See comment in do_hres().228 */229 while ((seq = READ_ONCE(vd->seq)) & 1) {230 if (IS_ENABLED(CONFIG_TIME_NS) &&231 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)232 return do_coarse_timens(vd, clk, ts);233 cpu_relax();234 }235 smp_rmb();236 237 ts->tv_sec = vdso_ts->sec;238 ts->tv_nsec = vdso_ts->nsec;239 } while (unlikely(vdso_read_retry(vd, seq)));240 241 return 0;242}243 244static __always_inline int245__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,246 struct __kernel_timespec *ts)247{248 u32 msk;249 250 /* Check for negative values or invalid clocks */251 if (unlikely((u32) clock >= MAX_CLOCKS))252 return -1;253 254 /*255 * Convert the clockid to a bitmask and use it to check which256 * clocks are handled in the VDSO directly.257 */258 msk = 1U << clock;259 if (likely(msk & VDSO_HRES))260 vd = &vd[CS_HRES_COARSE];261 else if (msk & VDSO_COARSE)262 return do_coarse(&vd[CS_HRES_COARSE], clock, ts);263 else if (msk & VDSO_RAW)264 vd = &vd[CS_RAW];265 else266 return -1;267 268 return do_hres(vd, clock, ts);269}270 271static __maybe_unused int272__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,273 struct __kernel_timespec *ts)274{275 int ret = __cvdso_clock_gettime_common(vd, clock, ts);276 277 if (unlikely(ret))278 return clock_gettime_fallback(clock, ts);279 return 0;280}281 282static __maybe_unused int283__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)284{285 return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);286}287 288#ifdef BUILD_VDSO32289static __maybe_unused int290__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,291 struct old_timespec32 *res)292{293 struct __kernel_timespec ts;294 int ret;295 296 ret = __cvdso_clock_gettime_common(vd, clock, &ts);297 298 if (unlikely(ret))299 return clock_gettime32_fallback(clock, res);300 301 /* For ret == 0 */302 res->tv_sec = ts.tv_sec;303 res->tv_nsec = ts.tv_nsec;304 305 return ret;306}307 308static __maybe_unused int309__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)310{311 return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);312}313#endif /* BUILD_VDSO32 */314 315static __maybe_unused int316__cvdso_gettimeofday_data(const struct vdso_data *vd,317 struct __kernel_old_timeval *tv, struct timezone *tz)318{319 320 if (likely(tv != NULL)) {321 struct __kernel_timespec ts;322 323 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))324 return gettimeofday_fallback(tv, tz);325 326 tv->tv_sec = ts.tv_sec;327 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;328 }329 330 if (unlikely(tz != NULL)) {331 if (IS_ENABLED(CONFIG_TIME_NS) &&332 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)333 vd = __arch_get_timens_vdso_data(vd);334 335 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;336 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;337 }338 339 return 0;340}341 342static __maybe_unused int343__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)344{345 return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);346}347 348#ifdef VDSO_HAS_TIME349static __maybe_unused __kernel_old_time_t350__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)351{352 __kernel_old_time_t t;353 354 if (IS_ENABLED(CONFIG_TIME_NS) &&355 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)356 vd = __arch_get_timens_vdso_data(vd);357 358 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);359 360 if (time)361 *time = t;362 363 return t;364}365 366static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)367{368 return __cvdso_time_data(__arch_get_vdso_data(), time);369}370#endif /* VDSO_HAS_TIME */371 372#ifdef VDSO_HAS_CLOCK_GETRES373static __maybe_unused374int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,375 struct __kernel_timespec *res)376{377 u32 msk;378 u64 ns;379 380 /* Check for negative values or invalid clocks */381 if (unlikely((u32) clock >= MAX_CLOCKS))382 return -1;383 384 if (IS_ENABLED(CONFIG_TIME_NS) &&385 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)386 vd = __arch_get_timens_vdso_data(vd);387 388 /*389 * Convert the clockid to a bitmask and use it to check which390 * clocks are handled in the VDSO directly.391 */392 msk = 1U << clock;393 if (msk & (VDSO_HRES | VDSO_RAW)) {394 /*395 * Preserves the behaviour of posix_get_hrtimer_res().396 */397 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);398 } else if (msk & VDSO_COARSE) {399 /*400 * Preserves the behaviour of posix_get_coarse_res().401 */402 ns = LOW_RES_NSEC;403 } else {404 return -1;405 }406 407 if (likely(res)) {408 res->tv_sec = 0;409 res->tv_nsec = ns;410 }411 return 0;412}413 414static __maybe_unused415int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,416 struct __kernel_timespec *res)417{418 int ret = __cvdso_clock_getres_common(vd, clock, res);419 420 if (unlikely(ret))421 return clock_getres_fallback(clock, res);422 return 0;423}424 425static __maybe_unused426int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)427{428 return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);429}430 431#ifdef BUILD_VDSO32432static __maybe_unused int433__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,434 struct old_timespec32 *res)435{436 struct __kernel_timespec ts;437 int ret;438 439 ret = __cvdso_clock_getres_common(vd, clock, &ts);440 441 if (unlikely(ret))442 return clock_getres32_fallback(clock, res);443 444 if (likely(res)) {445 res->tv_sec = ts.tv_sec;446 res->tv_nsec = ts.tv_nsec;447 }448 return ret;449}450 451static __maybe_unused int452__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)453{454 return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),455 clock, res);456}457#endif /* BUILD_VDSO32 */458#endif /* VDSO_HAS_CLOCK_GETRES */459