lkml.org 
[lkml]   [2018]   [Mar]   [21]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    From
    Subject[PATCH v4.16-rc6 1/1] x86/vdso: VDSO should handle clock_gettime(CLOCK_MONOTONIC_RAW) without syscall
    Date

    This patch implements clock_gettime(CLOCK_MONOTONIC_RAW,&ts) calls entirely in the
    vDSO, without calling vdso_fallback_gettime() .

    It has been augmented to support compilation with or without -DRETPOLINE / $(RETPOLINE_CFLAGS) ;
    when compiled with -DRETPOLINE, not all functions calls can be inlined within __vdso_clock_gettime,
    and all functions invoked by __vdso_clock_gettime must have 'indirect_branch("keep")' +
    'function_return("keep")' attributes to compile, otherwise thunk relocations will be generated ;
    and the functions cannot all be declared '__always_inline_', otherwise a compiler -Werror
    ('not all __always_inline__ functions can be inlined') is generated.
    Also, compared to previous version of same patch, the do_*_coarse functions are still
    not inlines, and not inadvertently changed to inline.

    I still think it might be better to apply H.J. Liu's patch from
    https://bugzilla.kernel.org/show_bug.cgi?id=199129 to disable
    -DRETPOLINE compilation for the vDSO .

    ---
    diff --git a/arch/x86/entry/vdso/vclock_gettime.c b/arch/x86/entry/vdso/vclock_gettime.c
    index f19856d..80d65d4 100644
    --- a/arch/x86/entry/vdso/vclock_gettime.c
    +++ b/arch/x86/entry/vdso/vclock_gettime.c
    @@ -182,29 +182,62 @@ notrace static u64 vread_tsc(void)
    return last;
    }

    -notrace static inline u64 vgetsns(int *mode)
    +notrace static inline u64 vgetcycles(int *mode)
    {
    - u64 v;
    - cycles_t cycles;
    -
    - if (gtod->vclock_mode == VCLOCK_TSC)
    - cycles = vread_tsc();
    + switch (gtod->vclock_mode) {
    + case VCLOCK_TSC:
    + return vread_tsc();
    #ifdef CONFIG_PARAVIRT_CLOCK
    - else if (gtod->vclock_mode == VCLOCK_PVCLOCK)
    - cycles = vread_pvclock(mode);
    + case VCLOCK_PVCLOCK:
    + return vread_pvclock(mode);
    #endif
    #ifdef CONFIG_HYPERV_TSCPAGE
    - else if (gtod->vclock_mode == VCLOCK_HVCLOCK)
    - cycles = vread_hvclock(mode);
    + case VCLOCK_HVCLOCK:
    + return vread_hvclock(mode);
    #endif
    - else
    + default:
    + break;
    + }
    + return 0;
    +}
    +
    +notrace static inline u64 vgetsns(int *mode)
    +{
    + u64 v;
    + cycles_t cycles = vgetcycles(mode);
    +
    + if (cycles == 0)
    return 0;
    +
    v = (cycles - gtod->cycle_last) & gtod->mask;
    return v * gtod->mult;
    }

    +notrace static inline u64 vgetsns_raw(int *mode)
    +{
    + u64 v;
    + cycles_t cycles = vgetcycles(mode);
    +
    + if (cycles == 0)
    + return 0;
    +
    + v = (cycles - gtod->cycle_last) & gtod->mask;
    + return v * gtod->raw_mult;
    +}
    +
    +#ifdef RETPOLINE
    +# define _NO_THUNK_RELOCS_()(indirect_branch("keep"),\
    + function_return("keep"))
    +# define _RETPOLINE_FUNC_ATTR_ __attribute__(_NO_THUNK_RELOCS_())
    +# define _RETPOLINE_INLINE_ inline
    +#else
    +# define _RETPOLINE_FUNC_ATTR_
    +# define _RETPOLINE_INLINE_ __always_inline
    +#endif
    +
    /* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
    -notrace static int __always_inline do_realtime(struct timespec *ts)
    +notrace static _RETPOLINE_INLINE_ _RETPOLINE_FUNC_ATTR_
    +int do_realtime(struct timespec *ts)
    {
    unsigned long seq;
    u64 ns;
    @@ -225,7 +258,8 @@ notrace static int __always_inline do_realtime(struct timespec *ts)
    return mode;
    }

    -notrace static int __always_inline do_monotonic(struct timespec *ts)
    +notrace static _RETPOLINE_INLINE_ _RETPOLINE_FUNC_ATTR_
    +int do_monotonic(struct timespec *ts)
    {
    unsigned long seq;
    u64 ns;
    @@ -246,7 +280,30 @@ notrace static int __always_inline do_monotonic(struct timespec *ts)
    return mode;
    }

    -notrace static void do_realtime_coarse(struct timespec *ts)
    +notrace static _RETPOLINE_INLINE_ _RETPOLINE_FUNC_ATTR_
    +int do_monotonic_raw(struct timespec *ts)
    +{
    + unsigned long seq;
    + u64 ns;
    + int mode;
    +
    + do {
    + seq = gtod_read_begin(gtod);
    + mode = gtod->vclock_mode;
    + ts->tv_sec = gtod->monotonic_time_raw_sec;
    + ns = gtod->monotonic_time_raw_nsec;
    + ns += vgetsns_raw(&mode);
    + ns >>= gtod->raw_shift;
    + } while (unlikely(gtod_read_retry(gtod, seq)));
    +
    + ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
    + ts->tv_nsec = ns;
    +
    + return mode;
    +}
    +
    +notrace static _RETPOLINE_FUNC_ATTR_
    +void do_realtime_coarse(struct timespec *ts)
    {
    unsigned long seq;
    do {
    @@ -256,7 +313,8 @@ notrace static void do_realtime_coarse(struct timespec *ts)
    } while (unlikely(gtod_read_retry(gtod, seq)));
    }

    -notrace static void do_monotonic_coarse(struct timespec *ts)
    +notrace static _RETPOLINE_FUNC_ATTR_
    +void do_monotonic_coarse(struct timespec *ts)
    {
    unsigned long seq;
    do {
    @@ -266,7 +324,8 @@ notrace static void do_monotonic_coarse(struct timespec *ts)
    } while (unlikely(gtod_read_retry(gtod, seq)));
    }

    -notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
    +notrace _RETPOLINE_FUNC_ATTR_
    +int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
    {
    switch (clock) {
    case CLOCK_REALTIME:
    @@ -277,6 +336,10 @@ notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
    if (do_monotonic(ts) == VCLOCK_NONE)
    goto fallback;
    break;
    + case CLOCK_MONOTONIC_RAW:
    + if (do_monotonic_raw(ts) == VCLOCK_NONE)
    + goto fallback;
    + break;
    case CLOCK_REALTIME_COARSE:
    do_realtime_coarse(ts);
    break;
    diff --git a/arch/x86/entry/vsyscall/vsyscall_gtod.c b/arch/x86/entry/vsyscall/vsyscall_gtod.c
    index e1216dd..c4d89b6 100644
    --- a/arch/x86/entry/vsyscall/vsyscall_gtod.c
    +++ b/arch/x86/entry/vsyscall/vsyscall_gtod.c
    @@ -44,6 +44,8 @@ void update_vsyscall(struct timekeeper *tk)
    vdata->mask = tk->tkr_mono.mask;
    vdata->mult = tk->tkr_mono.mult;
    vdata->shift = tk->tkr_mono.shift;
    + vdata->raw_mult = tk->tkr_raw.mult;
    + vdata->raw_shift = tk->tkr_raw.shift;

    vdata->wall_time_sec = tk->xtime_sec;
    vdata->wall_time_snsec = tk->tkr_mono.xtime_nsec;
    @@ -74,5 +76,8 @@ void update_vsyscall(struct timekeeper *tk)
    vdata->monotonic_time_coarse_sec++;
    }

    + vdata->monotonic_time_raw_sec = tk->raw_sec;
    + vdata->monotonic_time_raw_nsec = tk->tkr_raw.xtime_nsec;
    +
    gtod_write_end(vdata);
    }
    diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h
    index fb856c9..ec1a37c 100644
    --- a/arch/x86/include/asm/vgtod.h
    +++ b/arch/x86/include/asm/vgtod.h
    @@ -22,7 +22,8 @@ struct vsyscall_gtod_data {
    u64 mask;
    u32 mult;
    u32 shift;
    -
    + u32 raw_mult;
    + u32 raw_shift;
    /* open coded 'struct timespec' */
    u64 wall_time_snsec;
    gtod_long_t wall_time_sec;
    @@ -32,6 +33,8 @@ struct vsyscall_gtod_data {
    gtod_long_t wall_time_coarse_nsec;
    gtod_long_t monotonic_time_coarse_sec;
    gtod_long_t monotonic_time_coarse_nsec;
    + gtod_long_t monotonic_time_raw_sec;
    + gtod_long_t monotonic_time_raw_nsec;

    int tz_minuteswest;
    int tz_dsttime;
    \
     
     \ /
      Last update: 2018-03-21 17:36    [W:3.069 / U:0.284 seconds]
    ©2003-2020 Jasper Spaans|hosted at Digital Ocean and TransIP|Read the blog|Advertise on this site