lkml.org 
[lkml]   [2012]   [Jul]   [22]   [last100]   RSS Feed
Views: [wrap][no wrap]   [headers]  [forward] 
 
Messages in this thread
    Patch in this message
    /
    Date
    From
    Subject[GIT PULL] timer changes for v3.6
    Linus,

    Please pull the latest timers-core-for-linus git tree from:

    git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git timers-core-for-linus

    HEAD: eec19d1a0d04c80e66eef634f7b8f460f2ca5643 Merge branch 'linus' into timers/core

    Continued cleanups of the core time and NTP code, plus more nohz
    work preparing for tick-less userspace execution.

    Thanks,

    Ingo

    ------------------>
    Frederic Weisbecker (5):
    nohz: Separate idle sleeping time accounting from nohz logic
    nohz: Make nohz API agnostic against idle ticks cputime accounting
    nohz: Rename ts->idle_tick to ts->last_tick
    nohz: Move ts->idle_calls incrementation into strict idle logic
    nohz: Move next idle expiry time record into idle logic area

    John Stultz (8):
    ntp: Fix STA_INS/DEL clearing bug
    time: Whitespace cleanups per Ingo%27s requests
    time: Explicitly use u32 instead of int for shift values
    time: Condense timekeeper.xtime into xtime_sec
    time: Refactor accumulation of nsecs to secs
    time: Move arch_gettimeoffset() usage into timekeeping_get_ns()
    time: Move xtime_nsec adjustment underflow handling timekeeping_adjust
    time: Rework timekeeping functions to take timekeeper ptr as argument

    Thomas Gleixner (4):
    timers: Create detach_if_pending() and use it
    timers: Consolidate base->next_timer update
    timers: Add accounting of non deferrable timers
    timers: Improve get_next_timer_interrupt()


    include/linux/tick.h | 8 +-
    kernel/time/ntp.c | 8 +-
    kernel/time/tick-sched.c | 192 ++++++++++--------
    kernel/time/timekeeping.c | 487 ++++++++++++++++++++++++---------------------
    kernel/time/timer_list.c | 4 +-
    kernel/timer.c | 110 +++++-----
    6 files changed, 442 insertions(+), 367 deletions(-)

    diff --git a/include/linux/tick.h b/include/linux/tick.h
    index ab8be90..f37fceb 100644
    --- a/include/linux/tick.h
    +++ b/include/linux/tick.h
    @@ -31,10 +31,10 @@ enum tick_nohz_mode {
    * struct tick_sched - sched tick emulation and no idle tick control/stats
    * @sched_timer: hrtimer to schedule the periodic tick in high
    * resolution mode
    - * @idle_tick: Store the last idle tick expiry time when the tick
    - * timer is modified for idle sleeps. This is necessary
    + * @last_tick: Store the last tick expiry time when the tick
    + * timer is modified for nohz sleeps. This is necessary
    * to resume the tick timer operation in the timeline
    - * when the CPU returns from idle
    + * when the CPU returns from nohz sleep.
    * @tick_stopped: Indicator that the idle tick has been stopped
    * @idle_jiffies: jiffies at the entry to idle for idle time accounting
    * @idle_calls: Total number of idle calls
    @@ -51,7 +51,7 @@ struct tick_sched {
    struct hrtimer sched_timer;
    unsigned long check_clocks;
    enum tick_nohz_mode nohz_mode;
    - ktime_t idle_tick;
    + ktime_t last_tick;
    int inidle;
    int tick_stopped;
    unsigned long idle_jiffies;
    diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
    index 70b33ab..b7fbadc 100644
    --- a/kernel/time/ntp.c
    +++ b/kernel/time/ntp.c
    @@ -409,7 +409,9 @@ int second_overflow(unsigned long secs)
    time_state = TIME_DEL;
    break;
    case TIME_INS:
    - if (secs % 86400 == 0) {
    + if (!(time_status & STA_INS))
    + time_state = TIME_OK;
    + else if (secs % 86400 == 0) {
    leap = -1;
    time_state = TIME_OOP;
    time_tai++;
    @@ -418,7 +420,9 @@ int second_overflow(unsigned long secs)
    }
    break;
    case TIME_DEL:
    - if ((secs + 1) % 86400 == 0) {
    + if (!(time_status & STA_DEL))
    + time_state = TIME_OK;
    + else if ((secs + 1) % 86400 == 0) {
    leap = 1;
    time_tai--;
    time_state = TIME_WAIT;
    diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
    index 4a08472..41be022 100644
    --- a/kernel/time/tick-sched.c
    +++ b/kernel/time/tick-sched.c
    @@ -271,50 +271,15 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
    }
    EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);

    -static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
    +static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts,
    + ktime_t now, int cpu)
    {
    unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;
    + ktime_t last_update, expires, ret = { .tv64 = 0 };
    unsigned long rcu_delta_jiffies;
    - ktime_t last_update, expires, now;
    struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;
    u64 time_delta;
    - int cpu;
    -
    - cpu = smp_processor_id();
    - ts = &per_cpu(tick_cpu_sched, cpu);
    -
    - now = tick_nohz_start_idle(cpu, ts);
    -
    - /*
    - * If this cpu is offline and it is the one which updates
    - * jiffies, then give up the assignment and let it be taken by
    - * the cpu which runs the tick timer next. If we don't drop
    - * this here the jiffies might be stale and do_timer() never
    - * invoked.
    - */
    - if (unlikely(!cpu_online(cpu))) {
    - if (cpu == tick_do_timer_cpu)
    - tick_do_timer_cpu = TICK_DO_TIMER_NONE;
    - }
    -
    - if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
    - return;

    - if (need_resched())
    - return;
    -
    - if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
    - static int ratelimit;
    -
    - if (ratelimit < 10) {
    - printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
    - (unsigned int) local_softirq_pending());
    - ratelimit++;
    - }
    - return;
    - }
    -
    - ts->idle_calls++;
    /* Read jiffies and the time when jiffies were updated last */
    do {
    seq = read_seqbegin(&xtime_lock);
    @@ -397,6 +362,8 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
    if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
    goto out;

    + ret = expires;
    +
    /*
    * nohz_stop_sched_tick can be called several times before
    * the nohz_restart_sched_tick is called. This happens when
    @@ -408,16 +375,10 @@ static void tick_nohz_stop_sched_tick(struct tick_sched *ts)
    select_nohz_load_balancer(1);
    calc_load_enter_idle();

    - ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);
    + ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
    ts->tick_stopped = 1;
    - ts->idle_jiffies = last_jiffies;
    }

    - ts->idle_sleeps++;
    -
    - /* Mark expires */
    - ts->idle_expires = expires;
    -
    /*
    * If the expiration time == KTIME_MAX, then
    * in this case we simply stop the tick timer.
    @@ -448,6 +409,65 @@ out:
    ts->next_jiffies = next_jiffies;
    ts->last_jiffies = last_jiffies;
    ts->sleep_length = ktime_sub(dev->next_event, now);
    +
    + return ret;
    +}
    +
    +static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
    +{
    + /*
    + * If this cpu is offline and it is the one which updates
    + * jiffies, then give up the assignment and let it be taken by
    + * the cpu which runs the tick timer next. If we don't drop
    + * this here the jiffies might be stale and do_timer() never
    + * invoked.
    + */
    + if (unlikely(!cpu_online(cpu))) {
    + if (cpu == tick_do_timer_cpu)
    + tick_do_timer_cpu = TICK_DO_TIMER_NONE;
    + }
    +
    + if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
    + return false;
    +
    + if (need_resched())
    + return false;
    +
    + if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
    + static int ratelimit;
    +
    + if (ratelimit < 10) {
    + printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",
    + (unsigned int) local_softirq_pending());
    + ratelimit++;
    + }
    + return false;
    + }
    +
    + return true;
    +}
    +
    +static void __tick_nohz_idle_enter(struct tick_sched *ts)
    +{
    + ktime_t now, expires;
    + int cpu = smp_processor_id();
    +
    + now = tick_nohz_start_idle(cpu, ts);
    +
    + if (can_stop_idle_tick(cpu, ts)) {
    + int was_stopped = ts->tick_stopped;
    +
    + ts->idle_calls++;
    +
    + expires = tick_nohz_stop_sched_tick(ts, now, cpu);
    + if (expires.tv64 > 0LL) {
    + ts->idle_sleeps++;
    + ts->idle_expires = expires;
    + }
    +
    + if (!was_stopped && ts->tick_stopped)
    + ts->idle_jiffies = ts->last_jiffies;
    + }
    }

    /**
    @@ -485,7 +505,7 @@ void tick_nohz_idle_enter(void)
    * update of the idle time accounting in tick_nohz_start_idle().
    */
    ts->inidle = 1;
    - tick_nohz_stop_sched_tick(ts);
    + __tick_nohz_idle_enter(ts);

    local_irq_enable();
    }
    @@ -505,7 +525,7 @@ void tick_nohz_irq_exit(void)
    if (!ts->inidle)
    return;

    - tick_nohz_stop_sched_tick(ts);
    + __tick_nohz_idle_enter(ts);
    }

    /**
    @@ -523,7 +543,7 @@ ktime_t tick_nohz_get_sleep_length(void)
    static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
    {
    hrtimer_cancel(&ts->sched_timer);
    - hrtimer_set_expires(&ts->sched_timer, ts->idle_tick);
    + hrtimer_set_expires(&ts->sched_timer, ts->last_tick);

    while (1) {
    /* Forward the time to expire in the future */
    @@ -546,6 +566,41 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
    }
    }

    +static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
    +{
    + /* Update jiffies first */
    + select_nohz_load_balancer(0);
    + tick_do_update_jiffies64(now);
    + update_cpu_load_nohz();
    +
    + touch_softlockup_watchdog();
    + /*
    + * Cancel the scheduled timer and restore the tick
    + */
    + ts->tick_stopped = 0;
    + ts->idle_exittime = now;
    +
    + tick_nohz_restart(ts, now);
    +}
    +
    +static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
    +{
    +#ifndef CONFIG_VIRT_CPU_ACCOUNTING
    + unsigned long ticks;
    + /*
    + * We stopped the tick in idle. Update process times would miss the
    + * time we slept as update_process_times does only a 1 tick
    + * accounting. Enforce that this is accounted to idle !
    + */
    + ticks = jiffies - ts->idle_jiffies;
    + /*
    + * We might be one off. Do not randomly account a huge number of ticks!
    + */
    + if (ticks && ticks < LONG_MAX)
    + account_idle_ticks(ticks);
    +#endif
    +}
    +
    /**
    * tick_nohz_idle_exit - restart the idle tick from the idle task
    *
    @@ -557,9 +612,6 @@ void tick_nohz_idle_exit(void)
    {
    int cpu = smp_processor_id();
    struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
    -#ifndef CONFIG_VIRT_CPU_ACCOUNTING
    - unsigned long ticks;
    -#endif
    ktime_t now;

    local_irq_disable();
    @@ -574,40 +626,11 @@ void tick_nohz_idle_exit(void)
    if (ts->idle_active)
    tick_nohz_stop_idle(cpu, now);

    - if (!ts->tick_stopped) {
    - local_irq_enable();
    - return;
    + if (ts->tick_stopped) {
    + tick_nohz_restart_sched_tick(ts, now);
    + tick_nohz_account_idle_ticks(ts);
    }

    - /* Update jiffies first */
    - select_nohz_load_balancer(0);
    - tick_do_update_jiffies64(now);
    - update_cpu_load_nohz();
    -
    -#ifndef CONFIG_VIRT_CPU_ACCOUNTING
    - /*
    - * We stopped the tick in idle. Update process times would miss the
    - * time we slept as update_process_times does only a 1 tick
    - * accounting. Enforce that this is accounted to idle !
    - */
    - ticks = jiffies - ts->idle_jiffies;
    - /*
    - * We might be one off. Do not randomly account a huge number of ticks!
    - */
    - if (ticks && ticks < LONG_MAX)
    - account_idle_ticks(ticks);
    -#endif
    -
    - calc_load_exit_idle();
    - touch_softlockup_watchdog();
    - /*
    - * Cancel the scheduled timer and restore the tick
    - */
    - ts->tick_stopped = 0;
    - ts->idle_exittime = now;
    -
    - tick_nohz_restart(ts, now);
    -
    local_irq_enable();
    }

    @@ -811,7 +834,8 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
    */
    if (ts->tick_stopped) {
    touch_softlockup_watchdog();
    - ts->idle_jiffies++;
    + if (idle_cpu(cpu))
    + ts->idle_jiffies++;
    }
    update_process_times(user_mode(regs));
    profile_tick(CPU_PROFILING);
    diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
    index 3447cfa..f045cc5 100644
    --- a/kernel/time/timekeeping.c
    +++ b/kernel/time/timekeeping.c
    @@ -24,32 +24,32 @@
    /* Structure holding internal timekeeping values. */
    struct timekeeper {
    /* Current clocksource used for timekeeping. */
    - struct clocksource *clock;
    + struct clocksource *clock;
    /* NTP adjusted clock multiplier */
    - u32 mult;
    + u32 mult;
    /* The shift value of the current clocksource. */
    - int shift;
    -
    + u32 shift;
    /* Number of clock cycles in one NTP interval. */
    - cycle_t cycle_interval;
    + cycle_t cycle_interval;
    /* Number of clock shifted nano seconds in one NTP interval. */
    - u64 xtime_interval;
    + u64 xtime_interval;
    /* shifted nano seconds left over when rounding cycle_interval */
    - s64 xtime_remainder;
    + s64 xtime_remainder;
    /* Raw nano seconds accumulated per NTP interval. */
    - u32 raw_interval;
    + u32 raw_interval;
    +
    + /* Current CLOCK_REALTIME time in seconds */
    + u64 xtime_sec;
    + /* Clock shifted nano seconds */
    + u64 xtime_nsec;

    - /* Clock shifted nano seconds remainder not stored in xtime.tv_nsec. */
    - u64 xtime_nsec;
    /* Difference between accumulated time and NTP time in ntp
    * shifted nano seconds. */
    - s64 ntp_error;
    + s64 ntp_error;
    /* Shift conversion between clock shifted nano seconds and
    * ntp shifted nano seconds. */
    - int ntp_error_shift;
    + u32 ntp_error_shift;

    - /* The current time */
    - struct timespec xtime;
    /*
    * wall_to_monotonic is what we need to add to xtime (or xtime corrected
    * for sub jiffie times) to get to monotonic time. Monotonic is pegged
    @@ -64,20 +64,17 @@ struct timekeeper {
    * - wall_to_monotonic is no longer the boot time, getboottime must be
    * used instead.
    */
    - struct timespec wall_to_monotonic;
    + struct timespec wall_to_monotonic;
    /* time spent in suspend */
    - struct timespec total_sleep_time;
    + struct timespec total_sleep_time;
    /* The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock. */
    - struct timespec raw_time;
    -
    + struct timespec raw_time;
    /* Offset clock monotonic -> clock realtime */
    - ktime_t offs_real;
    -
    + ktime_t offs_real;
    /* Offset clock monotonic -> clock boottime */
    - ktime_t offs_boot;
    -
    + ktime_t offs_boot;
    /* Seqlock for all timekeeper values */
    - seqlock_t lock;
    + seqlock_t lock;
    };

    static struct timekeeper timekeeper;
    @@ -88,11 +85,37 @@ static struct timekeeper timekeeper;
    */
    __cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);

    -
    /* flag for if timekeeping is suspended */
    int __read_mostly timekeeping_suspended;

    +static inline void tk_normalize_xtime(struct timekeeper *tk)
    +{
    + while (tk->xtime_nsec >= ((u64)NSEC_PER_SEC << tk->shift)) {
    + tk->xtime_nsec -= (u64)NSEC_PER_SEC << tk->shift;
    + tk->xtime_sec++;
    + }
    +}

    +static struct timespec tk_xtime(struct timekeeper *tk)
    +{
    + struct timespec ts;
    +
    + ts.tv_sec = tk->xtime_sec;
    + ts.tv_nsec = (long)(tk->xtime_nsec >> tk->shift);
    + return ts;
    +}
    +
    +static void tk_set_xtime(struct timekeeper *tk, const struct timespec *ts)
    +{
    + tk->xtime_sec = ts->tv_sec;
    + tk->xtime_nsec = ts->tv_nsec << tk->shift;
    +}
    +
    +static void tk_xtime_add(struct timekeeper *tk, const struct timespec *ts)
    +{
    + tk->xtime_sec += ts->tv_sec;
    + tk->xtime_nsec += ts->tv_nsec << tk->shift;
    +}

    /**
    * timekeeper_setup_internals - Set up internals to use clocksource clock.
    @@ -104,12 +127,14 @@ int __read_mostly timekeeping_suspended;
    *
    * Unless you're the timekeeping code, you should not be using this!
    */
    -static void timekeeper_setup_internals(struct clocksource *clock)
    +static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
    {
    cycle_t interval;
    u64 tmp, ntpinterval;
    + struct clocksource *old_clock;

    - timekeeper.clock = clock;
    + old_clock = tk->clock;
    + tk->clock = clock;
    clock->cycle_last = clock->read(clock);

    /* Do the ns -> cycle conversion first, using original mult */
    @@ -122,80 +147,96 @@ static void timekeeper_setup_internals(struct clocksource *clock)
    tmp = 1;

    interval = (cycle_t) tmp;
    - timekeeper.cycle_interval = interval;
    + tk->cycle_interval = interval;

    /* Go back from cycles -> shifted ns */
    - timekeeper.xtime_interval = (u64) interval * clock->mult;
    - timekeeper.xtime_remainder = ntpinterval - timekeeper.xtime_interval;
    - timekeeper.raw_interval =
    + tk->xtime_interval = (u64) interval * clock->mult;
    + tk->xtime_remainder = ntpinterval - tk->xtime_interval;
    + tk->raw_interval =
    ((u64) interval * clock->mult) >> clock->shift;

    - timekeeper.xtime_nsec = 0;
    - timekeeper.shift = clock->shift;
    + /* if changing clocks, convert xtime_nsec shift units */
    + if (old_clock) {
    + int shift_change = clock->shift - old_clock->shift;
    + if (shift_change < 0)
    + tk->xtime_nsec >>= -shift_change;
    + else
    + tk->xtime_nsec <<= shift_change;
    + }
    + tk->shift = clock->shift;

    - timekeeper.ntp_error = 0;
    - timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
    + tk->ntp_error = 0;
    + tk->ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;

    /*
    * The timekeeper keeps its own mult values for the currently
    * active clocksource. These value will be adjusted via NTP
    * to counteract clock drifting.
    */
    - timekeeper.mult = clock->mult;
    + tk->mult = clock->mult;
    }

    /* Timekeeper helper functions. */
    -static inline s64 timekeeping_get_ns(void)
    +static inline s64 timekeeping_get_ns(struct timekeeper *tk)
    {
    cycle_t cycle_now, cycle_delta;
    struct clocksource *clock;
    + s64 nsec;

    /* read clocksource: */
    - clock = timekeeper.clock;
    + clock = tk->clock;
    cycle_now = clock->read(clock);

    /* calculate the delta since the last update_wall_time: */
    cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;

    - /* return delta convert to nanoseconds using ntp adjusted mult. */
    - return clocksource_cyc2ns(cycle_delta, timekeeper.mult,
    - timekeeper.shift);
    + nsec = cycle_delta * tk->mult + tk->xtime_nsec;
    + nsec >>= tk->shift;
    +
    + /* If arch requires, add in gettimeoffset() */
    + return nsec + arch_gettimeoffset();
    }

    -static inline s64 timekeeping_get_ns_raw(void)
    +static inline s64 timekeeping_get_ns_raw(struct timekeeper *tk)
    {
    cycle_t cycle_now, cycle_delta;
    struct clocksource *clock;
    + s64 nsec;

    /* read clocksource: */
    - clock = timekeeper.clock;
    + clock = tk->clock;
    cycle_now = clock->read(clock);

    /* calculate the delta since the last update_wall_time: */
    cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;

    - /* return delta convert to nanoseconds. */
    - return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
    + /* convert delta to nanoseconds. */
    + nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
    +
    + /* If arch requires, add in gettimeoffset() */
    + return nsec + arch_gettimeoffset();
    }

    -static void update_rt_offset(void)
    +static void update_rt_offset(struct timekeeper *tk)
    {
    - struct timespec tmp, *wtm = &timekeeper.wall_to_monotonic;
    + struct timespec tmp, *wtm = &tk->wall_to_monotonic;

    set_normalized_timespec(&tmp, -wtm->tv_sec, -wtm->tv_nsec);
    - timekeeper.offs_real = timespec_to_ktime(tmp);
    + tk->offs_real = timespec_to_ktime(tmp);
    }

    /* must hold write on timekeeper.lock */
    -static void timekeeping_update(bool clearntp)
    +static void timekeeping_update(struct timekeeper *tk, bool clearntp)
    {
    + struct timespec xt;
    +
    if (clearntp) {
    - timekeeper.ntp_error = 0;
    + tk->ntp_error = 0;
    ntp_clear();
    }
    - update_rt_offset();
    - update_vsyscall(&timekeeper.xtime, &timekeeper.wall_to_monotonic,
    - timekeeper.clock, timekeeper.mult);
    + update_rt_offset(tk);
    + xt = tk_xtime(tk);
    + update_vsyscall(&xt, &tk->wall_to_monotonic, tk->clock, tk->mult);
    }


    @@ -206,27 +247,26 @@ static void timekeeping_update(bool clearntp)
    * update_wall_time(). This is useful before significant clock changes,
    * as it avoids having to deal with this time offset explicitly.
    */
    -static void timekeeping_forward_now(void)
    +static void timekeeping_forward_now(struct timekeeper *tk)
    {
    cycle_t cycle_now, cycle_delta;
    struct clocksource *clock;
    s64 nsec;

    - clock = timekeeper.clock;
    + clock = tk->clock;
    cycle_now = clock->read(clock);
    cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
    clock->cycle_last = cycle_now;

    - nsec = clocksource_cyc2ns(cycle_delta, timekeeper.mult,
    - timekeeper.shift);
    + tk->xtime_nsec += cycle_delta * tk->mult;

    /* If arch requires, add in gettimeoffset() */
    - nsec += arch_gettimeoffset();
    + tk->xtime_nsec += arch_gettimeoffset() << tk->shift;

    - timespec_add_ns(&timekeeper.xtime, nsec);
    + tk_normalize_xtime(tk);

    nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
    - timespec_add_ns(&timekeeper.raw_time, nsec);
    + timespec_add_ns(&tk->raw_time, nsec);
    }

    /**
    @@ -238,18 +278,15 @@ static void timekeeping_forward_now(void)
    void getnstimeofday(struct timespec *ts)
    {
    unsigned long seq;
    - s64 nsecs;
    + s64 nsecs = 0;

    WARN_ON(timekeeping_suspended);

    do {
    seq = read_seqbegin(&timekeeper.lock);

    - *ts = timekeeper.xtime;
    - nsecs = timekeeping_get_ns();
    -
    - /* If arch requires, add in gettimeoffset() */
    - nsecs += arch_gettimeoffset();
    + ts->tv_sec = timekeeper.xtime_sec;
    + ts->tv_nsec = timekeeping_get_ns(&timekeeper);

    } while (read_seqretry(&timekeeper.lock, seq));

    @@ -266,13 +303,10 @@ ktime_t ktime_get(void)

    do {
    seq = read_seqbegin(&timekeeper.lock);
    - secs = timekeeper.xtime.tv_sec +
    + secs = timekeeper.xtime_sec +
    timekeeper.wall_to_monotonic.tv_sec;
    - nsecs = timekeeper.xtime.tv_nsec +
    + nsecs = timekeeping_get_ns(&timekeeper) +
    timekeeper.wall_to_monotonic.tv_nsec;
    - nsecs += timekeeping_get_ns();
    - /* If arch requires, add in gettimeoffset() */
    - nsecs += arch_gettimeoffset();

    } while (read_seqretry(&timekeeper.lock, seq));
    /*
    @@ -295,22 +329,19 @@ void ktime_get_ts(struct timespec *ts)
    {
    struct timespec tomono;
    unsigned int seq;
    - s64 nsecs;

    WARN_ON(timekeeping_suspended);

    do {
    seq = read_seqbegin(&timekeeper.lock);
    - *ts = timekeeper.xtime;
    + ts->tv_sec = timekeeper.xtime_sec;
    + ts->tv_nsec = timekeeping_get_ns(&timekeeper);
    tomono = timekeeper.wall_to_monotonic;
    - nsecs = timekeeping_get_ns();
    - /* If arch requires, add in gettimeoffset() */
    - nsecs += arch_gettimeoffset();

    } while (read_seqretry(&timekeeper.lock, seq));

    set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
    - ts->tv_nsec + tomono.tv_nsec + nsecs);
    + ts->tv_nsec + tomono.tv_nsec);
    }
    EXPORT_SYMBOL_GPL(ktime_get_ts);

    @@ -333,20 +364,14 @@ void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
    WARN_ON_ONCE(timekeeping_suspended);

    do {
    - u32 arch_offset;
    -
    seq = read_seqbegin(&timekeeper.lock);

    *ts_raw = timekeeper.raw_time;
    - *ts_real = timekeeper.xtime;
    -
    - nsecs_raw = timekeeping_get_ns_raw();
    - nsecs_real = timekeeping_get_ns();
    + ts_real->tv_sec = timekeeper.xtime_sec;
    + ts_real->tv_nsec = 0;

    - /* If arch requires, add in gettimeoffset() */
    - arch_offset = arch_gettimeoffset();
    - nsecs_raw += arch_offset;
    - nsecs_real += arch_offset;
    + nsecs_raw = timekeeping_get_ns_raw(&timekeeper);
    + nsecs_real = timekeeping_get_ns(&timekeeper);

    } while (read_seqretry(&timekeeper.lock, seq));

    @@ -381,7 +406,7 @@ EXPORT_SYMBOL(do_gettimeofday);
    */
    int do_settimeofday(const struct timespec *tv)
    {
    - struct timespec ts_delta;
    + struct timespec ts_delta, xt;
    unsigned long flags;

    if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
    @@ -389,15 +414,18 @@ int do_settimeofday(const struct timespec *tv)

    write_seqlock_irqsave(&timekeeper.lock, flags);

    - timekeeping_forward_now();
    + timekeeping_forward_now(&timekeeper);
    +
    + xt = tk_xtime(&timekeeper);
    + ts_delta.tv_sec = tv->tv_sec - xt.tv_sec;
    + ts_delta.tv_nsec = tv->tv_nsec - xt.tv_nsec;

    - ts_delta.tv_sec = tv->tv_sec - timekeeper.xtime.tv_sec;
    - ts_delta.tv_nsec = tv->tv_nsec - timekeeper.xtime.tv_nsec;
    timekeeper.wall_to_monotonic =
    timespec_sub(timekeeper.wall_to_monotonic, ts_delta);

    - timekeeper.xtime = *tv;
    - timekeeping_update(true);
    + tk_set_xtime(&timekeeper, tv);
    +
    + timekeeping_update(&timekeeper, true);

    write_sequnlock_irqrestore(&timekeeper.lock, flags);

    @@ -424,13 +452,14 @@ int timekeeping_inject_offset(struct timespec *ts)

    write_seqlock_irqsave(&timekeeper.lock, flags);

    - timekeeping_forward_now();
    + timekeeping_forward_now(&timekeeper);
    +

    - timekeeper.xtime = timespec_add(timekeeper.xtime, *ts);
    + tk_xtime_add(&timekeeper, ts);
    timekeeper.wall_to_monotonic =
    timespec_sub(timekeeper.wall_to_monotonic, *ts);

    - timekeeping_update(true);
    + timekeeping_update(&timekeeper, true);

    write_sequnlock_irqrestore(&timekeeper.lock, flags);

    @@ -455,14 +484,14 @@ static int change_clocksource(void *data)

    write_seqlock_irqsave(&timekeeper.lock, flags);

    - timekeeping_forward_now();
    + timekeeping_forward_now(&timekeeper);
    if (!new->enable || new->enable(new) == 0) {
    old = timekeeper.clock;
    - timekeeper_setup_internals(new);
    + tk_setup_internals(&timekeeper, new);
    if (old->disable)
    old->disable(old);
    }
    - timekeeping_update(true);
    + timekeeping_update(&timekeeper, true);

    write_sequnlock_irqrestore(&timekeeper.lock, flags);

    @@ -512,7 +541,7 @@ void getrawmonotonic(struct timespec *ts)

    do {
    seq = read_seqbegin(&timekeeper.lock);
    - nsecs = timekeeping_get_ns_raw();
    + nsecs = timekeeping_get_ns_raw(&timekeeper);
    *ts = timekeeper.raw_time;

    } while (read_seqretry(&timekeeper.lock, seq));
    @@ -547,6 +576,7 @@ u64 timekeeping_max_deferment(void)
    {
    unsigned long seq;
    u64 ret;
    +
    do {
    seq = read_seqbegin(&timekeeper.lock);

    @@ -607,19 +637,17 @@ void __init timekeeping_init(void)
    clock = clocksource_default_clock();
    if (clock->enable)
    clock->enable(clock);
    - timekeeper_setup_internals(clock);
    + tk_setup_internals(&timekeeper, clock);

    - timekeeper.xtime.tv_sec = now.tv_sec;
    - timekeeper.xtime.tv_nsec = now.tv_nsec;
    + tk_set_xtime(&timekeeper, &now);
    timekeeper.raw_time.tv_sec = 0;
    timekeeper.raw_time.tv_nsec = 0;
    - if (boot.tv_sec == 0 && boot.tv_nsec == 0) {
    - boot.tv_sec = timekeeper.xtime.tv_sec;
    - boot.tv_nsec = timekeeper.xtime.tv_nsec;
    - }
    + if (boot.tv_sec == 0 && boot.tv_nsec == 0)
    + boot = tk_xtime(&timekeeper);
    +
    set_normalized_timespec(&timekeeper.wall_to_monotonic,
    -boot.tv_sec, -boot.tv_nsec);
    - update_rt_offset();
    + update_rt_offset(&timekeeper);
    timekeeper.total_sleep_time.tv_sec = 0;
    timekeeper.total_sleep_time.tv_nsec = 0;
    write_sequnlock_irqrestore(&timekeeper.lock, flags);
    @@ -641,7 +669,8 @@ static void update_sleep_time(struct timespec t)
    * Takes a timespec offset measuring a suspend interval and properly
    * adds the sleep offset to the timekeeping variables.
    */
    -static void __timekeeping_inject_sleeptime(struct timespec *delta)
    +static void __timekeeping_inject_sleeptime(struct timekeeper *tk,
    + struct timespec *delta)
    {
    if (!timespec_valid(delta)) {
    printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid "
    @@ -649,10 +678,9 @@ static void __timekeeping_inject_sleeptime(struct timespec *delta)
    return;
    }

    - timekeeper.xtime = timespec_add(timekeeper.xtime, *delta);
    - timekeeper.wall_to_monotonic =
    - timespec_sub(timekeeper.wall_to_monotonic, *delta);
    - update_sleep_time(timespec_add(timekeeper.total_sleep_time, *delta));
    + tk_xtime_add(tk, delta);
    + tk->wall_to_monotonic = timespec_sub(tk->wall_to_monotonic, *delta);
    + update_sleep_time(timespec_add(tk->total_sleep_time, *delta));
    }


    @@ -678,11 +706,11 @@ void timekeeping_inject_sleeptime(struct timespec *delta)

    write_seqlock_irqsave(&timekeeper.lock, flags);

    - timekeeping_forward_now();
    + timekeeping_forward_now(&timekeeper);

    - __timekeeping_inject_sleeptime(delta);
    + __timekeeping_inject_sleeptime(&timekeeper, delta);

    - timekeeping_update(true);
    + timekeeping_update(&timekeeper, true);

    write_sequnlock_irqrestore(&timekeeper.lock, flags);

    @@ -711,13 +739,13 @@ static void timekeeping_resume(void)

    if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
    ts = timespec_sub(ts, timekeeping_suspend_time);
    - __timekeeping_inject_sleeptime(&ts);
    + __timekeeping_inject_sleeptime(&timekeeper, &ts);
    }
    /* re-base the last cycle value */
    timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
    timekeeper.ntp_error = 0;
    timekeeping_suspended = 0;
    - timekeeping_update(false);
    + timekeeping_update(&timekeeper, false);
    write_sequnlock_irqrestore(&timekeeper.lock, flags);

    touch_softlockup_watchdog();
    @@ -737,7 +765,7 @@ static int timekeeping_suspend(void)
    read_persistent_clock(&timekeeping_suspend_time);

    write_seqlock_irqsave(&timekeeper.lock, flags);
    - timekeeping_forward_now();
    + timekeeping_forward_now(&timekeeper);
    timekeeping_suspended = 1;

    /*
    @@ -746,7 +774,7 @@ static int timekeeping_suspend(void)
    * try to compensate so the difference in system time
    * and persistent_clock time stays close to constant.
    */
    - delta = timespec_sub(timekeeper.xtime, timekeeping_suspend_time);
    + delta = timespec_sub(tk_xtime(&timekeeper), timekeeping_suspend_time);
    delta_delta = timespec_sub(delta, old_delta);
    if (abs(delta_delta.tv_sec) >= 2) {
    /*
    @@ -785,7 +813,8 @@ device_initcall(timekeeping_init_ops);
    * If the error is already larger, we look ahead even further
    * to compensate for late or lost adjustments.
    */
    -static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
    +static __always_inline int timekeeping_bigadjust(struct timekeeper *tk,
    + s64 error, s64 *interval,
    s64 *offset)
    {
    s64 tick_error, i;
    @@ -801,7 +830,7 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
    * here. This is tuned so that an error of about 1 msec is adjusted
    * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
    */
    - error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
    + error2 = tk->ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
    error2 = abs(error2);
    for (look_ahead = 0; error2 > 0; look_ahead++)
    error2 >>= 2;
    @@ -810,8 +839,8 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
    * Now calculate the error in (1 << look_ahead) ticks, but first
    * remove the single look ahead already included in the error.
    */
    - tick_error = ntp_tick_length() >> (timekeeper.ntp_error_shift + 1);
    - tick_error -= timekeeper.xtime_interval >> 1;
    + tick_error = ntp_tick_length() >> (tk->ntp_error_shift + 1);
    + tick_error -= tk->xtime_interval >> 1;
    error = ((error - tick_error) >> look_ahead) + tick_error;

    /* Finally calculate the adjustment shift value. */
    @@ -836,9 +865,9 @@ static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
    * this is optimized for the most common adjustments of -1,0,1,
    * for other values we can do a bit more work.
    */
    -static void timekeeping_adjust(s64 offset)
    +static void timekeeping_adjust(struct timekeeper *tk, s64 offset)
    {
    - s64 error, interval = timekeeper.cycle_interval;
    + s64 error, interval = tk->cycle_interval;
    int adj;

    /*
    @@ -854,7 +883,7 @@ static void timekeeping_adjust(s64 offset)
    *
    * Note: It does not "save" on aggravation when reading the code.
    */
    - error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1);
    + error = tk->ntp_error >> (tk->ntp_error_shift - 1);
    if (error > interval) {
    /*
    * We now divide error by 4(via shift), which checks if
    @@ -876,7 +905,8 @@ static void timekeeping_adjust(s64 offset)
    if (likely(error <= interval))
    adj = 1;
    else
    - adj = timekeeping_bigadjust(error, &interval, &offset);
    + adj = timekeeping_bigadjust(tk, error, &interval,
    + &offset);
    } else if (error < -interval) {
    /* See comment above, this is just switched for the negative */
    error >>= 2;
    @@ -885,18 +915,17 @@ static void timekeeping_adjust(s64 offset)
    interval = -interval;
    offset = -offset;
    } else
    - adj = timekeeping_bigadjust(error, &interval, &offset);
    - } else /* No adjustment needed */
    + adj = timekeeping_bigadjust(tk, error, &interval,
    + &offset);
    + } else
    return;

    - if (unlikely(timekeeper.clock->maxadj &&
    - (timekeeper.mult + adj >
    - timekeeper.clock->mult + timekeeper.clock->maxadj))) {
    + if (unlikely(tk->clock->maxadj &&
    + (tk->mult + adj > tk->clock->mult + tk->clock->maxadj))) {
    printk_once(KERN_WARNING
    "Adjusting %s more than 11%% (%ld vs %ld)\n",
    - timekeeper.clock->name, (long)timekeeper.mult + adj,
    - (long)timekeeper.clock->mult +
    - timekeeper.clock->maxadj);
    + tk->clock->name, (long)tk->mult + adj,
    + (long)tk->clock->mult + tk->clock->maxadj);
    }
    /*
    * So the following can be confusing.
    @@ -947,11 +976,60 @@ static void timekeeping_adjust(s64 offset)
    *
    * XXX - TODO: Doc ntp_error calculation.
    */
    - timekeeper.mult += adj;
    - timekeeper.xtime_interval += interval;
    - timekeeper.xtime_nsec -= offset;
    - timekeeper.ntp_error -= (interval - offset) <<
    - timekeeper.ntp_error_shift;
    + tk->mult += adj;
    + tk->xtime_interval += interval;
    + tk->xtime_nsec -= offset;
    + tk->ntp_error -= (interval - offset) << tk->ntp_error_shift;
    +
    + /*
    + * It may be possible that when we entered this function, xtime_nsec
    + * was very small. Further, if we're slightly speeding the clocksource
    + * in the code above, its possible the required corrective factor to
    + * xtime_nsec could cause it to underflow.
    + *
    + * Now, since we already accumulated the second, cannot simply roll
    + * the accumulated second back, since the NTP subsystem has been
    + * notified via second_overflow. So instead we push xtime_nsec forward
    + * by the amount we underflowed, and add that amount into the error.
    + *
    + * We'll correct this error next time through this function, when
    + * xtime_nsec is not as small.
    + */
    + if (unlikely((s64)tk->xtime_nsec < 0)) {
    + s64 neg = -(s64)tk->xtime_nsec;
    + tk->xtime_nsec = 0;
    + tk->ntp_error += neg << tk->ntp_error_shift;
    + }
    +
    +}
    +
    +
    +/**
    + * accumulate_nsecs_to_secs - Accumulates nsecs into secs
    + *
    + * Helper function that accumulates a the nsecs greater then a second
    + * from the xtime_nsec field to the xtime_secs field.
    + * It also calls into the NTP code to handle leapsecond processing.
    + *
    + */
    +static inline void accumulate_nsecs_to_secs(struct timekeeper *tk)
    +{
    + u64 nsecps = (u64)NSEC_PER_SEC << tk->shift;
    +
    + while (tk->xtime_nsec >= nsecps) {
    + int leap;
    +
    + tk->xtime_nsec -= nsecps;
    + tk->xtime_sec++;
    +
    + /* Figure out if its a leap sec and apply if needed */
    + leap = second_overflow(tk->xtime_sec);
    + tk->xtime_sec += leap;
    + tk->wall_to_monotonic.tv_sec -= leap;
    + if (leap)
    + clock_was_set_delayed();
    +
    + }
    }


    @@ -964,46 +1042,36 @@ static void timekeeping_adjust(s64 offset)
    *
    * Returns the unconsumed cycles.
    */
    -static cycle_t logarithmic_accumulation(cycle_t offset, int shift)
    +static cycle_t logarithmic_accumulation(struct timekeeper *tk, cycle_t offset,
    + u32 shift)
    {
    - u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift;
    u64 raw_nsecs;

    - /* If the offset is smaller than a shifted interval, do nothing */
    - if (offset < timekeeper.cycle_interval<<shift)
    + /* If the offset is smaller then a shifted interval, do nothing */
    + if (offset < tk->cycle_interval<<shift)
    return offset;

    /* Accumulate one shifted interval */
    - offset -= timekeeper.cycle_interval << shift;
    - timekeeper.clock->cycle_last += timekeeper.cycle_interval << shift;
    + offset -= tk->cycle_interval << shift;
    + tk->clock->cycle_last += tk->cycle_interval << shift;

    - timekeeper.xtime_nsec += timekeeper.xtime_interval << shift;
    - while (timekeeper.xtime_nsec >= nsecps) {
    - int leap;
    - timekeeper.xtime_nsec -= nsecps;
    - timekeeper.xtime.tv_sec++;
    - leap = second_overflow(timekeeper.xtime.tv_sec);
    - timekeeper.xtime.tv_sec += leap;
    - timekeeper.wall_to_monotonic.tv_sec -= leap;
    - if (leap)
    - clock_was_set_delayed();
    - }
    + tk->xtime_nsec += tk->xtime_interval << shift;
    + accumulate_nsecs_to_secs(tk);

    /* Accumulate raw time */
    - raw_nsecs = timekeeper.raw_interval << shift;
    - raw_nsecs += timekeeper.raw_time.tv_nsec;
    + raw_nsecs = tk->raw_interval << shift;
    + raw_nsecs += tk->raw_time.tv_nsec;
    if (raw_nsecs >= NSEC_PER_SEC) {
    u64 raw_secs = raw_nsecs;
    raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
    - timekeeper.raw_time.tv_sec += raw_secs;
    + tk->raw_time.tv_sec += raw_secs;
    }
    - timekeeper.raw_time.tv_nsec = raw_nsecs;
    + tk->raw_time.tv_nsec = raw_nsecs;

    /* Accumulate error between NTP and clock interval */
    - timekeeper.ntp_error += ntp_tick_length() << shift;
    - timekeeper.ntp_error -=
    - (timekeeper.xtime_interval + timekeeper.xtime_remainder) <<
    - (timekeeper.ntp_error_shift + shift);
    + tk->ntp_error += ntp_tick_length() << shift;
    + tk->ntp_error -= (tk->xtime_interval + tk->xtime_remainder) <<
    + (tk->ntp_error_shift + shift);

    return offset;
    }
    @@ -1019,6 +1087,7 @@ static void update_wall_time(void)
    cycle_t offset;
    int shift = 0, maxshift;
    unsigned long flags;
    + s64 remainder;

    write_seqlock_irqsave(&timekeeper.lock, flags);

    @@ -1033,8 +1102,6 @@ static void update_wall_time(void)
    #else
    offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
    #endif
    - timekeeper.xtime_nsec = (s64)timekeeper.xtime.tv_nsec <<
    - timekeeper.shift;

    /*
    * With NO_HZ we may have to accumulate many cycle_intervals
    @@ -1050,64 +1117,36 @@ static void update_wall_time(void)
    maxshift = (64 - (ilog2(ntp_tick_length())+1)) - 1;
    shift = min(shift, maxshift);
    while (offset >= timekeeper.cycle_interval) {
    - offset = logarithmic_accumulation(offset, shift);
    + offset = logarithmic_accumulation(&timekeeper, offset, shift);
    if(offset < timekeeper.cycle_interval<<shift)
    shift--;
    }

    /* correct the clock when NTP error is too big */
    - timekeeping_adjust(offset);
    -
    - /*
    - * Since in the loop above, we accumulate any amount of time
    - * in xtime_nsec over a second into xtime.tv_sec, its possible for
    - * xtime_nsec to be fairly small after the loop. Further, if we're
    - * slightly speeding the clocksource up in timekeeping_adjust(),
    - * its possible the required corrective factor to xtime_nsec could
    - * cause it to underflow.
    - *
    - * Now, we cannot simply roll the accumulated second back, since
    - * the NTP subsystem has been notified via second_overflow. So
    - * instead we push xtime_nsec forward by the amount we underflowed,
    - * and add that amount into the error.
    - *
    - * We'll correct this error next time through this function, when
    - * xtime_nsec is not as small.
    - */
    - if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
    - s64 neg = -(s64)timekeeper.xtime_nsec;
    - timekeeper.xtime_nsec = 0;
    - timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
    - }
    + timekeeping_adjust(&timekeeper, offset);


    /*
    - * Store full nanoseconds into xtime after rounding it up and
    - * add the remainder to the error difference.
    - */
    - timekeeper.xtime.tv_nsec = ((s64)timekeeper.xtime_nsec >>
    - timekeeper.shift) + 1;
    - timekeeper.xtime_nsec -= (s64)timekeeper.xtime.tv_nsec <<
    - timekeeper.shift;
    - timekeeper.ntp_error += timekeeper.xtime_nsec <<
    - timekeeper.ntp_error_shift;
    + * Store only full nanoseconds into xtime_nsec after rounding
    + * it up and add the remainder to the error difference.
    + * XXX - This is necessary to avoid small 1ns inconsistnecies caused
    + * by truncating the remainder in vsyscalls. However, it causes
    + * additional work to be done in timekeeping_adjust(). Once
    + * the vsyscall implementations are converted to use xtime_nsec
    + * (shifted nanoseconds), this can be killed.
    + */
    + remainder = timekeeper.xtime_nsec & ((1 << timekeeper.shift) - 1);
    + timekeeper.xtime_nsec -= remainder;
    + timekeeper.xtime_nsec += 1 << timekeeper.shift;
    + timekeeper.ntp_error += remainder << timekeeper.ntp_error_shift;

    /*
    * Finally, make sure that after the rounding
    - * xtime.tv_nsec isn't larger than NSEC_PER_SEC
    + * xtime_nsec isn't larger than NSEC_PER_SEC
    */
    - if (unlikely(timekeeper.xtime.tv_nsec >= NSEC_PER_SEC)) {
    - int leap;
    - timekeeper.xtime.tv_nsec -= NSEC_PER_SEC;
    - timekeeper.xtime.tv_sec++;
    - leap = second_overflow(timekeeper.xtime.tv_sec);
    - timekeeper.xtime.tv_sec += leap;
    - timekeeper.wall_to_monotonic.tv_sec -= leap;
    - if (leap)
    - clock_was_set_delayed();
    - }
    + accumulate_nsecs_to_secs(&timekeeper);

    - timekeeping_update(false);
    + timekeeping_update(&timekeeper, false);

    out:
    write_sequnlock_irqrestore(&timekeeper.lock, flags);
    @@ -1152,21 +1191,20 @@ void get_monotonic_boottime(struct timespec *ts)
    {
    struct timespec tomono, sleep;
    unsigned int seq;
    - s64 nsecs;

    WARN_ON(timekeeping_suspended);

    do {
    seq = read_seqbegin(&timekeeper.lock);
    - *ts = timekeeper.xtime;
    + ts->tv_sec = timekeeper.xtime_sec;
    + ts->tv_nsec = timekeeping_get_ns(&timekeeper);
    tomono = timekeeper.wall_to_monotonic;
    sleep = timekeeper.total_sleep_time;
    - nsecs = timekeeping_get_ns();

    } while (read_seqretry(&timekeeper.lock, seq));

    set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec,
    - ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec + nsecs);
    + ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec);
    }
    EXPORT_SYMBOL_GPL(get_monotonic_boottime);

    @@ -1199,13 +1237,13 @@ EXPORT_SYMBOL_GPL(monotonic_to_bootbased);

    unsigned long get_seconds(void)
    {
    - return timekeeper.xtime.tv_sec;
    + return timekeeper.xtime_sec;
    }
    EXPORT_SYMBOL(get_seconds);

    struct timespec __current_kernel_time(void)
    {
    - return timekeeper.xtime;
    + return tk_xtime(&timekeeper);
    }

    struct timespec current_kernel_time(void)
    @@ -1216,7 +1254,7 @@ struct timespec current_kernel_time(void)
    do {
    seq = read_seqbegin(&timekeeper.lock);

    - now = timekeeper.xtime;
    + now = tk_xtime(&timekeeper);
    } while (read_seqretry(&timekeeper.lock, seq));

    return now;
    @@ -1231,7 +1269,7 @@ struct timespec get_monotonic_coarse(void)
    do {
    seq = read_seqbegin(&timekeeper.lock);

    - now = timekeeper.xtime;
    + now = tk_xtime(&timekeeper);
    mono = timekeeper.wall_to_monotonic;
    } while (read_seqretry(&timekeeper.lock, seq));

    @@ -1266,7 +1304,7 @@ void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,

    do {
    seq = read_seqbegin(&timekeeper.lock);
    - *xtim = timekeeper.xtime;
    + *xtim = tk_xtime(&timekeeper);
    *wtom = timekeeper.wall_to_monotonic;
    *sleep = timekeeper.total_sleep_time;
    } while (read_seqretry(&timekeeper.lock, seq));
    @@ -1290,11 +1328,8 @@ ktime_t ktime_get_update_offsets(ktime_t *offs_real, ktime_t *offs_boot)
    do {
    seq = read_seqbegin(&timekeeper.lock);

    - secs = timekeeper.xtime.tv_sec;
    - nsecs = timekeeper.xtime.tv_nsec;
    - nsecs += timekeeping_get_ns();
    - /* If arch requires, add in gettimeoffset() */
    - nsecs += arch_gettimeoffset();
    + secs = timekeeper.xtime_sec;
    + nsecs = timekeeping_get_ns(&timekeeper);

    *offs_real = timekeeper.offs_real;
    *offs_boot = timekeeper.offs_boot;
    diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
    index 3258455..af5a7e9 100644
    --- a/kernel/time/timer_list.c
    +++ b/kernel/time/timer_list.c
    @@ -167,7 +167,7 @@ static void print_cpu(struct seq_file *m, int cpu, u64 now)
    {
    struct tick_sched *ts = tick_get_tick_sched(cpu);
    P(nohz_mode);
    - P_ns(idle_tick);
    + P_ns(last_tick);
    P(tick_stopped);
    P(idle_jiffies);
    P(idle_calls);
    @@ -259,7 +259,7 @@ static int timer_list_show(struct seq_file *m, void *v)
    u64 now = ktime_to_ns(ktime_get());
    int cpu;

    - SEQ_printf(m, "Timer List Version: v0.6\n");
    + SEQ_printf(m, "Timer List Version: v0.7\n");
    SEQ_printf(m, "HRTIMER_MAX_CLOCK_BASES: %d\n", HRTIMER_MAX_CLOCK_BASES);
    SEQ_printf(m, "now at %Ld nsecs\n", (unsigned long long)now);

    diff --git a/kernel/timer.c b/kernel/timer.c
    index 6ec7e7e..a61c093 100644
    --- a/kernel/timer.c
    +++ b/kernel/timer.c
    @@ -77,6 +77,7 @@ struct tvec_base {
    struct timer_list *running_timer;
    unsigned long timer_jiffies;
    unsigned long next_timer;
    + unsigned long active_timers;
    struct tvec_root tv1;
    struct tvec tv2;
    struct tvec tv3;
    @@ -330,7 +331,8 @@ void set_timer_slack(struct timer_list *timer, int slack_hz)
    }
    EXPORT_SYMBOL_GPL(set_timer_slack);

    -static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
    +static void
    +__internal_add_timer(struct tvec_base *base, struct timer_list *timer)
    {
    unsigned long expires = timer->expires;
    unsigned long idx = expires - base->timer_jiffies;
    @@ -372,6 +374,19 @@ static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
    list_add_tail(&timer->entry, vec);
    }

    +static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
    +{
    + __internal_add_timer(base, timer);
    + /*
    + * Update base->active_timers and base->next_timer
    + */
    + if (!tbase_get_deferrable(timer->base)) {
    + if (time_before(timer->expires, base->next_timer))
    + base->next_timer = timer->expires;
    + base->active_timers++;
    + }
    +}
    +
    #ifdef CONFIG_TIMER_STATS
    void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr)
    {
    @@ -654,8 +669,7 @@ void init_timer_deferrable_key(struct timer_list *timer,
    }
    EXPORT_SYMBOL(init_timer_deferrable_key);

    -static inline void detach_timer(struct timer_list *timer,
    - int clear_pending)
    +static inline void detach_timer(struct timer_list *timer, bool clear_pending)
    {
    struct list_head *entry = &timer->entry;

    @@ -667,6 +681,29 @@ static inline void detach_timer(struct timer_list *timer,
    entry->prev = LIST_POISON2;
    }

    +static inline void
    +detach_expired_timer(struct timer_list *timer, struct tvec_base *base)
    +{
    + detach_timer(timer, true);
    + if (!tbase_get_deferrable(timer->base))
    + timer->base->active_timers--;
    +}
    +
    +static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,
    + bool clear_pending)
    +{
    + if (!timer_pending(timer))
    + return 0;
    +
    + detach_timer(timer, clear_pending);
    + if (!tbase_get_deferrable(timer->base)) {
    + timer->base->active_timers--;
    + if (timer->expires == base->next_timer)
    + base->next_timer = base->timer_jiffies;
    + }
    + return 1;
    +}
    +
    /*
    * We are using hashed locking: holding per_cpu(tvec_bases).lock
    * means that all timers which are tied to this base via timer->base are
    @@ -712,16 +749,9 @@ __mod_timer(struct timer_list *timer, unsigned long expires,

    base = lock_timer_base(timer, &flags);

    - if (timer_pending(timer)) {
    - detach_timer(timer, 0);
    - if (timer->expires == base->next_timer &&
    - !tbase_get_deferrable(timer->base))
    - base->next_timer = base->timer_jiffies;
    - ret = 1;
    - } else {
    - if (pending_only)
    - goto out_unlock;
    - }
    + ret = detach_if_pending(timer, base, false);
    + if (!ret && pending_only)
    + goto out_unlock;

    debug_activate(timer, expires);

    @@ -752,9 +782,6 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
    }

    timer->expires = expires;
    - if (time_before(timer->expires, base->next_timer) &&
    - !tbase_get_deferrable(timer->base))
    - base->next_timer = timer->expires;
    internal_add_timer(base, timer);

    out_unlock:
    @@ -920,9 +947,6 @@ void add_timer_on(struct timer_list *timer, int cpu)
    spin_lock_irqsave(&base->lock, flags);
    timer_set_base(timer, base);
    debug_activate(timer, timer->expires);
    - if (time_before(timer->expires, base->next_timer) &&
    - !tbase_get_deferrable(timer->base))
    - base->next_timer = timer->expires;
    internal_add_timer(base, timer);
    /*
    * Check whether the other CPU is idle and needs to be
    @@ -959,13 +983,7 @@ int del_timer(struct timer_list *timer)
    timer_stats_timer_clear_start_info(timer);
    if (timer_pending(timer)) {
    base = lock_timer_base(timer, &flags);
    - if (timer_pending(timer)) {
    - detach_timer(timer, 1);
    - if (timer->expires == base->next_timer &&
    - !tbase_get_deferrable(timer->base))
    - base->next_timer = base->timer_jiffies;
    - ret = 1;
    - }
    + ret = detach_if_pending(timer, base, true);
    spin_unlock_irqrestore(&base->lock, flags);
    }

    @@ -990,19 +1008,10 @@ int try_to_del_timer_sync(struct timer_list *timer)

    base = lock_timer_base(timer, &flags);

    - if (base->running_timer == timer)
    - goto out;
    -
    - timer_stats_timer_clear_start_info(timer);
    - ret = 0;
    - if (timer_pending(timer)) {
    - detach_timer(timer, 1);
    - if (timer->expires == base->next_timer &&
    - !tbase_get_deferrable(timer->base))
    - base->next_timer = base->timer_jiffies;
    - ret = 1;
    + if (base->running_timer != timer) {
    + timer_stats_timer_clear_start_info(timer);
    + ret = detach_if_pending(timer, base, true);
    }
    -out:
    spin_unlock_irqrestore(&base->lock, flags);

    return ret;
    @@ -1089,7 +1098,8 @@ static int cascade(struct tvec_base *base, struct tvec *tv, int index)
    */
    list_for_each_entry_safe(timer, tmp, &tv_list, entry) {
    BUG_ON(tbase_get_base(timer->base) != base);
    - internal_add_timer(base, timer);
    + /* No accounting, while moving them */
    + __internal_add_timer(base, timer);
    }

    return index;
    @@ -1178,7 +1188,7 @@ static inline void __run_timers(struct tvec_base *base)
    timer_stats_account_timer(timer);

    base->running_timer = timer;
    - detach_timer(timer, 1);
    + detach_expired_timer(timer, base);

    spin_unlock_irq(&base->lock);
    call_timer_fn(timer, fn, data);
    @@ -1316,18 +1326,21 @@ static unsigned long cmp_next_hrtimer_event(unsigned long now,
    unsigned long get_next_timer_interrupt(unsigned long now)
    {
    struct tvec_base *base = __this_cpu_read(tvec_bases);
    - unsigned long expires;
    + unsigned long expires = now + NEXT_TIMER_MAX_DELTA;

    /*
    * Pretend that there is no timer pending if the cpu is offline.
    * Possible pending timers will be migrated later to an active cpu.
    */
    if (cpu_is_offline(smp_processor_id()))
    - return now + NEXT_TIMER_MAX_DELTA;
    + return expires;
    +
    spin_lock(&base->lock);
    - if (time_before_eq(base->next_timer, base->timer_jiffies))
    - base->next_timer = __next_timer_interrupt(base);
    - expires = base->next_timer;
    + if (base->active_timers) {
    + if (time_before_eq(base->next_timer, base->timer_jiffies))
    + base->next_timer = __next_timer_interrupt(base);
    + expires = base->next_timer;
    + }
    spin_unlock(&base->lock);

    if (time_before_eq(expires, now))
    @@ -1704,6 +1717,7 @@ static int __cpuinit init_timers_cpu(int cpu)

    base->timer_jiffies = jiffies;
    base->next_timer = base->timer_jiffies;
    + base->active_timers = 0;
    return 0;
    }

    @@ -1714,11 +1728,9 @@ static void migrate_timer_list(struct tvec_base *new_base, struct list_head *hea

    while (!list_empty(head)) {
    timer = list_first_entry(head, struct timer_list, entry);
    - detach_timer(timer, 0);
    + /* We ignore the accounting on the dying cpu */
    + detach_timer(timer, false);
    timer_set_base(timer, new_base);
    - if (time_before(timer->expires, new_base->next_timer) &&
    - !tbase_get_deferrable(timer->base))
    - new_base->next_timer = timer->expires;
    internal_add_timer(new_base, timer);
    }
    }

    \
     
     \ /
      Last update: 2012-07-22 17:41    [W:0.119 / U:32.772 seconds]
    ©2003-2016 Jasper Spaans. hosted at Digital OceanAdvertise on this site