aboutsummaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-08-18 16:23:56 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-08-18 16:23:56 -0700
commit3b4128b9f374b4219eb716f4ad8a307bc7eb3d84 (patch)
treee9489db7a4ab524e7849fb70a3cc47a08700e1ad /kernel
parentb0239dd672306ad242545f793132938847f17e53 (diff)
parent4fa377c19e111c539a530a8200996b911ceff9ff (diff)
Merge tag 'timers-core-2026-08-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull timer and timekeeping core updates from Thomas Gleixner: - Fix a subtly inconsistency in the timekeeping code, which fails to account for the monotonicity adjustment in ntp_error. For small changes of the clocksource multiplicator (+/-1) which are typically used by the NTP PLL this is hard to observe. But for larger adjustments, e.g. caused by a direct frequency setting through adjtimex() the one-time uncompensated offset is significant. Cure this by adjusting ntp_error with the resulting offset so that the discrepancy is smoothed away over time - Make tick length calculations correct in NTP. The timekeeping core takes the quantisation of the clocksource into account when calculating the tick length to compensate for the deviation of the nominal NTP_INTERVAL_LENGTH. While timekeeping gets this right, NTP is not aware of that, which means it operates on the nominal value and not on the actual value which is determined by the clock source frequency. The rounding of a coarse clocksource like the ACPI PM timer results in a +127 PPM deviation. Cure this by exposing the deviation to the NTP code so that it can operate on the same data as the timekeeping core. This is purely kernel internal. User space still sees the nominal tick lenght via adjtimex(). - The accuracy of the NTP adjustments is fairly approximate as the code assumes that the invocations are precisely in NTP interval frequency ticks and the final adjustment can over and under-run. Cure this by adjusting ntp_error by the intended skew on each tick to achieve the desired rate. - Handle the two competing skews of time offset and time adjustment correctly by calculating the conflict portion between the skews and adjusting both accordingly. - A set of updates and improvements for the selftests - The usual small fixes and improvements all over the place * tag 'timers-core-2026-08-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (58 commits) selftests: timers: nsleep-lat: Check all calls to clock_nanosleep() and clock_gettime() selftests: timers: nsleep-lat: Reuse kselftest error numbers selftests: timers: nsleep-lat: Explicitly list the tested clocks selftests: timers: nsleep-lat: Use NSEC_PER_MSEC define for unreasonable latency selftests: timers: nanosleep: Report each test separately selftests: timers: nanosleep: Explicitly handle timer_delete() failure selftests: timers: nanosleep: Move all single clock tests out of the loop in main() selftests: timers: nanosleep: Reuse kselftest error numbers selftests: timers: nanosleep: Explicitly list the tested clocks selftests: timers: nanosleep: Drop output alignment selftests: timers: Use clock_name() and constants from clock-helpers.h selftests: Add clock-helpers.h timer_list: Use ktime_t over nanoseconds timer_list: Use standard 'long long' format placeholders hrtimer: Add a lockdep assertion to hrtimer_update_base() timekeeping: Use u32 for clock_was_set_seq timekeeping: Rename clockid_aux_valid() to clockid_is_aux_clock() hrtimer: Account nr_retries on recovered interrupt retries timers/itimer: Zero-init old itimerval before copy to userspace nohz: Replace dead select with choice default ...
Diffstat (limited to 'kernel')
-rw-r--r--kernel/sched/fair.c2
-rw-r--r--kernel/time/Kconfig1
-rw-r--r--kernel/time/clocksource.c9
-rw-r--r--kernel/time/hrtimer.c64
-rw-r--r--kernel/time/itimer.c2
-rw-r--r--kernel/time/namespace.c6
-rw-r--r--kernel/time/ntp.c310
-rw-r--r--kernel/time/ntp_internal.h4
-rw-r--r--kernel/time/posix-cpu-timers.c9
-rw-r--r--kernel/time/tick-internal.h1
-rw-r--r--kernel/time/time.c1
-rw-r--r--kernel/time/timekeeping.c204
-rw-r--r--kernel/time/timekeeping.h2
-rw-r--r--kernel/time/timekeeping_internal.h21
-rw-r--r--kernel/time/timer_list.c78
-rw-r--r--kernel/time/timer_migration.c4
-rw-r--r--kernel/time/timer_migration.h2
17 files changed, 546 insertions, 174 deletions
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 635f5775cc13..001140132a7d 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -26,6 +26,8 @@
#include <linux/jiffies.h>
#include <linux/mm_api.h>
#include <linux/highmem.h>
+#include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
#include <linux/spinlock_api.h>
#include <linux/cpumask_api.h>
#include <linux/lockdep_api.h>
diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig
index d098ac39bde4..ddfb6bee0745 100644
--- a/kernel/time/Kconfig
+++ b/kernel/time/Kconfig
@@ -133,7 +133,6 @@ config NO_HZ_FULL
depends on HAVE_VIRT_CPU_ACCOUNTING_GEN
select NO_HZ_COMMON
select RCU_NOCB_CPU
- select VIRT_CPU_ACCOUNTING_GEN
select IRQ_WORK
select CPU_ISOLATION
help
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index e48c4d379a7c..f1253f5795c6 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -123,7 +123,6 @@ static atomic_t watchdog_reset_pending;
/* Watchdog interval: 0.5sec. */
#define WATCHDOG_INTERVAL (HZ >> 1)
-#define WATCHDOG_INTERVAL_NS (WATCHDOG_INTERVAL * (NSEC_PER_SEC / HZ))
/* Maximum time between two reference watchdog readouts */
#define WATCHDOG_READOUT_MAX_NS (50U * NSEC_PER_USEC)
@@ -1566,8 +1565,12 @@ static int __init init_clocksource_sysfs(void)
{
int error = subsys_system_register(&clocksource_subsys, NULL);
- if (!error)
- error = device_register(&device_clocksource);
+ if (error)
+ return error;
+
+ error = device_register(&device_clocksource);
+ if (error)
+ bus_unregister(&clocksource_subsys);
return error;
}
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 313dcea127fe..530d61257b9a 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -26,6 +26,7 @@
#include <linux/export.h>
#include <linux/percpu.h>
#include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
#include <linux/notifier.h>
#include <linux/syscalls.h>
#include <linux/interrupt.h>
@@ -676,6 +677,8 @@ static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base)
static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
{
+ lockdep_assert_held(&base->lock);
+
ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
@@ -707,7 +710,7 @@ static inline void hrtimer_rearm_event(ktime_t expires_next, bool deferred)
tick_program_event(expires_next, 1);
}
-static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, struct hrtimer *next_timer,
+static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base,
ktime_t expires_next)
{
cpu_base->expires_next = expires_next;
@@ -743,7 +746,7 @@ static void hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, bool skip
if (skip_equal && expires_next == cpu_base->expires_next)
return;
- __hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next);
+ __hrtimer_reprogram(cpu_base, expires_next);
}
/* High resolution timer related functions */
@@ -896,14 +899,14 @@ static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
cpu_base->next_timer = timer;
- __hrtimer_reprogram(cpu_base, timer, expires);
+ __hrtimer_reprogram(cpu_base, expires);
}
static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, unsigned int active)
{
struct hrtimer_clock_base *base;
- unsigned int seq;
ktime_t expires;
+ u32 seq;
/*
* Update the base offsets unconditionally so the following
@@ -1040,6 +1043,30 @@ static inline void unlock_hrtimer_base(const struct hrtimer *timer, unsigned lon
}
/**
+ * hrtimer_update_function - Update the timer's callback function
+ * @timer: Timer to update
+ * @function: New callback function
+ *
+ * Only safe to call if the timer is not enqueued. Can be called in the callback function if the
+ * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED).
+ */
+void hrtimer_update_function(struct hrtimer *timer,
+ enum hrtimer_restart (*function)(struct hrtimer *))
+{
+#ifdef CONFIG_PROVE_LOCKING
+ guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock);
+
+ if (WARN_ON_ONCE(hrtimer_is_queued(timer)))
+ return;
+
+ if (WARN_ON_ONCE(!function))
+ return;
+#endif
+ ACCESS_PRIVATE(timer, function) = function;
+}
+EXPORT_SYMBOL_GPL(hrtimer_update_function);
+
+/**
* hrtimer_forward() - forward the timer expiry
* @timer: hrtimer to forward
* @now: forward past this time
@@ -1786,13 +1813,21 @@ EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
ktime_t hrtimer_get_next_event(void)
{
struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
- ktime_t expires = KTIME_MAX;
- guard(raw_spinlock_irqsave)(&cpu_base->lock);
- if (!hrtimer_hres_active(cpu_base))
- expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
+ /*
+ * When HRES is active cmp_next_hrtimer_event() expects KTIME_MAX.
+ *
+ * cpu_base->hres_active is written only by the local CPU in
+ * hrtimer_switch_to_hres() from hard interrupt context and in
+ * hrtimers_cpu_starting() during CPU bring-up, and all callers reach
+ * this with interrupts disabled on the same CPU, so an unlocked read is
+ * stable without holding the lock.
+ */
+ if (hrtimer_hres_active(cpu_base))
+ return KTIME_MAX;
- return expires;
+ guard(raw_spinlock_irqsave)(&cpu_base->lock);
+ return __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
}
/**
@@ -2060,13 +2095,6 @@ static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, struct hrtimer_cloc
base->running = NULL;
}
-static __always_inline struct hrtimer *clock_base_next_timer_safe(struct hrtimer_clock_base *base)
-{
- struct timerqueue_linked_node *next = timerqueue_linked_first(&base->active);
-
- return next ? hrtimer_from_timerqueue_node(next) : NULL;
-}
-
static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
unsigned long flags, unsigned int active_mask)
{
@@ -2228,8 +2256,10 @@ retry:
expires_next = hrtimer_update_next_event(cpu_base);
cpu_base->hang_detected = false;
if (expires_next < now) {
- if (++retries < 3)
+ if (++retries < 3) {
+ cpu_base->nr_retries++;
goto retry;
+ }
delta = ktime_sub(now, entry_time);
cpu_base->max_hang_time = max_t(unsigned int, cpu_base->max_hang_time, delta);
diff --git a/kernel/time/itimer.c b/kernel/time/itimer.c
index 7c6110e964e7..03a32dffc56c 100644
--- a/kernel/time/itimer.c
+++ b/kernel/time/itimer.c
@@ -100,7 +100,7 @@ static int do_getitimer(int which, struct itimerspec64 *value)
static int put_itimerval(struct __kernel_old_itimerval __user *o,
const struct itimerspec64 *i)
{
- struct __kernel_old_itimerval v;
+ struct __kernel_old_itimerval v = {};
v.it_interval.tv_sec = i->it_interval.tv_sec;
v.it_interval.tv_usec = i->it_interval.tv_nsec / NSEC_PER_USEC;
diff --git a/kernel/time/namespace.c b/kernel/time/namespace.c
index 5fa0af66cf3f..3aff27bb0a15 100644
--- a/kernel/time/namespace.c
+++ b/kernel/time/namespace.c
@@ -293,10 +293,12 @@ int proc_timens_set_offset(struct file *file, struct task_struct *p,
return -EINVAL;
}
- if (off->val.tv_sec > KTIME_SEC_MAX ||
- off->val.tv_sec < -KTIME_SEC_MAX)
+ if (off->val.tv_sec > KTIME_SEC_MAX || off->val.tv_sec < -KTIME_SEC_MAX)
return -ERANGE;
+ if (off->val.tv_nsec < 0 || off->val.tv_nsec >= NSEC_PER_SEC)
+ return -EINVAL;
+
tp = timespec64_add(tp, off->val);
/*
* KTIME_SEC_MAX is divided by 2 to be sure that KTIME_MAX is
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 97fa99b96dd0..d22b532ec536 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -26,11 +26,13 @@
/**
* struct ntp_data - Structure holding all NTP related state
* @tick_usec: USER_HZ period in microseconds
- * @tick_length: Adjusted tick length
- * @tick_length_base: Base value for @tick_length
+ * @tick_length: Tick length in ns << NTP_SCALE_SHIFT
* @time_state: State of the clock synchronization
* @time_status: Clock status bits
* @time_offset: Time adjustment in nanoseconds
+ * @skew_delta: Per-tick phase slew rate for the coming second, in
+ * @time_offset units (shifted-ns / HZ). Set by
+ * second_overflow().
* @time_constant: PLL time constant
* @time_maxerror: Maximum error in microseconds holding the NTP sync distance
* (NTP dispersion + delay / 2)
@@ -38,7 +40,13 @@
* @time_freq: Frequency offset scaled nsecs/secs
* @time_reftime: Time at last adjustment in seconds
* @time_adjust: Adjustment value
+ * @time_adjust_frac: Sub-microsecond remainder of @time_adjust being
+ * delivered, in ns << NTP_SCALE_SHIFT (not divided by HZ).
* @ntp_tick_adj: Constant boot-param configurable NTP tick adjustment (upscaled)
+ * @cs_tick_adj: Fixed per-second adjustment compensating for the difference
+ * between the nominal NTP interval and the real time taken
+ * by the clocksource's integer @cycle_interval (upscaled).
+ * Set by the timekeeping core via ntp_clear().
* @ntp_next_leap_sec: Second value of the next pending leapsecond, or TIME64_MAX if no leap
*
* @pps_valid: PPS signal watchdog counter
@@ -59,17 +67,19 @@
struct ntp_data {
unsigned long tick_usec;
u64 tick_length;
- u64 tick_length_base;
int time_state;
int time_status;
s64 time_offset;
+ s64 skew_delta;
long time_constant;
long time_maxerror;
long time_esterror;
s64 time_freq;
time64_t time_reftime;
long time_adjust;
+ s64 time_adjust_frac;
s64 ntp_tick_adj;
+ s64 cs_tick_adj;
time64_t ntp_next_leap_sec;
#ifdef CONFIG_NTP_PPS
int pps_valid;
@@ -101,6 +111,9 @@ static struct ntp_data tk_ntp_data[TIMEKEEPERS_MAX] = {
#define SECS_PER_DAY 86400
#define MAX_TICKADJ 500LL /* usecs */
+/* One microsecond of phase, in plain shifted-ns (ns << NTP_SCALE_SHIFT) */
+#define ONE_US_NS ((s64)NSEC_PER_USEC << NTP_SCALE_SHIFT)
+/* Per-tick MAX_TICKADJ slew, in plain shifted-ns */
#define MAX_TICKADJ_SCALED \
(((MAX_TICKADJ * NSEC_PER_USEC) << NTP_SCALE_SHIFT) / NTP_INTERVAL_FREQ)
#define MAX_TAI_OFFSET 100000
@@ -245,8 +258,7 @@ static inline void pps_fill_timex(struct ntp_data *ntpdata, struct __kernel_time
#endif /* CONFIG_NTP_PPS */
/*
- * Update tick_length and tick_length_base, based on tick_usec, ntp_tick_adj and
- * time_freq:
+ * Update tick_length based on tick_usec, ntp_tick_adj and time_freq:
*/
static void ntp_update_frequency(struct ntp_data *ntpdata)
{
@@ -255,6 +267,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata)
second_length = (u64)(tick_usec * NSEC_PER_USEC * USER_HZ) << NTP_SCALE_SHIFT;
second_length += ntpdata->ntp_tick_adj;
+ second_length += ntpdata->cs_tick_adj;
second_length += ntpdata->time_freq;
new_base = div_u64(second_length, NTP_INTERVAL_FREQ);
@@ -263,8 +276,7 @@ static void ntp_update_frequency(struct ntp_data *ntpdata)
* Don't wait for the next second_overflow, apply the change to the
* tick length immediately:
*/
- ntpdata->tick_length += new_base - ntpdata->tick_length_base;
- ntpdata->tick_length_base = new_base;
+ ntpdata->tick_length = new_base;
}
static inline s64 ntp_update_offset_fll(struct ntp_data *ntpdata, s64 offset64, long secs)
@@ -335,14 +347,15 @@ static void __ntp_clear(struct ntp_data *ntpdata)
{
/* Stop active adjtime() */
ntpdata->time_adjust = 0;
+ ntpdata->time_adjust_frac = 0;
ntpdata->time_status |= STA_UNSYNC;
ntpdata->time_maxerror = NTP_PHASE_LIMIT;
ntpdata->time_esterror = NTP_PHASE_LIMIT;
ntp_update_frequency(ntpdata);
- ntpdata->tick_length = ntpdata->tick_length_base;
ntpdata->time_offset = 0;
+ ntpdata->skew_delta = 0;
ntpdata->ntp_next_leap_sec = TIME64_MAX;
/* Clear PPS state variables */
@@ -350,11 +363,26 @@ static void __ntp_clear(struct ntp_data *ntpdata)
}
/**
- * ntp_clear - Clears the NTP state variables
- * @tkid: Timekeeper ID to be able to select proper ntp data array member
+ * ntp_clear - Clear NTP state and set the clocksource quantisation adjustment
+ * @tkid: Timekeeper ID
+ * @cs_tick_adj: Per-second adjustment in ns << NTP_SCALE_SHIFT
+ *
+ * The timekeeping core uses an integer number of cycles (@cycle_interval)
+ * per NTP interval, so the real time that interval represents differs from
+ * the nominal NTP_INTERVAL_LENGTH by up to half a counter period. Folding
+ * this fixed offset into @cs_tick_adj makes it an explicit part of the NTP
+ * tick_length computation in ntp.c, instead of being applied during
+ * timekeeping accumulation where the NTP code never saw it. Like
+ * @ntp_tick_adj it stays internal to the kernel; userspace still sees the
+ * nominal tick via adjtimex. NTP retains its full symmetric ±MAXFREQ range
+ * around the corrected base rate.
+ *
+ * Called whenever the clocksource is (re)configured, which is also when the
+ * rest of the NTP state must be cleared, so the two are done together.
*/
-void ntp_clear(unsigned int tkid)
+void ntp_clear(unsigned int tkid, s64 cs_tick_adj)
{
+ tk_ntp_data[tkid].cs_tick_adj = cs_tick_adj;
__ntp_clear(&tk_ntp_data[tkid]);
}
@@ -364,6 +392,186 @@ u64 ntp_tick_length(unsigned int tkid)
return tk_ntp_data[tkid].tick_length;
}
+s64 ntp_get_skew_delta(unsigned int tkid)
+{
+ return tk_ntp_data[tkid].skew_delta;
+}
+
+/* Sign of @x as +1 or -1 (zero counts as positive; callers pass nonzero). */
+static inline int signof(s64 x)
+{
+ return x < 0 ? -1 : 1;
+}
+
+static s64 ntp_drain_time_offset(unsigned int tkid, s64 amount)
+{
+ struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+
+ /* Only drain if amount and time_offset have the same sign */
+ if (!amount || signof(amount) != signof(ntpdata->time_offset))
+ return amount;
+
+ /* Clamp: don't overshoot zero */
+ if (abs(amount) > abs(ntpdata->time_offset)) {
+ s64 undrained = amount - ntpdata->time_offset;
+
+ ntpdata->time_offset = 0;
+ return undrained;
+ }
+
+ ntpdata->time_offset -= amount;
+ return 0;
+}
+
+/*
+ * Drain the legacy adjtime() correction (time_adjust) as it is delivered.
+ *
+ * @amount is the total intentional per-tick skew for this accumulation
+ * (skew_delta << shift), in time_offset units (shifted_ns / HZ); it covers
+ * both the exponential time_offset slew and the linear adjtime slew. This
+ * function claims only the adjtime share — capped at the MAX_TICKADJ rate —
+ * and returns the remainder for ntp_drain_time_offset().
+ *
+ * time_adjust is in whole µs. The sub-µs remainder being delivered lives in
+ * time_adjust_frac (plain shifted-ns, i.e. ns << NTP_SCALE_SHIFT -- unlike
+ * time_offset these are NOT pre-divided by HZ); we top it up by borrowing
+ * whole microseconds from time_adjust as the drain consumes it.
+ */
+static s64 ntp_drain_time_adjust(unsigned int tkid, s64 amount, unsigned int shift)
+{
+ struct ntp_data *ntpdata = &tk_ntp_data[tkid];
+ /* Sign reference: time_adjust if any whole us remain, else the drawer */
+ s64 ref = ntpdata->time_adjust ? (s64)ntpdata->time_adjust
+ : ntpdata->time_adjust_frac;
+ s64 deliver, deficit, claimed;
+
+ if (!amount || !ref || signof(amount) != signof(ref))
+ return amount;
+
+ /*
+ * Phase to deliver this accumulation, in plain shifted-ns. The drain
+ * @amount is in ÷HZ units, so multiply by HZ first, then clamp to the
+ * MAX_TICKADJ rate (MAX_TICKADJ_SCALED is the per-tick slew in
+ * shifted-ns). Multiply-then-clamp avoids an s64 divide for the cap.
+ */
+ deliver = min(abs(amount) * NTP_INTERVAL_FREQ,
+ (s64)MAX_TICKADJ_SCALED << shift);
+
+ /* Top up the sub-µs drawer from whole-µs time_adjust as needed */
+ deficit = deliver - abs(ntpdata->time_adjust_frac);
+ if (deficit > 0 && ntpdata->time_adjust) {
+ long borrow = div64_u64(deficit + ONE_US_NS - 1, ONE_US_NS);
+
+ if (ntpdata->time_adjust > 0) {
+ borrow = min(borrow, ntpdata->time_adjust);
+ ntpdata->time_adjust -= borrow;
+ ntpdata->time_adjust_frac += (s64)borrow * ONE_US_NS;
+ } else {
+ /* Clamp without negating time_adjust (UB for LONG_MIN) */
+ if (ntpdata->time_adjust > -borrow)
+ borrow = -ntpdata->time_adjust;
+ ntpdata->time_adjust += borrow;
+ ntpdata->time_adjust_frac -= (s64)borrow * ONE_US_NS;
+ }
+ }
+
+ /* Never deliver more than the drawer holds */
+ deliver = min(deliver, abs(ntpdata->time_adjust_frac));
+ if (ntpdata->time_adjust_frac > 0)
+ ntpdata->time_adjust_frac -= deliver;
+ else
+ ntpdata->time_adjust_frac += deliver;
+
+ /* Return the unclaimed remainder in ÷HZ drain units for time_offset */
+ claimed = div_s64(deliver, NTP_INTERVAL_FREQ);
+ return amount - signof(amount) * claimed;
+}
+
+/*
+ * Drain one accumulation's worth of intentional skew as it is delivered.
+ *
+ * @amount is the total intentional per-tick skew for this accumulation
+ * (skew_delta << shift), in time_offset units (shifted_ns / HZ). The
+ * adjtime() linear share is taken from time_adjust first (capped at the
+ * MAX_TICKADJ rate, hence @shift), then the exponential remainder from
+ * time_offset. Returns the amount actually claimed (same ÷HZ units).
+ */
+s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift)
+{
+ s64 unclaimed = ntp_drain_time_adjust(tkid, amount, shift);
+
+ unclaimed = ntp_drain_time_offset(tkid, unclaimed);
+
+ /*
+ * Return the amount actually drained from the intentional
+ * phase offset in time_offset and/or time_adjust.
+ */
+ return amount - unclaimed;
+}
+
+/*
+ * time_offset (drained exponentially) and time_adjust (drained linearly at the
+ * MAX_TICKADJ rate) can be asked to slew the clock in opposite directions.
+ * second_overflow() only folds their *net* into skew_delta, so the cancelling
+ * part would never be drained from either tracker via the per-tick code -- and
+ * if they cancel exactly, skew_delta is zero and neither converges at all.
+ *
+ * Settle that cancelling phase directly between the two here. No clock motion
+ * results (the opposing slews annihilate), but both move toward zero so neither
+ * stalls. @amount is the phase to take off time_offset, in its (÷HZ) units and
+ * with its sign; the same real magnitude comes off time_adjust in the opposite
+ * direction. Clamped so neither tracker is driven past zero.
+ */
+static void ntp_transfer_offset_adjust(struct ntp_data *ntpdata, s64 amount)
+{
+ s64 frac_delta, carry;
+
+ /*
+ * Don't drain time_offset past zero. @amount shares its sign and is
+ * normally bounded below it by ntp_offset_chunk(), but the ±1 skew_delta
+ * floor for a tiny time_offset can exceed it, so clamp.
+ */
+ if (abs(amount) > abs(ntpdata->time_offset))
+ amount = ntpdata->time_offset;
+ if (!amount)
+ return;
+
+ /*
+ * Remove the matching phase from time_adjust, in plain shifted-ns. No
+ * clamp against time_adjust's zero is needed: @amount is bounded by the
+ * adjtime chunk, which second_overflow() never lets exceed time_adjust's
+ * own pending phase, so this cannot overshoot.
+ */
+ frac_delta = amount * NTP_INTERVAL_FREQ;
+
+ ntpdata->time_offset -= amount;
+
+ /* Add the matching phase to time_adjust, carrying whole µs (O(1)). */
+ ntpdata->time_adjust_frac += frac_delta;
+ if (ntpdata->time_adjust_frac >= ONE_US_NS ||
+ ntpdata->time_adjust_frac <= -ONE_US_NS) {
+ carry = div64_s64(ntpdata->time_adjust_frac, ONE_US_NS);
+ ntpdata->time_adjust += carry;
+ ntpdata->time_adjust_frac -= carry * ONE_US_NS;
+ }
+
+ /*
+ * Keep time_adjust and its sub-µs remainder the same sign. The
+ * truncating carry above can leave them opposed (e.g. +4 µs paired
+ * with -250 ns), and ntp_drain_time_adjust() treats abs(time_adjust_frac)
+ * as same-direction drawer capacity -- an opposing remainder there makes
+ * it over-deliver phase that was never removed from the pile. Borrow or
+ * repay a single whole µs to realign; the total phase is unchanged.
+ */
+ if (ntpdata->time_adjust > 0 && ntpdata->time_adjust_frac < 0) {
+ ntpdata->time_adjust--;
+ ntpdata->time_adjust_frac += ONE_US_NS;
+ } else if (ntpdata->time_adjust < 0 && ntpdata->time_adjust_frac > 0) {
+ ntpdata->time_adjust++;
+ ntpdata->time_adjust_frac -= ONE_US_NS;
+ }
+}
+
/**
* ntp_get_next_leap - Returns the next leapsecond in CLOCK_REALTIME ktime_t
* @tkid: Timekeeper ID
@@ -398,7 +606,6 @@ ktime_t ntp_get_next_leap(unsigned int tkid)
int second_overflow(unsigned int tkid, time64_t secs)
{
struct ntp_data *ntpdata = &tk_ntp_data[tkid];
- s64 delta;
int leap = 0;
s32 rem;
@@ -458,35 +665,70 @@ int second_overflow(unsigned int tkid, time64_t secs)
}
/* Compute the phase adjustment for the next second */
- ntpdata->tick_length = ntpdata->tick_length_base;
-
- delta = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
- ntpdata->time_offset -= delta;
- ntpdata->tick_length += delta;
/* Check PPS signal */
pps_dec_valid(ntpdata);
- if (!ntpdata->time_adjust)
- goto out;
+ /*
+ * Set the per-tick skew rate for the next second. This is in
+ * the same units as time_offset: (ns << NTP_SCALE_SHIFT) / HZ.
+ * If the result is so low that the skew imparted would round
+ * to zero, pass the bare minimum ±1 to ensure that it *does*
+ * actually drain completely to zero. It won't overshoot because
+ * logarithmic_accumulation() only drains what it can from
+ * time_offset or time_adjust, and the rest ends up in ntp_error
+ * which drives the selection of 'mult' immediately each tick.
+ */
+ if (ntpdata->time_offset || ntpdata->time_adjust ||
+ ntpdata->time_adjust_frac) {
+ s64 off_chunk = ntp_offset_chunk(ntpdata, ntpdata->time_offset);
+ s64 adj_chunk = 0, net;
- if (ntpdata->time_adjust > MAX_TICKADJ) {
- ntpdata->time_adjust -= MAX_TICKADJ;
- ntpdata->tick_length += MAX_TICKADJ_SCALED;
- goto out;
- }
+ /*
+ * Once the exponential chunk rounds to zero, deliver the last
+ * remaining offset this second so it converges to zero instead
+ * of stalling just above it.
+ */
+ if (!off_chunk)
+ off_chunk = ntpdata->time_offset;
+
+ if (ntpdata->time_adjust || ntpdata->time_adjust_frac) {
+ s64 adj;
+
+ if (ntpdata->time_adjust >= MAX_TICKADJ)
+ adj = MAX_TICKADJ * ONE_US_NS;
+ else if (ntpdata->time_adjust <= -MAX_TICKADJ)
+ adj = -MAX_TICKADJ * ONE_US_NS;
+ else
+ adj = ntpdata->time_adjust * ONE_US_NS +
+ ntpdata->time_adjust_frac;
+
+ adj_chunk = div_s64(adj, NTP_INTERVAL_FREQ);
+ if (!adj_chunk)
+ adj_chunk = signof(ntpdata->time_adjust_frac);
+ }
- if (ntpdata->time_adjust < -MAX_TICKADJ) {
- ntpdata->time_adjust += MAX_TICKADJ;
- ntpdata->tick_length -= MAX_TICKADJ_SCALED;
- goto out;
- }
+ /*
+ * If the two slews oppose, only their net would drive the
+ * per-tick drain, so the cancelling part would never drain from
+ * either tracker and an exact cancellation would stall both.
+ * Settle that overlap directly between them (no clock motion).
+ */
+ if (off_chunk && adj_chunk && signof(off_chunk) != signof(adj_chunk)) {
+ s64 conflict = min(abs(off_chunk), abs(adj_chunk));
- ntpdata->tick_length += (s64)(ntpdata->time_adjust * NSEC_PER_USEC / NTP_INTERVAL_FREQ)
- << NTP_SCALE_SHIFT;
- ntpdata->time_adjust = 0;
+ ntp_transfer_offset_adjust(ntpdata, signof(off_chunk) * conflict);
+ }
+
+ /* Net is what the clock delivers; reduce to per-tick, then floor. */
+ net = off_chunk + adj_chunk;
+ ntpdata->skew_delta = div_s64(net, NTP_INTERVAL_FREQ);
+ if (!ntpdata->skew_delta && net)
+ ntpdata->skew_delta = signof(net);
+ } else {
+ ntpdata->skew_delta = 0;
+ }
-out:
return leap;
}
@@ -779,6 +1021,7 @@ int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct tim
if (!(txc->modes & ADJ_OFFSET_READONLY)) {
/* adjtime() is independent from ntp_adjtime() */
ntpdata->time_adjust = txc->offset;
+ ntpdata->time_adjust_frac = 0;
ntp_update_frequency(ntpdata);
audit_ntp_set_old(ad, AUDIT_NTP_ADJUST, save_adjust);
@@ -1020,6 +1263,7 @@ static void hardpps_update_phase(struct ntp_data *ntpdata, long error)
NTP_INTERVAL_FREQ);
/* Cancel running adjtime() */
ntpdata->time_adjust = 0;
+ ntpdata->time_adjust_frac = 0;
}
/* Update jitter */
ntpdata->pps_jitter += (jitter - ntpdata->pps_jitter) >> PPS_INTMIN;
diff --git a/kernel/time/ntp_internal.h b/kernel/time/ntp_internal.h
index 7084d839c207..0474a761bafc 100644
--- a/kernel/time/ntp_internal.h
+++ b/kernel/time/ntp_internal.h
@@ -3,9 +3,11 @@
#define _LINUX_NTP_INTERNAL_H
extern void ntp_init(void);
-extern void ntp_clear(unsigned int tkid);
+extern void ntp_clear(unsigned int tkid, s64 cs_tick_adj);
/* Returns how long ticks are at present, in ns / 2^NTP_SCALE_SHIFT. */
extern u64 ntp_tick_length(unsigned int tkid);
+extern s64 ntp_get_skew_delta(unsigned int tkid);
+extern s64 ntp_drain_skew(unsigned int tkid, s64 amount, unsigned int shift);
extern ktime_t ntp_get_next_leap(unsigned int tkid);
extern int second_overflow(unsigned int tkid, time64_t secs);
extern int ntp_adjtimex(unsigned int tkid, struct __kernel_timex *txc, const struct timespec64 *ts,
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index a7d3e8229c4b..d73d31c7994f 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -1357,8 +1357,11 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
unsigned long flags, start;
LIST_HEAD(firing);
- if (!lock_task_sighand(tsk, &flags))
- return;
+ /*
+ * tsk is current and ->sighand is stable, see the
+ * tsk->exit_state check in run_posix_cpu_timers()
+ */
+ spin_lock_irqsave(&tsk->sighand->siglock, flags);
do {
/*
@@ -1418,7 +1421,7 @@ static void handle_posix_cpu_timers(struct task_struct *tsk)
* that gets the timer lock before we do will give it up and
* spin until we've taken care of that timer below.
*/
- unlock_task_sighand(tsk, &flags);
+ spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
/*
* Now that all the timers on our list have the firing flag,
diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h
index 597d816d22e8..182974c4f21b 100644
--- a/kernel/time/tick-internal.h
+++ b/kernel/time/tick-internal.h
@@ -3,6 +3,7 @@
* tick internal variable and functions used by low/high res code
*/
#include <linux/hrtimer.h>
+#include <linux/hrtimer_bases.h>
#include <linux/tick.h>
#include "timekeeping.h"
diff --git a/kernel/time/time.c b/kernel/time/time.c
index 0dd63a91e7c5..d1a7efd80bf5 100644
--- a/kernel/time/time.c
+++ b/kernel/time/time.c
@@ -42,6 +42,7 @@
#include <generated/timeconst.h>
#include "timekeeping.h"
+#include "timekeeping_internal.h"
/*
* The timezone where the local system is located. Used as a default by some
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index b1b5ec43c0f2..ea2e6e55f37b 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -339,7 +339,6 @@ static inline void clocksource_enable_inline_read(void) { }
static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
{
u64 interval;
- u64 tmp, ntpinterval;
struct clocksource *old_clock;
++tk->cs_was_changed_seq;
@@ -353,20 +352,16 @@ static void tk_setup_internals(struct timekeeper *tk, struct clocksource *clock)
tk->tkr_raw.cycle_last = tk->tkr_mono.cycle_last;
/* Do the ns -> cycle conversion first, using original mult */
- tmp = NTP_INTERVAL_LENGTH;
- tmp <<= clock->shift;
- ntpinterval = tmp;
- tmp += clock->mult/2;
- do_div(tmp, clock->mult);
- if (tmp == 0)
- tmp = 1;
-
- interval = (u64) tmp;
+ interval = (u64)NTP_INTERVAL_LENGTH << clock->shift;
+ interval += clock->mult / 2;
+ do_div(interval, clock->mult);
+ if (interval == 0)
+ interval = 1;
+
tk->cycle_interval = interval;
/* Go back from cycles -> shifted ns */
tk->xtime_interval = interval * clock->mult;
- tk->xtime_remainder = ntpinterval - tk->xtime_interval;
tk->raw_interval = interval * clock->mult;
/* if changing clocks, convert xtime_nsec shift units */
@@ -386,7 +381,38 @@ static void tk_setup_internals(