include/linux/hrtimer_rearm.h

root/include/linux/hrtimer_rearm.h
// SPDX-License-Identifier: GPL-2.0
#ifndef _LINUX_HRTIMER_REARM_H
#define _LINUX_HRTIMER_REARM_H

#ifdef CONFIG_HRTIMER_REARM_DEFERRED
#include <linux/thread_info.h>

void __hrtimer_rearm_deferred(void);

/*
 * This is purely CPU local, so check the TIF bit first to avoid the overhead of
 * the atomic test_and_clear_bit() operation for the common case where the bit
 * is not set.
 */
static __always_inline bool hrtimer_test_and_clear_rearm_deferred_tif(unsigned long tif_work)
{
        lockdep_assert_irqs_disabled();

        if (unlikely(tif_work & _TIF_HRTIMER_REARM)) {
                clear_thread_flag(TIF_HRTIMER_REARM);
                return true;
        }
        return false;
}

#define TIF_REARM_MASK  (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_HRTIMER_REARM)

/* Invoked from the exit to user before invoking exit_to_user_mode_loop() */
static __always_inline bool
hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask)
{
        /* Help the compiler to optimize the function out for syscall returns */
        if (!(tif_mask & _TIF_HRTIMER_REARM))
                return false;
        /*
         * Rearm the timer if none of the resched flags is set before going into
         * the loop which re-enables interrupts.
         */
        if (unlikely((*tif_work & TIF_REARM_MASK) == _TIF_HRTIMER_REARM)) {
                clear_thread_flag(TIF_HRTIMER_REARM);
                __hrtimer_rearm_deferred();
                /* Don't go into the loop if HRTIMER_REARM was the only flag */
                *tif_work &= ~TIF_HRTIMER_REARM;
                return !*tif_work;
        }
        return false;
}

/* Invoked from the time slice extension decision function */
static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work)
{
        if (hrtimer_test_and_clear_rearm_deferred_tif(tif_work))
                __hrtimer_rearm_deferred();
}

/*
 * This is to be called on all irqentry_exit() paths that will enable
 * interrupts.
 */
static __always_inline void hrtimer_rearm_deferred(void)
{
        hrtimer_rearm_deferred_tif(read_thread_flags());
}

/*
 * Invoked from the scheduler on entry to __schedule() so it can defer
 * rearming after the load balancing callbacks which might change hrtick.
 */
static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void)
{
        return hrtimer_test_and_clear_rearm_deferred_tif(read_thread_flags());
}

#else  /* CONFIG_HRTIMER_REARM_DEFERRED */
static __always_inline void __hrtimer_rearm_deferred(void) { }
static __always_inline void hrtimer_rearm_deferred(void) { }
static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work) { }
static __always_inline bool
hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask) { return false; }
static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void) { return false; }
#endif  /* !CONFIG_HRTIMER_REARM_DEFERRED */

#endif
Linux
