#ifndef _RV_DA_MONITOR_H
#define _RV_DA_MONITOR_H
#include <rv/automata.h>
#include <linux/rv.h>
#include <linux/stringify.h>
#include <linux/bug.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/hashtable.h>
#define DA_MON_NAME CONCATENATE(da_mon_, MONITOR_NAME)
static struct rv_monitor rv_this;
#ifndef da_monitor_event_hook
#define da_monitor_event_hook(...) true
#endif
#ifndef da_monitor_init_hook
#define da_monitor_init_hook(da_mon)
#endif
#ifndef da_monitor_reset_hook
#define da_monitor_reset_hook(da_mon)
#endif
#ifndef da_monitor_sync_hook
#define da_monitor_sync_hook()
#endif
#ifndef da_id_type
#define da_id_type int
#endif
static void react(enum states curr_state, enum events event)
{
rv_react(&rv_this,
"rv: monitor %s does not allow event %s on state %s\n",
__stringify(MONITOR_NAME),
model_get_event_name(event),
model_get_state_name(curr_state));
}
static inline void da_monitor_reset_state(struct da_monitor *da_mon)
{
WRITE_ONCE(da_mon->monitoring, 0);
smp_store_release(&da_mon->curr_state, model_get_initial_state());
}
static inline void da_monitor_reset(struct da_monitor *da_mon)
{
da_monitor_reset_hook(da_mon);
da_monitor_reset_state(da_mon);
}
static inline void da_monitor_start(struct da_monitor *da_mon)
{
da_mon->curr_state = model_get_initial_state();
da_monitor_init_hook(da_mon);
smp_store_release(&da_mon->monitoring, 1);
}
static inline bool da_monitoring(struct da_monitor *da_mon)
{
return smp_load_acquire(&da_mon->monitoring);
}
static inline bool da_monitor_enabled(void)
{
if (unlikely(!rv_monitoring_on()))
return 0;
if (unlikely(!rv_this.enabled))
return 0;
return 1;
}
static inline bool da_monitor_handling_event(struct da_monitor *da_mon)
{
if (!da_monitor_enabled())
return 0;
if (unlikely(!da_monitoring(da_mon)))
return 0;
return 1;
}
#if RV_MON_TYPE == RV_MON_GLOBAL
static struct da_monitor DA_MON_NAME;
static struct da_monitor *da_get_monitor(void)
{
return &DA_MON_NAME;
}
static void __da_monitor_reset_all(void (*reset)(struct da_monitor *))
{
reset(da_get_monitor());
}
static void da_monitor_reset_all(void)
{
__da_monitor_reset_all(da_monitor_reset);
}
static inline void da_monitor_reset_state_all(void)
{
__da_monitor_reset_all(da_monitor_reset_state);
}
static inline int da_monitor_init(void)
{
da_monitor_reset_state_all();
return 0;
}
static inline void da_monitor_destroy(void)
{
da_monitor_reset_all();
da_monitor_sync_hook();
}
#ifndef da_implicit_guard
#define da_implicit_guard()
#endif
#elif RV_MON_TYPE == RV_MON_PER_CPU
static DEFINE_PER_CPU(struct da_monitor, DA_MON_NAME);
static struct da_monitor *da_get_monitor(void)
{
return this_cpu_ptr(&DA_MON_NAME);
}
static void __da_monitor_reset_all(void (*reset)(struct da_monitor *))
{
struct da_monitor *da_mon;
int cpu;
for_each_cpu(cpu, cpu_online_mask) {
da_mon = per_cpu_ptr(&DA_MON_NAME, cpu);
reset(da_mon);
}
}
static void da_monitor_reset_all(void)
{
__da_monitor_reset_all(da_monitor_reset);
}
static inline void da_monitor_reset_state_all(void)
{
__da_monitor_reset_all(da_monitor_reset_state);
}
static inline int da_monitor_init(void)
{
da_monitor_reset_state_all();
return 0;
}
static inline void da_monitor_destroy(void)
{
da_monitor_reset_all();
da_monitor_sync_hook();
}
#ifndef da_implicit_guard
#define da_implicit_guard() guard(migrate)()
#endif
#elif RV_MON_TYPE == RV_MON_PER_TASK
static int task_mon_slot = RV_PER_TASK_MONITOR_INIT;
static inline struct da_monitor *da_get_monitor(struct task_struct *tsk)
{
return &tsk->rv[task_mon_slot].da_mon;
}
static inline struct task_struct *da_get_target(struct da_monitor *da_mon)
{
return container_of(da_mon, struct task_struct, rv[task_mon_slot].da_mon);
}
static inline da_id_type da_get_id(struct da_monitor *da_mon)
{
return da_get_target(da_mon)->pid;
}
static void __da_monitor_reset_all(void (*reset)(struct da_monitor *))
{
struct task_struct *g, *p;
int cpu;
read_lock(&tasklist_lock);
for_each_process_thread(g, p)
reset(da_get_monitor(p));
for_each_present_cpu(cpu)
reset(da_get_monitor(idle_task(cpu)));
read_unlock(&tasklist_lock);
}
static void da_monitor_reset_all(void)
{
__da_monitor_reset_all(da_monitor_reset);
}
static inline void da_monitor_reset_state_all(void)
{
__da_monitor_reset_all(da_monitor_reset_state);
}
static int da_monitor_init(void)
{
int slot;
slot = rv_get_task_monitor_slot();
if (slot < 0 || slot >= RV_PER_TASK_MONITOR_INIT)
return slot;
task_mon_slot = slot;
da_monitor_reset_state_all();
return 0;
}
static inline void da_monitor_destroy(void)
{
if (task_mon_slot == RV_PER_TASK_MONITOR_INIT) {
WARN_ONCE(1, "Disabling a disabled monitor: " __stringify(MONITOR_NAME));
return;
}
tracepoint_synchronize_unregister();
da_monitor_reset_all();
da_monitor_sync_hook();
rv_put_task_monitor_slot(task_mon_slot);
task_mon_slot = RV_PER_TASK_MONITOR_INIT;
}
#elif RV_MON_TYPE == RV_MON_PER_OBJ
struct da_monitor_storage {
da_id_type id;
monitor_target target;
union rv_task_monitor rv;
struct hlist_node node;
struct rcu_head rcu;
};
#ifndef DA_MONITOR_HT_BITS
#define DA_MONITOR_HT_BITS 10
#endif
static DEFINE_HASHTABLE(da_monitor_ht, DA_MONITOR_HT_BITS);
static inline struct da_monitor_storage *da_create_empty_storage(da_id_type id)
{
struct da_monitor_storage *mon_storage;
mon_storage = kmalloc_nolock(sizeof(struct da_monitor_storage),
__GFP_ZERO, NUMA_NO_NODE);
if (!mon_storage)
return NULL;
hash_add_rcu(da_monitor_ht, &mon_storage->node, id);
mon_storage->id = id;
return mon_storage;
}
static inline struct da_monitor *da_create_storage(da_id_type id,
monitor_target target,
struct da_monitor *da_mon)
{
struct da_monitor_storage *mon_storage;
if (da_mon)
return da_mon;
mon_storage = da_create_empty_storage(id);
if (!mon_storage)
return NULL;
mon_storage->target = target;
return &mon_storage->rv.da_mon;
}
static inline struct da_monitor_storage *__da_get_mon_storage(da_id_type id)
{
struct da_monitor_storage *mon_storage;
lockdep_assert_in_rcu_read_lock();
hash_for_each_possible_rcu(da_monitor_ht, mon_storage, node, id) {
if (mon_storage->id == id)
return mon_storage;
}
return NULL;
}
static struct da_monitor *da_get_monitor(da_id_type id, monitor_target target)
{
struct da_monitor_storage *mon_storage;
mon_storage = __da_get_mon_storage(id);
return mon_storage ? &mon_storage->rv.da_mon : NULL;
}
static inline monitor_target da_get_target(struct da_monitor *da_mon)
{
return container_of(da_mon, struct da_monitor_storage, rv.da_mon)->target;
}
static inline da_id_type da_get_id(struct da_monitor *da_mon)
{
return container_of(da_mon, struct da_monitor_storage, rv.da_mon)->id;
}
static inline void da_create_or_get(da_id_type id, monitor_target target)
{
guard(rcu)();
da_create_storage(id, target, da_get_monitor(id, target));
}
static inline struct da_monitor *da_fill_empty_storage(da_id_type id,
monitor_target target,
struct da_monitor *da_mon)
{
if (unlikely(da_mon && !da_get_target(da_mon)))
container_of(da_mon, struct da_monitor_storage, rv.da_mon)->target = target;
return da_mon;
}
static inline monitor_target da_get_target_by_id(da_id_type id)
{
struct da_monitor_storage *mon_storage;
guard(rcu)();
mon_storage = __da_get_mon_storage(id);
if (unlikely(!mon_storage))
return NULL;
return mon_storage->target;
}
static inline void da_destroy_storage(da_id_type id)
{
struct da_monitor_storage *mon_storage;
guard(rcu)();
mon_storage = __da_get_mon_storage(id);
if (!mon_storage)
return;
da_monitor_reset_hook(&mon_storage->rv.da_mon);
hash_del_rcu(&mon_storage->node);
kfree_rcu(mon_storage, rcu);
}
static void __da_monitor_reset_all(void (*reset)(struct da_monitor *))
{
struct da_monitor_storage *mon_storage;
int bkt;
guard(rcu)();
hash_for_each_rcu(da_monitor_ht, bkt, mon_storage, node)
reset(&mon_storage->rv.da_mon);
}
static void da_monitor_reset_all(void)
{
__da_monitor_reset_all(da_monitor_reset);
}
static inline void da_monitor_reset_state_all(void)
{
__da_monitor_reset_all(da_monitor_reset_state);
}
static inline int da_monitor_init(void)
{
hash_init(da_monitor_ht);
return 0;
}
static inline void da_monitor_destroy(void)
{
struct da_monitor_storage *mon_storage;
struct hlist_node *tmp;
int bkt;
tracepoint_synchronize_unregister();
da_monitor_reset_all();
da_monitor_sync_hook();
hash_for_each_safe(da_monitor_ht, bkt, tmp, mon_storage, node) {
hash_del_rcu(&mon_storage->node);
kfree(mon_storage);
}
}
#ifdef DA_SKIP_AUTO_ALLOC
#define da_prepare_storage da_fill_empty_storage
#else
#define da_prepare_storage da_create_storage
#endif
#endif
#if RV_MON_TYPE == RV_MON_GLOBAL || RV_MON_TYPE == RV_MON_PER_CPU
static inline void da_trace_event(struct da_monitor *da_mon,
char *curr_state, char *event,
char *next_state, bool is_final,
da_id_type id)
{
CONCATENATE(trace_event_, MONITOR_NAME)(curr_state, event, next_state,
is_final);
}
static inline void da_trace_error(struct da_monitor *da_mon,
char *curr_state, char *event,
da_id_type id)
{
CONCATENATE(trace_error_, MONITOR_NAME)(curr_state, event);
}
static inline da_id_type da_get_id(struct da_monitor *da_mon)
{
return 0;
}
#elif RV_MON_TYPE == RV_MON_PER_TASK || RV_MON_TYPE == RV_MON_PER_OBJ
static inline void da_trace_event(struct da_monitor *da_mon,
char *curr_state, char *event,
char *next_state, bool is_final,
da_id_type id)
{
CONCATENATE(trace_event_, MONITOR_NAME)(id, curr_state, event,
next_state, is_final);
}
static inline void da_trace_error(struct da_monitor *da_mon,
char *curr_state, char *event,
da_id_type id)
{
CONCATENATE(trace_error_, MONITOR_NAME)(id, curr_state, event);
}
#endif
static inline bool da_event(struct da_monitor *da_mon, enum events event, da_id_type id)
{
enum states curr_state, next_state;
curr_state = READ_ONCE(da_mon->curr_state);
for (int i = 0; i < MAX_DA_RETRY_RACING_EVENTS; i++) {
next_state = model_get_next_state(curr_state, event);
if (next_state == INVALID_STATE) {
react(curr_state, event);
da_trace_error(da_mon, model_get_state_name(curr_state),
model_get_event_name(event), id);
return false;
}
if (likely(try_cmpxchg(&da_mon->curr_state, &curr_state, next_state))) {
if (!da_monitor_event_hook(da_mon, curr_state, event, next_state, id))
return false;
da_trace_event(da_mon, model_get_state_name(curr_state),
model_get_event_name(event),
model_get_state_name(next_state),
model_is_final_state(next_state), id);
return true;
}
}
trace_rv_retries_error(__stringify(MONITOR_NAME), model_get_event_name(event));
pr_warn("rv: " __stringify(MAX_DA_RETRY_RACING_EVENTS)
" retries reached for event %s, resetting monitor %s",
model_get_event_name(event), __stringify(MONITOR_NAME));
return false;
}
static inline void __da_handle_event_common(struct da_monitor *da_mon,
enum events event, da_id_type id)
{
if (!da_event(da_mon, event, id))
da_monitor_reset(da_mon);
}
static inline void __da_handle_event(struct da_monitor *da_mon,
enum events event, da_id_type id)
{
if (da_monitor_handling_event(da_mon))
__da_handle_event_common(da_mon, event, id);
}
static inline bool __da_handle_start_event(struct da_monitor *da_mon,
enum events event, da_id_type id)
{
if (!da_monitor_enabled())
return 0;
if (unlikely(!da_monitoring(da_mon))) {
da_monitor_start(da_mon);
return 0;
}
__da_handle_event_common(da_mon, event, id);
return 1;
}
static inline bool __da_handle_start_run_event(struct da_monitor *da_mon,
enum events event, da_id_type id)
{
if (!da_monitor_enabled())
return 0;
if (unlikely(!da_monitoring(da_mon)))
da_monitor_start(da_mon);
__da_handle_event_common(da_mon, event, id);
return 1;
}
#if RV_MON_TYPE == RV_MON_GLOBAL || RV_MON_TYPE == RV_MON_PER_CPU
static inline void da_handle_event(enum events event)
{
da_implicit_guard();
__da_handle_event(da_get_monitor(), event, 0);
}
static inline bool da_handle_start_event(enum events event)
{
da_implicit_guard();
return __da_handle_start_event(da_get_monitor(), event, 0);
}
static inline bool da_handle_start_run_event(enum events event)
{
da_implicit_guard();
return __da_handle_start_run_event(da_get_monitor(), event, 0);
}
#elif RV_MON_TYPE == RV_MON_PER_TASK
static inline void da_handle_event(struct task_struct *tsk, enum events event)
{
__da_handle_event(da_get_monitor(tsk), event, tsk->pid);
}
static inline bool da_handle_start_event(struct task_struct *tsk,
enum events event)
{
return __da_handle_start_event(da_get_monitor(tsk), event, tsk->pid);
}
static inline bool da_handle_start_run_event(struct task_struct *tsk,
enum events event)
{
return __da_handle_start_run_event(da_get_monitor(tsk), event, tsk->pid);
}
#elif RV_MON_TYPE == RV_MON_PER_OBJ
static inline void da_handle_event(da_id_type id, monitor_target target, enum events event)
{
struct da_monitor *da_mon;
guard(rcu)();
da_mon = da_get_monitor(id, target);
if (likely(da_mon))
__da_handle_event(da_mon, event, id);
}
static inline bool da_handle_start_event(da_id_type id, monitor_target target,
enum events event)
{
struct da_monitor *da_mon;
guard(rcu)();
da_mon = da_get_monitor(id, target);
da_mon = da_prepare_storage(id, target, da_mon);
if (unlikely(!da_mon))
return 0;
return __da_handle_start_event(da_mon, event, id);
}
static inline bool da_handle_start_run_event(da_id_type id, monitor_target target,
enum events event)
{
struct da_monitor *da_mon;
guard(rcu)();
da_mon = da_get_monitor(id, target);
da_mon = da_prepare_storage(id, target, da_mon);
if (unlikely(!da_mon))
return 0;
return __da_handle_start_run_event(da_mon, event, id);
}
static inline void da_reset(da_id_type id, monitor_target target)
{
struct da_monitor *da_mon;
guard(rcu)();
da_mon = da_get_monitor(id, target);
if (likely(da_mon))
da_monitor_reset(da_mon);
}
#endif
#endif