flusher
if ((zone->flusher == NULL) && attempt_increment(zone)) {
zone->flusher = page;
if (zone->flusher == page) {
zone->flusher = NULL;
} else if ((zone->flusher == NULL) && vdo_waitq_has_waiters(&zone->flush_waiters) &&
zone->flusher = container_of(vdo_waitq_dequeue_waiter(&zone->flush_waiters),
write_page(zone->flusher, pooled);
if (zone->flusher == tree_page)
if ((zone->flusher != tree_page) &&
if (zone->flusher != tree_page) {
if (waiting && (zone->flusher == page))
struct tree_page *flusher;
vdo_drain_flusher(vdo->flusher, completion);
vdo_resume_flusher(vdo->flusher, completion);
struct flusher *flusher = pool_data;
vdo_initialize_completion(&flush->completion, flusher->vdo,
int result = vdo_allocate(1, __func__, &vdo->flusher);
vdo->flusher->vdo = vdo;
vdo->flusher->thread_id = vdo->thread_config.packer_thread;
vdo_set_admin_state_code(&vdo->flusher->state, VDO_ADMIN_STATE_NORMAL_OPERATION);
vdo_initialize_completion(&vdo->flusher->completion, vdo,
spin_lock_init(&vdo->flusher->lock);
bio_list_init(&vdo->flusher->waiting_flush_bios);
vdo->flusher->flush_pool = mempool_create(1, allocate_flush, free_flush,
vdo->flusher);
return ((vdo->flusher->flush_pool == NULL) ? -ENOMEM : VDO_SUCCESS);
void vdo_free_flusher(struct flusher *flusher)
if (flusher == NULL)
if (flusher->flush_pool != NULL)
mempool_destroy(vdo_forget(flusher->flush_pool));
vdo_free(flusher);
thread_id_t vdo_get_flusher_thread_id(struct flusher *flusher)
return flusher->thread_id;
static void notify_flush(struct flusher *flusher);
struct flusher *flusher = as_flusher(completion);
assert_on_flusher_thread(flusher, __func__);
vdo_waitq_enqueue_waiter(&flusher->pending_flushes,
vdo_waitq_dequeue_waiter(&flusher->notifiers));
vdo_complete_flushes(flusher);
if (vdo_waitq_has_waiters(&flusher->notifiers))
notify_flush(flusher);
struct flusher *flusher = as_flusher(completion);
vdo_increment_packer_flush_generation(flusher->vdo->packer);
flusher->thread_id);
struct flusher *flusher = as_flusher(completion);
struct logical_zone *zone = flusher->logical_zone_to_notify;
vdo_increment_logical_zone_flush_generation(zone, flusher->notify_generation);
flusher->thread_id);
flusher->logical_zone_to_notify = zone->next;
flusher->logical_zone_to_notify->thread_id);
static void notify_flush(struct flusher *flusher)
vdo_waiter_as_flush(vdo_waitq_get_first_waiter(&flusher->notifiers));
flusher->notify_generation = flush->flush_generation;
flusher->logical_zone_to_notify = &flusher->vdo->logical_zones->zones[0];
flusher->completion.requeue = true;
vdo_launch_completion_callback(&flusher->completion, increment_generation,
flusher->logical_zone_to_notify->thread_id);
struct flusher *flusher = completion->vdo->flusher;
assert_on_flusher_thread(flusher, __func__);
result = VDO_ASSERT(vdo_is_state_normal(&flusher->state),
vdo_enter_read_only_mode(flusher->vdo, result);
flush->flush_generation = flusher->flush_generation++;
may_notify = !vdo_waitq_has_waiters(&flusher->notifiers);
vdo_waitq_enqueue_waiter(&flusher->notifiers, &flush->waiter);
notify_flush(flusher);
static void check_for_drain_complete(struct flusher *flusher)
if (!vdo_is_state_draining(&flusher->state) ||
vdo_waitq_has_waiters(&flusher->pending_flushes))
spin_lock(&flusher->lock);
drained = bio_list_empty(&flusher->waiting_flush_bios);
spin_unlock(&flusher->lock);
vdo_finish_draining(&flusher->state);
void vdo_complete_flushes(struct flusher *flusher)
assert_on_flusher_thread(flusher, __func__);
for (zone = &flusher->vdo->logical_zones->zones[0]; zone != NULL; zone = zone->next)
while (vdo_waitq_has_waiters(&flusher->pending_flushes)) {
vdo_waiter_as_flush(vdo_waitq_get_first_waiter(&flusher->pending_flushes));
flusher->first_unacknowledged_generation),
(unsigned long long) flusher->first_unacknowledged_generation,
vdo_waitq_dequeue_waiter(&flusher->pending_flushes);
flusher->first_unacknowledged_generation++;
check_for_drain_complete(flusher);
void vdo_dump_flusher(const struct flusher *flusher)
(unsigned long long) flusher->flush_generation,
(unsigned long long) flusher->first_unacknowledged_generation);
(vdo_waitq_has_waiters(&flusher->notifiers) ? "not empty" : "empty"),
(vdo_waitq_has_waiters(&flusher->pending_flushes) ? "not empty" : "empty"));
bio_list_merge_init(&flush->bios, &vdo->flusher->waiting_flush_bios);
struct vdo_flush *flush = mempool_alloc(vdo->flusher->flush_pool, GFP_NOWAIT);
struct flusher *flusher = vdo->flusher;
const struct admin_state_code *code = vdo_get_admin_state_code(&flusher->state);
spin_lock(&flusher->lock);
bio_list_add(&flusher->waiting_flush_bios, bio);
spin_unlock(&flusher->lock);
spin_unlock(&flusher->lock);
struct flusher *flusher = flush->completion.vdo->flusher;
spin_lock(&flusher->lock);
if (bio_list_empty(&flusher->waiting_flush_bios)) {
initialize_flush(flush, flusher->vdo);
spin_unlock(&flusher->lock);
mempool_free(flush, flusher->flush_pool);
static thread_id_t select_bio_queue(struct flusher *flusher)
struct vdo *vdo = flusher->vdo;
zone_count_t bio_threads = flusher->vdo->thread_config.bio_thread_count;
if (flusher->flush_count == interval) {
flusher->flush_count = 1;
flusher->bio_queue_rotor = ((flusher->bio_queue_rotor + 1) % bio_threads);
flusher->flush_count++;
return vdo->thread_config.bio_threads[flusher->bio_queue_rotor];
select_bio_queue(completion->vdo->flusher), NULL);
check_for_drain_complete(container_of(state, struct flusher, state));
void vdo_drain_flusher(struct flusher *flusher, struct vdo_completion *completion)
assert_on_flusher_thread(flusher, __func__);
vdo_start_draining(&flusher->state, VDO_ADMIN_STATE_SUSPENDING, completion,
void vdo_resume_flusher(struct flusher *flusher, struct vdo_completion *parent)
assert_on_flusher_thread(flusher, __func__);
vdo_continue_completion(parent, vdo_resume_if_quiescent(&flusher->state));
static inline void assert_on_flusher_thread(struct flusher *flusher, const char *caller)
VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == flusher->thread_id),
static struct flusher *as_flusher(struct vdo_completion *completion)
return container_of(completion, struct flusher, completion);
struct flusher;
void vdo_free_flusher(struct flusher *flusher);
thread_id_t __must_check vdo_get_flusher_thread_id(struct flusher *flusher);
void vdo_complete_flushes(struct flusher *flusher);
void vdo_dump_flusher(const struct flusher *flusher);
void vdo_drain_flusher(struct flusher *flusher, struct vdo_completion *completion);
void vdo_resume_flusher(struct flusher *flusher, struct vdo_completion *parent);
vdo_complete_flushes(zone->zones->vdo->flusher);
vdo_get_flusher_thread_id(zone->zones->vdo->flusher));
vdo_dump_flusher(vdo->flusher);
vdo_free_flusher(vdo_forget(vdo->flusher));
struct flusher *flusher;
int BPF_PROG(flusher, struct cgroup *cgrp, struct cgroup *parent, int cpu)