drivers/md/dm-vdo/block-map.c

1713

static void release_page_lock(struct data_vio *data_vio, char *what)

drivers/md/dm-vdo/block-map.c

1717

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1723

zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

1731

static void finish_lookup(struct data_vio *data_vio, int result)

drivers/md/dm-vdo/block-map.c

1733

data_vio->tree_lock.height = 0;

drivers/md/dm-vdo/block-map.c

1735

--data_vio->logical.zone->block_map_zone->active_lookups;

drivers/md/dm-vdo/block-map.c

1737

set_data_vio_logical_callback(data_vio, continue_data_vio_with_block_map_slot);

drivers/md/dm-vdo/block-map.c

1738

data_vio->vio.completion.error_handler = handle_data_vio_error;

drivers/md/dm-vdo/block-map.c

1739

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/block-map.c

1744

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/block-map.c

1747

if (!data_vio->write) {

drivers/md/dm-vdo/block-map.c

1754

finish_lookup(data_vio, result);

drivers/md/dm-vdo/block-map.c

1757

static void abort_lookup(struct data_vio *data_vio, int result, char *what)

drivers/md/dm-vdo/block-map.c

1760

enter_zone_read_only_mode(data_vio->logical.zone->block_map_zone, result);

drivers/md/dm-vdo/block-map.c

1762

if (data_vio->tree_lock.locked) {

drivers/md/dm-vdo/block-map.c

1763

release_page_lock(data_vio, what);

drivers/md/dm-vdo/block-map.c

1764

vdo_waitq_notify_all_waiters(&data_vio->tree_lock.waiters,

drivers/md/dm-vdo/block-map.c

1769

finish_lookup(data_vio, result);

drivers/md/dm-vdo/block-map.c

1772

static void abort_load(struct data_vio *data_vio, int result)

drivers/md/dm-vdo/block-map.c

1774

abort_lookup(data_vio, result, "load");

drivers/md/dm-vdo/block-map.c

1793

static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.c

1795

struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.c

1797

static void continue_with_loaded_page(struct data_vio *data_vio,

drivers/md/dm-vdo/block-map.c

1800

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1805

if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) {

drivers/md/dm-vdo/block-map.c

1811

abort_load(data_vio, VDO_BAD_MAPPING);

drivers/md/dm-vdo/block-map.c

1817

allocate_block_map_page(data_vio->logical.zone->block_map_zone,

drivers/md/dm-vdo/block-map.c

1818

data_vio);

drivers/md/dm-vdo/block-map.c

1824

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

1829

load_block_map_page(data_vio->logical.zone->block_map_zone, data_vio);

drivers/md/dm-vdo/block-map.c

1834

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/block-map.c

1836

data_vio->tree_lock.height--;

drivers/md/dm-vdo/block-map.c

1837

continue_with_loaded_page(data_vio, context);

drivers/md/dm-vdo/block-map.c

1848

struct data_vio *data_vio = completion->parent;

drivers/md/dm-vdo/block-map.c

1850

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1863

release_page_lock(data_vio, "load");

drivers/md/dm-vdo/block-map.c

1865

continue_with_loaded_page(data_vio, page);

drivers/md/dm-vdo/block-map.c

1873

struct data_vio *data_vio = completion->parent;

drivers/md/dm-vdo/block-map.c

1877

abort_load(data_vio, result);

drivers/md/dm-vdo/block-map.c

1883

struct data_vio *data_vio = vio->completion.parent;

drivers/md/dm-vdo/block-map.c

1886

data_vio->logical.zone->thread_id);

drivers/md/dm-vdo/block-map.c

1892

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/block-map.c

1893

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1896

pooled->vio.completion.parent = data_vio;

drivers/md/dm-vdo/block-map.c

1905

static int attempt_page_lock(struct block_map_zone *zone, struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

1909

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1929

data_vio->tree_lock.locked = true;

drivers/md/dm-vdo/block-map.c

1934

vdo_waitq_enqueue_waiter(&lock_holder->waiters, &data_vio->waiter);

drivers/md/dm-vdo/block-map.c

1939

static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

1943

result = attempt_page_lock(zone, data_vio);

drivers/md/dm-vdo/block-map.c

1945

abort_load(data_vio, result);

drivers/md/dm-vdo/block-map.c

1949

if (data_vio->tree_lock.locked) {

drivers/md/dm-vdo/block-map.c

1950

data_vio->waiter.callback = load_page;

drivers/md/dm-vdo/block-map.c

1951

acquire_vio_from_pool(zone->vio_pool, &data_vio->waiter);

drivers/md/dm-vdo/block-map.c

1957

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

1960

data_vio->logical.zone->thread_id))

drivers/md/dm-vdo/block-map.c

1963

abort_lookup(data_vio, completion->result, "allocation");

drivers/md/dm-vdo/block-map.c

1968

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/block-map.c

1969

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1973

data_vio->tree_lock.tree_slots[tree_lock->height].block_map_slot.pbn = pbn;

drivers/md/dm-vdo/block-map.c

1976

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

1980

allocate_block_map_page(data_vio->logical.zone->block_map_zone, data_vio);

drivers/md/dm-vdo/block-map.c

2090

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2091

struct block_map_zone *zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

2092

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

2095

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2103

vdo_update_block_map_page(page, data_vio, pbn,

drivers/md/dm-vdo/block-map.c

2134

release_page_lock(data_vio, "allocation");

drivers/md/dm-vdo/block-map.c

2138

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

2142

allocate_block_map_page(zone, data_vio);

drivers/md/dm-vdo/block-map.c

2147

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2149

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2151

release_data_vio_allocation_lock(data_vio, true);

drivers/md/dm-vdo/block-map.c

2152

launch_data_vio_logical_callback(data_vio, finish_block_map_allocation);

drivers/md/dm-vdo/block-map.c

2162

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2164

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2167

vdo_modify_reference_count(completion, &data_vio->increment_updater);

drivers/md/dm-vdo/block-map.c

2172

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2174

assert_data_vio_in_journal_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2176

set_data_vio_allocated_zone_callback(data_vio,

drivers/md/dm-vdo/block-map.c

2178

vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio);

drivers/md/dm-vdo/block-map.c

2183

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2184

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

2187

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2189

if (!vdo_allocate_block_in_zone(data_vio))

drivers/md/dm-vdo/block-map.c

2192

pbn = data_vio->allocation.pbn;

drivers/md/dm-vdo/block-map.c

2194

data_vio->increment_updater = (struct reference_updater) {

drivers/md/dm-vdo/block-map.c

2201

.lock = data_vio->allocation.lock,

drivers/md/dm-vdo/block-map.c

2204

launch_data_vio_journal_callback(data_vio, journal_block_map_allocation);

drivers/md/dm-vdo/block-map.c

2208

struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

2212

if (!data_vio->write || data_vio->is_discard) {

drivers/md/dm-vdo/block-map.c

2214

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

2218

result = attempt_page_lock(zone, data_vio);

drivers/md/dm-vdo/block-map.c

2220

abort_lookup(data_vio, result, "allocation");

drivers/md/dm-vdo/block-map.c

2224

if (!data_vio->tree_lock.locked)

drivers/md/dm-vdo/block-map.c

2227

data_vio_allocate_data_block(data_vio, VIO_BLOCK_MAP_WRITE_LOCK,

drivers/md/dm-vdo/block-map.c

2238

void vdo_find_block_map_slot(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

2244

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

2245

struct block_map_zone *zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

2249

finish_lookup(data_vio, VDO_SHUTTING_DOWN);

drivers/md/dm-vdo/block-map.c

2254

data_vio->logical.lbn % VDO_BLOCK_MAP_ENTRIES_PER_PAGE;

drivers/md/dm-vdo/block-map.c

2283

if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) {

drivers/md/dm-vdo/block-map.c

2289

abort_load(data_vio, VDO_BAD_MAPPING);

drivers/md/dm-vdo/block-map.c

2295

allocate_block_map_page(zone, data_vio);

drivers/md/dm-vdo/block-map.c

2302

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

2307

load_block_map_page(zone, data_vio);

drivers/md/dm-vdo/block-map.c

2977

zone_count_t vdo_compute_logical_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

2979

struct block_map *map = vdo_from_data_vio(data_vio)->block_map;

drivers/md/dm-vdo/block-map.c

2980

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

2981

page_number_t page_number = data_vio->logical.lbn / VDO_BLOCK_MAP_ENTRIES_PER_PAGE;

drivers/md/dm-vdo/block-map.c

3108

static void fetch_mapping_page(struct data_vio *data_vio, bool modifiable,

drivers/md/dm-vdo/block-map.c

3111

struct block_map_zone *zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

3114

continue_data_vio_with_error(data_vio, VDO_SHUTTING_DOWN);

drivers/md/dm-vdo/block-map.c

3118

vdo_get_page(&data_vio->page_completion, zone,

drivers/md/dm-vdo/block-map.c

3119

data_vio->tree_lock.tree_slots[0].block_map_slot.pbn,

drivers/md/dm-vdo/block-map.c

3120

modifiable, &data_vio->vio.completion,

drivers/md/dm-vdo/block-map.c

3130

static void clear_mapped_location(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

3132

data_vio->mapped = (struct zoned_pbn) {

drivers/md/dm-vdo/block-map.c

3146

static int __must_check set_mapped_location(struct data_vio *data_vio,

drivers/md/dm-vdo/block-map.c

3155

result = vdo_get_physical_zone(vdo_from_data_vio(data_vio),

drivers/md/dm-vdo/block-map.c

3156

mapped.pbn, &data_vio->mapped.zone);

drivers/md/dm-vdo/block-map.c

3158

data_vio->mapped.pbn = mapped.pbn;

drivers/md/dm-vdo/block-map.c

3159

data_vio->mapped.state = mapped.state;

drivers/md/dm-vdo/block-map.c

3183

if (!data_vio->write)

drivers/md/dm-vdo/block-map.c

3190

clear_mapped_location(data_vio);

drivers/md/dm-vdo/block-map.c

3201

struct data_vio *data_vio = as_data_vio(completion->parent);

drivers/md/dm-vdo/block-map.c

3216

tree_slot = &data_vio->tree_lock.tree_slots[0];

drivers/md/dm-vdo/block-map.c

3219

result = set_mapped_location(data_vio, entry);

drivers/md/dm-vdo/block-map.c

3223

void vdo_update_block_map_page(struct block_map_page *page, struct data_vio *data_vio,

drivers/md/dm-vdo/block-map.c

3228

struct block_map_zone *zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

3232

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

3240

new_locked = data_vio->recovery_sequence_number;

drivers/md/dm-vdo/block-map.c

3261

data_vio->recovery_sequence_number = 0;

drivers/md/dm-vdo/block-map.c

3266

struct data_vio *data_vio = as_data_vio(completion->parent);

drivers/md/dm-vdo/block-map.c

3287

data_vio, data_vio->new_mapped.pbn,

drivers/md/dm-vdo/block-map.c

3288

data_vio->new_mapped.state, &info->recovery_lock);

drivers/md/dm-vdo/block-map.c

3296

void vdo_get_mapped_block(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

3298

if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/block-map.c

3303

clear_mapped_location(data_vio);

drivers/md/dm-vdo/block-map.c

3304

continue_data_vio(data_vio);

drivers/md/dm-vdo/block-map.c

3308

fetch_mapping_page(data_vio, false, get_mapping_from_fetched_page);

drivers/md/dm-vdo/block-map.c

3312

void vdo_put_mapped_block(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

3314

fetch_mapping_page(data_vio, true, put_mapping_in_fetched_page);

drivers/md/dm-vdo/block-map.h

323

void vdo_find_block_map_slot(struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.h

358

zone_count_t vdo_compute_logical_zone(struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.h

363

void vdo_update_block_map_page(struct block_map_page *page, struct data_vio *data_vio,

drivers/md/dm-vdo/block-map.h

368

void vdo_get_mapped_block(struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.h

370

void vdo_put_mapped_block(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.c

1062

struct data_vio *data_vio = &pool->data_vios[i];

drivers/md/dm-vdo/data-vio.c

1064

if (!list_empty(&data_vio->pool_entry))

drivers/md/dm-vdo/data-vio.c

1067

dump_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1095

static void update_data_vio_error_stats(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1108

if (data_vio->read)

drivers/md/dm-vdo/data-vio.c

1111

if (data_vio->write)

drivers/md/dm-vdo/data-vio.c

1114

if (data_vio->fua)

drivers/md/dm-vdo/data-vio.c

1117

update_vio_error_stats(&data_vio->vio,

drivers/md/dm-vdo/data-vio.c

1120

(unsigned long long) data_vio->logical.lbn,

drivers/md/dm-vdo/data-vio.c

1121

get_data_vio_operation_name(data_vio));

drivers/md/dm-vdo/data-vio.c

1124

static void perform_cleanup_stage(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

1134

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1136

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1137

release_data_vio_allocation_lock(data_vio, false);

drivers/md/dm-vdo/data-vio.c

1138

perform_cleanup_stage(data_vio, VIO_RELEASE_RECOVERY_LOCKS);

drivers/md/dm-vdo/data-vio.c

1142

static void release_lock(struct data_vio *data_vio, struct lbn_lock *lock)

drivers/md/dm-vdo/data-vio.c

1145

struct data_vio *lock_holder;

drivers/md/dm-vdo/data-vio.c

1149

struct data_vio *lock_holder = vdo_int_map_get(lock_map, lock->lbn);

drivers/md/dm-vdo/data-vio.c

1151

VDO_ASSERT_LOG_ONLY((data_vio != lock_holder),

drivers/md/dm-vdo/data-vio.c

1159

VDO_ASSERT_LOG_ONLY((data_vio == lock_holder),

drivers/md/dm-vdo/data-vio.c

1166

static void transfer_lock(struct data_vio *data_vio, struct lbn_lock *lock)

drivers/md/dm-vdo/data-vio.c

1168

struct data_vio *lock_holder, *next_lock_holder;

drivers/md/dm-vdo/data-vio.c

1188

VDO_ASSERT_LOG_ONLY((lock_holder == data_vio),

drivers/md/dm-vdo/data-vio.c

1215

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1216

struct lbn_lock *lock = &data_vio->logical;

drivers/md/dm-vdo/data-vio.c

1218

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1221

transfer_lock(data_vio, lock);

drivers/md/dm-vdo/data-vio.c

1223

release_lock(data_vio, lock);

drivers/md/dm-vdo/data-vio.c

1225

vdo_release_flush_generation_lock(data_vio);

drivers/md/dm-vdo/data-vio.c

1226

perform_cleanup_stage(data_vio, VIO_CLEANUP_DONE);

drivers/md/dm-vdo/data-vio.c

1232

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1234

assert_data_vio_in_hash_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1236

vdo_clean_failed_hash_lock(data_vio);

drivers/md/dm-vdo/data-vio.c

1240

vdo_release_hash_lock(data_vio);

drivers/md/dm-vdo/data-vio.c

1241

perform_cleanup_stage(data_vio, VIO_RELEASE_LOGICAL);

drivers/md/dm-vdo/data-vio.c

1251

static void finish_cleanup(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1253

struct vdo_completion *completion = &data_vio->vio.completion;

drivers/md/dm-vdo/data-vio.c

1254

u32 discard_size = min_t(u32, data_vio->remaining_discard,

drivers/md/dm-vdo/data-vio.c

1255

VDO_BLOCK_SIZE - data_vio->offset);

drivers/md/dm-vdo/data-vio.c

1257

VDO_ASSERT_LOG_ONLY(data_vio->allocation.lock == NULL,

drivers/md/dm-vdo/data-vio.c

1259

VDO_ASSERT_LOG_ONLY(data_vio->hash_lock == NULL,

drivers/md/dm-vdo/data-vio.c

1261

if ((data_vio->remaining_discard <= discard_size) ||

drivers/md/dm-vdo/data-vio.c

1270

data_vio->remaining_discard -= discard_size;

drivers/md/dm-vdo/data-vio.c

1271

data_vio->is_partial = (data_vio->remaining_discard < VDO_BLOCK_SIZE);

drivers/md/dm-vdo/data-vio.c

1272

data_vio->read = data_vio->is_partial;

drivers/md/dm-vdo/data-vio.c

1273

data_vio->offset = 0;

drivers/md/dm-vdo/data-vio.c

1275

data_vio->first_reference_operation_complete = false;

drivers/md/dm-vdo/data-vio.c

1276

launch_data_vio(data_vio, data_vio->logical.lbn + 1);

drivers/md/dm-vdo/data-vio.c

1280

static void perform_cleanup_stage(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

1283

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1287

if (data_vio->hash_lock != NULL) {

drivers/md/dm-vdo/data-vio.c

1288

launch_data_vio_hash_zone_callback(data_vio, clean_hash_lock);

drivers/md/dm-vdo/data-vio.c

1294

if (data_vio_has_allocation(data_vio)) {

drivers/md/dm-vdo/data-vio.c

1295

launch_data_vio_allocated_zone_callback(data_vio,

drivers/md/dm-vdo/data-vio.c

1302

if ((data_vio->recovery_sequence_number > 0) &&

drivers/md/dm-vdo/data-vio.c

1304

(data_vio->vio.completion.result != VDO_READ_ONLY))

drivers/md/dm-vdo/data-vio.c

1309

launch_data_vio_logical_callback(data_vio, release_logical_lock);

drivers/md/dm-vdo/data-vio.c

1313

finish_cleanup(data_vio);

drivers/md/dm-vdo/data-vio.c

1319

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1322

data_vio->last_async_operation = VIO_ASYNC_OP_CLEANUP;

drivers/md/dm-vdo/data-vio.c

1323

perform_cleanup_stage(data_vio,

drivers/md/dm-vdo/data-vio.c

1324

(data_vio->write ? VIO_CLEANUP_START : VIO_RELEASE_LOGICAL));

drivers/md/dm-vdo/data-vio.c

1333

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1337

(unsigned long long) data_vio->logical.lbn,

drivers/md/dm-vdo/data-vio.c

1338

(unsigned long long) data_vio->new_mapped.pbn,

drivers/md/dm-vdo/data-vio.c

1339

(unsigned long long) data_vio->mapped.pbn,

drivers/md/dm-vdo/data-vio.c

1340

(unsigned long long) data_vio->allocation.pbn,

drivers/md/dm-vdo/data-vio.c

1341

get_data_vio_operation_name(data_vio));

drivers/md/dm-vdo/data-vio.c

1349

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1351

if ((completion->result == VDO_READ_ONLY) || (data_vio->user_bio == NULL))

drivers/md/dm-vdo/data-vio.c

1354

update_data_vio_error_stats(data_vio);

drivers/md/dm-vdo/data-vio.c

1363

const char *get_data_vio_operation_name(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1368

return ((data_vio->last_async_operation < MAX_VIO_ASYNC_OPERATION_NUMBER) ?

drivers/md/dm-vdo/data-vio.c

1369

ASYNC_OPERATION_NAMES[data_vio->last_async_operation] :

drivers/md/dm-vdo/data-vio.c

1381

void data_vio_allocate_data_block(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

1385

struct allocation *allocation = &data_vio->allocation;

drivers/md/dm-vdo/data-vio.c

1390

allocation->zone = vdo_get_next_allocation_zone(data_vio->logical.zone);

drivers/md/dm-vdo/data-vio.c

1393

data_vio->vio.completion.error_handler = error_handler;

drivers/md/dm-vdo/data-vio.c

1394

launch_data_vio_allocated_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.c

1404

void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset)

drivers/md/dm-vdo/data-vio.c

1406

struct allocation *allocation = &data_vio->allocation;

drivers/md/dm-vdo/data-vio.c

1409

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1424

int uncompress_data_vio(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

1429

struct compressed_block *block = data_vio->compression.block;

drivers/md/dm-vdo/data-vio.c

1456

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1457

char *data = data_vio->vio.data;

drivers/md/dm-vdo/data-vio.c

1458

struct bio *bio = data_vio->user_bio;

drivers/md/dm-vdo/data-vio.c

1460

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1463

memset(data + data_vio->offset, '\0', min_t(u32,

drivers/md/dm-vdo/data-vio.c

1464

data_vio->remaining_discard,

drivers/md/dm-vdo/data-vio.c

1465

VDO_BLOCK_SIZE - data_vio->offset));

drivers/md/dm-vdo/data-vio.c

1467

copy_from_bio(bio, data + data_vio->offset);

drivers/md/dm-vdo/data-vio.c

1470

data_vio->is_zero = mem_is_zero(data, VDO_BLOCK_SIZE);

drivers/md/dm-vdo/data-vio.c

1471

data_vio->read = false;

drivers/md/dm-vdo/data-vio.c

1472

launch_data_vio_logical_callback(data_vio,

drivers/md/dm-vdo/data-vio.c

1478

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1479

char *data = data_vio->vio.data;

drivers/md/dm-vdo/data-vio.c

1480

bool compressed = vdo_is_state_compressed(data_vio->mapped.state);

drivers/md/dm-vdo/data-vio.c

1482

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1485

int result = uncompress_data_vio(data_vio, data_vio->mapped.state, data);

drivers/md/dm-vdo/data-vio.c

1488

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

1493

if (data_vio->write) {

drivers/md/dm-vdo/data-vio.c

1498

if (compressed || data_vio->is_partial)

drivers/md/dm-vdo/data-vio.c

1499

copy_to_bio(data_vio->user_bio, data + data_vio->offset);

drivers/md/dm-vdo/data-vio.c

1501

acknowledge_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1507

struct data_vio *data_vio = vio_as_data_vio(bio->bi_private);

drivers/md/dm-vdo/data-vio.c

1512

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

1516

launch_data_vio_cpu_callback(data_vio, complete_read,

drivers/md/dm-vdo/data-vio.c

1522

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1524

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1526

if (data_vio->is_partial) {

drivers/md/dm-vdo/data-vio.c

1527

memset(data_vio->vio.data, 0, VDO_BLOCK_SIZE);

drivers/md/dm-vdo/data-vio.c

1528

if (data_vio->write) {

drivers/md/dm-vdo/data-vio.c

1533

zero_fill_bio(data_vio->user_bio);

drivers/md/dm-vdo/data-vio.c

1547

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1551

if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

1552

launch_data_vio_cpu_callback(data_vio, complete_zero_read,

drivers/md/dm-vdo/data-vio.c

1557

data_vio->last_async_operation = VIO_ASYNC_OP_READ_DATA_VIO;

drivers/md/dm-vdo/data-vio.c

1558

if (vdo_is_state_compressed(data_vio->mapped.state)) {

drivers/md/dm-vdo/data-vio.c

1559

result = vio_reset_bio(vio, (char *) data_vio->compression.block,

drivers/md/dm-vdo/data-vio.c

1560

read_endio, REQ_OP_READ, data_vio->mapped.pbn);

drivers/md/dm-vdo/data-vio.c

1562

blk_opf_t opf = ((data_vio->user_bio->bi_opf & PASSTHROUGH_FLAGS) | REQ_OP_READ);

drivers/md/dm-vdo/data-vio.c

1564

if (data_vio->is_partial) {

drivers/md/dm-vdo/data-vio.c

1566

data_vio->mapped.pbn);

drivers/md/dm-vdo/data-vio.c

1570

bio_init_clone(data_vio->user_bio->bi_bdev, vio->bio,

drivers/md/dm-vdo/data-vio.c

1571

data_vio->user_bio, GFP_KERNEL);

drivers/md/dm-vdo/data-vio.c

1575

data_vio->mapped.pbn);

drivers/md/dm-vdo/data-vio.c

1580

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

1584

vdo_submit_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1587

static inline struct data_vio *

drivers/md/dm-vdo/data-vio.c

1593

return container_of(completion, struct data_vio, decrement_completion);

drivers/md/dm-vdo/data-vio.c

1604

struct data_vio *data_vio = reference_count_update_completion_as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1606

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1608

if (!data_vio->first_reference_operation_complete) {

drivers/md/dm-vdo/data-vio.c

1610

data_vio->first_reference_operation_complete = true;

drivers/md/dm-vdo/data-vio.c

1614

completion = &data_vio->vio.completion;

drivers/md/dm-vdo/data-vio.c

1615

vdo_set_completion_result(completion, data_vio->decrement_completion.result);

drivers/md/dm-vdo/data-vio.c

1622

if (data_vio->hash_lock != NULL)

drivers/md/dm-vdo/data-vio.c

1623

set_data_vio_hash_zone_callback(data_vio, vdo_continue_hash_lock);

drivers/md/dm-vdo/data-vio.c

1627

data_vio->last_async_operation = VIO_ASYNC_OP_PUT_MAPPED_BLOCK;

drivers/md/dm-vdo/data-vio.c

1628

vdo_put_mapped_block(data_vio);

drivers/md/dm-vdo/data-vio.c

1633

struct data_vio *data_vio = container_of(completion, struct data_vio,

drivers/md/dm-vdo/data-vio.c

1636

assert_data_vio_in_mapped_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1639

data_vio->logical.zone->thread_id);

drivers/md/dm-vdo/data-vio.c

1641

vdo_modify_reference_count(completion, &data_vio->decrement_updater);

drivers/md/dm-vdo/data-vio.c

1646

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1648

assert_data_vio_in_new_mapped_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1650

if (data_vio->downgrade_allocation_lock) {

drivers/md/dm-vdo/data-vio.c

1658

vdo_downgrade_pbn_write_lock(data_vio->allocation.lock, false);

drivers/md/dm-vdo/data-vio.c

1661

set_data_vio_logical_callback(data_vio, update_block_map);

drivers/md/dm-vdo/data-vio.c

1663

vdo_modify_reference_count(completion, &data_vio->increment_updater);

drivers/md/dm-vdo/data-vio.c

1669

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1671

assert_data_vio_in_journal_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1673

data_vio->decrement_updater.operation = VDO_JOURNAL_DATA_REMAPPING;

drivers/md/dm-vdo/data-vio.c

1674

data_vio->decrement_updater.zpbn = data_vio->mapped;

drivers/md/dm-vdo/data-vio.c

1675

if (data_vio->new_mapped.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

1676

data_vio->first_reference_operation_complete = true;

drivers/md/dm-vdo/data-vio.c

1677

if (data_vio->mapped.pbn == VDO_ZERO_BLOCK)

drivers/md/dm-vdo/data-vio.c

1678

set_data_vio_logical_callback(data_vio, update_block_map);

drivers/md/dm-vdo/data-vio.c

1680

set_data_vio_new_mapped_zone_callback(data_vio,

drivers/md/dm-vdo/data-vio.c

1684

if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

1685

data_vio->first_reference_operation_complete = true;

drivers/md/dm-vdo/data-vio.c

1687

vdo_set_completion_callback(&data_vio->decrement_completion,

drivers/md/dm-vdo/data-vio.c

1689

data_vio->mapped.zone->thread_id);

drivers/md/dm-vdo/data-vio.c

1692

data_vio->last_async_operation = VIO_ASYNC_OP_JOURNAL_REMAPPING;

drivers/md/dm-vdo/data-vio.c

1693

vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio);

drivers/md/dm-vdo/data-vio.c

1705

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1707

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1709

data_vio->last_async_operation = VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_WRITE;

drivers/md/dm-vdo/data-vio.c

1710

set_data_vio_journal_callback(data_vio, journal_remapping);

drivers/md/dm-vdo/data-vio.c

1711

vdo_get_mapped_block(data_vio);

drivers/md/dm-vdo/data-vio.c

1714

void update_metadata_for_data_vio_write(struct data_vio *data_vio, struct pbn_lock *lock)

drivers/md/dm-vdo/data-vio.c

1716

data_vio->increment_updater = (struct reference_updater) {

drivers/md/dm-vdo/data-vio.c

1719

.zpbn = data_vio->new_mapped,

drivers/md/dm-vdo/data-vio.c

1723

launch_data_vio_logical_callback(data_vio, read_old_block_mapping);

drivers/md/dm-vdo/data-vio.c

1734

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1736

assert_data_vio_in_packer_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1738

if (!vdo_get_compressing(vdo_from_data_vio(data_vio)) ||

drivers/md/dm-vdo/data-vio.c

1739

get_data_vio_compression_status(data_vio).may_not_compress) {

drivers/md/dm-vdo/data-vio.c

1740

write_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1744

data_vio->last_async_operation = VIO_ASYNC_OP_ATTEMPT_PACKING;

drivers/md/dm-vdo/data-vio.c

1745

vdo_attempt_packing(data_vio);

drivers/md/dm-vdo/data-vio.c

1756

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1759

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1765

size = LZ4_compress_default(data_vio->vio.data,

drivers/md/dm-vdo/data-vio.c

1766

data_vio->compression.block->data, VDO_BLOCK_SIZE,

drivers/md/dm-vdo/data-vio.c

1770

data_vio->compression.size = size;

drivers/md/dm-vdo/data-vio.c

1771

launch_data_vio_packer_callback(data_vio, pack_compressed_data);

drivers/md/dm-vdo/data-vio.c

1775

write_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1784

void launch_compress_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1786

VDO_ASSERT_LOG_ONLY(!data_vio->is_duplicate, "compressing a non-duplicate block");

drivers/md/dm-vdo/data-vio.c

1787

VDO_ASSERT_LOG_ONLY(data_vio->hash_lock != NULL,

drivers/md/dm-vdo/data-vio.c

1789

VDO_ASSERT_LOG_ONLY(data_vio_has_allocation(data_vio),

drivers/md/dm-vdo/data-vio.c

179

struct data_vio data_vios[];

drivers/md/dm-vdo/data-vio.c

1807

if (data_vio->fua ||

drivers/md/dm-vdo/data-vio.c

1808

!vdo_get_compressing(vdo_from_data_vio(data_vio)) ||

drivers/md/dm-vdo/data-vio.c

1809

((data_vio->user_bio != NULL) && (bio_op(data_vio->user_bio) == REQ_OP_DISCARD)) ||

drivers/md/dm-vdo/data-vio.c

1810

(advance_data_vio_compression_stage(data_vio).stage != DATA_VIO_COMPRESSING)) {

drivers/md/dm-vdo/data-vio.c

1811

write_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1815

data_vio->last_async_operation = VIO_ASYNC_OP_COMPRESS_DATA_VIO;

drivers/md/dm-vdo/data-vio.c

1816

launch_data_vio_cpu_callback(data_vio, compress_data_vio,

drivers/md/dm-vdo/data-vio.c

1829

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1831

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1832

VDO_ASSERT_LOG_ONLY(!data_vio->is_zero, "zero blocks should not be hashed");

drivers/md/dm-vdo/data-vio.c

1834

murmurhash3_128(data_vio->vio.data, VDO_BLOCK_SIZE, 0x62ea60be,

drivers/md/dm-vdo/data-vio.c

1835

&data_vio->record_name);

drivers/md/dm-vdo/data-vio.c

1837

data_vio->hash_zone = vdo_select_hash_zone(vdo_from_data_vio(data_vio)->hash_zones,

drivers/md/dm-vdo/data-vio.c

1838

&data_vio->record_name);

drivers/md/dm-vdo/data-vio.c

1839

data_vio->last_async_operation = VIO_ASYNC_OP_ACQUIRE_VDO_HASH_LOCK;

drivers/md/dm-vdo/data-vio.c

1840

launch_data_vio_hash_zone_callback(data_vio, vdo_acquire_hash_lock);

drivers/md/dm-vdo/data-vio.c

1844

static void prepare_for_dedupe(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1847

VDO_ASSERT_LOG_ONLY(!data_vio->is_zero, "must not prepare to dedupe zero blocks");

drivers/md/dm-vdo/data-vio.c

1853

data_vio->last_async_operation = VIO_ASYNC_OP_HASH_DATA_VIO;

drivers/md/dm-vdo/data-vio.c

1854

launch_data_vio_cpu_callback(data_vio, hash_data_vio, CPU_Q_HASH_BLOCK_PRIORITY);

drivers/md/dm-vdo/data-vio.c

1864

struct data_vio *data_vio = vio_as_data_vio((struct vio *) bio->bi_private);

drivers/md/dm-vdo/data-vio.c

1867

vdo_set_completion_result(&data_vio->vio.completion,

drivers/md/dm-vdo/data-vio.c

1869

data_vio->downgrade_allocation_lock = true;

drivers/md/dm-vdo/data-vio.c

1870

update_metadata_for_data_vio_write(data_vio, data_vio->allocation.lock);

drivers/md/dm-vdo/data-vio.c

1874

void write_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1879

if (!data_vio_has_allocation(data_vio)) {

drivers/md/dm-vdo/data-vio.c

1884

continue_data_vio_with_error(data_vio, VDO_NO_SPACE);

drivers/md/dm-vdo/data-vio.c

1894

status = get_data_vio_compression_status(data_vio);

drivers/md/dm-vdo/data-vio.c

1896

!set_data_vio_compression_status(data_vio, status, new_status));

drivers/md/dm-vdo/data-vio.c

1899

result = vio_reset_bio(&data_vio->vio, data_vio->vio.data,

drivers/md/dm-vdo/data-vio.c

1901

data_vio->allocation.pbn);

drivers/md/dm-vdo/data-vio.c

1903

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

1907

data_vio->last_async_operation = VIO_ASYNC_OP_WRITE_DATA_VIO;

drivers/md/dm-vdo/data-vio.c

1908

vdo_submit_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1919

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1925

VDO_ASSERT_LOG_ONLY(data_vio_has_flush_generation_lock(data_vio),

drivers/md/dm-vdo/data-vio.c

1927

acknowledge_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1928

if (data_vio->new_mapped.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

1930

update_metadata_for_data_vio_write(data_vio, NULL);

drivers/md/dm-vdo/data-vio.c

1934

prepare_for_dedupe(data_vio);

drivers/md/dm-vdo/data-vio.c

1945

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1947

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1949

if (!vdo_allocate_block_in_zone(data_vio))

drivers/md/dm-vdo/data-vio.c

1953

WRITE_ONCE(data_vio->allocation_succeeded, true);

drivers/md/dm-vdo/data-vio.c

1954

data_vio->new_mapped = (struct zoned_pbn) {

drivers/md/dm-vdo/data-vio.c

1955

.zone = data_vio->allocation.zone,

drivers/md/dm-vdo/data-vio.c

1956

.pbn = data_vio->allocation.pbn,

drivers/md/dm-vdo/data-vio.c

1960

if (data_vio->fua ||

drivers/md/dm-vdo/data-vio.c

1961

data_vio->remaining_discard > (u32) (VDO_BLOCK_SIZE - data_vio->offset)) {

drivers/md/dm-vdo/data-vio.c

1962

prepare_for_dedupe(data_vio);

drivers/md/dm-vdo/data-vio.c

1966

data_vio->last_async_operation = VIO_ASYNC_OP_ACKNOWLEDGE_WRITE;

drivers/md/dm-vdo/data-vio.c

1967

launch_data_vio_on_bio_ack_queue(data_vio, acknowledge_write_callback);

drivers/md/dm-vdo/data-vio.c

1978

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1984

prepare_for_dedupe(data_vio);

drivers/md/dm-vdo/data-vio.c

1992

static int assert_is_discard(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1994

int result = VDO_ASSERT(data_vio->is_discard,

drivers/md/dm-vdo/data-vio.c

2008

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

2010

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

2011

if (data_vio->read) {

drivers/md/dm-vdo/data-vio.c

2012

set_data_vio_logical_callback(data_vio, read_block);

drivers/md/dm-vdo/data-vio.c

2013

data_vio->last_async_operation = VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_READ;

drivers/md/dm-vdo/data-vio.c

2014

vdo_get_mapped_block(data_vio);

drivers/md/dm-vdo/data-vio.c

2018

vdo_acquire_flush_generation_lock(data_vio);

drivers/md/dm-vdo/data-vio.c

2020

if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

2026

continue_data_vio_with_error(data_vio, assert_is_discard(data_vio));

drivers/md/dm-vdo/data-vio.c

2034

if (!data_vio->is_zero && (!data_vio->is_discard || data_vio->is_partial)) {

drivers/md/dm-vdo/data-vio.c

2035

data_vio_allocate_data_block(data_vio, VIO_WRITE_LOCK, allocate_block,

drivers/md/dm-vdo/data-vio.c

2044

data_vio->new_mapped.pbn = VDO_ZERO_BLOCK;

drivers/md/dm-vdo/data-vio.c

2045

if (data_vio->is_zero)

drivers/md/dm-vdo/data-vio.c

2046

data_vio->new_mapped.state = VDO_MAPPING_STATE_UNCOMPRESSED;

drivers/md/dm-vdo/data-vio.c

2048

if (data_vio->remaining_discard > (u32) (VDO_BLOCK_SIZE - data_vio->offset)) {

drivers/md/dm-vdo/data-vio.c

2050

update_metadata_for_data_vio_write(data_vio, NULL);

drivers/md/dm-vdo/data-vio.c

2054

data_vio->last_async_operation = VIO_ASYNC_OP_ACKNOWLEDGE_WRITE;

drivers/md/dm-vdo/data-vio.c

2055

launch_data_vio_on_bio_ack_queue(data_vio, acknowledge_write_callback);

drivers/md/dm-vdo/data-vio.c

244

static void initialize_lbn_lock(struct data_vio *data_vio, logical_block_number_t lbn)

drivers/md/dm-vdo/data-vio.c

246

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

248

struct lbn_lock *lock = &data_vio->logical;

drivers/md/dm-vdo/data-vio.c

253

zone_number = vdo_compute_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

257

static void launch_locked_request(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

259

data_vio->logical.locked = true;

drivers/md/dm-vdo/data-vio.c

260

if (data_vio->write) {

drivers/md/dm-vdo/data-vio.c

261

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

264

continue_data_vio_with_error(data_vio, VDO_READ_ONLY);

drivers/md/dm-vdo/data-vio.c

269

data_vio->last_async_operation = VIO_ASYNC_OP_FIND_BLOCK_MAP_SLOT;

drivers/md/dm-vdo/data-vio.c

270

vdo_find_block_map_slot(data_vio);

drivers/md/dm-vdo/data-vio.c

273

static void acknowledge_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

275

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

276

struct bio *bio = data_vio->user_bio;

drivers/md/dm-vdo/data-vio.c

277

int error = vdo_status_to_errno(data_vio->vio.completion.result);

drivers/md/dm-vdo/data-vio.c

282

VDO_ASSERT_LOG_ONLY((data_vio->remaining_discard <=

drivers/md/dm-vdo/data-vio.c

283

(u32) (VDO_BLOCK_SIZE - data_vio->offset)),

drivers/md/dm-vdo/data-vio.c

286

data_vio->user_bio = NULL;

drivers/md/dm-vdo/data-vio.c

288

if (data_vio->is_partial)

drivers/md/dm-vdo/data-vio.c

306

struct data_vio_compression_status get_data_vio_compression_status(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

308

u32 packed = atomic_read(&data_vio->compression.status);

drivers/md/dm-vdo/data-vio.c

340

set_data_vio_compression_status(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

353

actual = atomic_cmpxchg(&data_vio->compression.status, expected, replacement);

drivers/md/dm-vdo/data-vio.c

359

struct data_vio_compression_status advance_data_vio_compression_stage(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

363

get_data_vio_compression_status(data_vio);

drivers/md/dm-vdo/data-vio.c

382

if (set_data_vio_compression_status(data_vio, status, new_status))

drivers/md/dm-vdo/data-vio.c

395

bool cancel_data_vio_compression(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

400

status = get_data_vio_compression_status(data_vio);

drivers/md/dm-vdo/data-vio.c

409

if (set_data_vio_compression_status(data_vio, status, new_status))

drivers/md/dm-vdo/data-vio.c

424

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

425

struct lbn_lock *lock = &data_vio->logical;

drivers/md/dm-vdo/data-vio.c

426

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

427

struct data_vio *lock_holder;

drivers/md/dm-vdo/data-vio.c

430

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

432

if (data_vio->logical.lbn >= vdo->states.vdo.config.logical_blocks) {

drivers/md/dm-vdo/data-vio.c

433

continue_data_vio_with_error(data_vio, VDO_OUT_OF_RANGE);

drivers/md/dm-vdo/data-vio.c

438

data_vio, false, (void **) &lock_holder);

drivers/md/dm-vdo/data-vio.c

440

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

446

launch_locked_request(data_vio);

drivers/md/dm-vdo/data-vio.c

452

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

464

if (!data_vio->write && READ_ONCE(lock_holder->allocation_succeeded)) {

drivers/md/dm-vdo/data-vio.c

465

copy_to_bio(data_vio->user_bio, lock_holder->vio.data + data_vio->offset);

drivers/md/dm-vdo/data-vio.c

466

acknowledge_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

471

data_vio->last_async_operation = VIO_ASYNC_OP_ATTEMPT_LOGICAL_BLOCK_LOCK;

drivers/md/dm-vdo/data-vio.c

472

vdo_waitq_enqueue_waiter(&lock_holder->logical.waiters, &data_vio->waiter);

drivers/md/dm-vdo/data-vio.c

479

data_vio->compression.lock_holder = lock_holder;

drivers/md/dm-vdo/data-vio.c

480

launch_data_vio_packer_callback(data_vio,

drivers/md/dm-vdo/data-vio.c

491

static void launch_data_vio(struct data_vio *data_vio, logical_block_number_t lbn)

drivers/md/dm-vdo/data-vio.c

493

struct vdo_completion *completion = &data_vio->vio.completion;

drivers/md/dm-vdo/data-vio.c

499

memset(&data_vio->tree_lock, 0, sizeof(data_vio->tree_lock));

drivers/md/dm-vdo/data-vio.c

500

initialize_lbn_lock(data_vio, lbn);

drivers/md/dm-vdo/data-vio.c

501

INIT_LIST_HEAD(&data_vio->hash_lock_entry);

drivers/md/dm-vdo/data-vio.c

502

INIT_LIST_HEAD(&data_vio->write_entry);

drivers/md/dm-vdo/data-vio.c

504

memset(&data_vio->allocation, 0, sizeof(data_vio->allocation));

drivers/md/dm-vdo/data-vio.c

506

data_vio->is_duplicate = false;

drivers/md/dm-vdo/data-vio.c

508

memset(&data_vio->record_name, 0, sizeof(data_vio->record_name));

drivers/md/dm-vdo/data-vio.c

509

memset(&data_vio->duplicate, 0, sizeof(data_vio->duplicate));

drivers/md/dm-vdo/data-vio.c

510

vdo_reset_completion(&data_vio->decrement_completion);

drivers/md/dm-vdo/data-vio.c

513

set_data_vio_logical_callback(data_vio, attempt_logical_block_lock);

drivers/md/dm-vdo/data-vio.c

528

static void launch_bio(struct vdo *vdo, struct data_vio *data_vio, struct bio *bio)

drivers/md/dm-vdo/data-vio.c

535

memset(data_vio, 0, offsetof(struct data_vio, vio));

drivers/md/dm-vdo/data-vio.c

536

memset(&data_vio->compression, 0, offsetof(struct compression_state, block));

drivers/md/dm-vdo/data-vio.c

538

data_vio->user_bio = bio;

drivers/md/dm-vdo/data-vio.c

539

data_vio->offset = to_bytes(bio->bi_iter.bi_sector & VDO_SECTORS_PER_BLOCK_MASK);

drivers/md/dm-vdo/data-vio.c

540

data_vio->is_partial = (bio->bi_iter.bi_size < VDO_BLOCK_SIZE) || (data_vio->offset != 0);

drivers/md/dm-vdo/data-vio.c

548

data_vio->remaining_discard = bio->bi_iter.bi_size;

drivers/md/dm-vdo/data-vio.c

549

data_vio->write = true;

drivers/md/dm-vdo/data-vio.c

550

data_vio->is_discard = true;

drivers/md/dm-vdo/data-vio.c

551

if (data_vio->is_partial) {

drivers/md/dm-vdo/data-vio.c

553

data_vio->read = true;

drivers/md/dm-vdo/data-vio.c

555

} else if (data_vio->is_partial) {

drivers/md/dm-vdo/data-vio.c

557

data_vio->read = true;

drivers/md/dm-vdo/data-vio.c

559

data_vio->write = true;

drivers/md/dm-vdo/data-vio.c

561

data_vio->read = true;

drivers/md/dm-vdo/data-vio.c

567

copy_from_bio(bio, data_vio->vio.data);

drivers/md/dm-vdo/data-vio.c

568

data_vio->is_zero = mem_is_zero(data_vio->vio.data, VDO_BLOCK_SIZE);

drivers/md/dm-vdo/data-vio.c

569

data_vio->write = true;

drivers/md/dm-vdo/data-vio.c

572

if (data_vio->user_bio->bi_opf & REQ_FUA)

drivers/md/dm-vdo/data-vio.c

573

data_vio->fua = true;

drivers/md/dm-vdo/data-vio.c

576

launch_data_vio(data_vio, lbn);

drivers/md/dm-vdo/data-vio.c

579

static void assign_data_vio(struct limiter *limiter, struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

583

launch_bio(limiter->pool->completion.vdo, data_vio, bio);

drivers/md/dm-vdo/data-vio.c

605

static inline struct data_vio *get_available_data_vio(struct data_vio_pool *pool)

drivers/md/dm-vdo/data-vio.c

607

struct data_vio *data_vio =

drivers/md/dm-vdo/data-vio.c

608

list_first_entry(&pool->available, struct data_vio, pool_entry);

drivers/md/dm-vdo/data-vio.c

610

list_del_init(&data_vio->pool_entry);

drivers/md/dm-vdo/data-vio.c

611

return data_vio;

drivers/md/dm-vdo/data-vio.c

666

struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

669

if (data_vio->remaining_discard > 0) {

drivers/md/dm-vdo/data-vio.c

679

assign_data_vio(&pool->limiter, data_vio);

drivers/md/dm-vdo/data-vio.c

681

assign_data_vio(&pool->discard_limiter, data_vio);

drivers/md/dm-vdo/data-vio.c

683

list_add(&data_vio->pool_entry, returned);

drivers/md/dm-vdo/data-vio.c

715

struct data_vio *data_vio;

drivers/md/dm-vdo/data-vio.c

721

data_vio = as_data_vio(container_of(entry, struct vdo_completion,

drivers/md/dm-vdo/data-vio.c

723

acknowledge_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

724

reuse_or_release_resources(pool, data_vio, &returned);

drivers/md/dm-vdo/data-vio.c

783

static int initialize_data_vio(struct data_vio *data_vio, struct vdo *vdo)

drivers/md/dm-vdo/data-vio.c

790

&data_vio->vio.data);

drivers/md/dm-vdo/data-vio.c

796

&data_vio->compression.block);

drivers/md/dm-vdo/data-vio.c

803

&data_vio->scratch_block);

drivers/md/dm-vdo/data-vio.c

813

vdo_initialize_completion(&data_vio->decrement_completion, vdo,

drivers/md/dm-vdo/data-vio.c

815

initialize_vio(&data_vio->vio, bio, 1, VIO_TYPE_DATA, VIO_PRIORITY_DATA, vdo);

drivers/md/dm-vdo/data-vio.c

820

static void destroy_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

822

if (data_vio == NULL)

drivers/md/dm-vdo/data-vio.c

825

vdo_free_bio(vdo_forget(data_vio->vio.bio));

drivers/md/dm-vdo/data-vio.c

826

vdo_free(vdo_forget(data_vio->vio.data));

drivers/md/dm-vdo/data-vio.c

827

vdo_free(vdo_forget(data_vio->compression.block));

drivers/md/dm-vdo/data-vio.c

828

vdo_free(vdo_forget(data_vio->scratch_block));

drivers/md/dm-vdo/data-vio.c

845

result = vdo_allocate_extended(struct data_vio_pool, pool_size, struct data_vio,

drivers/md/dm-vdo/data-vio.c

872

struct data_vio *data_vio = &pool->data_vios[i];

drivers/md/dm-vdo/data-vio.c

874

result = initialize_data_vio(data_vio, vdo);

drivers/md/dm-vdo/data-vio.c

876

destroy_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

881

list_add(&data_vio->pool_entry, &pool->available);

drivers/md/dm-vdo/data-vio.c

896

struct data_vio *data_vio, *tmp;

drivers/md/dm-vdo/data-vio.c

920

list_for_each_entry_safe(data_vio, tmp, &pool->available, pool_entry) {

drivers/md/dm-vdo/data-vio.c

921

list_del_init(&data_vio->pool_entry);

drivers/md/dm-vdo/data-vio.c

922

destroy_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

962

struct data_vio *data_vio;

drivers/md/dm-vdo/data-vio.c

980

data_vio = get_available_data_vio(pool);

drivers/md/dm-vdo/data-vio.c

982

launch_bio(pool->completion.vdo, data_vio, bio);

drivers/md/dm-vdo/data-vio.h

131

struct data_vio *next_in_batch;

drivers/md/dm-vdo/data-vio.h

134

struct data_vio *lock_holder;

drivers/md/dm-vdo/data-vio.h

281

static inline struct data_vio *vio_as_data_vio(struct vio *vio)

drivers/md/dm-vdo/data-vio.h

284

return container_of(vio, struct data_vio, vio);

drivers/md/dm-vdo/data-vio.h

287

static inline struct data_vio *as_data_vio(struct vdo_completion *completion)

drivers/md/dm-vdo/data-vio.h

292

static inline struct data_vio *vdo_waiter_as_data_vio(struct vdo_waiter *waiter)

drivers/md/dm-vdo/data-vio.h

297

return container_of(waiter, struct data_vio, waiter);

drivers/md/dm-vdo/data-vio.h

300

static inline struct data_vio *data_vio_from_reference_updater(struct reference_updater *updater)

drivers/md/dm-vdo/data-vio.h

303

return container_of(updater, struct data_vio, increment_updater);

drivers/md/dm-vdo/data-vio.h

305

return container_of(updater, struct data_vio, decrement_updater);

drivers/md/dm-vdo/data-vio.h

308

static inline bool data_vio_has_flush_generation_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

310

return !list_empty(&data_vio->write_entry);

drivers/md/dm-vdo/data-vio.h

313

static inline struct vdo *vdo_from_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

315

return data_vio->vio.completion.vdo;

drivers/md/dm-vdo/data-vio.h

318

static inline bool data_vio_has_allocation(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

320

return (data_vio->allocation.pbn != VDO_ZERO_BLOCK);

drivers/md/dm-vdo/data-vio.h

324

advance_data_vio_compression_stage(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

326

get_data_vio_compression_status(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

327

bool cancel_data_vio_compression(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

346

static inline void continue_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

348

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

357

static inline void continue_data_vio_with_error(struct data_vio *data_vio, int result)

drivers/md/dm-vdo/data-vio.h

359

vdo_continue_completion(&data_vio->vio.completion, result);

drivers/md/dm-vdo/data-vio.h

362

const char * __must_check get_data_vio_operation_name(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

364

static inline void assert_data_vio_in_hash_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

366

thread_id_t expected = data_vio->hash_zone->thread_id;

drivers/md/dm-vdo/data-vio.h

374

(unsigned long long) data_vio->logical.lbn, thread_id, expected);

drivers/md/dm-vdo/data-vio.h

377

static inline void set_data_vio_hash_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

380

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

381

data_vio->hash_zone->thread_id);

drivers/md/dm-vdo/data-vio.h

388

static inline void launch_data_vio_hash_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

391

set_data_vio_hash_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

392

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

395

static inline void assert_data_vio_in_logical_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

397

thread_id_t expected = data_vio->logical.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

402

(unsigned long long) data_vio->logical.lbn, thread_id, expected);

drivers/md/dm-vdo/data-vio.h

405

static inline void set_data_vio_logical_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

408

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

409

data_vio->logical.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

416

static inline void launch_data_vio_logical_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

419

set_data_vio_logical_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

420

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

423

static inline void assert_data_vio_in_allocated_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

425

thread_id_t expected = data_vio->allocation.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

430

(unsigned long long) data_vio->allocation.pbn, thread_id,

drivers/md/dm-vdo/data-vio.h

434

static inline void set_data_vio_allocated_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

437

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

438

data_vio->allocation.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

446

static inline void launch_data_vio_allocated_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

449

set_data_vio_allocated_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

450

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

453

static inline void assert_data_vio_in_duplicate_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

455

thread_id_t expected = data_vio->duplicate.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

460

(unsigned long long) data_vio->duplicate.pbn, thread_id,

drivers/md/dm-vdo/data-vio.h

464

static inline void set_data_vio_duplicate_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

467

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

468

data_vio->duplicate.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

476

static inline void launch_data_vio_duplicate_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

479

set_data_vio_duplicate_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

480

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

483

static inline void assert_data_vio_in_mapped_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

485

thread_id_t expected = data_vio->mapped.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

490

(unsigned long long) data_vio->mapped.pbn, thread_id, expected);

drivers/md/dm-vdo/data-vio.h

493

static inline void set_data_vio_mapped_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

496

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

497

data_vio->mapped.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

500

static inline void assert_data_vio_in_new_mapped_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

502

thread_id_t expected = data_vio->new_mapped.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

507

(unsigned long long) data_vio->new_mapped.pbn, thread_id,

drivers/md/dm-vdo/data-vio.h

511

static inline void set_data_vio_new_mapped_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

514

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

515

data_vio->new_mapped.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

518

static inline void assert_data_vio_in_journal_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

520

thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread;

drivers/md/dm-vdo/data-vio.h

525

(unsigned long long) data_vio->logical.lbn, thread_id,

drivers/md/dm-vdo/data-vio.h

529

static inline void set_data_vio_journal_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

532

thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread;

drivers/md/dm-vdo/data-vio.h

534

vdo_set_completion_callback(&data_vio->vio.completion, callback, journal_thread);

drivers/md/dm-vdo/data-vio.h

541

static inline void launch_data_vio_journal_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

544

set_data_vio_journal_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

545

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

548

static inline void assert_data_vio_in_packer_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

550

thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread;

drivers/md/dm-vdo/data-vio.h

555

(unsigned long long) data_vio->logical.lbn, thread_id,

drivers/md/dm-vdo/data-vio.h

559

static inline void set_data_vio_packer_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

562

thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread;

drivers/md/dm-vdo/data-vio.h

564

vdo_set_completion_callback(&data_vio->vio.completion, callback, packer_thread);

drivers/md/dm-vdo/data-vio.h

571

static inline void launch_data_vio_packer_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

574

set_data_vio_packer_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

575

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

578

static inline void assert_data_vio_on_cpu_thread(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

580

thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread;

drivers/md/dm-vdo/data-vio.h

585

(unsigned long long) data_vio->logical.lbn, thread_id,

drivers/md/dm-vdo/data-vio.h

589

static inline void set_data_vio_cpu_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

592

thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread;

drivers/md/dm-vdo/data-vio.h

594

vdo_set_completion_callback(&data_vio->vio.completion, callback, cpu_thread);

drivers/md/dm-vdo/data-vio.h

601

static inline void launch_data_vio_cpu_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

605

set_data_vio_cpu_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

606

vdo_launch_completion_with_priority(&data_vio->vio.completion, priority);

drivers/md/dm-vdo/data-vio.h

609

static inline void set_data_vio_bio_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

612

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

613

get_vio_bio_zone_thread_id(&data_vio->vio));

drivers/md/dm-vdo/data-vio.h

620

static inline void launch_data_vio_bio_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

623

set_data_vio_bio_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

624

vdo_launch_completion_with_priority(&data_vio->vio.completion,

drivers/md/dm-vdo/data-vio.h

633

static inline void launch_data_vio_on_bio_ack_queue(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

636

struct vdo_completion *completion = &data_vio->vio.completion;

drivers/md/dm-vdo/data-vio.h

649

void data_vio_allocate_data_block(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

653

void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset);

drivers/md/dm-vdo/data-vio.h

655

int __must_check uncompress_data_vio(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

659

void update_metadata_for_data_vio_write(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

661

void write_data_vio(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

662

void launch_compress_data_vio(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.c

1006

static void start_deduping(struct hash_lock *lock, struct data_vio *agent,

drivers/md/dm-vdo/dedupe.c

1077

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1138

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1146

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1162

struct data_vio *agent = vio_as_data_vio(bio->bi_private);

drivers/md/dm-vdo/dedupe.c

1193

static void start_verifying(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1226

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1278

static bool acquire_provisional_reference(struct data_vio *agent, struct pbn_lock *lock,

drivers/md/dm-vdo/dedupe.c

1313

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1414

static void start_locking(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1444

static void finish_writing(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1508

static struct data_vio *select_writing_agent(struct hash_lock *lock)

drivers/md/dm-vdo/dedupe.c

1511

struct data_vio *data_vio;

drivers/md/dm-vdo/dedupe.c

1519

while (((data_vio = dequeue_lock_waiter(lock)) != NULL) &&

drivers/md/dm-vdo/dedupe.c

1520

!data_vio_has_allocation(data_vio)) {

drivers/md/dm-vdo/dedupe.c

1522

vdo_waitq_enqueue_waiter(&temp_queue, &data_vio->waiter);

drivers/md/dm-vdo/dedupe.c

1525

if (data_vio != NULL) {

drivers/md/dm-vdo/dedupe.c

1537

lock->agent = data_vio;

drivers/md/dm-vdo/dedupe.c

1540

data_vio = lock->agent;

drivers/md/dm-vdo/dedupe.c

1545

return data_vio;

drivers/md/dm-vdo/dedupe.c

1557

static void start_writing(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1602

struct data_vio *data_vio = context->requestor;

drivers/md/dm-vdo/dedupe.c

1605

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

1606

struct zoned_pbn *advice = &data_vio->duplicate;

drivers/md/dm-vdo/dedupe.c

1628

(unsigned long long) data_vio->logical.lbn);

drivers/md/dm-vdo/dedupe.c

1637

(unsigned long long) data_vio->logical.lbn);

drivers/md/dm-vdo/dedupe.c

1645

static void process_query_result(struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1667

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1705

static void start_querying(struct hash_lock *lock, struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1707

lock->agent = data_vio;

drivers/md/dm-vdo/dedupe.c

1709

data_vio->last_async_operation = VIO_ASYNC_OP_CHECK_FOR_DUPLICATION;

drivers/md/dm-vdo/dedupe.c

1710

set_data_vio_hash_zone_callback(data_vio, finish_querying);

drivers/md/dm-vdo/dedupe.c

1711

query_index(data_vio,

drivers/md/dm-vdo/dedupe.c

1712

(data_vio_has_allocation(data_vio) ? UDS_POST : UDS_QUERY));

drivers/md/dm-vdo/dedupe.c

1722

static void report_bogus_lock_state(struct hash_lock *lock, struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1726

continue_data_vio_with_error(data_vio, VDO_LOCK_ERROR);

drivers/md/dm-vdo/dedupe.c

1742

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1743

struct hash_lock *lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

1747

VDO_ASSERT_LOG_ONLY(data_vio == lock->agent,

drivers/md/dm-vdo/dedupe.c

1749

finish_writing(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1753

finish_deduping(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1758

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

1768

report_bogus_lock_state(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1772

report_bogus_lock_state(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1788

static bool is_hash_collision(struct hash_lock *lock, struct data_vio *candidate)

drivers/md/dm-vdo/dedupe.c

1790

struct data_vio *lock_holder;

drivers/md/dm-vdo/dedupe.c

1797

lock_holder = list_first_entry(&lock->duplicate_vios, struct data_vio,

drivers/md/dm-vdo/dedupe.c

1809

static inline int assert_hash_lock_preconditions(const struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1814

result = VDO_ASSERT(data_vio->hash_lock == NULL,

drivers/md/dm-vdo/dedupe.c

1819

result = VDO_ASSERT(list_empty(&data_vio->hash_lock_entry),

drivers/md/dm-vdo/dedupe.c

1824

return VDO_ASSERT(data_vio->recovery_sequence_number == 0,

drivers/md/dm-vdo/dedupe.c

1839

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1843

assert_data_vio_in_hash_zone(data_vio);

drivers/md/dm-vdo/dedupe.c

1845

result = assert_hash_lock_preconditions(data_vio);

drivers/md/dm-vdo/dedupe.c

1847

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/dedupe.c

1851

result = acquire_lock(data_vio->hash_zone, &data_vio->record_name, NULL, &lock);

drivers/md/dm-vdo/dedupe.c

1853

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/dedupe.c

1857

if (is_hash_collision(lock, data_vio)) {

drivers/md/dm-vdo/dedupe.c

1864

write_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

1868

set_hash_lock(data_vio, lock);

drivers/md/dm-vdo/dedupe.c

1871

start_querying(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1881

wait_on_hash_lock(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1886

vdo_release_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

1887

write_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

1891

launch_dedupe(lock, data_vio, false);

drivers/md/dm-vdo/dedupe.c

1896

report_bogus_lock_state(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1911

void vdo_release_hash_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1914

struct hash_lock *lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

1915

struct hash_zone *zone = data_vio->hash_zone;

drivers/md/dm-vdo/dedupe.c

1920

set_hash_lock(data_vio, NULL);

drivers/md/dm-vdo/dedupe.c

1959

static void transfer_allocation_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1961

struct allocation *allocation = &data_vio->allocation;

drivers/md/dm-vdo/dedupe.c

1962

struct hash_lock *hash_lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

1964

VDO_ASSERT_LOG_ONLY(data_vio->new_mapped.pbn == allocation->pbn,

drivers/md/dm-vdo/dedupe.c

1972

hash_lock->duplicate = data_vio->new_mapped;

drivers/md/dm-vdo/dedupe.c

1973

data_vio->duplicate = data_vio->new_mapped;

drivers/md/dm-vdo/dedupe.c

1991

void vdo_share_compressed_write_lock(struct data_vio *data_vio,

drivers/md/dm-vdo/dedupe.c

1996

VDO_ASSERT_LOG_ONLY(vdo_get_duplicate_lock(data_vio) == NULL,

drivers/md/dm-vdo/dedupe.c

1998

VDO_ASSERT_LOG_ONLY(vdo_is_state_compressed(data_vio->new_mapped.state),

drivers/md/dm-vdo/dedupe.c

2000

assert_data_vio_in_new_mapped_zone(data_vio);

drivers/md/dm-vdo/dedupe.c

2010

data_vio->duplicate = data_vio->new_mapped;

drivers/md/dm-vdo/dedupe.c

2011

data_vio->hash_lock->duplicate = data_vio->new_mapped;

drivers/md/dm-vdo/dedupe.c

2012

set_duplicate_lock(data_vio->hash_lock, pbn_lock);

drivers/md/dm-vdo/dedupe.c

262

struct data_vio *agent;

drivers/md/dm-vdo/dedupe.c

2877

static void prepare_uds_request(struct uds_request *request, struct data_vio *data_vio,

drivers/md/dm-vdo/dedupe.c

2880

request->record_name = data_vio->record_name;

drivers/md/dm-vdo/dedupe.c

2887

encoding->data[offset++] = data_vio->new_mapped.state;

drivers/md/dm-vdo/dedupe.c

2888

put_unaligned_le64(data_vio->new_mapped.pbn, &encoding->data[offset]);

drivers/md/dm-vdo/dedupe.c

2900

static void query_index(struct data_vio *data_vio, enum uds_request_type operation)

drivers/md/dm-vdo/dedupe.c

2904

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

2905

struct hash_zone *zone = data_vio->hash_zone;

drivers/md/dm-vdo/dedupe.c

2907

assert_data_vio_in_hash_zone(data_vio);

drivers/md/dm-vdo/dedupe.c

2910

continue_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

2917

continue_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

2921

data_vio->dedupe_context = context;

drivers/md/dm-vdo/dedupe.c

2922

context->requestor = data_vio;

drivers/md/dm-vdo/dedupe.c

2924

prepare_uds_request(&context->request, data_vio, operation);

drivers/md/dm-vdo/dedupe.c

358

struct pbn_lock *vdo_get_duplicate_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

360

if (data_vio->hash_lock == NULL)

drivers/md/dm-vdo/dedupe.c

363

return data_vio->hash_lock->duplicate_lock;

drivers/md/dm-vdo/dedupe.c

396

static void assert_hash_lock_agent(struct data_vio *data_vio, const char *where)

drivers/md/dm-vdo/dedupe.c

399

assert_data_vio_in_hash_zone(data_vio);

drivers/md/dm-vdo/dedupe.c

400

VDO_ASSERT_LOG_ONLY(data_vio == data_vio->hash_lock->agent,

drivers/md/dm-vdo/dedupe.c

424

static inline struct data_vio *dequeue_lock_waiter(struct hash_lock *lock)

drivers/md/dm-vdo/dedupe.c

436

static void set_hash_lock(struct data_vio *data_vio, struct hash_lock *new_lock)

drivers/md/dm-vdo/dedupe.c

438

struct hash_lock *old_lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

441

VDO_ASSERT_LOG_ONLY(data_vio->hash_zone != NULL,

drivers/md/dm-vdo/dedupe.c

443

VDO_ASSERT_LOG_ONLY(!list_empty(&data_vio->hash_lock_entry),

drivers/md/dm-vdo/dedupe.c

459

list_del_init(&data_vio->hash_lock_entry);

drivers/md/dm-vdo/dedupe.c

462

data_vio->hash_lock = NULL;

drivers/md/dm-vdo/dedupe.c

470

list_move_tail(&data_vio->hash_lock_entry, &new_lock->duplicate_vios);

drivers/md/dm-vdo/dedupe.c

475

data_vio->hash_lock = new_lock;

drivers/md/dm-vdo/dedupe.c

480

static void start_deduping(struct hash_lock *lock, struct data_vio *agent,

drivers/md/dm-vdo/dedupe.c

482

static void start_locking(struct hash_lock *lock, struct data_vio *agent);

drivers/md/dm-vdo/dedupe.c

483

static void start_writing(struct hash_lock *lock, struct data_vio *agent);

drivers/md/dm-vdo/dedupe.c

485

static void transfer_allocation_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.c

492

static void exit_hash_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

495

vdo_release_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

498

data_vio->vio.completion.callback = complete_data_vio;

drivers/md/dm-vdo/dedupe.c

500

continue_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

509

static void set_duplicate_location(struct data_vio *data_vio,

drivers/md/dm-vdo/dedupe.c

512

data_vio->is_duplicate = (source.pbn != VDO_ZERO_BLOCK);

drivers/md/dm-vdo/dedupe.c

513

data_vio->duplicate = source;

drivers/md/dm-vdo/dedupe.c

523

static struct data_vio *retire_lock_agent(struct hash_lock *lock)

drivers/md/dm-vdo/dedupe.c

525

struct data_vio *old_agent = lock->agent;

drivers/md/dm-vdo/dedupe.c

526

struct data_vio *new_agent = dequeue_lock_waiter(lock);

drivers/md/dm-vdo/dedupe.c

540

static void wait_on_hash_lock(struct hash_lock *lock, struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

542

vdo_waitq_enqueue_waiter(&lock->waiters, &data_vio->waiter);

drivers/md/dm-vdo/dedupe.c

558

data_vio->compression.lock_holder = lock->agent;

drivers/md/dm-vdo/dedupe.c

559

launch_data_vio_packer_callback(data_vio, vdo_remove_lock_holder_from_packer);

drivers/md/dm-vdo/dedupe.c

581

static void start_bypassing(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

587

void vdo_clean_failed_hash_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

589

struct hash_lock *lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

592

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

597

lock->agent = data_vio;

drivers/md/dm-vdo/dedupe.c

598

} else if (data_vio != lock->agent) {

drivers/md/dm-vdo/dedupe.c

599

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

612

data_vio->duplicate = lock->duplicate;

drivers/md/dm-vdo/dedupe.c

613

launch_data_vio_duplicate_zone_callback(data_vio, unlock_duplicate_pbn);

drivers/md/dm-vdo/dedupe.c

618

data_vio->is_duplicate = false;

drivers/md/dm-vdo/dedupe.c

619

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

631

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

687

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

710

static void start_unlocking(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

724

static void process_update_result(struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

744

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

783

static void query_index(struct data_vio *data_vio, enum uds_request_type operation);

drivers/md/dm-vdo/dedupe.c

791

static void start_updating(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

814

static void finish_deduping(struct hash_lock *lock, struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

816

struct data_vio *agent = data_vio;

drivers/md/dm-vdo/dedupe.c

824

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

928

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/dedupe.c

931

set_hash_lock(data_vio, new_lock);

drivers/md/dm-vdo/dedupe.c

932

wait_on_hash_lock(new_lock, data_vio);

drivers/md/dm-vdo/dedupe.c

943

static void fork_hash_lock(struct hash_lock *old_lock, struct data_vio *new_agent)

drivers/md/dm-vdo/dedupe.c

981

static void launch_dedupe(struct hash_lock *lock, struct data_vio *data_vio,

drivers/md/dm-vdo/dedupe.c

986

fork_hash_lock(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

991

set_duplicate_location(data_vio, lock->duplicate);

drivers/md/dm-vdo/dedupe.c

992

data_vio->new_mapped = data_vio->duplicate;

drivers/md/dm-vdo/dedupe.c

993

update_metadata_for_data_vio_write(data_vio, lock->duplicate_lock);

drivers/md/dm-vdo/dedupe.h

26

struct data_vio *requestor;

drivers/md/dm-vdo/dedupe.h

72

struct pbn_lock * __must_check vdo_get_duplicate_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.h

76

void vdo_release_hash_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.h

77

void vdo_clean_failed_hash_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.h

78

void vdo_share_compressed_write_lock(struct data_vio *data_vio,

drivers/md/dm-vdo/dump.c

155

struct data_vio *data_vio;

drivers/md/dm-vdo/dump.c

160

data_vio = vdo_waiter_as_data_vio(first);

drivers/md/dm-vdo/dump.c

163

wait_on, data_vio, data_vio->allocation.pbn, data_vio->logical.lbn,

drivers/md/dm-vdo/dump.c

164

data_vio->duplicate.pbn, get_data_vio_operation_name(data_vio));

drivers/md/dm-vdo/dump.c

167

data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/dump.c

169

data_vio, data_vio->allocation.pbn, data_vio->logical.lbn,

drivers/md/dm-vdo/dump.c

170

data_vio->duplicate.pbn,

drivers/md/dm-vdo/dump.c

171

get_data_vio_operation_name(data_vio));

drivers/md/dm-vdo/dump.c

189

static void encode_vio_dump_flags(struct data_vio *data_vio, char buffer[8])

drivers/md/dm-vdo/dump.c

193

if (data_vio->vio.completion.result != VDO_SUCCESS)

drivers/md/dm-vdo/dump.c

195

if (data_vio->waiter.next_waiter != NULL)

drivers/md/dm-vdo/dump.c

197

if (data_vio->is_duplicate)

drivers/md/dm-vdo/dump.c

199

if (data_vio->is_partial)

drivers/md/dm-vdo/dump.c

201

if (data_vio->is_zero)

drivers/md/dm-vdo/dump.c

203

if (data_vio->remaining_discard > 0)

drivers/md/dm-vdo/dump.c

215

struct data_vio *data_vio = data;

drivers/md/dm-vdo/dump.c

233

vdo_dump_completion_to_buffer(&data_vio->vio.completion,

drivers/md/dm-vdo/dump.c

236

if (data_vio->is_duplicate) {

drivers/md/dm-vdo/dump.c

239

data_vio->allocation.pbn, data_vio->logical.lbn,

drivers/md/dm-vdo/dump.c

240

data_vio->duplicate.pbn);

drivers/md/dm-vdo/dump.c

241

} else if (data_vio_has_allocation(data_vio)) {

drivers/md/dm-vdo/dump.c

244

data_vio->allocation.pbn, data_vio->logical.lbn);

drivers/md/dm-vdo/dump.c

248

data_vio->logical.lbn);

drivers/md/dm-vdo/dump.c

251

if (data_vio->flush_generation != 0) {

drivers/md/dm-vdo/dump.c

254

data_vio->flush_generation);

drivers/md/dm-vdo/dump.c

259

encode_vio_dump_flags(data_vio, flags_dump_buffer);

drivers/md/dm-vdo/dump.c

261

vdo_log_info(" vio %px %s%s %s %s%s", data_vio,

drivers/md/dm-vdo/dump.c

264

get_data_vio_operation_name(data_vio),

drivers/md/dm-vdo/dump.c

272

dump_vio_waiters(&data_vio->logical.waiters, "lbn");

drivers/md/dm-vdo/io-submitter.c

316

void vdo_submit_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/io-submitter.c

318

if (try_bio_map_merge(&data_vio->vio))

drivers/md/dm-vdo/io-submitter.c

321

launch_data_vio_bio_zone_callback(data_vio, submit_data_vio);

drivers/md/dm-vdo/io-submitter.h

26

void vdo_submit_data_vio(struct data_vio *data_vio);

drivers/md/dm-vdo/logical-zone.c

215

struct data_vio *data_vio =

drivers/md/dm-vdo/logical-zone.c

216

list_first_entry_or_null(&zone->write_vios, struct data_vio,

drivers/md/dm-vdo/logical-zone.c

219

(data_vio == NULL) ? zone->flush_generation : data_vio->flush_generation;

drivers/md/dm-vdo/logical-zone.c

253

void vdo_acquire_flush_generation_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/logical-zone.c

255

struct logical_zone *zone = data_vio->logical.zone;

drivers/md/dm-vdo/logical-zone.c

260

data_vio->flush_generation = zone->flush_generation;

drivers/md/dm-vdo/logical-zone.c

261

list_add_tail(&data_vio->write_entry, &zone->write_vios);

drivers/md/dm-vdo/logical-zone.c

313

void vdo_release_flush_generation_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/logical-zone.c

315

struct logical_zone *zone = data_vio->logical.zone;

drivers/md/dm-vdo/logical-zone.c

319

if (!data_vio_has_flush_generation_lock(data_vio))

drivers/md/dm-vdo/logical-zone.c

322

list_del_init(&data_vio->write_entry);

drivers/md/dm-vdo/logical-zone.c

323

VDO_ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation),

drivers/md/dm-vdo/logical-zone.c

325

(unsigned long long) data_vio->flush_generation,

drivers/md/dm-vdo/logical-zone.h

81

void vdo_acquire_flush_generation_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/logical-zone.h

83

void vdo_release_flush_generation_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/packer.c

214

static inline struct packer *get_packer_from_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

216

return vdo_from_data_vio(data_vio)->packer;

drivers/md/dm-vdo/packer.c

240

static void abort_packing(struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

242

struct packer *packer = get_packer_from_data_vio(data_vio);

drivers/md/dm-vdo/packer.c

247

write_data_vio(data_vio);

drivers/md/dm-vdo/packer.c

256

static void release_compressed_write_waiter(struct data_vio *data_vio,

drivers/md/dm-vdo/packer.c

259

data_vio->new_mapped = (struct zoned_pbn) {

drivers/md/dm-vdo/packer.c

262

.state = data_vio->compression.slot + VDO_MAPPING_STATE_COMPRESSED_BASE,

drivers/md/dm-vdo/packer.c

265

vdo_share_compressed_write_lock(data_vio, allocation->lock);

drivers/md/dm-vdo/packer.c

266

update_metadata_for_data_vio_write(data_vio, allocation->lock);

drivers/md/dm-vdo/packer.c

277

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/packer.c

278

struct data_vio *client, *next;

drivers/md/dm-vdo/packer.c

297

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/packer.c

299

struct data_vio *client, *next;

drivers/md/dm-vdo/packer.c

324

static void add_to_bin(struct packer_bin *bin, struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

326

data_vio->compression.bin = bin;

drivers/md/dm-vdo/packer.c

327

data_vio->compression.slot = bin->slots_used;

drivers/md/dm-vdo/packer.c

328

bin->incoming[bin->slots_used++] = data_vio;

drivers/md/dm-vdo/packer.c

339

static struct data_vio *remove_from_bin(struct packer *packer, struct packer_bin *bin)

drivers/md/dm-vdo/packer.c

342

struct data_vio *data_vio = bin->incoming[--bin->slots_used];

drivers/md/dm-vdo/packer.c

344

if (!advance_data_vio_compression_stage(data_vio).may_not_compress) {

drivers/md/dm-vdo/packer.c

345

data_vio->compression.bin = NULL;

drivers/md/dm-vdo/packer.c

346

return data_vio;

drivers/md/dm-vdo/packer.c

349

add_to_bin(packer->canceled_bin, data_vio);

drivers/md/dm-vdo/packer.c

391

struct data_vio *data_vio,

drivers/md/dm-vdo/packer.c

395

struct compression_state *to_pack = &data_vio->compression;

drivers/md/dm-vdo/packer.c

399

compression->next_in_batch = data_vio;

drivers/md/dm-vdo/packer.c

412

struct data_vio *data_vio = vio_as_data_vio(bio->bi_private);

drivers/md/dm-vdo/packer.c

415

set_data_vio_allocated_zone_callback(data_vio, finish_compressed_write);

drivers/md/dm-vdo/packer.c

416

continue_data_vio_with_error(data_vio, blk_status_to_errno(bio->bi_status));

drivers/md/dm-vdo/packer.c

431

struct data_vio *agent = remove_from_bin(packer, bin);

drivers/md/dm-vdo/packer.c

432

struct data_vio *client;

drivers/md/dm-vdo/packer.c

500

struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

503

if (bin->free_space < data_vio->compression.size)

drivers/md/dm-vdo/packer.c

506

add_to_bin(bin, data_vio);

drivers/md/dm-vdo/packer.c

507

bin->free_space -= data_vio->compression.size;

drivers/md/dm-vdo/packer.c

525

struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

534

if (bin->free_space >= data_vio->compression.size)

drivers/md/dm-vdo/packer.c

547

if (data_vio->compression.size >=

drivers/md/dm-vdo/packer.c

563

void vdo_attempt_packing(struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

567

struct data_vio_compression_status status = get_data_vio_compression_status(data_vio);

drivers/md/dm-vdo/packer.c

568

struct packer *packer = get_packer_from_data_vio(data_vio);

drivers/md/dm-vdo/packer.c

590

(data_vio->flush_generation < packer->flush_generation)) {

drivers/md/dm-vdo/packer.c

591

abort_packing(data_vio);

drivers/md/dm-vdo/packer.c

604

bin = select_bin(packer, data_vio);

drivers/md/dm-vdo/packer.c

606

(advance_data_vio_compression_stage(data_vio).stage != DATA_VIO_PACKING)) {

drivers/md/dm-vdo/packer.c

607

abort_packing(data_vio);

drivers/md/dm-vdo/packer.c

611

add_data_vio_to_packer_bin(packer, bin, data_vio);

drivers/md/dm-vdo/packer.c

664

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/packer.c

665

struct packer *packer = get_packer_from_data_vio(data_vio);

drivers/md/dm-vdo/packer.c

666

struct data_vio *lock_holder;

drivers/md/dm-vdo/packer.c

670

assert_data_vio_in_packer_zone(data_vio);

drivers/md/dm-vdo/packer.c

672

lock_holder = vdo_forget(data_vio->compression.lock_holder);

drivers/md/dm-vdo/packer.h

108

void vdo_attempt_packing(struct data_vio *data_vio);

drivers/md/dm-vdo/packer.h

71

struct data_vio *incoming[];

drivers/md/dm-vdo/physical-zone.c

523

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/physical-zone.c

526

data_vio->allocation.wait_for_clean_slab = false;

drivers/md/dm-vdo/physical-zone.c

527

data_vio->allocation.first_allocation_zone = data_vio->allocation.zone->zone_number;

drivers/md/dm-vdo/physical-zone.c

528

continue_data_vio(data_vio);

drivers/md/dm-vdo/physical-zone.c

540

static bool continue_allocating(struct data_vio *data_vio)

drivers/md/dm-vdo/physical-zone.c

542

struct allocation *allocation = &data_vio->allocation;

drivers/md/dm-vdo/physical-zone.c

544

struct vdo_completion *completion = &data_vio->vio.completion;

drivers/md/dm-vdo/physical-zone.c

561

data_vio->waiter.callback = retry_allocation;

drivers/md/dm-vdo/physical-zone.c

563

&data_vio->waiter);

drivers/md/dm-vdo/physical-zone.c

589

bool vdo_allocate_block_in_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/physical-zone.c

591

int result = allocate_and_lock_block(&data_vio->allocation);

drivers/md/dm-vdo/physical-zone.c

596

if ((result != VDO_NO_SPACE) || !continue_allocating(data_vio))

drivers/md/dm-vdo/physical-zone.c

597

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/physical-zone.h

107

bool __must_check vdo_allocate_block_in_zone(struct data_vio *data_vio);

drivers/md/dm-vdo/recovery-journal.c

1067

static void update_usages(struct recovery_journal *journal, struct data_vio *data_vio)

drivers/md/dm-vdo/recovery-journal.c

1069

if (data_vio->increment_updater.operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) {

drivers/md/dm-vdo/recovery-journal.c

1074

if (data_vio->new_mapped.state != VDO_MAPPING_STATE_UNMAPPED)

drivers/md/dm-vdo/recovery-journal.c

1077

if (data_vio->mapped.state != VDO_MAPPING_STATE_UNMAPPED)

drivers/md/dm-vdo/recovery-journal.c

1090

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/recovery-journal.c

1095

data_vio->recovery_journal_point = (struct journal_point) {

drivers/md/dm-vdo/recovery-journal.c

1100

update_usages(journal, data_vio);

drivers/md/dm-vdo/recovery-journal.c

1106

vdo_waitq_enqueue_waiter(&block->entry_waiters, &data_vio->waiter);

drivers/md/dm-vdo/recovery-journal.c

1179

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/recovery-journal.c

1185

&data_vio->recovery_journal_point),

drivers/md/dm-vdo/recovery-journal.c

1189

(unsigned long long) data_vio->recovery_journal_point.sequence_number,

drivers/md/dm-vdo/recovery-journal.c

1190

data_vio->recovery_journal_point.entry_count);

drivers/md/dm-vdo/recovery-journal.c

1192

journal->commit_point = data_vio->recovery_journal_point;

drivers/md/dm-vdo/recovery-journal.c

1193

data_vio->last_async_operation = VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS;

drivers/md/dm-vdo/recovery-journal.c

1195

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/recovery-journal.c

1203

has_decrement = (data_vio->decrement_updater.zpbn.pbn != VDO_ZERO_BLOCK);

drivers/md/dm-vdo/recovery-journal.c

1204

if ((data_vio->increment_updater.zpbn.pbn != VDO_ZERO_BLOCK) || !has_decrement)

drivers/md/dm-vdo/recovery-journal.c

1205

continue_data_vio(data_vio);

drivers/md/dm-vdo/recovery-journal.c

1208

vdo_launch_completion(&data_vio->decrement_completion);

drivers/md/dm-vdo/recovery-journal.c

1337

struct data_vio *data_vio =

drivers/md/dm-vdo/recovery-journal.c

1339

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/recovery-journal.c

1351

.pbn = data_vio->increment_updater.zpbn.pbn,

drivers/md/dm-vdo/recovery-journal.c

1352

.state = data_vio->increment_updater.zpbn.state,

drivers/md/dm-vdo/recovery-journal.c

1355

.pbn = data_vio->decrement_updater.zpbn.pbn,

drivers/md/dm-vdo/recovery-journal.c

1356

.state = data_vio->decrement_updater.zpbn.state,

drivers/md/dm-vdo/recovery-journal.c

1358

.operation = data_vio->increment_updater.operation,

drivers/md/dm-vdo/recovery-journal.c

1362

data_vio->recovery_sequence_number = block->sequence_number;

drivers/md/dm-vdo/recovery-journal.c

1365

vdo_waitq_enqueue_waiter(&block->commit_waiters, &data_vio->waiter);

drivers/md/dm-vdo/recovery-journal.c

1454

struct data_vio *data_vio)

drivers/md/dm-vdo/recovery-journal.c

1458

continue_data_vio_with_error(data_vio, VDO_INVALID_ADMIN_STATE);

drivers/md/dm-vdo/recovery-journal.c

1463

continue_data_vio_with_error(data_vio, VDO_READ_ONLY);

drivers/md/dm-vdo/recovery-journal.c

1467

VDO_ASSERT_LOG_ONLY(data_vio->recovery_sequence_number == 0,

drivers/md/dm-vdo/recovery-journal.c

1471

vdo_waitq_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter);

drivers/md/dm-vdo/recovery-journal.h

286

struct data_vio *data_vio);

drivers/md/dm-vdo/slab-depot.c

1696

struct data_vio *data_vio = data_vio_from_reference_updater(updater);

drivers/md/dm-vdo/slab-depot.c

1703

sequence_number_t recovery_block = data_vio->recovery_journal_point.sequence_number;

drivers/md/dm-vdo/slab-depot.c

1725

expand_journal_point(data_vio->recovery_journal_point,

drivers/md/dm-vdo/slab-depot.c

1743

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/slab-depot.c

1745

vdo_continue_completion(&data_vio->decrement_completion, result);

drivers/md/dm-vdo/slab-depot.c

3154

struct data_vio *data_vio = data_vio_from_reference_updater(updater);

drivers/md/dm-vdo/slab-depot.c

3157

continue_data_vio_with_error(data_vio, VDO_READ_ONLY);

drivers/md/dm-vdo/slab-depot.c

3161

vdo_continue_completion(&data_vio->decrement_completion, VDO_READ_ONLY);

drivers/md/dm-vdo/types.h

332

struct data_vio;