drivers/md/dm-vdo/block-map.c

1712

static void release_page_lock(struct data_vio *data_vio, char *what)

drivers/md/dm-vdo/block-map.c

1716

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1722

zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

1730

static void finish_lookup(struct data_vio *data_vio, int result)

drivers/md/dm-vdo/block-map.c

1732

data_vio->tree_lock.height = 0;

drivers/md/dm-vdo/block-map.c

1734

--data_vio->logical.zone->block_map_zone->active_lookups;

drivers/md/dm-vdo/block-map.c

1736

set_data_vio_logical_callback(data_vio, continue_data_vio_with_block_map_slot);

drivers/md/dm-vdo/block-map.c

1737

data_vio->vio.completion.error_handler = handle_data_vio_error;

drivers/md/dm-vdo/block-map.c

1738

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/block-map.c

1743

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/block-map.c

1746

if (!data_vio->write) {

drivers/md/dm-vdo/block-map.c

1753

finish_lookup(data_vio, result);

drivers/md/dm-vdo/block-map.c

1756

static void abort_lookup(struct data_vio *data_vio, int result, char *what)

drivers/md/dm-vdo/block-map.c

1759

enter_zone_read_only_mode(data_vio->logical.zone->block_map_zone, result);

drivers/md/dm-vdo/block-map.c

1761

if (data_vio->tree_lock.locked) {

drivers/md/dm-vdo/block-map.c

1762

release_page_lock(data_vio, what);

drivers/md/dm-vdo/block-map.c

1763

vdo_waitq_notify_all_waiters(&data_vio->tree_lock.waiters,

drivers/md/dm-vdo/block-map.c

1768

finish_lookup(data_vio, result);

drivers/md/dm-vdo/block-map.c

1771

static void abort_load(struct data_vio *data_vio, int result)

drivers/md/dm-vdo/block-map.c

1773

abort_lookup(data_vio, result, "load");

drivers/md/dm-vdo/block-map.c

1792

static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.c

1794

struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.c

1796

static void continue_with_loaded_page(struct data_vio *data_vio,

drivers/md/dm-vdo/block-map.c

1799

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1804

if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) {

drivers/md/dm-vdo/block-map.c

1810

abort_load(data_vio, VDO_BAD_MAPPING);

drivers/md/dm-vdo/block-map.c

1816

allocate_block_map_page(data_vio->logical.zone->block_map_zone,

drivers/md/dm-vdo/block-map.c

1817

data_vio);

drivers/md/dm-vdo/block-map.c

1823

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

1828

load_block_map_page(data_vio->logical.zone->block_map_zone, data_vio);

drivers/md/dm-vdo/block-map.c

1833

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/block-map.c

1835

data_vio->tree_lock.height--;

drivers/md/dm-vdo/block-map.c

1836

continue_with_loaded_page(data_vio, context);

drivers/md/dm-vdo/block-map.c

1847

struct data_vio *data_vio = completion->parent;

drivers/md/dm-vdo/block-map.c

1849

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1862

release_page_lock(data_vio, "load");

drivers/md/dm-vdo/block-map.c

1864

continue_with_loaded_page(data_vio, page);

drivers/md/dm-vdo/block-map.c

1872

struct data_vio *data_vio = completion->parent;

drivers/md/dm-vdo/block-map.c

1876

abort_load(data_vio, result);

drivers/md/dm-vdo/block-map.c

1882

struct data_vio *data_vio = vio->completion.parent;

drivers/md/dm-vdo/block-map.c

1885

data_vio->logical.zone->thread_id);

drivers/md/dm-vdo/block-map.c

1891

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/block-map.c

1892

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1895

pooled->vio.completion.parent = data_vio;

drivers/md/dm-vdo/block-map.c

1904

static int attempt_page_lock(struct block_map_zone *zone, struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

1908

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1928

data_vio->tree_lock.locked = true;

drivers/md/dm-vdo/block-map.c

1933

vdo_waitq_enqueue_waiter(&lock_holder->waiters, &data_vio->waiter);

drivers/md/dm-vdo/block-map.c

1938

static void load_block_map_page(struct block_map_zone *zone, struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

1942

result = attempt_page_lock(zone, data_vio);

drivers/md/dm-vdo/block-map.c

1944

abort_load(data_vio, result);

drivers/md/dm-vdo/block-map.c

1948

if (data_vio->tree_lock.locked) {

drivers/md/dm-vdo/block-map.c

1949

data_vio->waiter.callback = load_page;

drivers/md/dm-vdo/block-map.c

1950

acquire_vio_from_pool(zone->vio_pool, &data_vio->waiter);

drivers/md/dm-vdo/block-map.c

1956

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

1959

data_vio->logical.zone->thread_id))

drivers/md/dm-vdo/block-map.c

1962

abort_lookup(data_vio, completion->result, "allocation");

drivers/md/dm-vdo/block-map.c

1967

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/block-map.c

1968

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

1972

data_vio->tree_lock.tree_slots[tree_lock->height].block_map_slot.pbn = pbn;

drivers/md/dm-vdo/block-map.c

1975

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

1979

allocate_block_map_page(data_vio->logical.zone->block_map_zone, data_vio);

drivers/md/dm-vdo/block-map.c

2089

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2090

struct block_map_zone *zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

2091

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

2094

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2102

vdo_update_block_map_page(page, data_vio, pbn,

drivers/md/dm-vdo/block-map.c

2133

release_page_lock(data_vio, "allocation");

drivers/md/dm-vdo/block-map.c

2137

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

2141

allocate_block_map_page(zone, data_vio);

drivers/md/dm-vdo/block-map.c

2146

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2148

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2150

release_data_vio_allocation_lock(data_vio, true);

drivers/md/dm-vdo/block-map.c

2151

launch_data_vio_logical_callback(data_vio, finish_block_map_allocation);

drivers/md/dm-vdo/block-map.c

2161

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2163

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2166

vdo_modify_reference_count(completion, &data_vio->increment_updater);

drivers/md/dm-vdo/block-map.c

2171

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2173

assert_data_vio_in_journal_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2175

set_data_vio_allocated_zone_callback(data_vio,

drivers/md/dm-vdo/block-map.c

2177

vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio);

drivers/md/dm-vdo/block-map.c

2182

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/block-map.c

2183

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

2186

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/block-map.c

2188

if (!vdo_allocate_block_in_zone(data_vio))

drivers/md/dm-vdo/block-map.c

2191

pbn = data_vio->allocation.pbn;

drivers/md/dm-vdo/block-map.c

2193

data_vio->increment_updater = (struct reference_updater) {

drivers/md/dm-vdo/block-map.c

2200

.lock = data_vio->allocation.lock,

drivers/md/dm-vdo/block-map.c

2203

launch_data_vio_journal_callback(data_vio, journal_block_map_allocation);

drivers/md/dm-vdo/block-map.c

2207

struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

2211

if (!data_vio->write || data_vio->is_discard) {

drivers/md/dm-vdo/block-map.c

2213

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

2217

result = attempt_page_lock(zone, data_vio);

drivers/md/dm-vdo/block-map.c

2219

abort_lookup(data_vio, result, "allocation");

drivers/md/dm-vdo/block-map.c

2223

if (!data_vio->tree_lock.locked)

drivers/md/dm-vdo/block-map.c

2226

data_vio_allocate_data_block(data_vio, VIO_BLOCK_MAP_WRITE_LOCK,

drivers/md/dm-vdo/block-map.c

2237

void vdo_find_block_map_slot(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

2243

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

2244

struct block_map_zone *zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

2248

finish_lookup(data_vio, VDO_SHUTTING_DOWN);

drivers/md/dm-vdo/block-map.c

2253

data_vio->logical.lbn % VDO_BLOCK_MAP_ENTRIES_PER_PAGE;

drivers/md/dm-vdo/block-map.c

2282

if (is_invalid_tree_entry(vdo_from_data_vio(data_vio), &mapping, lock->height)) {

drivers/md/dm-vdo/block-map.c

2288

abort_load(data_vio, VDO_BAD_MAPPING);

drivers/md/dm-vdo/block-map.c

2294

allocate_block_map_page(zone, data_vio);

drivers/md/dm-vdo/block-map.c

2301

finish_lookup(data_vio, VDO_SUCCESS);

drivers/md/dm-vdo/block-map.c

2306

load_block_map_page(zone, data_vio);

drivers/md/dm-vdo/block-map.c

2965

zone_count_t vdo_compute_logical_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

2967

struct block_map *map = vdo_from_data_vio(data_vio)->block_map;

drivers/md/dm-vdo/block-map.c

2968

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

2969

page_number_t page_number = data_vio->logical.lbn / VDO_BLOCK_MAP_ENTRIES_PER_PAGE;

drivers/md/dm-vdo/block-map.c

3096

static void fetch_mapping_page(struct data_vio *data_vio, bool modifiable,

drivers/md/dm-vdo/block-map.c

3099

struct block_map_zone *zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

3102

continue_data_vio_with_error(data_vio, VDO_SHUTTING_DOWN);

drivers/md/dm-vdo/block-map.c

3106

vdo_get_page(&data_vio->page_completion, zone,

drivers/md/dm-vdo/block-map.c

3107

data_vio->tree_lock.tree_slots[0].block_map_slot.pbn,

drivers/md/dm-vdo/block-map.c

3108

modifiable, &data_vio->vio.completion,

drivers/md/dm-vdo/block-map.c

3118

static void clear_mapped_location(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

3120

data_vio->mapped = (struct zoned_pbn) {

drivers/md/dm-vdo/block-map.c

3134

static int __must_check set_mapped_location(struct data_vio *data_vio,

drivers/md/dm-vdo/block-map.c

3143

result = vdo_get_physical_zone(vdo_from_data_vio(data_vio),

drivers/md/dm-vdo/block-map.c

3144

mapped.pbn, &data_vio->mapped.zone);

drivers/md/dm-vdo/block-map.c

3146

data_vio->mapped.pbn = mapped.pbn;

drivers/md/dm-vdo/block-map.c

3147

data_vio->mapped.state = mapped.state;

drivers/md/dm-vdo/block-map.c

3171

if (!data_vio->write)

drivers/md/dm-vdo/block-map.c

3178

clear_mapped_location(data_vio);

drivers/md/dm-vdo/block-map.c

3189

struct data_vio *data_vio = as_data_vio(completion->parent);

drivers/md/dm-vdo/block-map.c

3204

tree_slot = &data_vio->tree_lock.tree_slots[0];

drivers/md/dm-vdo/block-map.c

3207

result = set_mapped_location(data_vio, entry);

drivers/md/dm-vdo/block-map.c

3211

void vdo_update_block_map_page(struct block_map_page *page, struct data_vio *data_vio,

drivers/md/dm-vdo/block-map.c

3216

struct block_map_zone *zone = data_vio->logical.zone->block_map_zone;

drivers/md/dm-vdo/block-map.c

3220

struct tree_lock *tree_lock = &data_vio->tree_lock;

drivers/md/dm-vdo/block-map.c

3228

new_locked = data_vio->recovery_sequence_number;

drivers/md/dm-vdo/block-map.c

3249

data_vio->recovery_sequence_number = 0;

drivers/md/dm-vdo/block-map.c

3254

struct data_vio *data_vio = as_data_vio(completion->parent);

drivers/md/dm-vdo/block-map.c

3275

data_vio, data_vio->new_mapped.pbn,

drivers/md/dm-vdo/block-map.c

3276

data_vio->new_mapped.state, &info->recovery_lock);

drivers/md/dm-vdo/block-map.c

3284

void vdo_get_mapped_block(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

3286

if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/block-map.c

3291

clear_mapped_location(data_vio);

drivers/md/dm-vdo/block-map.c

3292

continue_data_vio(data_vio);

drivers/md/dm-vdo/block-map.c

3296

fetch_mapping_page(data_vio, false, get_mapping_from_fetched_page);

drivers/md/dm-vdo/block-map.c

3300

void vdo_put_mapped_block(struct data_vio *data_vio)

drivers/md/dm-vdo/block-map.c

3302

fetch_mapping_page(data_vio, true, put_mapping_in_fetched_page);

drivers/md/dm-vdo/block-map.h

323

void vdo_find_block_map_slot(struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.h

358

zone_count_t vdo_compute_logical_zone(struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.h

363

void vdo_update_block_map_page(struct block_map_page *page, struct data_vio *data_vio,

drivers/md/dm-vdo/block-map.h

368

void vdo_get_mapped_block(struct data_vio *data_vio);

drivers/md/dm-vdo/block-map.h

370

void vdo_put_mapped_block(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.c

1061

struct data_vio *data_vio = &pool->data_vios[i];

drivers/md/dm-vdo/data-vio.c

1063

if (!list_empty(&data_vio->pool_entry))

drivers/md/dm-vdo/data-vio.c

1066

dump_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1094

static void update_data_vio_error_stats(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1107

if (data_vio->read)

drivers/md/dm-vdo/data-vio.c

1110

if (data_vio->write)

drivers/md/dm-vdo/data-vio.c

1113

if (data_vio->fua)

drivers/md/dm-vdo/data-vio.c

1116

update_vio_error_stats(&data_vio->vio,

drivers/md/dm-vdo/data-vio.c

1119

(unsigned long long) data_vio->logical.lbn,

drivers/md/dm-vdo/data-vio.c

1120

get_data_vio_operation_name(data_vio));

drivers/md/dm-vdo/data-vio.c

1123

static void perform_cleanup_stage(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

1133

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1135

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1136

release_data_vio_allocation_lock(data_vio, false);

drivers/md/dm-vdo/data-vio.c

1137

perform_cleanup_stage(data_vio, VIO_RELEASE_RECOVERY_LOCKS);

drivers/md/dm-vdo/data-vio.c

1141

static void release_lock(struct data_vio *data_vio, struct lbn_lock *lock)

drivers/md/dm-vdo/data-vio.c

1144

struct data_vio *lock_holder;

drivers/md/dm-vdo/data-vio.c

1148

struct data_vio *lock_holder = vdo_int_map_get(lock_map, lock->lbn);

drivers/md/dm-vdo/data-vio.c

1150

VDO_ASSERT_LOG_ONLY((data_vio != lock_holder),

drivers/md/dm-vdo/data-vio.c

1158

VDO_ASSERT_LOG_ONLY((data_vio == lock_holder),

drivers/md/dm-vdo/data-vio.c

1165

static void transfer_lock(struct data_vio *data_vio, struct lbn_lock *lock)

drivers/md/dm-vdo/data-vio.c

1167

struct data_vio *lock_holder, *next_lock_holder;

drivers/md/dm-vdo/data-vio.c

1187

VDO_ASSERT_LOG_ONLY((lock_holder == data_vio),

drivers/md/dm-vdo/data-vio.c

1214

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1215

struct lbn_lock *lock = &data_vio->logical;

drivers/md/dm-vdo/data-vio.c

1217

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1220

transfer_lock(data_vio, lock);

drivers/md/dm-vdo/data-vio.c

1222

release_lock(data_vio, lock);

drivers/md/dm-vdo/data-vio.c

1224

vdo_release_flush_generation_lock(data_vio);

drivers/md/dm-vdo/data-vio.c

1225

perform_cleanup_stage(data_vio, VIO_CLEANUP_DONE);

drivers/md/dm-vdo/data-vio.c

1231

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1233

assert_data_vio_in_hash_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1235

vdo_clean_failed_hash_lock(data_vio);

drivers/md/dm-vdo/data-vio.c

1239

vdo_release_hash_lock(data_vio);

drivers/md/dm-vdo/data-vio.c

1240

perform_cleanup_stage(data_vio, VIO_RELEASE_LOGICAL);

drivers/md/dm-vdo/data-vio.c

1250

static void finish_cleanup(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1252

struct vdo_completion *completion = &data_vio->vio.completion;

drivers/md/dm-vdo/data-vio.c

1253

u32 discard_size = min_t(u32, data_vio->remaining_discard,

drivers/md/dm-vdo/data-vio.c

1254

VDO_BLOCK_SIZE - data_vio->offset);

drivers/md/dm-vdo/data-vio.c

1256

VDO_ASSERT_LOG_ONLY(data_vio->allocation.lock == NULL,

drivers/md/dm-vdo/data-vio.c

1258

VDO_ASSERT_LOG_ONLY(data_vio->hash_lock == NULL,

drivers/md/dm-vdo/data-vio.c

1260

if ((data_vio->remaining_discard <= discard_size) ||

drivers/md/dm-vdo/data-vio.c

1269

data_vio->remaining_discard -= discard_size;

drivers/md/dm-vdo/data-vio.c

1270

data_vio->is_partial = (data_vio->remaining_discard < VDO_BLOCK_SIZE);

drivers/md/dm-vdo/data-vio.c

1271

data_vio->read = data_vio->is_partial;

drivers/md/dm-vdo/data-vio.c

1272

data_vio->offset = 0;

drivers/md/dm-vdo/data-vio.c

1274

data_vio->first_reference_operation_complete = false;

drivers/md/dm-vdo/data-vio.c

1275

launch_data_vio(data_vio, data_vio->logical.lbn + 1);

drivers/md/dm-vdo/data-vio.c

1279

static void perform_cleanup_stage(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

1282

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1286

if (data_vio->hash_lock != NULL) {

drivers/md/dm-vdo/data-vio.c

1287

launch_data_vio_hash_zone_callback(data_vio, clean_hash_lock);

drivers/md/dm-vdo/data-vio.c

1293

if (data_vio_has_allocation(data_vio)) {

drivers/md/dm-vdo/data-vio.c

1294

launch_data_vio_allocated_zone_callback(data_vio,

drivers/md/dm-vdo/data-vio.c

1301

if ((data_vio->recovery_sequence_number > 0) &&

drivers/md/dm-vdo/data-vio.c

1303

(data_vio->vio.completion.result != VDO_READ_ONLY))

drivers/md/dm-vdo/data-vio.c

1308

launch_data_vio_logical_callback(data_vio, release_logical_lock);

drivers/md/dm-vdo/data-vio.c

1312

finish_cleanup(data_vio);

drivers/md/dm-vdo/data-vio.c

1318

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1321

data_vio->last_async_operation = VIO_ASYNC_OP_CLEANUP;

drivers/md/dm-vdo/data-vio.c

1322

perform_cleanup_stage(data_vio,

drivers/md/dm-vdo/data-vio.c

1323

(data_vio->write ? VIO_CLEANUP_START : VIO_RELEASE_LOGICAL));

drivers/md/dm-vdo/data-vio.c

1332

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1336

(unsigned long long) data_vio->logical.lbn,

drivers/md/dm-vdo/data-vio.c

1337

(unsigned long long) data_vio->new_mapped.pbn,

drivers/md/dm-vdo/data-vio.c

1338

(unsigned long long) data_vio->mapped.pbn,

drivers/md/dm-vdo/data-vio.c

1339

(unsigned long long) data_vio->allocation.pbn,

drivers/md/dm-vdo/data-vio.c

1340

get_data_vio_operation_name(data_vio));

drivers/md/dm-vdo/data-vio.c

1348

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1350

if ((completion->result == VDO_READ_ONLY) || (data_vio->user_bio == NULL))

drivers/md/dm-vdo/data-vio.c

1353

update_data_vio_error_stats(data_vio);

drivers/md/dm-vdo/data-vio.c

1362

const char *get_data_vio_operation_name(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1367

return ((data_vio->last_async_operation < MAX_VIO_ASYNC_OPERATION_NUMBER) ?

drivers/md/dm-vdo/data-vio.c

1368

ASYNC_OPERATION_NAMES[data_vio->last_async_operation] :

drivers/md/dm-vdo/data-vio.c

1380

void data_vio_allocate_data_block(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

1384

struct allocation *allocation = &data_vio->allocation;

drivers/md/dm-vdo/data-vio.c

1389

allocation->zone = vdo_get_next_allocation_zone(data_vio->logical.zone);

drivers/md/dm-vdo/data-vio.c

1392

data_vio->vio.completion.error_handler = error_handler;

drivers/md/dm-vdo/data-vio.c

1393

launch_data_vio_allocated_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.c

1403

void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset)

drivers/md/dm-vdo/data-vio.c

1405

struct allocation *allocation = &data_vio->allocation;

drivers/md/dm-vdo/data-vio.c

1408

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1423

int uncompress_data_vio(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

1428

struct compressed_block *block = data_vio->compression.block;

drivers/md/dm-vdo/data-vio.c

1455

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1456

char *data = data_vio->vio.data;

drivers/md/dm-vdo/data-vio.c

1457

struct bio *bio = data_vio->user_bio;

drivers/md/dm-vdo/data-vio.c

1459

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1462

memset(data + data_vio->offset, '\0', min_t(u32,

drivers/md/dm-vdo/data-vio.c

1463

data_vio->remaining_discard,

drivers/md/dm-vdo/data-vio.c

1464

VDO_BLOCK_SIZE - data_vio->offset));

drivers/md/dm-vdo/data-vio.c

1466

copy_from_bio(bio, data + data_vio->offset);

drivers/md/dm-vdo/data-vio.c

1469

data_vio->is_zero = mem_is_zero(data, VDO_BLOCK_SIZE);

drivers/md/dm-vdo/data-vio.c

1470

data_vio->read = false;

drivers/md/dm-vdo/data-vio.c

1471

launch_data_vio_logical_callback(data_vio,

drivers/md/dm-vdo/data-vio.c

1477

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1478

char *data = data_vio->vio.data;

drivers/md/dm-vdo/data-vio.c

1479

bool compressed = vdo_is_state_compressed(data_vio->mapped.state);

drivers/md/dm-vdo/data-vio.c

1481

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1484

int result = uncompress_data_vio(data_vio, data_vio->mapped.state, data);

drivers/md/dm-vdo/data-vio.c

1487

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

1492

if (data_vio->write) {

drivers/md/dm-vdo/data-vio.c

1497

if (compressed || data_vio->is_partial)

drivers/md/dm-vdo/data-vio.c

1498

copy_to_bio(data_vio->user_bio, data + data_vio->offset);

drivers/md/dm-vdo/data-vio.c

1500

acknowledge_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1506

struct data_vio *data_vio = vio_as_data_vio(bio->bi_private);

drivers/md/dm-vdo/data-vio.c

1511

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

1515

launch_data_vio_cpu_callback(data_vio, complete_read,

drivers/md/dm-vdo/data-vio.c

1521

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1523

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1525

if (data_vio->is_partial) {

drivers/md/dm-vdo/data-vio.c

1526

memset(data_vio->vio.data, 0, VDO_BLOCK_SIZE);

drivers/md/dm-vdo/data-vio.c

1527

if (data_vio->write) {

drivers/md/dm-vdo/data-vio.c

1532

zero_fill_bio(data_vio->user_bio);

drivers/md/dm-vdo/data-vio.c

1546

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1550

if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

1551

launch_data_vio_cpu_callback(data_vio, complete_zero_read,

drivers/md/dm-vdo/data-vio.c

1556

data_vio->last_async_operation = VIO_ASYNC_OP_READ_DATA_VIO;

drivers/md/dm-vdo/data-vio.c

1557

if (vdo_is_state_compressed(data_vio->mapped.state)) {

drivers/md/dm-vdo/data-vio.c

1558

result = vio_reset_bio(vio, (char *) data_vio->compression.block,

drivers/md/dm-vdo/data-vio.c

1559

read_endio, REQ_OP_READ, data_vio->mapped.pbn);

drivers/md/dm-vdo/data-vio.c

1561

blk_opf_t opf = ((data_vio->user_bio->bi_opf & PASSTHROUGH_FLAGS) | REQ_OP_READ);

drivers/md/dm-vdo/data-vio.c

1563

if (data_vio->is_partial) {

drivers/md/dm-vdo/data-vio.c

1565

data_vio->mapped.pbn);

drivers/md/dm-vdo/data-vio.c

1569

bio_init_clone(data_vio->user_bio->bi_bdev, vio->bio,

drivers/md/dm-vdo/data-vio.c

1570

data_vio->user_bio, GFP_KERNEL);

drivers/md/dm-vdo/data-vio.c

1574

data_vio->mapped.pbn);

drivers/md/dm-vdo/data-vio.c

1579

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

1583

vdo_submit_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1586

static inline struct data_vio *

drivers/md/dm-vdo/data-vio.c

1592

return container_of(completion, struct data_vio, decrement_completion);

drivers/md/dm-vdo/data-vio.c

1603

struct data_vio *data_vio = reference_count_update_completion_as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1605

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1607

if (!data_vio->first_reference_operation_complete) {

drivers/md/dm-vdo/data-vio.c

1609

data_vio->first_reference_operation_complete = true;

drivers/md/dm-vdo/data-vio.c

1613

completion = &data_vio->vio.completion;

drivers/md/dm-vdo/data-vio.c

1614

vdo_set_completion_result(completion, data_vio->decrement_completion.result);

drivers/md/dm-vdo/data-vio.c

1621

if (data_vio->hash_lock != NULL)

drivers/md/dm-vdo/data-vio.c

1622

set_data_vio_hash_zone_callback(data_vio, vdo_continue_hash_lock);

drivers/md/dm-vdo/data-vio.c

1626

data_vio->last_async_operation = VIO_ASYNC_OP_PUT_MAPPED_BLOCK;

drivers/md/dm-vdo/data-vio.c

1627

vdo_put_mapped_block(data_vio);

drivers/md/dm-vdo/data-vio.c

1632

struct data_vio *data_vio = container_of(completion, struct data_vio,

drivers/md/dm-vdo/data-vio.c

1635

assert_data_vio_in_mapped_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1638

data_vio->logical.zone->thread_id);

drivers/md/dm-vdo/data-vio.c

1640

vdo_modify_reference_count(completion, &data_vio->decrement_updater);

drivers/md/dm-vdo/data-vio.c

1645

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1647

assert_data_vio_in_new_mapped_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1649

if (data_vio->downgrade_allocation_lock) {

drivers/md/dm-vdo/data-vio.c

1657

vdo_downgrade_pbn_write_lock(data_vio->allocation.lock, false);

drivers/md/dm-vdo/data-vio.c

1660

set_data_vio_logical_callback(data_vio, update_block_map);

drivers/md/dm-vdo/data-vio.c

1662

vdo_modify_reference_count(completion, &data_vio->increment_updater);

drivers/md/dm-vdo/data-vio.c

1668

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1670

assert_data_vio_in_journal_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1672

data_vio->decrement_updater.operation = VDO_JOURNAL_DATA_REMAPPING;

drivers/md/dm-vdo/data-vio.c

1673

data_vio->decrement_updater.zpbn = data_vio->mapped;

drivers/md/dm-vdo/data-vio.c

1674

if (data_vio->new_mapped.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

1675

data_vio->first_reference_operation_complete = true;

drivers/md/dm-vdo/data-vio.c

1676

if (data_vio->mapped.pbn == VDO_ZERO_BLOCK)

drivers/md/dm-vdo/data-vio.c

1677

set_data_vio_logical_callback(data_vio, update_block_map);

drivers/md/dm-vdo/data-vio.c

1679

set_data_vio_new_mapped_zone_callback(data_vio,

drivers/md/dm-vdo/data-vio.c

1683

if (data_vio->mapped.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

1684

data_vio->first_reference_operation_complete = true;

drivers/md/dm-vdo/data-vio.c

1686

vdo_set_completion_callback(&data_vio->decrement_completion,

drivers/md/dm-vdo/data-vio.c

1688

data_vio->mapped.zone->thread_id);

drivers/md/dm-vdo/data-vio.c

1691

data_vio->last_async_operation = VIO_ASYNC_OP_JOURNAL_REMAPPING;

drivers/md/dm-vdo/data-vio.c

1692

vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio);

drivers/md/dm-vdo/data-vio.c

1704

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1706

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1708

data_vio->last_async_operation = VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_WRITE;

drivers/md/dm-vdo/data-vio.c

1709

set_data_vio_journal_callback(data_vio, journal_remapping);

drivers/md/dm-vdo/data-vio.c

1710

vdo_get_mapped_block(data_vio);

drivers/md/dm-vdo/data-vio.c

1713

void update_metadata_for_data_vio_write(struct data_vio *data_vio, struct pbn_lock *lock)

drivers/md/dm-vdo/data-vio.c

1715

data_vio->increment_updater = (struct reference_updater) {

drivers/md/dm-vdo/data-vio.c

1718

.zpbn = data_vio->new_mapped,

drivers/md/dm-vdo/data-vio.c

1722

launch_data_vio_logical_callback(data_vio, read_old_block_mapping);

drivers/md/dm-vdo/data-vio.c

1733

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1735

assert_data_vio_in_packer_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1737

if (!vdo_get_compressing(vdo_from_data_vio(data_vio)) ||

drivers/md/dm-vdo/data-vio.c

1738

get_data_vio_compression_status(data_vio).may_not_compress) {

drivers/md/dm-vdo/data-vio.c

1739

write_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1743

data_vio->last_async_operation = VIO_ASYNC_OP_ATTEMPT_PACKING;

drivers/md/dm-vdo/data-vio.c

1744

vdo_attempt_packing(data_vio);

drivers/md/dm-vdo/data-vio.c

1755

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1758

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1764

size = LZ4_compress_default(data_vio->vio.data,

drivers/md/dm-vdo/data-vio.c

1765

data_vio->compression.block->data, VDO_BLOCK_SIZE,

drivers/md/dm-vdo/data-vio.c

1769

data_vio->compression.size = size;

drivers/md/dm-vdo/data-vio.c

1770

launch_data_vio_packer_callback(data_vio, pack_compressed_data);

drivers/md/dm-vdo/data-vio.c

1774

write_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1783

void launch_compress_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1785

VDO_ASSERT_LOG_ONLY(!data_vio->is_duplicate, "compressing a non-duplicate block");

drivers/md/dm-vdo/data-vio.c

1786

VDO_ASSERT_LOG_ONLY(data_vio->hash_lock != NULL,

drivers/md/dm-vdo/data-vio.c

1788

VDO_ASSERT_LOG_ONLY(data_vio_has_allocation(data_vio),

drivers/md/dm-vdo/data-vio.c

179

struct data_vio data_vios[];

drivers/md/dm-vdo/data-vio.c

1806

if (data_vio->fua ||

drivers/md/dm-vdo/data-vio.c

1807

!vdo_get_compressing(vdo_from_data_vio(data_vio)) ||

drivers/md/dm-vdo/data-vio.c

1808

((data_vio->user_bio != NULL) && (bio_op(data_vio->user_bio) == REQ_OP_DISCARD)) ||

drivers/md/dm-vdo/data-vio.c

1809

(advance_data_vio_compression_stage(data_vio).stage != DATA_VIO_COMPRESSING)) {

drivers/md/dm-vdo/data-vio.c

1810

write_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1814

data_vio->last_async_operation = VIO_ASYNC_OP_COMPRESS_DATA_VIO;

drivers/md/dm-vdo/data-vio.c

1815

launch_data_vio_cpu_callback(data_vio, compress_data_vio,

drivers/md/dm-vdo/data-vio.c

1828

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1830

assert_data_vio_on_cpu_thread(data_vio);

drivers/md/dm-vdo/data-vio.c

1831

VDO_ASSERT_LOG_ONLY(!data_vio->is_zero, "zero blocks should not be hashed");

drivers/md/dm-vdo/data-vio.c

1833

murmurhash3_128(data_vio->vio.data, VDO_BLOCK_SIZE, 0x62ea60be,

drivers/md/dm-vdo/data-vio.c

1834

&data_vio->record_name);

drivers/md/dm-vdo/data-vio.c

1836

data_vio->hash_zone = vdo_select_hash_zone(vdo_from_data_vio(data_vio)->hash_zones,

drivers/md/dm-vdo/data-vio.c

1837

&data_vio->record_name);

drivers/md/dm-vdo/data-vio.c

1838

data_vio->last_async_operation = VIO_ASYNC_OP_ACQUIRE_VDO_HASH_LOCK;

drivers/md/dm-vdo/data-vio.c

1839

launch_data_vio_hash_zone_callback(data_vio, vdo_acquire_hash_lock);

drivers/md/dm-vdo/data-vio.c

1843

static void prepare_for_dedupe(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1846

VDO_ASSERT_LOG_ONLY(!data_vio->is_zero, "must not prepare to dedupe zero blocks");

drivers/md/dm-vdo/data-vio.c

1852

data_vio->last_async_operation = VIO_ASYNC_OP_HASH_DATA_VIO;

drivers/md/dm-vdo/data-vio.c

1853

launch_data_vio_cpu_callback(data_vio, hash_data_vio, CPU_Q_HASH_BLOCK_PRIORITY);

drivers/md/dm-vdo/data-vio.c

1863

struct data_vio *data_vio = vio_as_data_vio((struct vio *) bio->bi_private);

drivers/md/dm-vdo/data-vio.c

1866

vdo_set_completion_result(&data_vio->vio.completion,

drivers/md/dm-vdo/data-vio.c

1868

data_vio->downgrade_allocation_lock = true;

drivers/md/dm-vdo/data-vio.c

1869

update_metadata_for_data_vio_write(data_vio, data_vio->allocation.lock);

drivers/md/dm-vdo/data-vio.c

1873

void write_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1878

if (!data_vio_has_allocation(data_vio)) {

drivers/md/dm-vdo/data-vio.c

1883

continue_data_vio_with_error(data_vio, VDO_NO_SPACE);

drivers/md/dm-vdo/data-vio.c

1893

status = get_data_vio_compression_status(data_vio);

drivers/md/dm-vdo/data-vio.c

1895

!set_data_vio_compression_status(data_vio, status, new_status));

drivers/md/dm-vdo/data-vio.c

1898

result = vio_reset_bio(&data_vio->vio, data_vio->vio.data,

drivers/md/dm-vdo/data-vio.c

1900

data_vio->allocation.pbn);

drivers/md/dm-vdo/data-vio.c

1902

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

1906

data_vio->last_async_operation = VIO_ASYNC_OP_WRITE_DATA_VIO;

drivers/md/dm-vdo/data-vio.c

1907

vdo_submit_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1918

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1924

VDO_ASSERT_LOG_ONLY(data_vio_has_flush_generation_lock(data_vio),

drivers/md/dm-vdo/data-vio.c

1926

acknowledge_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

1927

if (data_vio->new_mapped.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

1929

update_metadata_for_data_vio_write(data_vio, NULL);

drivers/md/dm-vdo/data-vio.c

1933

prepare_for_dedupe(data_vio);

drivers/md/dm-vdo/data-vio.c

1944

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1946

assert_data_vio_in_allocated_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

1948

if (!vdo_allocate_block_in_zone(data_vio))

drivers/md/dm-vdo/data-vio.c

1952

WRITE_ONCE(data_vio->allocation_succeeded, true);

drivers/md/dm-vdo/data-vio.c

1953

data_vio->new_mapped = (struct zoned_pbn) {

drivers/md/dm-vdo/data-vio.c

1954

.zone = data_vio->allocation.zone,

drivers/md/dm-vdo/data-vio.c

1955

.pbn = data_vio->allocation.pbn,

drivers/md/dm-vdo/data-vio.c

1959

if (data_vio->fua ||

drivers/md/dm-vdo/data-vio.c

1960

data_vio->remaining_discard > (u32) (VDO_BLOCK_SIZE - data_vio->offset)) {

drivers/md/dm-vdo/data-vio.c

1961

prepare_for_dedupe(data_vio);

drivers/md/dm-vdo/data-vio.c

1965

data_vio->last_async_operation = VIO_ASYNC_OP_ACKNOWLEDGE_WRITE;

drivers/md/dm-vdo/data-vio.c

1966

launch_data_vio_on_bio_ack_queue(data_vio, acknowledge_write_callback);

drivers/md/dm-vdo/data-vio.c

1977

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

1983

prepare_for_dedupe(data_vio);

drivers/md/dm-vdo/data-vio.c

1991

static int assert_is_discard(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

1993

int result = VDO_ASSERT(data_vio->is_discard,

drivers/md/dm-vdo/data-vio.c

2007

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

2009

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

2010

if (data_vio->read) {

drivers/md/dm-vdo/data-vio.c

2011

set_data_vio_logical_callback(data_vio, read_block);

drivers/md/dm-vdo/data-vio.c

2012

data_vio->last_async_operation = VIO_ASYNC_OP_GET_MAPPED_BLOCK_FOR_READ;

drivers/md/dm-vdo/data-vio.c

2013

vdo_get_mapped_block(data_vio);

drivers/md/dm-vdo/data-vio.c

2017

vdo_acquire_flush_generation_lock(data_vio);

drivers/md/dm-vdo/data-vio.c

2019

if (data_vio->tree_lock.tree_slots[0].block_map_slot.pbn == VDO_ZERO_BLOCK) {

drivers/md/dm-vdo/data-vio.c

2025

continue_data_vio_with_error(data_vio, assert_is_discard(data_vio));

drivers/md/dm-vdo/data-vio.c

2033

if (!data_vio->is_zero && (!data_vio->is_discard || data_vio->is_partial)) {

drivers/md/dm-vdo/data-vio.c

2034

data_vio_allocate_data_block(data_vio, VIO_WRITE_LOCK, allocate_block,

drivers/md/dm-vdo/data-vio.c

2043

data_vio->new_mapped.pbn = VDO_ZERO_BLOCK;

drivers/md/dm-vdo/data-vio.c

2044

if (data_vio->is_zero)

drivers/md/dm-vdo/data-vio.c

2045

data_vio->new_mapped.state = VDO_MAPPING_STATE_UNCOMPRESSED;

drivers/md/dm-vdo/data-vio.c

2047

if (data_vio->remaining_discard > (u32) (VDO_BLOCK_SIZE - data_vio->offset)) {

drivers/md/dm-vdo/data-vio.c

2049

update_metadata_for_data_vio_write(data_vio, NULL);

drivers/md/dm-vdo/data-vio.c

2053

data_vio->last_async_operation = VIO_ASYNC_OP_ACKNOWLEDGE_WRITE;

drivers/md/dm-vdo/data-vio.c

2054

launch_data_vio_on_bio_ack_queue(data_vio, acknowledge_write_callback);

drivers/md/dm-vdo/data-vio.c

244

static void initialize_lbn_lock(struct data_vio *data_vio, logical_block_number_t lbn)

drivers/md/dm-vdo/data-vio.c

246

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

248

struct lbn_lock *lock = &data_vio->logical;

drivers/md/dm-vdo/data-vio.c

253

zone_number = vdo_compute_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

257

static void launch_locked_request(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

259

data_vio->logical.locked = true;

drivers/md/dm-vdo/data-vio.c

260

if (data_vio->write) {

drivers/md/dm-vdo/data-vio.c

261

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

264

continue_data_vio_with_error(data_vio, VDO_READ_ONLY);

drivers/md/dm-vdo/data-vio.c

269

data_vio->last_async_operation = VIO_ASYNC_OP_FIND_BLOCK_MAP_SLOT;

drivers/md/dm-vdo/data-vio.c

270

vdo_find_block_map_slot(data_vio);

drivers/md/dm-vdo/data-vio.c

273

static void acknowledge_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

275

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

276

struct bio *bio = data_vio->user_bio;

drivers/md/dm-vdo/data-vio.c

277

int error = vdo_status_to_errno(data_vio->vio.completion.result);

drivers/md/dm-vdo/data-vio.c

282

VDO_ASSERT_LOG_ONLY((data_vio->remaining_discard <=

drivers/md/dm-vdo/data-vio.c

283

(u32) (VDO_BLOCK_SIZE - data_vio->offset)),

drivers/md/dm-vdo/data-vio.c

286

data_vio->user_bio = NULL;

drivers/md/dm-vdo/data-vio.c

288

if (data_vio->is_partial)

drivers/md/dm-vdo/data-vio.c

306

struct data_vio_compression_status get_data_vio_compression_status(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

308

u32 packed = atomic_read(&data_vio->compression.status);

drivers/md/dm-vdo/data-vio.c

340

set_data_vio_compression_status(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

353

actual = atomic_cmpxchg(&data_vio->compression.status, expected, replacement);

drivers/md/dm-vdo/data-vio.c

359

struct data_vio_compression_status advance_data_vio_compression_stage(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

363

get_data_vio_compression_status(data_vio);

drivers/md/dm-vdo/data-vio.c

382

if (set_data_vio_compression_status(data_vio, status, new_status))

drivers/md/dm-vdo/data-vio.c

395

bool cancel_data_vio_compression(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

400

status = get_data_vio_compression_status(data_vio);

drivers/md/dm-vdo/data-vio.c

409

if (set_data_vio_compression_status(data_vio, status, new_status))

drivers/md/dm-vdo/data-vio.c

424

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/data-vio.c

425

struct lbn_lock *lock = &data_vio->logical;

drivers/md/dm-vdo/data-vio.c

426

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

427

struct data_vio *lock_holder;

drivers/md/dm-vdo/data-vio.c

430

assert_data_vio_in_logical_zone(data_vio);

drivers/md/dm-vdo/data-vio.c

432

if (data_vio->logical.lbn >= vdo->states.vdo.config.logical_blocks) {

drivers/md/dm-vdo/data-vio.c

433

continue_data_vio_with_error(data_vio, VDO_OUT_OF_RANGE);

drivers/md/dm-vdo/data-vio.c

438

data_vio, false, (void **) &lock_holder);

drivers/md/dm-vdo/data-vio.c

440

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

446

launch_locked_request(data_vio);

drivers/md/dm-vdo/data-vio.c

452

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/data-vio.c

464

if (!data_vio->write && READ_ONCE(lock_holder->allocation_succeeded)) {

drivers/md/dm-vdo/data-vio.c

465

copy_to_bio(data_vio->user_bio, lock_holder->vio.data + data_vio->offset);

drivers/md/dm-vdo/data-vio.c

466

acknowledge_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

471

data_vio->last_async_operation = VIO_ASYNC_OP_ATTEMPT_LOGICAL_BLOCK_LOCK;

drivers/md/dm-vdo/data-vio.c

472

vdo_waitq_enqueue_waiter(&lock_holder->logical.waiters, &data_vio->waiter);

drivers/md/dm-vdo/data-vio.c

479

data_vio->compression.lock_holder = lock_holder;

drivers/md/dm-vdo/data-vio.c

480

launch_data_vio_packer_callback(data_vio,

drivers/md/dm-vdo/data-vio.c

491

static void launch_data_vio(struct data_vio *data_vio, logical_block_number_t lbn)

drivers/md/dm-vdo/data-vio.c

493

struct vdo_completion *completion = &data_vio->vio.completion;

drivers/md/dm-vdo/data-vio.c

499

memset(&data_vio->tree_lock, 0, sizeof(data_vio->tree_lock));

drivers/md/dm-vdo/data-vio.c

500

initialize_lbn_lock(data_vio, lbn);

drivers/md/dm-vdo/data-vio.c

501

INIT_LIST_HEAD(&data_vio->hash_lock_entry);

drivers/md/dm-vdo/data-vio.c

502

INIT_LIST_HEAD(&data_vio->write_entry);

drivers/md/dm-vdo/data-vio.c

504

memset(&data_vio->allocation, 0, sizeof(data_vio->allocation));

drivers/md/dm-vdo/data-vio.c

506

data_vio->is_duplicate = false;

drivers/md/dm-vdo/data-vio.c

508

memset(&data_vio->record_name, 0, sizeof(data_vio->record_name));

drivers/md/dm-vdo/data-vio.c

509

memset(&data_vio->duplicate, 0, sizeof(data_vio->duplicate));

drivers/md/dm-vdo/data-vio.c

510

vdo_reset_completion(&data_vio->decrement_completion);

drivers/md/dm-vdo/data-vio.c

513

set_data_vio_logical_callback(data_vio, attempt_logical_block_lock);

drivers/md/dm-vdo/data-vio.c

528

static void launch_bio(struct vdo *vdo, struct data_vio *data_vio, struct bio *bio)

drivers/md/dm-vdo/data-vio.c

535

memset(data_vio, 0, offsetof(struct data_vio, vio));

drivers/md/dm-vdo/data-vio.c

536

memset(&data_vio->compression, 0, offsetof(struct compression_state, block));

drivers/md/dm-vdo/data-vio.c

538

data_vio->user_bio = bio;

drivers/md/dm-vdo/data-vio.c

539

data_vio->offset = to_bytes(bio->bi_iter.bi_sector & VDO_SECTORS_PER_BLOCK_MASK);

drivers/md/dm-vdo/data-vio.c

540

data_vio->is_partial = (bio->bi_iter.bi_size < VDO_BLOCK_SIZE) || (data_vio->offset != 0);

drivers/md/dm-vdo/data-vio.c

548

data_vio->remaining_discard = bio->bi_iter.bi_size;

drivers/md/dm-vdo/data-vio.c

549

data_vio->write = true;

drivers/md/dm-vdo/data-vio.c

550

data_vio->is_discard = true;

drivers/md/dm-vdo/data-vio.c

551

if (data_vio->is_partial) {

drivers/md/dm-vdo/data-vio.c

553

data_vio->read = true;

drivers/md/dm-vdo/data-vio.c

555

} else if (data_vio->is_partial) {

drivers/md/dm-vdo/data-vio.c

557

data_vio->read = true;

drivers/md/dm-vdo/data-vio.c

559

data_vio->write = true;

drivers/md/dm-vdo/data-vio.c

561

data_vio->read = true;

drivers/md/dm-vdo/data-vio.c

567

copy_from_bio(bio, data_vio->vio.data);

drivers/md/dm-vdo/data-vio.c

568

data_vio->is_zero = mem_is_zero(data_vio->vio.data, VDO_BLOCK_SIZE);

drivers/md/dm-vdo/data-vio.c

569

data_vio->write = true;

drivers/md/dm-vdo/data-vio.c

572

if (data_vio->user_bio->bi_opf & REQ_FUA)

drivers/md/dm-vdo/data-vio.c

573

data_vio->fua = true;

drivers/md/dm-vdo/data-vio.c

576

launch_data_vio(data_vio, lbn);

drivers/md/dm-vdo/data-vio.c

579

static void assign_data_vio(struct limiter *limiter, struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

583

launch_bio(limiter->pool->completion.vdo, data_vio, bio);

drivers/md/dm-vdo/data-vio.c

605

static inline struct data_vio *get_available_data_vio(struct data_vio_pool *pool)

drivers/md/dm-vdo/data-vio.c

607

struct data_vio *data_vio =

drivers/md/dm-vdo/data-vio.c

608

list_first_entry(&pool->available, struct data_vio, pool_entry);

drivers/md/dm-vdo/data-vio.c

610

list_del_init(&data_vio->pool_entry);

drivers/md/dm-vdo/data-vio.c

611

return data_vio;

drivers/md/dm-vdo/data-vio.c

666

struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.c

669

if (data_vio->remaining_discard > 0) {

drivers/md/dm-vdo/data-vio.c

679

assign_data_vio(&pool->limiter, data_vio);

drivers/md/dm-vdo/data-vio.c

681

assign_data_vio(&pool->discard_limiter, data_vio);

drivers/md/dm-vdo/data-vio.c

683

list_add(&data_vio->pool_entry, returned);

drivers/md/dm-vdo/data-vio.c

715

struct data_vio *data_vio;

drivers/md/dm-vdo/data-vio.c

721

data_vio = as_data_vio(container_of(entry, struct vdo_completion,

drivers/md/dm-vdo/data-vio.c

723

acknowledge_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

724

reuse_or_release_resources(pool, data_vio, &returned);

drivers/md/dm-vdo/data-vio.c

783

static int initialize_data_vio(struct data_vio *data_vio, struct vdo *vdo)

drivers/md/dm-vdo/data-vio.c

790

&data_vio->vio.data);

drivers/md/dm-vdo/data-vio.c

796

&data_vio->compression.block);

drivers/md/dm-vdo/data-vio.c

803

&data_vio->scratch_block);

drivers/md/dm-vdo/data-vio.c

813

vdo_initialize_completion(&data_vio->decrement_completion, vdo,

drivers/md/dm-vdo/data-vio.c

815

initialize_vio(&data_vio->vio, bio, 1, VIO_TYPE_DATA, VIO_PRIORITY_DATA, vdo);

drivers/md/dm-vdo/data-vio.c

820

static void destroy_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.c

822

if (data_vio == NULL)

drivers/md/dm-vdo/data-vio.c

825

vdo_free_bio(vdo_forget(data_vio->vio.bio));

drivers/md/dm-vdo/data-vio.c

826

vdo_free(vdo_forget(data_vio->vio.data));

drivers/md/dm-vdo/data-vio.c

827

vdo_free(vdo_forget(data_vio->compression.block));

drivers/md/dm-vdo/data-vio.c

828

vdo_free(vdo_forget(data_vio->scratch_block));

drivers/md/dm-vdo/data-vio.c

871

struct data_vio *data_vio = &pool->data_vios[i];

drivers/md/dm-vdo/data-vio.c

873

result = initialize_data_vio(data_vio, vdo);

drivers/md/dm-vdo/data-vio.c

875

destroy_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

880

list_add(&data_vio->pool_entry, &pool->available);

drivers/md/dm-vdo/data-vio.c

895

struct data_vio *data_vio, *tmp;

drivers/md/dm-vdo/data-vio.c

919

list_for_each_entry_safe(data_vio, tmp, &pool->available, pool_entry) {

drivers/md/dm-vdo/data-vio.c

920

list_del_init(&data_vio->pool_entry);

drivers/md/dm-vdo/data-vio.c

921

destroy_data_vio(data_vio);

drivers/md/dm-vdo/data-vio.c

961

struct data_vio *data_vio;

drivers/md/dm-vdo/data-vio.c

979

data_vio = get_available_data_vio(pool);

drivers/md/dm-vdo/data-vio.c

981

launch_bio(pool->completion.vdo, data_vio, bio);

drivers/md/dm-vdo/data-vio.h

131

struct data_vio *next_in_batch;

drivers/md/dm-vdo/data-vio.h

134

struct data_vio *lock_holder;

drivers/md/dm-vdo/data-vio.h

281

static inline struct data_vio *vio_as_data_vio(struct vio *vio)

drivers/md/dm-vdo/data-vio.h

284

return container_of(vio, struct data_vio, vio);

drivers/md/dm-vdo/data-vio.h

287

static inline struct data_vio *as_data_vio(struct vdo_completion *completion)

drivers/md/dm-vdo/data-vio.h

292

static inline struct data_vio *vdo_waiter_as_data_vio(struct vdo_waiter *waiter)

drivers/md/dm-vdo/data-vio.h

297

return container_of(waiter, struct data_vio, waiter);

drivers/md/dm-vdo/data-vio.h

300

static inline struct data_vio *data_vio_from_reference_updater(struct reference_updater *updater)

drivers/md/dm-vdo/data-vio.h

303

return container_of(updater, struct data_vio, increment_updater);

drivers/md/dm-vdo/data-vio.h

305

return container_of(updater, struct data_vio, decrement_updater);

drivers/md/dm-vdo/data-vio.h

308

static inline bool data_vio_has_flush_generation_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

310

return !list_empty(&data_vio->write_entry);

drivers/md/dm-vdo/data-vio.h

313

static inline struct vdo *vdo_from_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

315

return data_vio->vio.completion.vdo;

drivers/md/dm-vdo/data-vio.h

318

static inline bool data_vio_has_allocation(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

320

return (data_vio->allocation.pbn != VDO_ZERO_BLOCK);

drivers/md/dm-vdo/data-vio.h

324

advance_data_vio_compression_stage(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

326

get_data_vio_compression_status(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

327

bool cancel_data_vio_compression(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

346

static inline void continue_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

348

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

357

static inline void continue_data_vio_with_error(struct data_vio *data_vio, int result)

drivers/md/dm-vdo/data-vio.h

359

vdo_continue_completion(&data_vio->vio.completion, result);

drivers/md/dm-vdo/data-vio.h

362

const char * __must_check get_data_vio_operation_name(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

364

static inline void assert_data_vio_in_hash_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

366

thread_id_t expected = data_vio->hash_zone->thread_id;

drivers/md/dm-vdo/data-vio.h

374

(unsigned long long) data_vio->logical.lbn, thread_id, expected);

drivers/md/dm-vdo/data-vio.h

377

static inline void set_data_vio_hash_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

380

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

381

data_vio->hash_zone->thread_id);

drivers/md/dm-vdo/data-vio.h

388

static inline void launch_data_vio_hash_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

391

set_data_vio_hash_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

392

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

395

static inline void assert_data_vio_in_logical_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

397

thread_id_t expected = data_vio->logical.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

402

(unsigned long long) data_vio->logical.lbn, thread_id, expected);

drivers/md/dm-vdo/data-vio.h

405

static inline void set_data_vio_logical_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

408

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

409

data_vio->logical.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

416

static inline void launch_data_vio_logical_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

419

set_data_vio_logical_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

420

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

423

static inline void assert_data_vio_in_allocated_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

425

thread_id_t expected = data_vio->allocation.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

430

(unsigned long long) data_vio->allocation.pbn, thread_id,

drivers/md/dm-vdo/data-vio.h

434

static inline void set_data_vio_allocated_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

437

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

438

data_vio->allocation.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

446

static inline void launch_data_vio_allocated_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

449

set_data_vio_allocated_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

450

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

453

static inline void assert_data_vio_in_duplicate_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

455

thread_id_t expected = data_vio->duplicate.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

460

(unsigned long long) data_vio->duplicate.pbn, thread_id,

drivers/md/dm-vdo/data-vio.h

464

static inline void set_data_vio_duplicate_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

467

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

468

data_vio->duplicate.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

476

static inline void launch_data_vio_duplicate_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

479

set_data_vio_duplicate_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

480

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

483

static inline void assert_data_vio_in_mapped_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

485

thread_id_t expected = data_vio->mapped.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

490

(unsigned long long) data_vio->mapped.pbn, thread_id, expected);

drivers/md/dm-vdo/data-vio.h

493

static inline void set_data_vio_mapped_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

496

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

497

data_vio->mapped.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

500

static inline void assert_data_vio_in_new_mapped_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

502

thread_id_t expected = data_vio->new_mapped.zone->thread_id;

drivers/md/dm-vdo/data-vio.h

507

(unsigned long long) data_vio->new_mapped.pbn, thread_id,

drivers/md/dm-vdo/data-vio.h

511

static inline void set_data_vio_new_mapped_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

514

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

515

data_vio->new_mapped.zone->thread_id);

drivers/md/dm-vdo/data-vio.h

518

static inline void assert_data_vio_in_journal_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

520

thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread;

drivers/md/dm-vdo/data-vio.h

525

(unsigned long long) data_vio->logical.lbn, thread_id,

drivers/md/dm-vdo/data-vio.h

529

static inline void set_data_vio_journal_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

532

thread_id_t journal_thread = vdo_from_data_vio(data_vio)->thread_config.journal_thread;

drivers/md/dm-vdo/data-vio.h

534

vdo_set_completion_callback(&data_vio->vio.completion, callback, journal_thread);

drivers/md/dm-vdo/data-vio.h

541

static inline void launch_data_vio_journal_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

544

set_data_vio_journal_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

545

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

548

static inline void assert_data_vio_in_packer_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

550

thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread;

drivers/md/dm-vdo/data-vio.h

555

(unsigned long long) data_vio->logical.lbn, thread_id,

drivers/md/dm-vdo/data-vio.h

559

static inline void set_data_vio_packer_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

562

thread_id_t packer_thread = vdo_from_data_vio(data_vio)->thread_config.packer_thread;

drivers/md/dm-vdo/data-vio.h

564

vdo_set_completion_callback(&data_vio->vio.completion, callback, packer_thread);

drivers/md/dm-vdo/data-vio.h

571

static inline void launch_data_vio_packer_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

574

set_data_vio_packer_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

575

vdo_launch_completion(&data_vio->vio.completion);

drivers/md/dm-vdo/data-vio.h

578

static inline void assert_data_vio_on_cpu_thread(struct data_vio *data_vio)

drivers/md/dm-vdo/data-vio.h

580

thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread;

drivers/md/dm-vdo/data-vio.h

585

(unsigned long long) data_vio->logical.lbn, thread_id,

drivers/md/dm-vdo/data-vio.h

589

static inline void set_data_vio_cpu_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

592

thread_id_t cpu_thread = vdo_from_data_vio(data_vio)->thread_config.cpu_thread;

drivers/md/dm-vdo/data-vio.h

594

vdo_set_completion_callback(&data_vio->vio.completion, callback, cpu_thread);

drivers/md/dm-vdo/data-vio.h

601

static inline void launch_data_vio_cpu_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

605

set_data_vio_cpu_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

606

vdo_launch_completion_with_priority(&data_vio->vio.completion, priority);

drivers/md/dm-vdo/data-vio.h

609

static inline void set_data_vio_bio_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

612

vdo_set_completion_callback(&data_vio->vio.completion, callback,

drivers/md/dm-vdo/data-vio.h

613

get_vio_bio_zone_thread_id(&data_vio->vio));

drivers/md/dm-vdo/data-vio.h

620

static inline void launch_data_vio_bio_zone_callback(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

623

set_data_vio_bio_zone_callback(data_vio, callback);

drivers/md/dm-vdo/data-vio.h

624

vdo_launch_completion_with_priority(&data_vio->vio.completion,

drivers/md/dm-vdo/data-vio.h

633

static inline void launch_data_vio_on_bio_ack_queue(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

636

struct vdo_completion *completion = &data_vio->vio.completion;

drivers/md/dm-vdo/data-vio.h

649

void data_vio_allocate_data_block(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

653

void release_data_vio_allocation_lock(struct data_vio *data_vio, bool reset);

drivers/md/dm-vdo/data-vio.h

655

int __must_check uncompress_data_vio(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

659

void update_metadata_for_data_vio_write(struct data_vio *data_vio,

drivers/md/dm-vdo/data-vio.h

661

void write_data_vio(struct data_vio *data_vio);

drivers/md/dm-vdo/data-vio.h

662

void launch_compress_data_vio(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.c

1006

static void start_deduping(struct hash_lock *lock, struct data_vio *agent,

drivers/md/dm-vdo/dedupe.c

1077

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1138

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1146

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1162

struct data_vio *agent = vio_as_data_vio(bio->bi_private);

drivers/md/dm-vdo/dedupe.c

1193

static void start_verifying(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1226

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1278

static bool acquire_provisional_reference(struct data_vio *agent, struct pbn_lock *lock,

drivers/md/dm-vdo/dedupe.c

1313

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1414

static void start_locking(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1444

static void finish_writing(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1508

static struct data_vio *select_writing_agent(struct hash_lock *lock)

drivers/md/dm-vdo/dedupe.c

1511

struct data_vio *data_vio;

drivers/md/dm-vdo/dedupe.c

1519

while (((data_vio = dequeue_lock_waiter(lock)) != NULL) &&

drivers/md/dm-vdo/dedupe.c

1520

!data_vio_has_allocation(data_vio)) {

drivers/md/dm-vdo/dedupe.c

1522

vdo_waitq_enqueue_waiter(&temp_queue, &data_vio->waiter);

drivers/md/dm-vdo/dedupe.c

1525

if (data_vio != NULL) {

drivers/md/dm-vdo/dedupe.c

1537

lock->agent = data_vio;

drivers/md/dm-vdo/dedupe.c

1540

data_vio = lock->agent;

drivers/md/dm-vdo/dedupe.c

1545

return data_vio;

drivers/md/dm-vdo/dedupe.c

1557

static void start_writing(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1602

struct data_vio *data_vio = context->requestor;

drivers/md/dm-vdo/dedupe.c

1605

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

1606

struct zoned_pbn *advice = &data_vio->duplicate;

drivers/md/dm-vdo/dedupe.c

1628

(unsigned long long) data_vio->logical.lbn);

drivers/md/dm-vdo/dedupe.c

1637

(unsigned long long) data_vio->logical.lbn);

drivers/md/dm-vdo/dedupe.c

1645

static void process_query_result(struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

1667

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1705

static void start_querying(struct hash_lock *lock, struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1707

lock->agent = data_vio;

drivers/md/dm-vdo/dedupe.c

1709

data_vio->last_async_operation = VIO_ASYNC_OP_CHECK_FOR_DUPLICATION;

drivers/md/dm-vdo/dedupe.c

1710

set_data_vio_hash_zone_callback(data_vio, finish_querying);

drivers/md/dm-vdo/dedupe.c

1711

query_index(data_vio,

drivers/md/dm-vdo/dedupe.c

1712

(data_vio_has_allocation(data_vio) ? UDS_POST : UDS_QUERY));

drivers/md/dm-vdo/dedupe.c

1722

static void report_bogus_lock_state(struct hash_lock *lock, struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1726

continue_data_vio_with_error(data_vio, VDO_LOCK_ERROR);

drivers/md/dm-vdo/dedupe.c

1742

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1743

struct hash_lock *lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

1747

VDO_ASSERT_LOG_ONLY(data_vio == lock->agent,

drivers/md/dm-vdo/dedupe.c

1749

finish_writing(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1753

finish_deduping(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1758

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

1768

report_bogus_lock_state(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1772

report_bogus_lock_state(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1788

static bool is_hash_collision(struct hash_lock *lock, struct data_vio *candidate)

drivers/md/dm-vdo/dedupe.c

1790

struct data_vio *lock_holder;

drivers/md/dm-vdo/dedupe.c

1797

lock_holder = list_first_entry(&lock->duplicate_vios, struct data_vio,

drivers/md/dm-vdo/dedupe.c

1809

static inline int assert_hash_lock_preconditions(const struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1814

result = VDO_ASSERT(data_vio->hash_lock == NULL,

drivers/md/dm-vdo/dedupe.c

1819

result = VDO_ASSERT(list_empty(&data_vio->hash_lock_entry),

drivers/md/dm-vdo/dedupe.c

1824

return VDO_ASSERT(data_vio->recovery_sequence_number == 0,

drivers/md/dm-vdo/dedupe.c

1839

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

1843

assert_data_vio_in_hash_zone(data_vio);

drivers/md/dm-vdo/dedupe.c

1845

result = assert_hash_lock_preconditions(data_vio);

drivers/md/dm-vdo/dedupe.c

1847

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/dedupe.c

1851

result = acquire_lock(data_vio->hash_zone, &data_vio->record_name, NULL, &lock);

drivers/md/dm-vdo/dedupe.c

1853

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/dedupe.c

1857

if (is_hash_collision(lock, data_vio)) {

drivers/md/dm-vdo/dedupe.c

1864

write_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

1868

set_hash_lock(data_vio, lock);

drivers/md/dm-vdo/dedupe.c

1871

start_querying(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1881

wait_on_hash_lock(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1886

vdo_release_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

1887

write_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

1891

launch_dedupe(lock, data_vio, false);

drivers/md/dm-vdo/dedupe.c

1896

report_bogus_lock_state(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

1911

void vdo_release_hash_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1914

struct hash_lock *lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

1915

struct hash_zone *zone = data_vio->hash_zone;

drivers/md/dm-vdo/dedupe.c

1920

set_hash_lock(data_vio, NULL);

drivers/md/dm-vdo/dedupe.c

1959

static void transfer_allocation_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

1961

struct allocation *allocation = &data_vio->allocation;

drivers/md/dm-vdo/dedupe.c

1962

struct hash_lock *hash_lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

1964

VDO_ASSERT_LOG_ONLY(data_vio->new_mapped.pbn == allocation->pbn,

drivers/md/dm-vdo/dedupe.c

1972

hash_lock->duplicate = data_vio->new_mapped;

drivers/md/dm-vdo/dedupe.c

1973

data_vio->duplicate = data_vio->new_mapped;

drivers/md/dm-vdo/dedupe.c

1991

void vdo_share_compressed_write_lock(struct data_vio *data_vio,

drivers/md/dm-vdo/dedupe.c

1996

VDO_ASSERT_LOG_ONLY(vdo_get_duplicate_lock(data_vio) == NULL,

drivers/md/dm-vdo/dedupe.c

1998

VDO_ASSERT_LOG_ONLY(vdo_is_state_compressed(data_vio->new_mapped.state),

drivers/md/dm-vdo/dedupe.c

2000

assert_data_vio_in_new_mapped_zone(data_vio);

drivers/md/dm-vdo/dedupe.c

2010

data_vio->duplicate = data_vio->new_mapped;

drivers/md/dm-vdo/dedupe.c

2011

data_vio->hash_lock->duplicate = data_vio->new_mapped;

drivers/md/dm-vdo/dedupe.c

2012

set_duplicate_lock(data_vio->hash_lock, pbn_lock);

drivers/md/dm-vdo/dedupe.c

262

struct data_vio *agent;

drivers/md/dm-vdo/dedupe.c

2875

static void prepare_uds_request(struct uds_request *request, struct data_vio *data_vio,

drivers/md/dm-vdo/dedupe.c

2878

request->record_name = data_vio->record_name;

drivers/md/dm-vdo/dedupe.c

2885

encoding->data[offset++] = data_vio->new_mapped.state;

drivers/md/dm-vdo/dedupe.c

2886

put_unaligned_le64(data_vio->new_mapped.pbn, &encoding->data[offset]);

drivers/md/dm-vdo/dedupe.c

2898

static void query_index(struct data_vio *data_vio, enum uds_request_type operation)

drivers/md/dm-vdo/dedupe.c

2902

struct vdo *vdo = vdo_from_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

2903

struct hash_zone *zone = data_vio->hash_zone;

drivers/md/dm-vdo/dedupe.c

2905

assert_data_vio_in_hash_zone(data_vio);

drivers/md/dm-vdo/dedupe.c

2908

continue_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

2915

continue_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

2919

data_vio->dedupe_context = context;

drivers/md/dm-vdo/dedupe.c

2920

context->requestor = data_vio;

drivers/md/dm-vdo/dedupe.c

2922

prepare_uds_request(&context->request, data_vio, operation);

drivers/md/dm-vdo/dedupe.c

358

struct pbn_lock *vdo_get_duplicate_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

360

if (data_vio->hash_lock == NULL)

drivers/md/dm-vdo/dedupe.c

363

return data_vio->hash_lock->duplicate_lock;

drivers/md/dm-vdo/dedupe.c

396

static void assert_hash_lock_agent(struct data_vio *data_vio, const char *where)

drivers/md/dm-vdo/dedupe.c

399

assert_data_vio_in_hash_zone(data_vio);

drivers/md/dm-vdo/dedupe.c

400

VDO_ASSERT_LOG_ONLY(data_vio == data_vio->hash_lock->agent,

drivers/md/dm-vdo/dedupe.c

424

static inline struct data_vio *dequeue_lock_waiter(struct hash_lock *lock)

drivers/md/dm-vdo/dedupe.c

436

static void set_hash_lock(struct data_vio *data_vio, struct hash_lock *new_lock)

drivers/md/dm-vdo/dedupe.c

438

struct hash_lock *old_lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

441

VDO_ASSERT_LOG_ONLY(data_vio->hash_zone != NULL,

drivers/md/dm-vdo/dedupe.c

443

VDO_ASSERT_LOG_ONLY(!list_empty(&data_vio->hash_lock_entry),

drivers/md/dm-vdo/dedupe.c

459

list_del_init(&data_vio->hash_lock_entry);

drivers/md/dm-vdo/dedupe.c

462

data_vio->hash_lock = NULL;

drivers/md/dm-vdo/dedupe.c

470

list_move_tail(&data_vio->hash_lock_entry, &new_lock->duplicate_vios);

drivers/md/dm-vdo/dedupe.c

475

data_vio->hash_lock = new_lock;

drivers/md/dm-vdo/dedupe.c

480

static void start_deduping(struct hash_lock *lock, struct data_vio *agent,

drivers/md/dm-vdo/dedupe.c

482

static void start_locking(struct hash_lock *lock, struct data_vio *agent);

drivers/md/dm-vdo/dedupe.c

483

static void start_writing(struct hash_lock *lock, struct data_vio *agent);

drivers/md/dm-vdo/dedupe.c

485

static void transfer_allocation_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.c

492

static void exit_hash_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

495

vdo_release_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

498

data_vio->vio.completion.callback = complete_data_vio;

drivers/md/dm-vdo/dedupe.c

500

continue_data_vio(data_vio);

drivers/md/dm-vdo/dedupe.c

509

static void set_duplicate_location(struct data_vio *data_vio,

drivers/md/dm-vdo/dedupe.c

512

data_vio->is_duplicate = (source.pbn != VDO_ZERO_BLOCK);

drivers/md/dm-vdo/dedupe.c

513

data_vio->duplicate = source;

drivers/md/dm-vdo/dedupe.c

523

static struct data_vio *retire_lock_agent(struct hash_lock *lock)

drivers/md/dm-vdo/dedupe.c

525

struct data_vio *old_agent = lock->agent;

drivers/md/dm-vdo/dedupe.c

526

struct data_vio *new_agent = dequeue_lock_waiter(lock);

drivers/md/dm-vdo/dedupe.c

540

static void wait_on_hash_lock(struct hash_lock *lock, struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

542

vdo_waitq_enqueue_waiter(&lock->waiters, &data_vio->waiter);

drivers/md/dm-vdo/dedupe.c

558

data_vio->compression.lock_holder = lock->agent;

drivers/md/dm-vdo/dedupe.c

559

launch_data_vio_packer_callback(data_vio, vdo_remove_lock_holder_from_packer);

drivers/md/dm-vdo/dedupe.c

581

static void start_bypassing(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

587

void vdo_clean_failed_hash_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

589

struct hash_lock *lock = data_vio->hash_lock;

drivers/md/dm-vdo/dedupe.c

592

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

597

lock->agent = data_vio;

drivers/md/dm-vdo/dedupe.c

598

} else if (data_vio != lock->agent) {

drivers/md/dm-vdo/dedupe.c

599

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

612

data_vio->duplicate = lock->duplicate;

drivers/md/dm-vdo/dedupe.c

613

launch_data_vio_duplicate_zone_callback(data_vio, unlock_duplicate_pbn);

drivers/md/dm-vdo/dedupe.c

618

data_vio->is_duplicate = false;

drivers/md/dm-vdo/dedupe.c

619

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

631

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

687

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

710

static void start_unlocking(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

724

static void process_update_result(struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

744

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/dedupe.c

783

static void query_index(struct data_vio *data_vio, enum uds_request_type operation);

drivers/md/dm-vdo/dedupe.c

791

static void start_updating(struct hash_lock *lock, struct data_vio *agent)

drivers/md/dm-vdo/dedupe.c

814

static void finish_deduping(struct hash_lock *lock, struct data_vio *data_vio)

drivers/md/dm-vdo/dedupe.c

816

struct data_vio *agent = data_vio;

drivers/md/dm-vdo/dedupe.c

824

exit_hash_lock(data_vio);

drivers/md/dm-vdo/dedupe.c

928

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/dedupe.c

931

set_hash_lock(data_vio, new_lock);

drivers/md/dm-vdo/dedupe.c

932

wait_on_hash_lock(new_lock, data_vio);

drivers/md/dm-vdo/dedupe.c

943

static void fork_hash_lock(struct hash_lock *old_lock, struct data_vio *new_agent)

drivers/md/dm-vdo/dedupe.c

981

static void launch_dedupe(struct hash_lock *lock, struct data_vio *data_vio,

drivers/md/dm-vdo/dedupe.c

986

fork_hash_lock(lock, data_vio);

drivers/md/dm-vdo/dedupe.c

991

set_duplicate_location(data_vio, lock->duplicate);

drivers/md/dm-vdo/dedupe.c

992

data_vio->new_mapped = data_vio->duplicate;

drivers/md/dm-vdo/dedupe.c

993

update_metadata_for_data_vio_write(data_vio, lock->duplicate_lock);

drivers/md/dm-vdo/dedupe.h

26

struct data_vio *requestor;

drivers/md/dm-vdo/dedupe.h

72

struct pbn_lock * __must_check vdo_get_duplicate_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.h

76

void vdo_release_hash_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.h

77

void vdo_clean_failed_hash_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/dedupe.h

78

void vdo_share_compressed_write_lock(struct data_vio *data_vio,

drivers/md/dm-vdo/dump.c

155

struct data_vio *data_vio;

drivers/md/dm-vdo/dump.c

160

data_vio = vdo_waiter_as_data_vio(first);

drivers/md/dm-vdo/dump.c

163

wait_on, data_vio, data_vio->allocation.pbn, data_vio->logical.lbn,

drivers/md/dm-vdo/dump.c

164

data_vio->duplicate.pbn, get_data_vio_operation_name(data_vio));

drivers/md/dm-vdo/dump.c

167

data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/dump.c

169

data_vio, data_vio->allocation.pbn, data_vio->logical.lbn,

drivers/md/dm-vdo/dump.c

170

data_vio->duplicate.pbn,

drivers/md/dm-vdo/dump.c

171

get_data_vio_operation_name(data_vio));

drivers/md/dm-vdo/dump.c

189

static void encode_vio_dump_flags(struct data_vio *data_vio, char buffer[8])

drivers/md/dm-vdo/dump.c

193

if (data_vio->vio.completion.result != VDO_SUCCESS)

drivers/md/dm-vdo/dump.c

195

if (data_vio->waiter.next_waiter != NULL)

drivers/md/dm-vdo/dump.c

197

if (data_vio->is_duplicate)

drivers/md/dm-vdo/dump.c

199

if (data_vio->is_partial)

drivers/md/dm-vdo/dump.c

201

if (data_vio->is_zero)

drivers/md/dm-vdo/dump.c

203

if (data_vio->remaining_discard > 0)

drivers/md/dm-vdo/dump.c

215

struct data_vio *data_vio = data;

drivers/md/dm-vdo/dump.c

233

vdo_dump_completion_to_buffer(&data_vio->vio.completion,

drivers/md/dm-vdo/dump.c

236

if (data_vio->is_duplicate) {

drivers/md/dm-vdo/dump.c

239

data_vio->allocation.pbn, data_vio->logical.lbn,

drivers/md/dm-vdo/dump.c

240

data_vio->duplicate.pbn);

drivers/md/dm-vdo/dump.c

241

} else if (data_vio_has_allocation(data_vio)) {

drivers/md/dm-vdo/dump.c

244

data_vio->allocation.pbn, data_vio->logical.lbn);

drivers/md/dm-vdo/dump.c

248

data_vio->logical.lbn);

drivers/md/dm-vdo/dump.c

251

if (data_vio->flush_generation != 0) {

drivers/md/dm-vdo/dump.c

254

data_vio->flush_generation);

drivers/md/dm-vdo/dump.c

259

encode_vio_dump_flags(data_vio, flags_dump_buffer);

drivers/md/dm-vdo/dump.c

261

vdo_log_info(" vio %px %s%s %s %s%s", data_vio,

drivers/md/dm-vdo/dump.c

264

get_data_vio_operation_name(data_vio),

drivers/md/dm-vdo/dump.c

272

dump_vio_waiters(&data_vio->logical.waiters, "lbn");

drivers/md/dm-vdo/io-submitter.c

316

void vdo_submit_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/io-submitter.c

318

if (try_bio_map_merge(&data_vio->vio))

drivers/md/dm-vdo/io-submitter.c

321

launch_data_vio_bio_zone_callback(data_vio, submit_data_vio);

drivers/md/dm-vdo/io-submitter.h

26

void vdo_submit_data_vio(struct data_vio *data_vio);

drivers/md/dm-vdo/logical-zone.c

214

struct data_vio *data_vio =

drivers/md/dm-vdo/logical-zone.c

215

list_first_entry_or_null(&zone->write_vios, struct data_vio,

drivers/md/dm-vdo/logical-zone.c

218

(data_vio == NULL) ? zone->flush_generation : data_vio->flush_generation;

drivers/md/dm-vdo/logical-zone.c

252

void vdo_acquire_flush_generation_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/logical-zone.c

254

struct logical_zone *zone = data_vio->logical.zone;

drivers/md/dm-vdo/logical-zone.c

259

data_vio->flush_generation = zone->flush_generation;

drivers/md/dm-vdo/logical-zone.c

260

list_add_tail(&data_vio->write_entry, &zone->write_vios);

drivers/md/dm-vdo/logical-zone.c

312

void vdo_release_flush_generation_lock(struct data_vio *data_vio)

drivers/md/dm-vdo/logical-zone.c

314

struct logical_zone *zone = data_vio->logical.zone;

drivers/md/dm-vdo/logical-zone.c

318

if (!data_vio_has_flush_generation_lock(data_vio))

drivers/md/dm-vdo/logical-zone.c

321

list_del_init(&data_vio->write_entry);

drivers/md/dm-vdo/logical-zone.c

322

VDO_ASSERT_LOG_ONLY((zone->oldest_active_generation <= data_vio->flush_generation),

drivers/md/dm-vdo/logical-zone.c

324

(unsigned long long) data_vio->flush_generation,

drivers/md/dm-vdo/logical-zone.h

81

void vdo_acquire_flush_generation_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/logical-zone.h

83

void vdo_release_flush_generation_lock(struct data_vio *data_vio);

drivers/md/dm-vdo/packer.c

213

static inline struct packer *get_packer_from_data_vio(struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

215

return vdo_from_data_vio(data_vio)->packer;

drivers/md/dm-vdo/packer.c

239

static void abort_packing(struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

241

struct packer *packer = get_packer_from_data_vio(data_vio);

drivers/md/dm-vdo/packer.c

246

write_data_vio(data_vio);

drivers/md/dm-vdo/packer.c

255

static void release_compressed_write_waiter(struct data_vio *data_vio,

drivers/md/dm-vdo/packer.c

258

data_vio->new_mapped = (struct zoned_pbn) {

drivers/md/dm-vdo/packer.c

261

.state = data_vio->compression.slot + VDO_MAPPING_STATE_COMPRESSED_BASE,

drivers/md/dm-vdo/packer.c

264

vdo_share_compressed_write_lock(data_vio, allocation->lock);

drivers/md/dm-vdo/packer.c

265

update_metadata_for_data_vio_write(data_vio, allocation->lock);

drivers/md/dm-vdo/packer.c

276

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/packer.c

277

struct data_vio *client, *next;

drivers/md/dm-vdo/packer.c

296

struct data_vio *agent = as_data_vio(completion);

drivers/md/dm-vdo/packer.c

298

struct data_vio *client, *next;

drivers/md/dm-vdo/packer.c

323

static void add_to_bin(struct packer_bin *bin, struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

325

data_vio->compression.bin = bin;

drivers/md/dm-vdo/packer.c

326

data_vio->compression.slot = bin->slots_used;

drivers/md/dm-vdo/packer.c

327

bin->incoming[bin->slots_used++] = data_vio;

drivers/md/dm-vdo/packer.c

338

static struct data_vio *remove_from_bin(struct packer *packer, struct packer_bin *bin)

drivers/md/dm-vdo/packer.c

341

struct data_vio *data_vio = bin->incoming[--bin->slots_used];

drivers/md/dm-vdo/packer.c

343

if (!advance_data_vio_compression_stage(data_vio).may_not_compress) {

drivers/md/dm-vdo/packer.c

344

data_vio->compression.bin = NULL;

drivers/md/dm-vdo/packer.c

345

return data_vio;

drivers/md/dm-vdo/packer.c

348

add_to_bin(packer->canceled_bin, data_vio);

drivers/md/dm-vdo/packer.c

390

struct data_vio *data_vio,

drivers/md/dm-vdo/packer.c

394

struct compression_state *to_pack = &data_vio->compression;

drivers/md/dm-vdo/packer.c

398

compression->next_in_batch = data_vio;

drivers/md/dm-vdo/packer.c

411

struct data_vio *data_vio = vio_as_data_vio(bio->bi_private);

drivers/md/dm-vdo/packer.c

414

set_data_vio_allocated_zone_callback(data_vio, finish_compressed_write);

drivers/md/dm-vdo/packer.c

415

continue_data_vio_with_error(data_vio, blk_status_to_errno(bio->bi_status));

drivers/md/dm-vdo/packer.c

430

struct data_vio *agent = remove_from_bin(packer, bin);

drivers/md/dm-vdo/packer.c

431

struct data_vio *client;

drivers/md/dm-vdo/packer.c

499

struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

502

if (bin->free_space < data_vio->compression.size)

drivers/md/dm-vdo/packer.c

505

add_to_bin(bin, data_vio);

drivers/md/dm-vdo/packer.c

506

bin->free_space -= data_vio->compression.size;

drivers/md/dm-vdo/packer.c

524

struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

533

if (bin->free_space >= data_vio->compression.size)

drivers/md/dm-vdo/packer.c

546

if (data_vio->compression.size >=

drivers/md/dm-vdo/packer.c

562

void vdo_attempt_packing(struct data_vio *data_vio)

drivers/md/dm-vdo/packer.c

566

struct data_vio_compression_status status = get_data_vio_compression_status(data_vio);

drivers/md/dm-vdo/packer.c

567

struct packer *packer = get_packer_from_data_vio(data_vio);

drivers/md/dm-vdo/packer.c

589

(data_vio->flush_generation < packer->flush_generation)) {

drivers/md/dm-vdo/packer.c

590

abort_packing(data_vio);

drivers/md/dm-vdo/packer.c

603

bin = select_bin(packer, data_vio);

drivers/md/dm-vdo/packer.c

605

(advance_data_vio_compression_stage(data_vio).stage != DATA_VIO_PACKING)) {

drivers/md/dm-vdo/packer.c

606

abort_packing(data_vio);

drivers/md/dm-vdo/packer.c

610

add_data_vio_to_packer_bin(packer, bin, data_vio);

drivers/md/dm-vdo/packer.c

663

struct data_vio *data_vio = as_data_vio(completion);

drivers/md/dm-vdo/packer.c

664

struct packer *packer = get_packer_from_data_vio(data_vio);

drivers/md/dm-vdo/packer.c

665

struct data_vio *lock_holder;

drivers/md/dm-vdo/packer.c

669

assert_data_vio_in_packer_zone(data_vio);

drivers/md/dm-vdo/packer.c

671

lock_holder = vdo_forget(data_vio->compression.lock_holder);

drivers/md/dm-vdo/packer.h

108

void vdo_attempt_packing(struct data_vio *data_vio);

drivers/md/dm-vdo/packer.h

71

struct data_vio *incoming[];

drivers/md/dm-vdo/physical-zone.c

521

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/physical-zone.c

524

data_vio->allocation.wait_for_clean_slab = false;

drivers/md/dm-vdo/physical-zone.c

525

data_vio->allocation.first_allocation_zone = data_vio->allocation.zone->zone_number;

drivers/md/dm-vdo/physical-zone.c

526

continue_data_vio(data_vio);

drivers/md/dm-vdo/physical-zone.c

538

static bool continue_allocating(struct data_vio *data_vio)

drivers/md/dm-vdo/physical-zone.c

540

struct allocation *allocation = &data_vio->allocation;

drivers/md/dm-vdo/physical-zone.c

542

struct vdo_completion *completion = &data_vio->vio.completion;

drivers/md/dm-vdo/physical-zone.c

559

data_vio->waiter.callback = retry_allocation;

drivers/md/dm-vdo/physical-zone.c

561

&data_vio->waiter);

drivers/md/dm-vdo/physical-zone.c

587

bool vdo_allocate_block_in_zone(struct data_vio *data_vio)

drivers/md/dm-vdo/physical-zone.c

589

int result = allocate_and_lock_block(&data_vio->allocation);

drivers/md/dm-vdo/physical-zone.c

594

if ((result != VDO_NO_SPACE) || !continue_allocating(data_vio))

drivers/md/dm-vdo/physical-zone.c

595

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/physical-zone.h

107

bool __must_check vdo_allocate_block_in_zone(struct data_vio *data_vio);

drivers/md/dm-vdo/recovery-journal.c

1062

static void update_usages(struct recovery_journal *journal, struct data_vio *data_vio)

drivers/md/dm-vdo/recovery-journal.c

1064

if (data_vio->increment_updater.operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) {

drivers/md/dm-vdo/recovery-journal.c

1069

if (data_vio->new_mapped.state != VDO_MAPPING_STATE_UNMAPPED)

drivers/md/dm-vdo/recovery-journal.c

1072

if (data_vio->mapped.state != VDO_MAPPING_STATE_UNMAPPED)

drivers/md/dm-vdo/recovery-journal.c

1085

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/recovery-journal.c

1090

data_vio->recovery_journal_point = (struct journal_point) {

drivers/md/dm-vdo/recovery-journal.c

1095

update_usages(journal, data_vio);

drivers/md/dm-vdo/recovery-journal.c

1101

vdo_waitq_enqueue_waiter(&block->entry_waiters, &data_vio->waiter);

drivers/md/dm-vdo/recovery-journal.c

1174

struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);

drivers/md/dm-vdo/recovery-journal.c

1180

&data_vio->recovery_journal_point),

drivers/md/dm-vdo/recovery-journal.c

1184

(unsigned long long) data_vio->recovery_journal_point.sequence_number,

drivers/md/dm-vdo/recovery-journal.c

1185

data_vio->recovery_journal_point.entry_count);

drivers/md/dm-vdo/recovery-journal.c

1187

journal->commit_point = data_vio->recovery_journal_point;

drivers/md/dm-vdo/recovery-journal.c

1188

data_vio->last_async_operation = VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS;

drivers/md/dm-vdo/recovery-journal.c

1190

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/recovery-journal.c

1198

has_decrement = (data_vio->decrement_updater.zpbn.pbn != VDO_ZERO_BLOCK);

drivers/md/dm-vdo/recovery-journal.c

1199

if ((data_vio->increment_updater.zpbn.pbn != VDO_ZERO_BLOCK) || !has_decrement)

drivers/md/dm-vdo/recovery-journal.c

1200

continue_data_vio(data_vio);

drivers/md/dm-vdo/recovery-journal.c

1203

vdo_launch_completion(&data_vio->decrement_completion);

drivers/md/dm-vdo/recovery-journal.c

1332

struct data_vio *data_vio =

drivers/md/dm-vdo/recovery-journal.c

1334

struct tree_lock *lock = &data_vio->tree_lock;

drivers/md/dm-vdo/recovery-journal.c

1346

.pbn = data_vio->increment_updater.zpbn.pbn,

drivers/md/dm-vdo/recovery-journal.c

1347

.state = data_vio->increment_updater.zpbn.state,

drivers/md/dm-vdo/recovery-journal.c

1350

.pbn = data_vio->decrement_updater.zpbn.pbn,

drivers/md/dm-vdo/recovery-journal.c

1351

.state = data_vio->decrement_updater.zpbn.state,

drivers/md/dm-vdo/recovery-journal.c

1353

.operation = data_vio->increment_updater.operation,

drivers/md/dm-vdo/recovery-journal.c

1357

data_vio->recovery_sequence_number = block->sequence_number;

drivers/md/dm-vdo/recovery-journal.c

1360

vdo_waitq_enqueue_waiter(&block->commit_waiters, &data_vio->waiter);

drivers/md/dm-vdo/recovery-journal.c

1449

struct data_vio *data_vio)

drivers/md/dm-vdo/recovery-journal.c

1453

continue_data_vio_with_error(data_vio, VDO_INVALID_ADMIN_STATE);

drivers/md/dm-vdo/recovery-journal.c

1458

continue_data_vio_with_error(data_vio, VDO_READ_ONLY);

drivers/md/dm-vdo/recovery-journal.c

1462

VDO_ASSERT_LOG_ONLY(data_vio->recovery_sequence_number == 0,

drivers/md/dm-vdo/recovery-journal.c

1466

vdo_waitq_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter);

drivers/md/dm-vdo/recovery-journal.h

286

struct data_vio *data_vio);

drivers/md/dm-vdo/slab-depot.c

1696

struct data_vio *data_vio = data_vio_from_reference_updater(updater);

drivers/md/dm-vdo/slab-depot.c

1703

sequence_number_t recovery_block = data_vio->recovery_journal_point.sequence_number;

drivers/md/dm-vdo/slab-depot.c

1725

expand_journal_point(data_vio->recovery_journal_point,

drivers/md/dm-vdo/slab-depot.c

1743

continue_data_vio_with_error(data_vio, result);

drivers/md/dm-vdo/slab-depot.c

1745

vdo_continue_completion(&data_vio->decrement_completion, result);

drivers/md/dm-vdo/slab-depot.c

3152

struct data_vio *data_vio = data_vio_from_reference_updater(updater);

drivers/md/dm-vdo/slab-depot.c

3155

continue_data_vio_with_error(data_vio, VDO_READ_ONLY);

drivers/md/dm-vdo/slab-depot.c

3159

vdo_continue_completion(&data_vio->decrement_completion, VDO_READ_ONLY);

drivers/md/dm-vdo/types.h

335

struct data_vio;