block/fops.c

133

struct blkdev_dio *dio = bio->bi_private;

block/fops.c

134

bool should_dirty = dio->flags & DIO_SHOULD_DIRTY;

block/fops.c

135

bool is_sync = dio->flags & DIO_IS_SYNC;

block/fops.c

137

if (bio->bi_status && !dio->bio.bi_status)

block/fops.c

138

dio->bio.bi_status = bio->bi_status;

block/fops.c

143

if (atomic_dec_and_test(&dio->ref)) {

block/fops.c

145

struct kiocb *iocb = dio->iocb;

block/fops.c

150

if (likely(!dio->bio.bi_status)) {

block/fops.c

151

ret = dio->size;

block/fops.c

154

ret = blk_status_to_errno(dio->bio.bi_status);

block/fops.c

157

dio->iocb->ki_complete(iocb, ret);

block/fops.c

158

bio_put(&dio->bio);

block/fops.c

160

struct task_struct *waiter = dio->waiter;

block/fops.c

162

WRITE_ONCE(dio->waiter, NULL);

block/fops.c

179

struct blkdev_dio *dio;

block/fops.c

188

dio = container_of(bio, struct blkdev_dio, bio);

block/fops.c

189

atomic_set(&dio->ref, 1);

block/fops.c

198

dio->flags = DIO_IS_SYNC;

block/fops.c

199

dio->waiter = current;

block/fops.c

201

dio->flags = 0;

block/fops.c

202

dio->iocb = iocb;

block/fops.c

205

dio->size = 0;

block/fops.c

207

dio->flags |= DIO_SHOULD_DIRTY;

block/fops.c

215

bio->bi_private = dio;

block/fops.c

246

if (dio->flags & DIO_SHOULD_DIRTY)

block/fops.c

251

dio->size += bio->bi_iter.bi_size;

block/fops.c

259

atomic_inc(&dio->ref);

block/fops.c

271

if (!READ_ONCE(dio->waiter))

block/fops.c

278

ret = blk_status_to_errno(dio->bio.bi_status);

block/fops.c

280

ret = dio->size;

block/fops.c

282

bio_put(&dio->bio);

block/fops.c

294

struct blkdev_dio *dio = container_of(bio, struct blkdev_dio, bio);

block/fops.c

295

struct kiocb *iocb = dio->iocb;

block/fops.c

301

ret = dio->size;

block/fops.c

312

if (dio->flags & DIO_SHOULD_DIRTY) {

block/fops.c

327

struct blkdev_dio *dio;

block/fops.c

334

dio = container_of(bio, struct blkdev_dio, bio);

block/fops.c

335

dio->flags = 0;

block/fops.c

336

dio->iocb = iocb;

block/fops.c

356

dio->size = bio->bi_iter.bi_size;

block/fops.c

360

dio->flags |= DIO_SHOULD_DIRTY;

drivers/block/loop.c

698

int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);

drivers/block/loop.c

700

return sysfs_emit(buf, "%s\n", dio ? "1" : "0");

drivers/block/loop.c

708

LOOP_ATTR_RO(dio);

drivers/comedi/drivers/ni_670x.c

68

int dio;

drivers/comedi/drivers/ni_pcidio.c

286

int dio;

drivers/gpu/drm/amd/display/dc/dc.h

890

bool dio : 1; /* Display output */

drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_dio.c

19

static void dcn10_dio_mem_pwr_ctrl(struct dio *dio, bool enable_i2c_light_sleep)

drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_dio.c

21

struct dcn10_dio *dio10 = TO_DCN10_DIO(dio);

drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_dio.h

29

struct dio base;

drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c

1930

if (dc->res_pool->dio && dc->res_pool->dio->funcs->mem_pwr_ctrl)

drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c

1931

dc->res_pool->dio->funcs->mem_pwr_ctrl(dc->res_pool->dio, false);

drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c

365

if (dc->res_pool->dio && dc->res_pool->dio->funcs->mem_pwr_ctrl)

drivers/gpu/drm/amd/display/dc/hwss/dcn201/dcn201_hwseq.c

366

dc->res_pool->dio->funcs->mem_pwr_ctrl(dc->res_pool->dio, false);

drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c

802

if (dc->res_pool->dio && dc->res_pool->dio->funcs->mem_pwr_ctrl)

drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c

803

dc->res_pool->dio->funcs->mem_pwr_ctrl(dc->res_pool->dio, false);

drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c

244

if (dc->res_pool->dio && dc->res_pool->dio->funcs->mem_pwr_ctrl)

drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c

245

dc->res_pool->dio->funcs->mem_pwr_ctrl(dc->res_pool->dio, dc->debug.enable_mem_low_power.bits.i2c);

drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c

966

if (dc->res_pool->dio && dc->res_pool->dio->funcs->mem_pwr_ctrl)

drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c

967

dc->res_pool->dio->funcs->mem_pwr_ctrl(dc->res_pool->dio, false);

drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c

279

if (dc->res_pool->dio && dc->res_pool->dio->funcs->mem_pwr_ctrl)

drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c

280

dc->res_pool->dio->funcs->mem_pwr_ctrl(dc->res_pool->dio, !dc->debug.enable_mem_low_power.bits.i2c);

drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c

357

if (dc->res_pool->dio && dc->res_pool->dio->funcs->mem_pwr_ctrl)

drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c

358

dc->res_pool->dio->funcs->mem_pwr_ctrl(dc->res_pool->dio, false);

drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c

239

if (dc->res_pool->dio && dc->res_pool->dio->funcs->mem_pwr_ctrl)

drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c

240

dc->res_pool->dio->funcs->mem_pwr_ctrl(dc->res_pool->dio, false);

drivers/gpu/drm/amd/display/dc/inc/core_types.h

258

struct dio *dio;

drivers/gpu/drm/amd/display/dc/inc/hw/dio.h

11

struct dio;

drivers/gpu/drm/amd/display/dc/inc/hw/dio.h

14

void (*mem_pwr_ctrl)(struct dio *dio, bool enable_i2c_light_sleep);

drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c

1701

pool->base.dio = dcn10_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c

1702

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c

463

static struct dio *dcn10_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c

949

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c

950

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c

951

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c

1137

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c

1138

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c

1139

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c

2749

pool->base.dio = dcn20_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c

2750

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c

569

static struct dio *dcn20_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c

1314

pool->base.dio = dcn201_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c

1315

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c

774

static struct dio *dcn201_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c

961

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c

962

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn201/dcn201_resource.c

963

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c

1701

pool->base.dio = dcn21_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c

1702

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c

341

static struct dio *dcn21_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c

701

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c

702

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c

703

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c

1251

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c

1252

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c

1253

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c

2635

pool->base.dio = dcn30_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c

2636

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c

947

static struct dio *dcn30_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c

1100

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c

1101

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c

1102

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c

1624

pool->base.dio = dcn301_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c

1625

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c

863

static struct dio *dcn301_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c

1160

if (pool->dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c

1161

kfree(TO_DCN10_DIO(pool->dio));

drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c

1162

pool->dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c

1520

pool->dio = dcn302_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c

1521

if (pool->dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c

275

static struct dio *dcn302_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c

1104

if (pool->dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c

1105

kfree(TO_DCN10_DIO(pool->dio));

drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c

1106

pool->dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c

1452

pool->dio = dcn303_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c

1453

if (pool->dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c

271

static struct dio *dcn303_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c

1083

static struct dio *dcn31_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c

1552

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c

1553

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c

1554

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c

2237

pool->base.dio = dcn31_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c

2238

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c

1140

static struct dio *dcn314_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c

1612

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c

1613

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c

1614

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c

2155

pool->base.dio = dcn314_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c

2156

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c

1082

static struct dio *dcn315_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c

1554

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c

1555

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c

1556

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c

2189

pool->base.dio = dcn315_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c

2190

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c

1075

static struct dio *dcn316_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c

1547

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c

1548

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c

1549

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c

2061

pool->base.dio = dcn316_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c

2062

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c

1660

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c

1661

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c

1662

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c

2551

pool->base.dio = dcn32_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c

2552

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c

892

static struct dio *dcn32_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c

1634

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c

1635

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c

1636

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c

2044

pool->base.dio = dcn321_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c

2045

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c

880

static struct dio *dcn321_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c

1026

static struct dio *dcn35_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c

1715

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c

1716

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c

1717

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c

2205

pool->base.dio = dcn35_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c

2206

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c

1006

static struct dio *dcn351_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c

1695

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c

1696

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c

1697

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c

2177

pool->base.dio = dcn351_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c

2178

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c

1013

static struct dio *dcn36_dio_create(struct dc_context *ctx)

drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c

1702

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c

1703

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c

1704

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c

2175

pool->base.dio = dcn36_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c

2176

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c

1650

if (pool->base.dio != NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c

1651

kfree(TO_DCN10_DIO(pool->base.dio));

drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c

1652

pool->base.dio = NULL;

drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c

2251

pool->base.dio = dcn401_dio_create(ctx);

drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c

2252

if (pool->base.dio == NULL) {

drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c

936

static struct dio *dcn401_dio_create(struct dc_context *ctx)

drivers/md/dm-integrity.c

1563

static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)

drivers/md/dm-integrity.c

1569

bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

1595

static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)

drivers/md/dm-integrity.c

1597

struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

1599

if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))

drivers/md/dm-integrity.c

1600

submit_flush_bio(ic, dio);

drivers/md/dm-integrity.c

1605

static void dec_in_flight(struct dm_integrity_io *dio)

drivers/md/dm-integrity.c

1607

if (atomic_dec_and_test(&dio->in_flight)) {

drivers/md/dm-integrity.c

1608

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

1611

remove_range(ic, &dio->range);

drivers/md/dm-integrity.c

1613

if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))

drivers/md/dm-integrity.c

1616

bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

1617

if (unlikely(dio->bi_status) && !bio->bi_status)

drivers/md/dm-integrity.c

1618

bio->bi_status = dio->bi_status;

drivers/md/dm-integrity.c

1619

if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {

drivers/md/dm-integrity.c

1620

dio->range.logical_sector += dio->range.n_sectors;

drivers/md/dm-integrity.c

1621

bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);

drivers/md/dm-integrity.c

1622

INIT_WORK(&dio->work, integrity_bio_wait);

drivers/md/dm-integrity.c

1623

queue_work(ic->offload_wq, &dio->work);

drivers/md/dm-integrity.c

1626

do_endio_flush(ic, dio);

drivers/md/dm-integrity.c

1632

struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

1634

dm_bio_restore(&dio->bio_details, bio);

drivers/md/dm-integrity.c

1638

if (dio->completion)

drivers/md/dm-integrity.c

1639

complete(dio->completion);

drivers/md/dm-integrity.c

1641

dec_in_flight(dio);

drivers/md/dm-integrity.c

1795

static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum)

drivers/md/dm-integrity.c

1797

struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

1798

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

1804

get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);

drivers/md/dm-integrity.c

1805

dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset,

drivers/md/dm-integrity.c

1806

&dio->metadata_offset);

drivers/md/dm-integrity.c

1808

logical_sector = dio->range.logical_sector;

drivers/md/dm-integrity.c

1812

__bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {

drivers/md/dm-integrity.c

1833

alignment = dio->range.logical_sector | bio_sectors(bio) | (PAGE_SIZE >> SECTOR_SHIFT);

drivers/md/dm-integrity.c

1842

dio->bi_status = errno_to_blk_status(r);

drivers/md/dm-integrity.c

1846

integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum);

drivers/md/dm-integrity.c

1847

r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block,

drivers/md/dm-integrity.c

1848

&dio->metadata_offset, ic->tag_size, TAG_CMP);

drivers/md/dm-integrity.c

1858

dio->bi_status = errno_to_blk_status(r);

drivers/md/dm-integrity.c

1877

struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);

drivers/md/dm-integrity.c

1878

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

1886

struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

1896

if (likely(dio->op != REQ_OP_DISCARD))

drivers/md/dm-integrity.c

1910

if (unlikely(dio->op == REQ_OP_DISCARD)) {

drivers/md/dm-integrity.c

1911

unsigned int bi_size = dio->bio_details.bi_iter.bi_size;

drivers/md/dm-integrity.c

1921

r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,

drivers/md/dm-integrity.c

1937

sector = dio->range.logical_sector;

drivers/md/dm-integrity.c

1938

sectors_to_process = dio->range.n_sectors;

drivers/md/dm-integrity.c

1940

__bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {

drivers/md/dm-integrity.c

1950

integrity_sector_checksum(ic, &dio->ahash_req, sector, mem, bv_copy.bv_offset + pos, checksums_ptr);

drivers/md/dm-integrity.c

1958

r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,

drivers/md/dm-integrity.c

1959

checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);

drivers/md/dm-integrity.c

1964

integrity_recheck(dio, checksums_onstack);

drivers/md/dm-integrity.c

1983

struct bio_integrity_payload *bip = dio->bio_details.bi_integrity;

drivers/md/dm-integrity.c

1988

unsigned int data_to_process = dio->range.n_sectors;

drivers/md/dm-integrity.c

2000

r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,

drivers/md/dm-integrity.c

2001

this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE);

drivers/md/dm-integrity.c

2011

dec_in_flight(dio);

drivers/md/dm-integrity.c

2014

dio->bi_status = errno_to_blk_status(r);

drivers/md/dm-integrity.c

2015

dec_in_flight(dio);

drivers/md/dm-integrity.c

2050

struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

2055

dio->ic = ic;

drivers/md/dm-integrity.c

2056

dio->bi_status = 0;

drivers/md/dm-integrity.c

2057

dio->op = bio_op(bio);

drivers/md/dm-integrity.c

2058

dio->ahash_req = NULL;

drivers/md/dm-integrity.c

2062

dio->integrity_payload = NULL;

drivers/md/dm-integrity.c

2063

dio->integrity_payload_from_mempool = false;

drivers/md/dm-integrity.c

2064

dio->integrity_range_locked = false;

drivers/md/dm-integrity.c

2065

return dm_integrity_map_inline(dio, true);

drivers/md/dm-integrity.c

2068

if (unlikely(dio->op == REQ_OP_DISCARD)) {

drivers/md/dm-integrity.c

2084

submit_flush_bio(ic, dio);

drivers/md/dm-integrity.c

2088

dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);

drivers/md/dm-integrity.c

2089

dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA;

drivers/md/dm-integrity.c

2090

if (unlikely(dio->fua)) {

drivers/md/dm-integrity.c

2097

if (unlikely(!dm_integrity_check_limits(ic, dio->range.logical_sector, bio)))

drivers/md/dm-integrity.c

2122

if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ))

drivers/md/dm-integrity.c

2125

get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);

drivers/md/dm-integrity.c

2126

dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);

drivers/md/dm-integrity.c

2129

dm_integrity_map_continue(dio, true);

drivers/md/dm-integrity.c

2133

static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,

drivers/md/dm-integrity.c

2136

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

2140

logical_sector = dio->range.logical_sector;

drivers/md/dm-integrity.c

2141

n_sectors = dio->range.n_sectors;

drivers/md/dm-integrity.c

2152

if (likely(dio->op == REQ_OP_WRITE))

drivers/md/dm-integrity.c

2158

if (unlikely(dio->op == REQ_OP_READ)) {

drivers/md/dm-integrity.c

2196

if (likely(dio->op == REQ_OP_WRITE))

drivers/md/dm-integrity.c

2204

} else if (likely(dio->op == REQ_OP_WRITE))

drivers/md/dm-integrity.c

2208

if (likely(dio->op == REQ_OP_WRITE)) {

drivers/md/dm-integrity.c

2228

integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, js_page, js_offset, checksums_onstack);

drivers/md/dm-integrity.c

2231

integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, js_page, js_offset, journal_entry_tag(ic, je));

drivers/md/dm-integrity.c

2248

if (unlikely(dio->op == REQ_OP_READ))

drivers/md/dm-integrity.c

2253

if (likely(dio->op == REQ_OP_WRITE)) {

drivers/md/dm-integrity.c

2262

remove_range(ic, &dio->range);

drivers/md/dm-integrity.c

2267

dio->range.logical_sector = logical_sector;

drivers/md/dm-integrity.c

2268

get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);

drivers/md/dm-integrity.c

2269

dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);

drivers/md/dm-integrity.c

2276

static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)

drivers/md/dm-integrity.c

2278

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

2279

struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

2285

bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;

drivers/md/dm-integrity.c

2287

if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')

drivers/md/dm-integrity.c

2291

INIT_WORK(&dio->work, integrity_bio_wait);

drivers/md/dm-integrity.c

2292

queue_work(ic->offload_wq, &dio->work);

drivers/md/dm-integrity.c

2304

dio->range.n_sectors = bio_sectors(bio);

drivers/md/dm-integrity.c

2306

if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {

drivers/md/dm-integrity.c

2307

if (dio->op == REQ_OP_WRITE) {

drivers/md/dm-integrity.c

2311

dio->range.n_sectors = min(dio->range.n_sectors,

drivers/md/dm-integrity.c

2313

if (unlikely(!dio->range.n_sectors)) {

drivers/md/dm-integrity.c

2319

range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;

drivers/md/dm-integrity.c

2337

add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);

drivers/md/dm-integrity.c

2351

} while ((i += ic->sectors_per_block) < dio->range.n_sectors);

drivers/md/dm-integrity.c

2358

journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);

drivers/md/dm-integrity.c

2360

if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))

drivers/md/dm-integrity.c

2361

dio->range.n_sectors = next_sector - dio->range.logical_sector;

drivers/md/dm-integrity.c

2366

for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {

drivers/md/dm-integrity.c

2367

if (!test_journal_node(ic, jp, dio->range.logical_sector + i))

drivers/md/dm-integrity.c

2370

dio->range.n_sectors = i;

drivers/md/dm-integrity.c

2374

if (unlikely(!add_new_range(ic, &dio->range, true))) {

drivers/md/dm-integrity.c

2383

INIT_WORK(&dio->work, integrity_bio_wait);

drivers/md/dm-integrity.c

2384

queue_work(ic->wait_wq, &dio->work);

drivers/md/dm-integrity.c

2388

dio->range.n_sectors = ic->sectors_per_block;

drivers/md/dm-integrity.c

2389

wait_and_add_new_range(ic, &dio->range);

drivers/md/dm-integrity.c

2399

new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);

drivers/md/dm-integrity.c

2401

remove_range_unlocked(ic, &dio->range);

drivers/md/dm-integrity.c

2406

if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {

drivers/md/dm-integrity.c

2410

new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);

drivers/md/dm-integrity.c

2412

unlikely(next_sector < dio->range.logical_sector + dio->range.n_sectors)) {

drivers/md/dm-integrity.c

2413

remove_range_unlocked(ic, &dio->range);

drivers/md/dm-integrity.c

2432

if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {

drivers/md/dm-integrity.c

2433

if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,

drivers/md/dm-integrity.c

2434

dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {

drivers/md/dm-integrity.c

2437

bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);

drivers/md/dm-integrity.c

2446

dio->in_flight = (atomic_t)ATOMIC_INIT(2);

drivers/md/dm-integrity.c

2450

dio->completion = &read_comp;

drivers/md/dm-integrity.c

2452

dio->completion = NULL;

drivers/md/dm-integrity.c

2454

dm_bio_record(&dio->bio_details, bio);

drivers/md/dm-integrity.c

2459

bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;

drivers/md/dm-integrity.c

2461

if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {

drivers/md/dm-integrity.c

2462

integrity_metadata(&dio->work);

drivers/md/dm-integrity.c

2465

dio->in_flight = (atomic_t)ATOMIC_INIT(1);

drivers/md/dm-integrity.c

2466

dio->completion = NULL;

drivers/md/dm-integrity.c

2478

dio->range.logical_sector + dio->range.n_sectors > recalc_sector)

drivers/md/dm-integrity.c

2481

if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,

drivers/md/dm-integrity.c

2482

dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))

drivers/md/dm-integrity.c

2487

integrity_metadata(&dio->work);

drivers/md/dm-integrity.c

2490

dec_in_flight(dio);

drivers/md/dm-integrity.c

2492

INIT_WORK(&dio->work, integrity_metadata);

drivers/md/dm-integrity.c

2493

queue_work(ic->metadata_wq, &dio->work);

drivers/md/dm-integrity.c

2499

if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))

drivers/md/dm-integrity.c

2502

do_endio_flush(ic, dio);

drivers/md/dm-integrity.c

2505

static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map)

drivers/md/dm-integrity.c

2507

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

2508

struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

2527

if (!dio->integrity_payload) {

drivers/md/dm-integrity.c

2529

dio->payload_len = ic->tuple_size * (bio_sectors(bio) >> ic->sb->log2_sectors_per_block);

drivers/md/dm-integrity.c

2532

dio->payload_len += extra_size;

drivers/md/dm-integrity.c

2533

dio->integrity_payload = kmalloc(dio->payload_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);

drivers/md/dm-integrity.c

2534

if (unlikely(!dio->integrity_payload)) {

drivers/md/dm-integrity.c

2536

if (dio->payload_len > x_size) {

drivers/md/dm-integrity.c

2549

dio->range.logical_sector = bio->bi_iter.bi_sector;

drivers/md/dm-integrity.c

2550

dio->range.n_sectors = bio_sectors(bio);

drivers/md/dm-integrity.c

2562

if (likely(dio->range.logical_sector + dio->range.n_sectors <= recalc_sector))

drivers/md/dm-integrity.c

2567

if (dio->range.logical_sector + dio->range.n_sectors <= recalc_sector)

drivers/md/dm-integrity.c

2569

if (unlikely(!add_new_range(ic, &dio->range, true))) {

drivers/md/dm-integrity.c

2572

INIT_WORK(&dio->work, integrity_bio_wait);

drivers/md/dm-integrity.c

2573

queue_work(ic->wait_wq, &dio->work);

drivers/md/dm-integrity.c

2576

wait_and_add_new_range(ic, &dio->range);

drivers/md/dm-integrity.c

2578

dio->integrity_range_locked = true;

drivers/md/dm-integrity.c

2583

if (unlikely(!dio->integrity_payload)) {

drivers/md/dm-integrity.c

2584

dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO));

drivers/md/dm-integrity.c

2585

dio->integrity_payload_from_mempool = true;

drivers/md/dm-integrity.c

2588

dio->bio_details.bi_iter = bio->bi_iter;

drivers/md/dm-integrity.c

2599

if (dio->op == REQ_OP_WRITE) {

drivers/md/dm-integrity.c

2601

while (dio->bio_details.bi_iter.bi_size) {

drivers/md/dm-integrity.c

2602

struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);

drivers/md/dm-integrity.c

2605

memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tag_size);

drivers/md/dm-integrity.c

2606

integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos);

drivers/md/dm-integrity.c

2609

bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);

drivers/md/dm-integrity.c

2613

ret = bio_integrity_add_page(bio, virt_to_page(dio->integrity_payload),

drivers/md/dm-integrity.c

2614

dio->payload_len, offset_in_page(dio->integrity_payload));

drivers/md/dm-integrity.c

2615

if (unlikely(ret != dio->payload_len)) {

drivers/md/dm-integrity.c

2624

static inline void dm_integrity_free_payload(struct dm_integrity_io *dio)

drivers/md/dm-integrity.c

2626

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

2627

if (unlikely(dio->integrity_payload_from_mempool))

drivers/md/dm-integrity.c

2628

mempool_free(virt_to_page(dio->integrity_payload), &ic->recheck_pool);

drivers/md/dm-integrity.c

2630

kfree(dio->integrity_payload);

drivers/md/dm-integrity.c

2631

dio->integrity_payload = NULL;

drivers/md/dm-integrity.c

2632

dio->integrity_payload_from_mempool = false;

drivers/md/dm-integrity.c

2637

struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);

drivers/md/dm-integrity.c

2638

struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

2639

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

2643

dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO));

drivers/md/dm-integrity.c

2644

dio->integrity_payload_from_mempool = true;

drivers/md/dm-integrity.c

2646

outgoing_data = dio->integrity_payload + PAGE_SIZE;

drivers/md/dm-integrity.c

2648

while (dio->bio_details.bi_iter.bi_size) {

drivers/md/dm-integrity.c

2667

r = bio_integrity_add_page(outgoing_bio, virt_to_page(dio->integrity_payload), ic->tuple_size, 0);

drivers/md/dm-integrity.c

2675

outgoing_bio->bi_iter.bi_sector = dio->bio_details.bi_iter.bi_sector + ic->start + SB_SECTORS;

drivers/md/dm-integrity.c

2686

integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, integrity_identity(ic, outgoing_data), 0, digest);

drivers/md/dm-integrity.c

2687

if (unlikely(crypto_memneq(digest, dio->integrity_payload, min(ic->internal_hash_digestsize, ic->tag_size)))) {

drivers/md/dm-integrity.c

2689

ic->dev->bdev, dio->bio_details.bi_iter.bi_sector);

drivers/md/dm-integrity.c

2692

bio, dio->bio_details.bi_iter.bi_sector, 0);

drivers/md/dm-integrity.c

2699

bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);

drivers/md/dm-integrity.c

2704

bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);

drivers/md/dm-integrity.c

2710

static inline bool dm_integrity_check(struct dm_integrity_c *ic, struct dm_integrity_io *dio)

drivers/md/dm-integrity.c

2712

struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

2715

while (dio->bio_details.bi_iter.bi_size) {

drivers/md/dm-integrity.c

2717

struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);

drivers/md/dm-integrity.c

2719

integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest);

drivers/md/dm-integrity.c

2720

if (unlikely(crypto_memneq(digest, dio->integrity_payload + pos,

drivers/md/dm-integrity.c

2723

dm_integrity_free_payload(dio);

drivers/md/dm-integrity.c

2724

INIT_WORK(&dio->work, dm_integrity_inline_recheck);

drivers/md/dm-integrity.c

2725

queue_work(ic->offload_wq, &dio->work);

drivers/md/dm-integrity.c

2730

bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);

drivers/md/dm-integrity.c

2738

struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);

drivers/md/dm-integrity.c

2739

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

2740

struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

2742

if (likely(dm_integrity_check(ic, dio)))

drivers/md/dm-integrity.c

2749

struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

2751

if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK) && likely(dio->bio_details.bi_iter.bi_size != 0)) {

drivers/md/dm-integrity.c

2753

unlikely(dio->integrity_range_locked))

drivers/md/dm-integrity.c

2756

if (unlikely(!dm_integrity_check(ic, dio)))

drivers/md/dm-integrity.c

2759

INIT_WORK(&dio->work, dm_integrity_inline_async_check);

drivers/md/dm-integrity.c

2760

queue_work(ic->offload_wq, &dio->work);

drivers/md/dm-integrity.c

2765

dm_integrity_free_payload(dio);

drivers/md/dm-integrity.c

2766

if (unlikely(dio->integrity_range_locked))

drivers/md/dm-integrity.c

2767

remove_range(ic, &dio->range);

drivers/md/dm-integrity.c

2769

if (unlikely(dio->ahash_req))

drivers/md/dm-integrity.c

2770

mempool_free(dio->ahash_req, &ic->ahash_req_pool);

drivers/md/dm-integrity.c

2776

struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);

drivers/md/dm-integrity.c

2777

struct dm_integrity_c *ic = dio->ic;

drivers/md/dm-integrity.c

2780

struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

2781

int r = dm_integrity_map_inline(dio, false);

drivers/md/dm-integrity.c

2795

dm_integrity_map_continue(dio, false);

drivers/md/dm-integrity.c

3426

struct dm_integrity_io *dio;

drivers/md/dm-integrity.c

3428

dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

3430

if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,

drivers/md/dm-integrity.c

3431

dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {

drivers/md/dm-integrity.c

3432

remove_range(ic, &dio->range);

drivers/md/dm-integrity.c

3433

INIT_WORK(&dio->work, integrity_bio_wait);

drivers/md/dm-integrity.c

3434

queue_work(ic->offload_wq, &dio->work);

drivers/md/dm-integrity.c

3436

block_bitmap_op(ic, ic->journal, dio->range.logical_sector,

drivers/md/dm-integrity.c

3437

dio->range.n_sectors, BITMAP_OP_SET);

drivers/md/dm-integrity.c

3450

struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));

drivers/md/dm-integrity.c

3452

block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,

drivers/md/dm-integrity.c

3453

dio->range.n_sectors, BITMAP_OP_SET);

drivers/md/dm-integrity.c

3455

remove_range(ic, &dio->range);

drivers/md/dm-integrity.c

3456

INIT_WORK(&dio->work, integrity_bio_wait);

drivers/md/dm-integrity.c

3457

queue_work(ic->offload_wq, &dio->work);

drivers/md/dm-integrity.c

375

static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);

drivers/md/dm-integrity.c

376

static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map);

drivers/scsi/bfa/bfa_fcpim.c

2076

bfa_cb_ioim_good_comp(ioim->bfa->bfad, ioim->dio);

drivers/scsi/bfa/bfa_fcpim.c

2119

bfa_cb_ioim_done(ioim->bfa->bfad, ioim->dio, m->io_status,

drivers/scsi/bfa/bfa_fcpim.c

2345

bfa_cb_ioim_done(ioim->bfa->bfad, ioim->dio, BFI_IOIM_STS_ABORTED,

drivers/scsi/bfa/bfa_fcpim.c

2360

bfa_cb_ioim_done(ioim->bfa->bfad, ioim->dio, BFI_IOIM_STS_PATHTOV,

drivers/scsi/bfa/bfa_fcpim.c

2374

bfa_cb_ioim_abort(ioim->bfa->bfad, ioim->dio);

drivers/scsi/bfa/bfa_fcpim.c

2403

struct scsi_cmnd *cmnd = (struct scsi_cmnd *) ioim->dio;

drivers/scsi/bfa/bfa_fcpim.c

2879

bfa_ioim_alloc(struct bfa_s *bfa, struct bfad_ioim_s *dio,

drivers/scsi/bfa/bfa_fcpim.c

2897

ioim->dio = dio;

drivers/scsi/bfa/bfa_fcpim.c

3267

cmnd = (struct scsi_cmnd *) ioim->dio;

drivers/scsi/bfa/bfa_fcpim.c

3280

cmnd = (struct scsi_cmnd *) ioim->dio;

drivers/scsi/bfa/bfa_fcpim.c

396

idx = bfa_ioim_get_index(scsi_bufflen((struct scsi_cmnd *)ioim->dio));

drivers/scsi/bfa/bfa_fcpim.h

193

struct bfad_ioim_s *dio; /* driver IO handle */

drivers/scsi/bfa/bfa_fcpim.h

418

struct bfad_ioim_s *dio,

drivers/scsi/bfa/bfa_fcpim.h

439

void bfa_cb_ioim_done(void *bfad, struct bfad_ioim_s *dio,

drivers/scsi/bfa/bfa_fcpim.h

447

void bfa_cb_ioim_good_comp(void *bfad, struct bfad_ioim_s *dio);

drivers/scsi/bfa/bfa_fcpim.h

452

void bfa_cb_ioim_abort(void *bfad, struct bfad_ioim_s *dio);

drivers/scsi/bfa/bfad_im.c

104

bfa_cb_ioim_good_comp(void *drv, struct bfad_ioim_s *dio)

drivers/scsi/bfa/bfad_im.c

106

struct scsi_cmnd *cmnd = (struct scsi_cmnd *)dio;

drivers/scsi/bfa/bfad_im.c

132

bfa_cb_ioim_abort(void *drv, struct bfad_ioim_s *dio)

drivers/scsi/bfa/bfad_im.c

134

struct scsi_cmnd *cmnd = (struct scsi_cmnd *)dio;

drivers/scsi/bfa/bfad_im.c

209

if (hal_io->dio != (struct bfad_ioim_s *) cmnd) {

drivers/scsi/bfa/bfad_im.c

34

bfa_cb_ioim_done(void *drv, struct bfad_ioim_s *dio,

drivers/scsi/bfa/bfad_im.c

38

struct scsi_cmnd *cmnd = (struct scsi_cmnd *)dio;

fs/btrfs/direct-io.c

854

struct iomap_dio *dio;

fs/btrfs/direct-io.c

957

dio = btrfs_dio_write(iocb, from, written);

fs/btrfs/direct-io.c

960

if (IS_ERR_OR_NULL(dio)) {

fs/btrfs/direct-io.c

961

ret = PTR_ERR_OR_ZERO(dio);

fs/btrfs/direct-io.c

973

ret = iomap_dio_complete(dio);

fs/direct-io.c

1008

dio_zero_block(dio, sdio, 0, map_bh);

fs/direct-io.c

1026

ret = submit_page_section(dio, sdio, page,

fs/direct-io.c

1032

dio_unpin_page(dio, page);

fs/direct-io.c

1039

dio->result += this_chunk_bytes;

fs/direct-io.c

1048

dio_unpin_page(dio, page);

fs/direct-io.c

1054

static inline int drop_refcount(struct dio *dio)

fs/direct-io.c

1070

spin_lock_irqsave(&dio->bio_lock, flags);

fs/direct-io.c

1071

ret2 = --dio->refcount;

fs/direct-io.c

1072

spin_unlock_irqrestore(&dio->bio_lock, flags);

fs/direct-io.c

1113

struct dio *dio;

fs/direct-io.c

1123

dio = kmem_cache_alloc(dio_cache, GFP_KERNEL);

fs/direct-io.c

1124

if (!dio)

fs/direct-io.c

1131

memset(dio, 0, offsetof(struct dio, pages));

fs/direct-io.c

1133

dio->flags = flags;

fs/direct-io.c

1134

if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) {

fs/direct-io.c

1138

dio->is_pinned = iov_iter_extract_will_pin(iter);

fs/direct-io.c

1141

dio->i_size = i_size_read(inode);

fs/direct-io.c

1142

if (iov_iter_rw(iter) == READ && offset >= dio->i_size) {

fs/direct-io.c

1155

if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) {

fs/direct-io.c

1170

dio->is_async = false;

fs/direct-io.c

1172

dio->is_async = false;

fs/direct-io.c

1174

dio->is_async = true;

fs/direct-io.c

1176

dio->inode = inode;

fs/direct-io.c

1178

dio->opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;

fs/direct-io.c

1180

dio->opf = REQ_OP_READ;

fs/direct-io.c

1186

if (dio->is_async && iov_iter_rw(iter) == WRITE) {

fs/direct-io.c

1189

retval = dio_set_defer_completion(dio);

fs/direct-io.c

1190

else if (!dio->inode->i_sb->s_dio_done_wq) {

fs/direct-io.c

1196

retval = sb_init_dio_done_wq(dio->inode->i_sb);

fs/direct-io.c

1212

dio->end_io = end_io;

fs/direct-io.c

1216

dio->iocb = iocb;

fs/direct-io.c

1218

spin_lock_init(&dio->bio_lock);

fs/direct-io.c

1219

dio->refcount = 1;

fs/direct-io.c

1221

dio->should_dirty = user_backed_iter(iter) && iov_iter_rw(iter) == READ;

fs/direct-io.c

1236

retval = do_direct_IO(dio, &sdio, &map_bh);

fs/direct-io.c

1238

dio_cleanup(dio, &sdio);

fs/direct-io.c

1251

dio_zero_block(dio, &sdio, 1, &map_bh);

fs/direct-io.c

1256

ret2 = dio_send_cur_page(dio, &sdio, &map_bh);

fs/direct-io.c

1259

dio_unpin_page(dio, sdio.cur_page);

fs/direct-io.c

1263

dio_bio_submit(dio, &sdio);

fs/direct-io.c

1271

dio_cleanup(dio, &sdio);

fs/direct-io.c

1278

if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING))

fs/direct-io.c

1279

inode_unlock(dio->inode);

fs/direct-io.c

1289

if (dio->is_async && retval == 0 && dio->result &&

fs/direct-io.c

1290

(iov_iter_rw(iter) == READ || dio->result == count))

fs/direct-io.c

1293

dio_await_completion(dio);

fs/direct-io.c

1295

if (drop_refcount(dio) == 0) {

fs/direct-io.c

1296

retval = dio_complete(dio, retval, DIO_COMPLETE_INVALIDATE);

fs/direct-io.c

1303

if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ)

fs/direct-io.c

1306

kmem_cache_free(dio_cache, dio);

fs/direct-io.c

1313

dio_cache = KMEM_CACHE(dio, SLAB_PANIC);

fs/direct-io.c

166

static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)

fs/direct-io.c

168

struct page **pages = dio->pages;

fs/direct-io.c

169

const enum req_op dio_op = dio->opf & REQ_OP_MASK;

fs/direct-io.c

181

if (dio->page_errors == 0)

fs/direct-io.c

182

dio->page_errors = ret;

fs/direct-io.c

183

dio->pages[0] = ZERO_PAGE(0);

fs/direct-io.c

207

static inline struct page *dio_get_page(struct dio *dio,

fs/direct-io.c

213

ret = dio_refill_pages(dio, sdio);

fs/direct-io.c

218

return dio->pages[sdio->head];

fs/direct-io.c

221

static void dio_pin_page(struct dio *dio, struct page *page)

fs/direct-io.c

223

if (dio->is_pinned)

fs/direct-io.c

227

static void dio_unpin_page(struct dio *dio, struct page *page)

fs/direct-io.c

229

if (dio->is_pinned)

fs/direct-io.c

244

static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags)

fs/direct-io.c

246

const enum req_op dio_op = dio->opf & REQ_OP_MASK;

fs/direct-io.c

247

loff_t offset = dio->iocb->ki_pos;

fs/direct-io.c

260

if (dio->result) {

fs/direct-io.c

261

transferred = dio->result;

fs/direct-io.c

265

((offset + transferred) > dio->i_size))

fs/direct-io.c

266

transferred = dio->i_size - offset;

fs/direct-io.c

273

ret = dio->page_errors;

fs/direct-io.c

275

ret = dio->io_error;

fs/direct-io.c

279

if (dio->end_io) {

fs/direct-io.c

281

err = dio->end_io(dio->iocb, offset, ret, dio->private);

fs/direct-io.c

300

kiocb_invalidate_post_direct_write(dio->iocb, ret);

fs/direct-io.c

302

inode_dio_end(dio->inode);

fs/direct-io.c

310

dio->iocb->ki_pos += transferred;

fs/direct-io.c

313

ret = generic_write_sync(dio->iocb, ret);

fs/direct-io.c

314

dio->iocb->ki_complete(dio->iocb, ret);

fs/direct-io.c

317

kmem_cache_free(dio_cache, dio);

fs/direct-io.c

323

struct dio *dio = container_of(work, struct dio, complete_work);

fs/direct-io.c

325

dio_complete(dio, 0, DIO_COMPLETE_ASYNC | DIO_COMPLETE_INVALIDATE);

fs/direct-io.c

328

static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio);

fs/direct-io.c

335

struct dio *dio = bio->bi_private;

fs/direct-io.c

336

const enum req_op dio_op = dio->opf & REQ_OP_MASK;

fs/direct-io.c

342

dio_bio_complete(dio, bio);

fs/direct-io.c

344

spin_lock_irqsave(&dio->bio_lock, flags);

fs/direct-io.c

345

remaining = --dio->refcount;

fs/direct-io.c

346

if (remaining == 1 && dio->waiter)

fs/direct-io.c

347

wake_up_process(dio->waiter);

fs/direct-io.c

348

spin_unlock_irqrestore(&dio->bio_lock, flags);

fs/direct-io.c

359

if (dio->result)

fs/direct-io.c

360

defer_completion = dio->defer_completion ||

fs/direct-io.c

362

dio->inode->i_mapping->nrpages);

fs/direct-io.c

364

INIT_WORK(&dio->complete_work, dio_aio_complete_work);

fs/direct-io.c

365

queue_work(dio->inode->i_sb->s_dio_done_wq,

fs/direct-io.c

366

&dio->complete_work);

fs/direct-io.c

368

dio_complete(dio, 0, DIO_COMPLETE_ASYNC);

fs/direct-io.c

382

struct dio *dio = bio->bi_private;

fs/direct-io.c

385

spin_lock_irqsave(&dio->bio_lock, flags);

fs/direct-io.c

386

bio->bi_private = dio->bio_list;

fs/direct-io.c

387

dio->bio_list = bio;

fs/direct-io.c

388

if (--dio->refcount == 1 && dio->waiter)

fs/direct-io.c

389

wake_up_process(dio->waiter);

fs/direct-io.c

390

spin_unlock_irqrestore(&dio->bio_lock, flags);

fs/direct-io.c

394

dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,

fs/direct-io.c

404

bio = bio_alloc(bdev, nr_vecs, dio->opf, GFP_KERNEL);

fs/direct-io.c

406

if (dio->is_async)

fs/direct-io.c

410

if (dio->is_pinned)

fs/direct-io.c

412

bio->bi_write_hint = file_inode(dio->iocb->ki_filp)->i_write_hint;

fs/direct-io.c

425

static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)

fs/direct-io.c

427

const enum req_op dio_op = dio->opf & REQ_OP_MASK;

fs/direct-io.c

431

bio->bi_private = dio;

fs/direct-io.c

433

spin_lock_irqsave(&dio->bio_lock, flags);

fs/direct-io.c

434

dio->refcount++;

fs/direct-io.c

435

spin_unlock_irqrestore(&dio->bio_lock, flags);

fs/direct-io.c

437

if (dio->is_async && dio_op == REQ_OP_READ && dio->should_dirty)

fs/direct-io.c

440

dio->bio_disk = bio->bi_bdev->bd_disk;

fs/direct-io.c

452

static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)

fs/direct-io.c

454

if (dio->is_pinned)

fs/direct-io.c

455

unpin_user_pages(dio->pages + sdio->head,

fs/direct-io.c

466

static struct bio *dio_await_one(struct dio *dio)

fs/direct-io.c

471

spin_lock_irqsave(&dio->bio_lock, flags);

fs/direct-io.c

479

while (dio->refcount > 1 && dio->bio_list == NULL) {

fs/direct-io.c

481

dio->waiter = current;

fs/direct-io.c

482

spin_unlock_irqrestore(&dio->bio_lock, flags);

fs/direct-io.c

485

spin_lock_irqsave(&dio->bio_lock, flags);

fs/direct-io.c

486

dio->waiter = NULL;

fs/direct-io.c

488

if (dio->bio_list) {

fs/direct-io.c

489

bio = dio->bio_list;

fs/direct-io.c

490

dio->bio_list = bio->bi_private;

fs/direct-io.c

492

spin_unlock_irqrestore(&dio->bio_lock, flags);

fs/direct-io.c

499

static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio)

fs/direct-io.c

502

const enum req_op dio_op = dio->opf & REQ_OP_MASK;

fs/direct-io.c

503

bool should_dirty = dio_op == REQ_OP_READ && dio->should_dirty;

fs/direct-io.c

506

dio->io_error = -EIO;

fs/direct-io.c

508

if (dio->is_async && should_dirty) {

fs/direct-io.c

524

static void dio_await_completion(struct dio *dio)

fs/direct-io.c

528

bio = dio_await_one(dio);

fs/direct-io.c

530

dio_bio_complete(dio, bio);

fs/direct-io.c

541

static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)

fs/direct-io.c

546

while (dio->bio_list) {

fs/direct-io.c

551

spin_lock_irqsave(&dio->bio_lock, flags);

fs/direct-io.c

552

bio = dio->bio_list;

fs/direct-io.c

553

dio->bio_list = bio->bi_private;

fs/direct-io.c

554

spin_unlock_irqrestore(&dio->bio_lock, flags);

fs/direct-io.c

555

ret2 = blk_status_to_errno(dio_bio_complete(dio, bio));

fs/direct-io.c

564

static int dio_set_defer_completion(struct dio *dio)

fs/direct-io.c

566

struct super_block *sb = dio->inode->i_sb;

fs/direct-io.c

568

if (dio->defer_completion)

fs/direct-io.c

570

dio->defer_completion = true;

fs/direct-io.c

599

static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,

fs/direct-io.c

602

const enum req_op dio_op = dio->opf & REQ_OP_MASK;

fs/direct-io.c

615

ret = dio->page_errors;

fs/direct-io.c

638

if (dio->flags & DIO_SKIP_HOLES) {

fs/direct-io.c

639

i_size = i_size_read(dio->inode);

fs/direct-io.c

644

ret = (*sdio->get_block)(dio->inode, fs_startblk,

fs/direct-io.c

648

dio->private = map_bh->b_private;

fs/direct-io.c

651

ret = dio_set_defer_completion(dio);

fs/direct-io.c

659

static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,

fs/direct-io.c

665

ret = dio_bio_reap(dio, sdio);

fs/direct-io.c

671

dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);

fs/direct-io.c

684

static inline int dio_bio_add_page(struct dio *dio, struct dio_submit *sdio)

fs/direct-io.c

696

dio_pin_page(dio, sdio->cur_page);

fs/direct-io.c

716

static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,

fs/direct-io.c

742

dio_bio_submit(dio, sdio);

fs/direct-io.c

746

ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);

fs/direct-io.c

751

if (dio_bio_add_page(dio, sdio) != 0) {

fs/direct-io.c

752

dio_bio_submit(dio, sdio);

fs/direct-io.c

753

ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);

fs/direct-io.c

755

ret = dio_bio_add_page(dio, sdio);

fs/direct-io.c

781

submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,

fs/direct-io.c

785

const enum req_op dio_op = dio->opf & REQ_OP_MASK;

fs/direct-io.c

811

ret = dio_send_cur_page(dio, sdio, map_bh);

fs/direct-io.c

812

dio_unpin_page(dio, sdio->cur_page);

fs/direct-io.c

818

dio_pin_page(dio, page); /* It is in dio */

fs/direct-io.c

830

ret = dio_send_cur_page(dio, sdio, map_bh);

fs/direct-io.c

832

dio_bio_submit(dio, sdio);

fs/direct-io.c

833

dio_unpin_page(dio, sdio->cur_page);

fs/direct-io.c

848

static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,

fs/direct-io.c

876

if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,

fs/direct-io.c

899

static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,

fs/direct-io.c

902

const enum req_op dio_op = dio->opf & REQ_OP_MASK;

fs/direct-io.c

911

page = dio_get_page(dio, sdio);

fs/direct-io.c

932

ret = get_more_blocks(dio, sdio, map_bh);

fs/direct-io.c

934

dio_unpin_page(dio, page);

fs/direct-io.c

979

dio_unpin_page(dio, page);

fs/direct-io.c

987

i_size_aligned = ALIGN(i_size_read(dio->inode),

fs/direct-io.c

992

dio_unpin_page(dio, page);

fs/direct-io.c

998

dio->result += 1 << blkbits;

fs/f2fs/file.c

4857

struct iomap_dio *dio;

fs/f2fs/file.c

4887

dio = __iomap_dio_rw(iocb, to, &f2fs_iomap_ops,

fs/f2fs/file.c

4889

if (IS_ERR_OR_NULL(dio)) {

fs/f2fs/file.c

4890

ret = PTR_ERR_OR_ZERO(dio);

fs/f2fs/file.c

4894

ret = iomap_dio_complete(dio);

fs/f2fs/file.c

4932

bool dio;

fs/f2fs/file.c

4941

dio = f2fs_should_use_dio(inode, iocb, to);

fs/f2fs/file.c

4946

(!f2fs_is_pinned_file(inode) || !dio))

fs/f2fs/file.c

4949

if (dio) {

fs/f2fs/file.c

5018

bool dio)

fs/f2fs/file.c

5029

if (dio && f2fs_lfs_mode(sbi))

fs/f2fs/file.c

5035

if (dio && i_size_read(inode) &&

fs/f2fs/file.c

5066

if (dio) {

fs/f2fs/file.c

5177

struct iomap_dio *dio;

fs/f2fs/file.c

5223

dio = __iomap_dio_rw(iocb, from, &f2fs_iomap_ops,

fs/f2fs/file.c

5225

if (IS_ERR_OR_NULL(dio)) {

fs/f2fs/file.c

5226

ret = PTR_ERR_OR_ZERO(dio);

fs/f2fs/file.c

5232

ret = iomap_dio_complete(dio);

fs/f2fs/file.c

5289

bool dio;

fs/f2fs/file.c

5326

dio = f2fs_should_use_dio(inode, iocb, from);

fs/f2fs/file.c

5329

if (dio && f2fs_is_atomic_file(inode)) {

fs/f2fs/file.c

5336

preallocated = f2fs_preallocate_blocks(iocb, from, dio);

fs/f2fs/file.c

5345

ret = dio ?

fs/f2fs/file.c

5376

if (ret > 0 && !dio && (iocb->ki_flags & IOCB_DIRECT))

fs/iomap/direct-io.c

104

ssize_t iomap_dio_complete(struct iomap_dio *dio)

fs/iomap/direct-io.c

106

const struct iomap_dio_ops *dops = dio->dops;

fs/iomap/direct-io.c

107

struct kiocb *iocb = dio->iocb;

fs/iomap/direct-io.c

109

ssize_t ret = dio->error;

fs/iomap/direct-io.c

112

ret = dops->end_io(iocb, dio->size, ret, dio->flags);

fs/iomap/direct-io.c

113

if (should_report_dio_fserror(dio))

fs/iomap/direct-io.c

115

iomap_dio_err_type(dio), offset, dio->size,

fs/iomap/direct-io.c

116

dio->error, GFP_NOFS);

fs/iomap/direct-io.c

119

ret = dio->size;

fs/iomap/direct-io.c

121

if (offset + ret > dio->i_size &&

fs/iomap/direct-io.c

122

!(dio->flags & IOMAP_DIO_WRITE))

fs/iomap/direct-io.c

123

ret = dio->i_size - offset;

fs/iomap/direct-io.c

138

if (!dio->error && dio->size && (dio->flags & IOMAP_DIO_WRITE) &&

fs/iomap/direct-io.c

139

!(dio->flags & IOMAP_DIO_NO_INVALIDATE))

fs/iomap/direct-io.c

140

kiocb_invalidate_post_direct_write(iocb, dio->size);

fs/iomap/direct-io.c

151

if (dio->flags & IOMAP_DIO_NEED_SYNC)

fs/iomap/direct-io.c

154

ret += dio->done_before;

fs/iomap/direct-io.c

156

trace_iomap_dio_complete(iocb, dio->error, ret);

fs/iomap/direct-io.c

157

kfree(dio);

fs/iomap/direct-io.c

164

struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);

fs/iomap/direct-io.c

165

struct kiocb *iocb = dio->iocb;

fs/iomap/direct-io.c

167

iocb->ki_complete(iocb, iomap_dio_complete(dio));

fs/iomap/direct-io.c

175

static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)

fs/iomap/direct-io.c

177

cmpxchg(&dio->error, 0, ret);

fs/iomap/direct-io.c

183

static void iomap_dio_done(struct iomap_dio *dio)

fs/iomap/direct-io.c

185

struct kiocb *iocb = dio->iocb;

fs/iomap/direct-io.c

187

if (dio->wait_for_completion) {

fs/iomap/direct-io.c

192

struct task_struct *waiter = dio->submit.waiter;

fs/iomap/direct-io.c

194

WRITE_ONCE(dio->submit.waiter, NULL);

fs/iomap/direct-io.c

204

if (dio->error)

fs/iomap/direct-io.c

205

dio->flags |= IOMAP_DIO_COMP_WORK;

fs/iomap/direct-io.c

214

if ((dio->flags & IOMAP_DIO_WRITE) &&

fs/iomap/direct-io.c

215

!(dio->flags & IOMAP_DIO_COMP_WORK)) {

fs/iomap/direct-io.c

216

if (dio->iocb->ki_filp->f_mapping->nrpages)

fs/iomap/direct-io.c

217

dio->flags |= IOMAP_DIO_COMP_WORK;

fs/iomap/direct-io.c

219

dio->flags |= IOMAP_DIO_NO_INVALIDATE;

fs/iomap/direct-io.c

222

if (dio->flags & IOMAP_DIO_COMP_WORK) {

fs/iomap/direct-io.c

231

INIT_WORK(&dio->aio.work, iomap_dio_complete_work);

fs/iomap/direct-io.c

232

queue_work(inode->i_sb->s_dio_done_wq, &dio->aio.work);

fs/iomap/direct-io.c

237

iomap_dio_complete_work(&dio->aio.work);

fs/iomap/direct-io.c

242

struct iomap_dio *dio = bio->bi_private;

fs/iomap/direct-io.c

247

if (dio->flags & IOMAP_DIO_BOUNCE) {

fs/iomap/direct-io.c

248

bio_iov_iter_unbounce(bio, !!dio->error,

fs/iomap/direct-io.c

249

dio->flags & IOMAP_DIO_USER_BACKED);

fs/iomap/direct-io.c

251

} else if (dio->flags & IOMAP_DIO_USER_BACKED) {

fs/iomap/direct-io.c

260

if (atomic_dec_and_test(&dio->ref)) {

fs/iomap/direct-io.c

266

dio->flags &= ~IOMAP_DIO_COMP_WORK;

fs/iomap/direct-io.c

267

iomap_dio_done(dio);

fs/iomap/direct-io.c

273

struct iomap_dio *dio = bio->bi_private;

fs/iomap/direct-io.c

276

iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));

fs/iomap/direct-io.c

283

struct iomap_dio *dio = ioend->io_bio.bi_private;

fs/iomap/direct-io.c

287

iomap_dio_set_error(dio, ioend->io_error);

fs/iomap/direct-io.c

298

static int iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio,

fs/iomap/direct-io.c

301

struct inode *inode = file_inode(dio->iocb->ki_filp);

fs/iomap/direct-io.c

316

bio = iomap_dio_alloc_bio(iter, dio, nr_vecs,

fs/iomap/direct-io.c

320

bio->bi_private = dio;

fs/iomap/direct-io.c

329

iomap_dio_submit_bio(iter, dio, bio, pos);

fs/iomap/direct-io.c

335

struct iomap_dio *dio, loff_t pos, unsigned int alignment,

fs/iomap/direct-io.c

342

if (dio->flags & IOMAP_DIO_BOUNCE)

fs/iomap/direct-io.c

343

nr_vecs = bio_iov_bounce_nr_vecs(dio->submit.iter, op);

fs/iomap/direct-io.c

345

nr_vecs = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS);

fs/iomap/direct-io.c

347

bio = iomap_dio_alloc_bio(iter, dio, nr_vecs, op);

fs/iomap/direct-io.c

351

bio->bi_ioprio = dio->iocb->ki_ioprio;

fs/iomap/direct-io.c

352

bio->bi_private = dio;

fs/iomap/direct-io.c

356

if (dio->flags & IOMAP_DIO_BOUNCE)

fs/iomap/direct-io.c

357

ret = bio_iov_iter_bounce(bio, dio->submit.iter,

fs/iomap/direct-io.c

360

ret = bio_iov_iter_get_pages(bio, dio->submit.iter,

fs/iomap/direct-io.c

376

if (dio->flags & IOMAP_DIO_WRITE)

fs/iomap/direct-io.c

382

if (dio->flags & IOMAP_DIO_WRITE)

fs/iomap/direct-io.c

384

else if ((dio->flags & IOMAP_DIO_USER_BACKED) &&

fs/iomap/direct-io.c

385

!(dio->flags & IOMAP_DIO_BOUNCE))

fs/iomap/direct-io.c

391

if (iov_iter_count(dio->submit.iter))

fs/iomap/direct-io.c

392

dio->iocb->ki_flags &= ~IOCB_HIPRI;

fs/iomap/direct-io.c

393

iomap_dio_submit_bio(iter, dio, bio, pos);

fs/iomap/direct-io.c

397

if (dio->flags & IOMAP_DIO_BOUNCE)

fs/iomap/direct-io.c

406

static int iomap_dio_bio_iter(struct iomap_iter *iter, struct iomap_dio *dio)

fs/iomap/direct-io.c

424

if (dio->flags & IOMAP_DIO_FSBLOCK_ALIGNED)

fs/iomap/direct-io.c

432

if (dio->flags & IOMAP_DIO_WRITE) {

fs/iomap/direct-io.c

445

dio->flags |= IOMAP_DIO_UNWRITTEN;

fs/iomap/direct-io.c

469

dio->flags |= IOMAP_DIO_COW;

fs/iomap/direct-io.c

487

if (dio->flags & IOMAP_DIO_WRITE_THROUGH) {

fs/iomap/direct-io.c

494

dio->flags &= ~IOMAP_DIO_WRITE_THROUGH;

fs/iomap/direct-io.c

508

dio->flags |= IOMAP_DIO_COMP_WORK;

fs/iomap/direct-io.c

520

orig_count = iov_iter_count(dio->submit.iter);

fs/iomap/direct-io.c

521

iov_iter_truncate(dio->submit.iter, length);

fs/iomap/direct-io.c

523

if (!iov_iter_count(dio->submit.iter))

fs/iomap/direct-io.c

531

if (dio->flags & IOMAP_DIO_COMP_WORK)

fs/iomap/direct-io.c

532

dio->iocb->ki_flags &= ~IOCB_HIPRI;

fs/iomap/direct-io.c

538

ret = iomap_dio_zero(iter, dio, pos - pad, pad);

fs/iomap/direct-io.c

548

if (unlikely(data_race(dio->error)))

fs/iomap/direct-io.c

55

struct iomap_dio *dio, unsigned short nr_vecs, blk_opf_t opf)

fs/iomap/direct-io.c

551

ret = iomap_dio_bio_iter_one(iter, dio, pos, alignment, bio_opf);

fs/iomap/direct-io.c

561

dio->size += ret;

fs/iomap/direct-io.c

565

} while (iov_iter_count(dio->submit.iter));

fs/iomap/direct-io.c

57

if (dio->dops && dio->dops->bio_set)

fs/iomap/direct-io.c

574

((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {

fs/iomap/direct-io.c

578

ret = iomap_dio_zero(iter, dio, pos,

fs/iomap/direct-io.c

583

iov_iter_reexpand(dio->submit.iter, orig_count - copied);

fs/iomap/direct-io.c

589

static int iomap_dio_hole_iter(struct iomap_iter *iter, struct iomap_dio *dio)

fs/iomap/direct-io.c

59

GFP_KERNEL, dio->dops->bio_set);

fs/iomap/direct-io.c

591

loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter);

fs/iomap/direct-io.c

593

dio->size += length;

fs/iomap/direct-io.c

599

static int iomap_dio_inline_iter(struct iomap_iter *iomi, struct iomap_dio *dio)

fs/iomap/direct-io.c

602

struct iov_iter *iter = dio->submit.iter;

fs/iomap/direct-io.c

611

if (dio->flags & IOMAP_DIO_WRITE) {

fs/iomap/direct-io.c

625

dio->size += copied;

fs/iomap/direct-io.c

631

static int iomap_dio_iter(struct iomap_iter *iter, struct iomap_dio *dio)

fs/iomap/direct-io.c

635

if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))

fs/iomap/direct-io.c

637

return iomap_dio_hole_iter(iter, dio);

fs/iomap/direct-io.c

639

if (!(dio->flags & IOMAP_DIO_WRITE))

fs/iomap/direct-io.c

64

struct iomap_dio *dio, struct bio *bio, loff_t pos)

fs/iomap/direct-io.c

640

return iomap_dio_hole_iter(iter, dio);

fs/iomap/direct-io.c

641

return iomap_dio_bio_iter(iter, dio);

fs/iomap/direct-io.c

643

return iomap_dio_bio_iter(iter, dio);

fs/iomap/direct-io.c

645

return iomap_dio_inline_iter(iter, dio);

fs/iomap/direct-io.c

654

dio->iocb->ki_filp, current->comm);

fs/iomap/direct-io.c

66

struct kiocb *iocb = dio->iocb;

fs/iomap/direct-io.c

68

atomic_inc(&dio->ref);

fs/iomap/direct-io.c

698

struct iomap_dio *dio;

fs/iomap/direct-io.c

706

dio = kmalloc_obj(*dio);

fs/iomap/direct-io.c

707

if (!dio)

fs/iomap/direct-io.c

710

dio->iocb = iocb;

fs/iomap/direct-io.c

711

atomic_set(&dio->ref, 1);

fs/iomap/direct-io.c

712

dio->size = 0;

fs/iomap/direct-io.c

713

dio->i_size = i_size_read(inode);

fs/iomap/direct-io.c

714

dio->dops = dops;

fs/iomap/direct-io.c

715

dio->error = 0;

fs/iomap/direct-io.c

716

dio->flags = dio_flags & (IOMAP_DIO_FSBLOCK_ALIGNED | IOMAP_DIO_BOUNCE);

fs/iomap/direct-io.c

717

dio->done_before = done_before;

fs/iomap/direct-io.c

719

dio->submit.iter = iter;

fs/iomap/direct-io.c

720

dio->submit.waiter = current;

fs/iomap/direct-io.c

726

if (iomi.pos >= dio->i_size)

fs/iomap/direct-io.c

730

dio->flags |= IOMAP_DIO_USER_BACKED;

fs/iomap/direct-io.c

737

dio->flags |= IOMAP_DIO_WRITE;

fs/iomap/direct-io.c

741

if (iomi.pos >= dio->i_size ||

fs/iomap/direct-io.c

742

iomi.pos + iomi.len > dio->i_size)

fs/iomap/direct-io.c

752

dio->flags |= IOMAP_DIO_NEED_SYNC;

fs/iomap/direct-io.c

76

if (dio->dops && dio->dops->submit_io) {

fs/iomap/direct-io.c

764

dio->flags |= IOMAP_DIO_WRITE_THROUGH;

fs/iomap/direct-io.c

77

dio->dops->submit_io(iter, bio, pos);

fs/iomap/direct-io.c

770

if (iomi.pos + iomi.len > dio->i_size)

fs/iomap/direct-io.c

771

dio->flags |= IOMAP_DIO_COMP_WORK;

fs/iomap/direct-io.c

809

iomi.status = iomap_dio_iter(&iomi, dio);

fs/iomap/direct-io.c

824

if (iov_iter_rw(iter) == READ && iomi.pos >= dio->i_size)

fs/iomap/direct-io.c

825

iov_iter_revert(iter, iomi.pos - dio->i_size);

fs/iomap/direct-io.c

827

if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) {

fs/iomap/direct-io.c

839

iomap_dio_set_error(dio, ret);

fs/iomap/direct-io.c

84

static inline enum fserror_type iomap_dio_err_type(const struct iomap_dio *dio)

fs/iomap/direct-io.c

849

if (dio->flags & IOMAP_DIO_WRITE_THROUGH)

fs/iomap/direct-io.c

850

dio->flags &= ~IOMAP_DIO_NEED_SYNC;

fs/iomap/direct-io.c

851

else if (dio->flags & IOMAP_DIO_NEED_SYNC)

fs/iomap/direct-io.c

852

dio->flags |= IOMAP_DIO_COMP_WORK;

fs/iomap/direct-io.c

86

if (dio->flags & IOMAP_DIO_WRITE)

fs/iomap/direct-io.c

869

dio->wait_for_completion = wait_for_completion;

fs/iomap/direct-io.c

870

if (!atomic_dec_and_test(&dio->ref)) {

fs/iomap/direct-io.c

878

if (!READ_ONCE(dio->submit.waiter))

fs/iomap/direct-io.c

886

return dio;

fs/iomap/direct-io.c

889

kfree(dio);

fs/iomap/direct-io.c

901

struct iomap_dio *dio;

fs/iomap/direct-io.c

903

dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private,

fs/iomap/direct-io.c

905

if (IS_ERR_OR_NULL(dio))

fs/iomap/direct-io.c

906

return PTR_ERR_OR_ZERO(dio);

fs/iomap/direct-io.c

907

return iomap_dio_complete(dio);

fs/iomap/direct-io.c

91

static inline bool should_report_dio_fserror(const struct iomap_dio *dio)

fs/iomap/direct-io.c

93

switch (dio->error) {

fs/ocfs2/namei.c

2151

bool dio)

fs/ocfs2/namei.c

2155

int namelen = dio ?

fs/ocfs2/namei.c

2159

if (dio) {

fs/ocfs2/namei.c

2209

bool dio)

fs/ocfs2/namei.c

2223

blkno, name, lookup, dio);

fs/ocfs2/namei.c

2252

bool dio)

fs/ocfs2/namei.c

2258

int namelen = dio ?

fs/ocfs2/namei.c

2312

if (dio) {

fs/ocfs2/namei.c

2352

bool dio)

fs/ocfs2/namei.c

2359

if (dio) {

fs/ocfs2/namei.c

70

bool dio);

fs/ocfs2/namei.c

79

bool dio);

fs/ocfs2/namei.h

25

bool dio);

include/linux/iomap.h

606

ssize_t iomap_dio_complete(struct iomap_dio *dio);