drivers/block/zram/zram_drv.c

100

lock_acquired(slot_dep_map(zram, index), _RET_IP_);

drivers/block/zram/zram_drv.c

1007

err = zram_writeback_complete(zram, req);

drivers/block/zram/zram_drv.c

1012

release_pp_slot(zram, req->pps);

drivers/block/zram/zram_drv.c

103

static void slot_unlock(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

1032

static int zram_writeback_slots(struct zram *zram,

drivers/block/zram/zram_drv.c

1043

if (zram->wb_limit_enable && !zram->bd_wb_limit) {

drivers/block/zram/zram_drv.c

105

unsigned long *lock = &zram->table[index].__lock;

drivers/block/zram/zram_drv.c

1056

err = zram_complete_done_reqs(zram, wb_ctl);

drivers/block/zram/zram_drv.c

107

mutex_release(slot_dep_map(zram, index), _RET_IP_);

drivers/block/zram/zram_drv.c

1070

blk_idx = zram_reserve_bdev_block(zram);

drivers/block/zram/zram_drv.c

1078

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

1085

if (!test_slot_flag(zram, index, ZRAM_PP_SLOT))

drivers/block/zram/zram_drv.c

1087

if (zram->compressed_wb)

drivers/block/zram/zram_drv.c

1088

err = read_from_zspool_raw(zram, req->page, index);

drivers/block/zram/zram_drv.c

1090

err = read_from_zspool(zram, req->page, index);

drivers/block/zram/zram_drv.c

1093

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

1103

bio_init(&req->bio, zram->bdev, &req->bio_vec, 1, REQ_OP_WRITE);

drivers/block/zram/zram_drv.c

1108

zram_submit_wb_request(zram, wb_ctl, req);

drivers/block/zram/zram_drv.c

111

static inline bool init_done(struct zram *zram)

drivers/block/zram/zram_drv.c

1115

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

1116

release_pp_slot(zram, pps);

drivers/block/zram/zram_drv.c

1128

err = zram_complete_done_reqs(zram, wb_ctl);

drivers/block/zram/zram_drv.c

113

return zram->disksize;

drivers/block/zram/zram_drv.c

116

static inline struct zram *dev_to_zram(struct device *dev)

drivers/block/zram/zram_drv.c

118

return (struct zram *)dev_to_disk(dev)->private_data;

drivers/block/zram/zram_drv.c

1199

static int scan_slots_for_writeback(struct zram *zram, u32 mode,

drivers/block/zram/zram_drv.c

1208

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

1209

if (!slot_allocated(zram, index))

drivers/block/zram/zram_drv.c

121

static unsigned long get_slot_handle(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

1212

if (test_slot_flag(zram, index, ZRAM_WB) ||

drivers/block/zram/zram_drv.c

1213

test_slot_flag(zram, index, ZRAM_SAME))

drivers/block/zram/zram_drv.c

1217

!test_slot_flag(zram, index, ZRAM_IDLE))

drivers/block/zram/zram_drv.c

1220

!test_slot_flag(zram, index, ZRAM_HUGE))

drivers/block/zram/zram_drv.c

1223

!test_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE))

drivers/block/zram/zram_drv.c

1226

ok = place_pp_slot(zram, ctl, index);

drivers/block/zram/zram_drv.c

1228

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

123

return zram->table[index].handle;

drivers/block/zram/zram_drv.c

1241

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

1242

u64 nr_pages = zram->disksize >> PAGE_SHIFT;

drivers/block/zram/zram_drv.c

1250

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

1251

if (!init_done(zram))

drivers/block/zram/zram_drv.c

1254

if (!zram->backing_dev)

drivers/block/zram/zram_drv.c

126

static void set_slot_handle(struct zram *zram, u32 index, unsigned long handle)

drivers/block/zram/zram_drv.c

1261

wb_ctl = init_wb_ctl(zram);

drivers/block/zram/zram_drv.c

128

zram->table[index].handle = handle;

drivers/block/zram/zram_drv.c

1291

scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);

drivers/block/zram/zram_drv.c

1302

scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);

drivers/block/zram/zram_drv.c

131

static bool test_slot_flag(struct zram *zram, u32 index,

drivers/block/zram/zram_drv.c

1313

scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);

drivers/block/zram/zram_drv.c

1324

scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);

drivers/block/zram/zram_drv.c

1329

err = zram_writeback_slots(zram, pp_ctl, wb_ctl);

drivers/block/zram/zram_drv.c

1334

release_pp_ctl(zram, pp_ctl);

drivers/block/zram/zram_drv.c

134

return zram->table[index].attr.flags & BIT(flag);

drivers/block/zram/zram_drv.c

1340

static int decompress_bdev_page(struct zram *zram, struct page *page, u32 index)

drivers/block/zram/zram_drv.c

1347

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

1349

if (!test_slot_flag(zram, index, ZRAM_WB)) {

drivers/block/zram/zram_drv.c

1350

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

1356

if (test_slot_flag(zram, index, ZRAM_HUGE)) {

drivers/block/zram/zram_drv.c

1357

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

1361

size = get_slot_size(zram, index);

drivers/block/zram/zram_drv.c

1362

prio = get_slot_comp_priority(zram, index);

drivers/block/zram/zram_drv.c

1364

zstrm = zcomp_stream_get(zram->comps[prio]);

drivers/block/zram/zram_drv.c

1366

ret = zcomp_decompress(zram->comps[prio], zstrm, src, size,

drivers/block/zram/zram_drv.c

137

static void set_slot_flag(struct zram *zram, u32 index,

drivers/block/zram/zram_drv.c

1372

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

1381

struct zram *zram = req->zram;

drivers/block/zram/zram_drv.c

1385

ret = decompress_bdev_page(zram, page, index);

drivers/block/zram/zram_drv.c

1398

struct zram *zram = req->zram;

drivers/block/zram/zram_drv.c

140

zram->table[index].attr.flags |= BIT(flag);

drivers/block/zram/zram_drv.c

1416

if (zram->compressed_wb == false) {

drivers/block/zram/zram_drv.c

143

static void clear_slot_flag(struct zram *zram, u32 index,

drivers/block/zram/zram_drv.c

1432

static void read_from_bdev_async(struct zram *zram, struct page *page,

drivers/block/zram/zram_drv.c

1443

bio = bio_alloc(zram->bdev, 1, parent->bi_opf, GFP_NOIO);

drivers/block/zram/zram_drv.c

1449

req->zram = zram;

drivers/block/zram/zram_drv.c

146

zram->table[index].attr.flags &= ~BIT(flag);

drivers/block/zram/zram_drv.c

1470

bio_init(&bio, req->zram->bdev, &bv, 1, REQ_OP_READ);

drivers/block/zram/zram_drv.c

1481

static int read_from_bdev_sync(struct zram *zram, struct page *page, u32 index,

drivers/block/zram/zram_drv.c

1487

req.zram = zram;

drivers/block/zram/zram_drv.c

149

static size_t get_slot_size(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

1495

if (req.error || zram->compressed_wb == false)

drivers/block/zram/zram_drv.c

1498

return decompress_bdev_page(zram, page, index);

drivers/block/zram/zram_drv.c

1501

static int read_from_bdev(struct zram *zram, struct page *page, u32 index,

drivers/block/zram/zram_drv.c

1504

atomic64_inc(&zram->stats.bd_reads);

drivers/block/zram/zram_drv.c

1508

return read_from_bdev_sync(zram, page, index, blk_idx);

drivers/block/zram/zram_drv.c

151

return zram->table[index].attr.flags & (BIT(ZRAM_FLAG_SHIFT) - 1);

drivers/block/zram/zram_drv.c

1510

read_from_bdev_async(zram, page, index, blk_idx, parent);

drivers/block/zram/zram_drv.c

1514

static inline void reset_bdev(struct zram *zram) {};

drivers/block/zram/zram_drv.c

1515

static int read_from_bdev(struct zram *zram, struct page *page, u32 index,

drivers/block/zram/zram_drv.c

1521

static void zram_release_bdev_block(struct zram *zram, unsigned long blk_idx)

drivers/block/zram/zram_drv.c

154

static void set_slot_size(struct zram *zram, u32 index, size_t size)

drivers/block/zram/zram_drv.c

1545

struct zram *zram = file->private_data;

drivers/block/zram/zram_drv.c

1546

unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;

drivers/block/zram/zram_drv.c

1552

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

1553

if (!init_done(zram)) {

drivers/block/zram/zram_drv.c

156

unsigned long flags = zram->table[index].attr.flags >> ZRAM_FLAG_SHIFT;

drivers/block/zram/zram_drv.c

1561

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

1562

if (!slot_allocated(zram, index))

drivers/block/zram/zram_drv.c

1567

index, zram->table[index].attr.ac_time, 0,

drivers/block/zram/zram_drv.c

1568

test_slot_flag(zram, index, ZRAM_SAME) ? 's' : '.',

drivers/block/zram/zram_drv.c

1569

test_slot_flag(zram, index, ZRAM_WB) ? 'w' : '.',

drivers/block/zram/zram_drv.c

1570

test_slot_flag(zram, index, ZRAM_HUGE) ? 'h' : '.',

drivers/block/zram/zram_drv.c

1571

test_slot_flag(zram, index, ZRAM_IDLE) ? 'i' : '.',

drivers/block/zram/zram_drv.c

1572

get_slot_comp_priority(zram, index) ? 'r' : '.',

drivers/block/zram/zram_drv.c

1573

test_slot_flag(zram, index,

drivers/block/zram/zram_drv.c

1577

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

158

zram->table[index].attr.flags = (flags << ZRAM_FLAG_SHIFT) | size;

drivers/block/zram/zram_drv.c

1583

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

1600

static void zram_debugfs_register(struct zram *zram)

drivers/block/zram/zram_drv.c

1605

zram->debugfs_dir = debugfs_create_dir(zram->disk->disk_name,

drivers/block/zram/zram_drv.c

1607

debugfs_create_file("block_state", 0400, zram->debugfs_dir,

drivers/block/zram/zram_drv.c

1608

zram, &proc_zram_block_state_op);

drivers/block/zram/zram_drv.c

161

static inline bool slot_allocated(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

1611

static void zram_debugfs_unregister(struct zram *zram)

drivers/block/zram/zram_drv.c

1613

debugfs_remove_recursive(zram->debugfs_dir);

drivers/block/zram/zram_drv.c

1618

static void zram_debugfs_register(struct zram *zram) {};

drivers/block/zram/zram_drv.c

1619

static void zram_debugfs_unregister(struct zram *zram) {};

drivers/block/zram/zram_drv.c

1622

static void comp_algorithm_set(struct zram *zram, u32 prio, const char *alg)

drivers/block/zram/zram_drv.c

1625

if (zram->comp_algs[prio] != default_compressor)

drivers/block/zram/zram_drv.c

1626

kfree(zram->comp_algs[prio]);

drivers/block/zram/zram_drv.c

1628

zram->comp_algs[prio] = alg;

drivers/block/zram/zram_drv.c

163

return get_slot_size(zram, index) ||

drivers/block/zram/zram_drv.c

1631

static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf)

drivers/block/zram/zram_drv.c

164

test_slot_flag(zram, index, ZRAM_SAME) ||

drivers/block/zram/zram_drv.c

165

test_slot_flag(zram, index, ZRAM_WB);

drivers/block/zram/zram_drv.c

1653

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

1654

if (init_done(zram)) {

drivers/block/zram/zram_drv.c

1660

comp_algorithm_set(zram, prio, compressor);

drivers/block/zram/zram_drv.c

1664

static void comp_params_reset(struct zram *zram, u32 prio)

drivers/block/zram/zram_drv.c

1666

struct zcomp_params *params = &zram->params[prio];

drivers/block/zram/zram_drv.c

1675

static int comp_params_store(struct zram *zram, u32 prio, s32 level,

drivers/block/zram/zram_drv.c

168

static inline void set_slot_comp_priority(struct zram *zram, u32 index,

drivers/block/zram/zram_drv.c

1681

comp_params_reset(zram, prio);

drivers/block/zram/zram_drv.c

1685

&zram->params[prio].dict,

drivers/block/zram/zram_drv.c

1693

zram->params[prio].dict_sz = sz;

drivers/block/zram/zram_drv.c

1694

zram->params[prio].level = level;

drivers/block/zram/zram_drv.c

1695

zram->params[prio].deflate.winbits = deflate_params->winbits;

drivers/block/zram/zram_drv.c

1707

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

1757

if (!zram->comp_algs[p])

drivers/block/zram/zram_drv.c

176

zram->table[index].attr.flags &= ~(ZRAM_COMP_PRIORITY_MASK <<

drivers/block/zram/zram_drv.c

1760

if (!strcmp(zram->comp_algs[p], algo)) {

drivers/block/zram/zram_drv.c

1770

ret = comp_params_store(zram, prio, level, dict_path, &deflate_params);

drivers/block/zram/zram_drv.c

1778

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

178

zram->table[index].attr.flags |= (prio << ZRAM_COMP_PRIORITY_BIT1);

drivers/block/zram/zram_drv.c

1781

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

1782

sz = zcomp_available_show(zram->comp_algs[ZRAM_PRIMARY_COMP], buf, 0);

drivers/block/zram/zram_drv.c

1791

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

1794

ret = __comp_algorithm_store(zram, ZRAM_PRIMARY_COMP, buf);

drivers/block/zram/zram_drv.c

1803

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

1807

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

1809

if (!zram->comp_algs[prio])

drivers/block/zram/zram_drv.c

181

static inline u32 get_slot_comp_priority(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

1813

sz += zcomp_available_show(zram->comp_algs[prio], buf, sz);

drivers/block/zram/zram_drv.c

1823

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

183

u32 prio = zram->table[index].attr.flags >> ZRAM_COMP_PRIORITY_BIT1;

drivers/block/zram/zram_drv.c

1855

ret = __comp_algorithm_store(zram, prio, alg);

drivers/block/zram/zram_drv.c

1863

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

1865

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

1866

if (!init_done(zram))

drivers/block/zram/zram_drv.c

1869

zs_compact(zram->mem_pool);

drivers/block/zram/zram_drv.c

1877

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

188

static void mark_slot_accessed(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

1880

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

1883

(u64)atomic64_read(&zram->stats.failed_reads),

drivers/block/zram/zram_drv.c

1884

(u64)atomic64_read(&zram->stats.failed_writes),

drivers/block/zram/zram_drv.c

1885

(u64)atomic64_read(&zram->stats.notify_free));

drivers/block/zram/zram_drv.c

1893

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

190

clear_slot_flag(zram, index, ZRAM_IDLE);

drivers/block/zram/zram_drv.c

1901

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

1902

if (init_done(zram)) {

drivers/block/zram/zram_drv.c

1903

mem_used = zs_get_total_pages(zram->mem_pool);

drivers/block/zram/zram_drv.c

1904

zs_pool_stats(zram->mem_pool, &pool_stats);

drivers/block/zram/zram_drv.c

1907

orig_size = atomic64_read(&zram->stats.pages_stored);

drivers/block/zram/zram_drv.c

1908

max_used = atomic_long_read(&zram->stats.max_used_pages);

drivers/block/zram/zram_drv.c

191

clear_slot_flag(zram, index, ZRAM_PP_SLOT);

drivers/block/zram/zram_drv.c

1913

(u64)atomic64_read(&zram->stats.compr_data_size),

drivers/block/zram/zram_drv.c

1915

zram->limit_pages << PAGE_SHIFT,

drivers/block/zram/zram_drv.c

1917

(u64)atomic64_read(&zram->stats.same_pages),

drivers/block/zram/zram_drv.c

1919

(u64)atomic64_read(&zram->stats.huge_pages),

drivers/block/zram/zram_drv.c

1920

(u64)atomic64_read(&zram->stats.huge_pages_since));

drivers/block/zram/zram_drv.c

1929

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

193

zram->table[index].attr.ac_time = (u32)ktime_get_boottime_seconds();

drivers/block/zram/zram_drv.c

1932

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

1936

(u64)atomic64_read(&zram->stats.miss_free));

drivers/block/zram/zram_drv.c

1941

static void zram_meta_free(struct zram *zram, u64 disksize)

drivers/block/zram/zram_drv.c

1946

if (!zram->table)

drivers/block/zram/zram_drv.c

1951

slot_free(zram, index);

drivers/block/zram/zram_drv.c

1953

zs_destroy_pool(zram->mem_pool);

drivers/block/zram/zram_drv.c

1954

vfree(zram->table);

drivers/block/zram/zram_drv.c

1955

zram->table = NULL;

drivers/block/zram/zram_drv.c

1958

static bool zram_meta_alloc(struct zram *zram, u64 disksize)

drivers/block/zram/zram_drv.c

1963

zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table)));

drivers/block/zram/zram_drv.c

1964

if (!zram->table)

drivers/block/zram/zram_drv.c

1967

zram->mem_pool = zs_create_pool(zram->disk->disk_name);

drivers/block/zram/zram_drv.c

1968

if (!zram->mem_pool) {

drivers/block/zram/zram_drv.c

1969

vfree(zram->table);

drivers/block/zram/zram_drv.c

197

static inline void update_used_max(struct zram *zram, const unsigned long pages)

drivers/block/zram/zram_drv.c

1970

zram->table = NULL;

drivers/block/zram/zram_drv.c

1975

huge_class_size = zs_huge_class_size(zram->mem_pool);

drivers/block/zram/zram_drv.c

1978

slot_lock_init(zram, index);

drivers/block/zram/zram_drv.c

1983

static void slot_free(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

1988

zram->table[index].attr.ac_time = 0;

drivers/block/zram/zram_drv.c

199

unsigned long cur_max = atomic_long_read(&zram->stats.max_used_pages);

drivers/block/zram/zram_drv.c

1991

clear_slot_flag(zram, index, ZRAM_IDLE);

drivers/block/zram/zram_drv.c

1992

clear_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE);

drivers/block/zram/zram_drv.c

1993

clear_slot_flag(zram, index, ZRAM_PP_SLOT);

drivers/block/zram/zram_drv.c

1994

set_slot_comp_priority(zram, index, 0);

drivers/block/zram/zram_drv.c

1996

if (test_slot_flag(zram, index, ZRAM_HUGE)) {

drivers/block/zram/zram_drv.c

2001

if (!test_slot_flag(zram, index, ZRAM_WB))

drivers/block/zram/zram_drv.c

2002

atomic64_dec(&zram->stats.huge_pages);

drivers/block/zram/zram_drv.c

2003

clear_slot_flag(zram, index, ZRAM_HUGE);

drivers/block/zram/zram_drv.c

2006

if (test_slot_flag(zram, index, ZRAM_WB)) {

drivers/block/zram/zram_drv.c

2007

clear_slot_flag(zram, index, ZRAM_WB);

drivers/block/zram/zram_drv.c

2008

zram_release_bdev_block(zram, get_slot_handle(zram, index));

drivers/block/zram/zram_drv.c

2016

if (test_slot_flag(zram, index, ZRAM_SAME)) {

drivers/block/zram/zram_drv.c

2017

clear_slot_flag(zram, index, ZRAM_SAME);

drivers/block/zram/zram_drv.c

2018

atomic64_dec(&zram->stats.same_pages);

drivers/block/zram/zram_drv.c

2022

handle = get_slot_handle(zram, index);

drivers/block/zram/zram_drv.c

2026

zs_free(zram->mem_pool, handle);

drivers/block/zram/zram_drv.c

2028

atomic64_sub(get_slot_size(zram, index),

drivers/block/zram/zram_drv.c

2029

&zram->stats.compr_data_size);

drivers/block/zram/zram_drv.c

2031

atomic64_dec(&zram->stats.pages_stored);

drivers/block/zram/zram_drv.c

2032

set_slot_handle(zram, index, 0);

drivers/block/zram/zram_drv.c

2033

set_slot_size(zram, index, 0);

drivers/block/zram/zram_drv.c

2036

static int read_same_filled_page(struct zram *zram, struct page *page,

drivers/block/zram/zram_drv.c

204

} while (!atomic_long_try_cmpxchg(&zram->stats.max_used_pages,

drivers/block/zram/zram_drv.c

2042

zram_fill_page(mem, PAGE_SIZE, get_slot_handle(zram, index));

drivers/block/zram/zram_drv.c

2047

static int read_incompressible_page(struct zram *zram, struct page *page,

drivers/block/zram/zram_drv.c

2053

handle = get_slot_handle(zram, index);

drivers/block/zram/zram_drv.c

2054

src = zs_obj_read_begin(zram->mem_pool, handle, PAGE_SIZE, NULL);

drivers/block/zram/zram_drv.c

2058

zs_obj_read_end(zram->mem_pool, handle, PAGE_SIZE, src);

drivers/block/zram/zram_drv.c

2063

static int read_compressed_page(struct zram *zram, struct page *page, u32 index)

drivers/block/zram/zram_drv.c

2071

handle = get_slot_handle(zram, index);

drivers/block/zram/zram_drv.c

2072

size = get_slot_size(zram, index);

drivers/block/zram/zram_drv.c

2073

prio = get_slot_comp_priority(zram, index);

drivers/block/zram/zram_drv.c

2075

zstrm = zcomp_stream_get(zram->comps[prio]);

drivers/block/zram/zram_drv.c

2076

src = zs_obj_read_begin(zram->mem_pool, handle, size,

drivers/block/zram/zram_drv.c

2079

ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, dst);

drivers/block/zram/zram_drv.c

208

static bool zram_can_store_page(struct zram *zram)

drivers/block/zram/zram_drv.c

2081

zs_obj_read_end(zram->mem_pool, handle, size, src);

drivers/block/zram/zram_drv.c

2088

static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index)

drivers/block/zram/zram_drv.c

2095

handle = get_slot_handle(zram, index);

drivers/block/zram/zram_drv.c

2096

size = get_slot_size(zram, index);

drivers/block/zram/zram_drv.c

2103

zstrm = zcomp_stream_get(zram->comps[ZRAM_PRIMARY_COMP]);

drivers/block/zram/zram_drv.c

2104

src = zs_obj_read_begin(zram->mem_pool, handle, size,

drivers/block/zram/zram_drv.c

2107

zs_obj_read_end(zram->mem_pool, handle, size, src);

drivers/block/zram/zram_drv.c

2118

static int read_from_zspool(struct zram *zram, struct page *page, u32 index)

drivers/block/zram/zram_drv.c

212

alloced_pages = zs_get_total_pages(zram->mem_pool);

drivers/block/zram/zram_drv.c

2120

if (test_slot_flag(zram, index, ZRAM_SAME) ||

drivers/block/zram/zram_drv.c

2121

!get_slot_handle(zram, index))

drivers/block/zram/zram_drv.c

2122

return read_same_filled_page(zram, page, index);

drivers/block/zram/zram_drv.c

2124

if (!test_slot_flag(zram, index, ZRAM_HUGE))

drivers/block/zram/zram_drv.c

2125

return read_compressed_page(zram, page, index);

drivers/block/zram/zram_drv.c

2127

return read_incompressible_page(zram, page, index);

drivers/block/zram/zram_drv.c

213

update_used_max(zram, alloced_pages);

drivers/block/zram/zram_drv.c

2130

static int zram_read_page(struct zram *zram, struct page *page, u32 index,

drivers/block/zram/zram_drv.c

2135

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

2136

if (!test_slot_flag(zram, index, ZRAM_WB)) {

drivers/block/zram/zram_drv.c

2138

ret = read_from_zspool(zram, page, index);

drivers/block/zram/zram_drv.c

2139

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2141

unsigned long blk_idx = get_slot_handle(zram, index);

drivers/block/zram/zram_drv.c

2147

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2148

ret = read_from_bdev(zram, page, index, blk_idx, parent);

drivers/block/zram/zram_drv.c

215

return !zram->limit_pages || alloced_pages <= zram->limit_pages;

drivers/block/zram/zram_drv.c

2162

static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec,

drivers/block/zram/zram_drv.c

2170

ret = zram_read_page(zram, page, index, NULL);

drivers/block/zram/zram_drv.c

2177

static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,

drivers/block/zram/zram_drv.c

2181

return zram_bvec_read_partial(zram, bvec, index, offset);

drivers/block/zram/zram_drv.c

2182

return zram_read_page(zram, bvec->bv_page, index, bio);

drivers/block/zram/zram_drv.c

2185

static int write_same_filled_page(struct zram *zram, unsigned long fill,

drivers/block/zram/zram_drv.c

2188

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

2189

slot_free(zram, index);

drivers/block/zram/zram_drv.c

2190

set_slot_flag(zram, index, ZRAM_SAME);

drivers/block/zram/zram_drv.c

2191

set_slot_handle(zram, index, fill);

drivers/block/zram/zram_drv.c

2192

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2194

atomic64_inc(&zram->stats.same_pages);

drivers/block/zram/zram_drv.c

2195

atomic64_inc(&zram->stats.pages_stored);

drivers/block/zram/zram_drv.c

2200

static int write_incompressible_page(struct zram *zram, struct page *page,

drivers/block/zram/zram_drv.c

2211

handle = zs_malloc(zram->mem_pool, PAGE_SIZE,

drivers/block/zram/zram_drv.c

2217

if (!zram_can_store_page(zram)) {

drivers/block/zram/zram_drv.c

2218

zs_free(zram->mem_pool, handle);

drivers/block/zram/zram_drv.c

2223

zs_obj_write(zram->mem_pool, handle, src, PAGE_SIZE);

drivers/block/zram/zram_drv.c

2226

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

2227

slot_free(zram, index);

drivers/block/zram/zram_drv.c

2228

set_slot_flag(zram, index, ZRAM_HUGE);

drivers/block/zram/zram_drv.c

2229

set_slot_handle(zram, index, handle);

drivers/block/zram/zram_drv.c

2230

set_slot_size(zram, index, PAGE_SIZE);

drivers/block/zram/zram_drv.c

2231

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2233

atomic64_add(PAGE_SIZE, &zram->stats.compr_data_size);

drivers/block/zram/zram_drv.c

2234

atomic64_inc(&zram->stats.huge_pages);

drivers/block/zram/zram_drv.c

2235

atomic64_inc(&zram->stats.huge_pages_since);

drivers/block/zram/zram_drv.c

2236

atomic64_inc(&zram->stats.pages_stored);

drivers/block/zram/zram_drv.c

2241

static int zram_write_page(struct zram *zram, struct page *page, u32 index)

drivers/block/zram/zram_drv.c

2255

return write_same_filled_page(zram, element, index);

drivers/block/zram/zram_drv.c

2257

zstrm = zcomp_stream_get(zram->comps[ZRAM_PRIMARY_COMP]);

drivers/block/zram/zram_drv.c

2259

ret = zcomp_compress(zram->comps[ZRAM_PRIMARY_COMP], zstrm,

drivers/block/zram/zram_drv.c

2271

return write_incompressible_page(zram, page, index);

drivers/block/zram/zram_drv.c

2274

handle = zs_malloc(zram->mem_pool, comp_len,

drivers/block/zram/zram_drv.c

2282

if (!zram_can_store_page(zram)) {

drivers/block/zram/zram_drv.c

2284

zs_free(zram->mem_pool, handle);

drivers/block/zram/zram_drv.c

2288

zs_obj_write(zram->mem_pool, handle, zstrm->buffer, comp_len);

drivers/block/zram/zram_drv.c

2291

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

2292

slot_free(zram, index);

drivers/block/zram/zram_drv.c

2293

set_slot_handle(zram, index, handle);

drivers/block/zram/zram_drv.c

2294

set_slot_size(zram, index, comp_len);

drivers/block/zram/zram_drv.c

2295

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2298

atomic64_inc(&zram->stats.pages_stored);

drivers/block/zram/zram_drv.c

2299

atomic64_add(comp_len, &zram->stats.compr_data_size);

drivers/block/zram/zram_drv.c

2307

static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec,

drivers/block/zram/zram_drv.c

2316

ret = zram_read_page(zram, page, index, bio);

drivers/block/zram/zram_drv.c

2319

ret = zram_write_page(zram, page, index);

drivers/block/zram/zram_drv.c

2325

static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,

drivers/block/zram/zram_drv.c

2329

return zram_bvec_write_partial(zram, bvec, index, offset, bio);

drivers/block/zram/zram_drv.c

2330

return zram_write_page(zram, bvec->bv_page, index);

drivers/block/zram/zram_drv.c

2337

static int scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio_max,

drivers/block/zram/zram_drv.c

2340

unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;

drivers/block/zram/zram_drv.c

2346

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

2347

if (!slot_allocated(zram, index))

drivers/block/zram/zram_drv.c

2351

!test_slot_flag(zram, index, ZRAM_IDLE))

drivers/block/zram/zram_drv.c

2355

!test_slot_flag(zram, index, ZRAM_HUGE))

drivers/block/zram/zram_drv.c

2358

if (test_slot_flag(zram, index, ZRAM_WB) ||

drivers/block/zram/zram_drv.c

2359

test_slot_flag(zram, index, ZRAM_SAME) ||

drivers/block/zram/zram_drv.c

2360

test_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE))

drivers/block/zram/zram_drv.c

2364

if (get_slot_comp_priority(zram, index) + 1 >= prio_max)

drivers/block/zram/zram_drv.c

2367

ok = place_pp_slot(zram, ctl, index);

drivers/block/zram/zram_drv.c

2369

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2384

static int recompress_slot(struct zram *zram, u32 index, struct page *page,

drivers/block/zram/zram_drv.c

2398

handle_old = get_slot_handle(zram, index);

drivers/block/zram/zram_drv.c

2402

comp_len_old = get_slot_size(zram, index);

drivers/block/zram/zram_drv.c

2409

ret = read_from_zspool(zram, page, index);

drivers/block/zram/zram_drv.c

2418

clear_slot_flag(zram, index, ZRAM_IDLE);

drivers/block/zram/zram_drv.c

2420

class_index_old = zs_lookup_class_index(zram->mem_pool, comp_len_old);

drivers/block/zram/zram_drv.c

2422

prio = max(prio, get_slot_comp_priority(zram, index) + 1);

drivers/block/zram/zram_drv.c

2436

if (!zram->comps[prio])

drivers/block/zram/zram_drv.c

2439

zstrm = zcomp_stream_get(zram->comps[prio]);

drivers/block/zram/zram_drv.c

2441

ret = zcomp_compress(zram->comps[prio], zstrm,

drivers/block/zram/zram_drv.c

2451

class_index_new = zs_lookup_class_index(zram->mem_pool,

drivers/block/zram/zram_drv.c

2487

if (prio < zram->num_active_comps)

drivers/block/zram/zram_drv.c

2489

set_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE);

drivers/block/zram/zram_drv.c

2503

handle_new = zs_malloc(zram->mem_pool, comp_len_new,

drivers/block/zram/zram_drv.c

2512

zs_obj_write(zram->mem_pool, handle_new, zstrm->buffer, comp_len_new);

drivers/block/zram/zram_drv.c

2515

slot_free(zram, index);

drivers/block/zram/zram_drv.c

2516

set_slot_handle(zram, index, handle_new);

drivers/block/zram/zram_drv.c

2517

set_slot_size(zram, index, comp_len_new);

drivers/block/zram/zram_drv.c

2518

set_slot_comp_priority(zram, index, prio);

drivers/block/zram/zram_drv.c

2520

atomic64_add(comp_len_new, &zram->stats.compr_data_size);

drivers/block/zram/zram_drv.c

2521

atomic64_inc(&zram->stats.pages_stored);

drivers/block/zram/zram_drv.c

2530

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

2541

prio_max = zram->num_active_comps;

drivers/block/zram/zram_drv.c

2603

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

2604

if (!init_done(zram))

drivers/block/zram/zram_drv.c

2611

if (!zram->comp_algs[prio])

drivers/block/zram/zram_drv.c

2614

if (!strcmp(zram->comp_algs[prio], algo)) {

drivers/block/zram/zram_drv.c

262

static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)

drivers/block/zram/zram_drv.c

2627

prio_max = min(prio_max, (u32)zram->num_active_comps);

drivers/block/zram/zram_drv.c

2645

scan_slots_for_recompress(zram, mode, prio_max, ctl);

drivers/block/zram/zram_drv.c

2654

slot_lock(zram, pps->index);

drivers/block/zram/zram_drv.c

2655

if (!test_slot_flag(zram, pps->index, ZRAM_PP_SLOT))

drivers/block/zram/zram_drv.c

2658

err = recompress_slot(zram, pps->index, page,

drivers/block/zram/zram_drv.c

266

slot_lock(zram, pps->index);

drivers/block/zram/zram_drv.c

2662

slot_unlock(zram, pps->index);

drivers/block/zram/zram_drv.c

2663

release_pp_slot(zram, pps);

drivers/block/zram/zram_drv.c

267

clear_slot_flag(zram, pps->index, ZRAM_PP_SLOT);

drivers/block/zram/zram_drv.c

2676

release_pp_ctl(zram, ctl);

drivers/block/zram/zram_drv.c

268

slot_unlock(zram, pps->index);

drivers/block/zram/zram_drv.c

2681

static void zram_bio_discard(struct zram *zram, struct bio *bio)

drivers/block/zram/zram_drv.c

2707

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

2708

slot_free(zram, index);

drivers/block/zram/zram_drv.c

2709

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2710

atomic64_inc(&zram->stats.notify_free);

drivers/block/zram/zram_drv.c

2718

static void zram_bio_read(struct zram *zram, struct bio *bio)

drivers/block/zram/zram_drv.c

273

static void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl)

drivers/block/zram/zram_drv.c

2731

if (zram_bvec_read(zram, &bv, index, offset, bio) < 0) {

drivers/block/zram/zram_drv.c

2732

atomic64_inc(&zram->stats.failed_reads);

drivers/block/zram/zram_drv.c

2738

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

2739

mark_slot_accessed(zram, index);

drivers/block/zram/zram_drv.c

2740

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2749

static void zram_bio_write(struct zram *zram, struct bio *bio)

drivers/block/zram/zram_drv.c

2762

if (zram_bvec_write(zram, &bv, index, offset, bio) < 0) {

drivers/block/zram/zram_drv.c

2763

atomic64_inc(&zram->stats.failed_writes);

drivers/block/zram/zram_drv.c

2768

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

2769

mark_slot_accessed(zram, index);

drivers/block/zram/zram_drv.c

2770

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2784

struct zram *zram = bio->bi_bdev->bd_disk->private_data;

drivers/block/zram/zram_drv.c

2788

zram_bio_read(zram, bio);

drivers/block/zram/zram_drv.c

2791

zram_bio_write(zram, bio);

drivers/block/zram/zram_drv.c

2795

zram_bio_discard(zram, bio);

drivers/block/zram/zram_drv.c

2806

struct zram *zram;

drivers/block/zram/zram_drv.c

2808

zram = bdev->bd_disk->private_data;

drivers/block/zram/zram_drv.c

2810

atomic64_inc(&zram->stats.notify_free);

drivers/block/zram/zram_drv.c

2811

if (!slot_trylock(zram, index)) {

drivers/block/zram/zram_drv.c

2812

atomic64_inc(&zram->stats.miss_free);

drivers/block/zram/zram_drv.c

2816

slot_free(zram, index);

drivers/block/zram/zram_drv.c

2817

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

2820

static void zram_comp_params_reset(struct zram *zram)

drivers/block/zram/zram_drv.c

2825

comp_params_reset(zram, prio);

drivers/block/zram/zram_drv.c

2829

static void zram_destroy_comps(struct zram *zram)

drivers/block/zram/zram_drv.c

2834

struct zcomp *comp = zram->comps[prio];

drivers/block/zram/zram_drv.c

2836

zram->comps[prio] = NULL;

drivers/block/zram/zram_drv.c

2840

zram->num_active_comps--;

drivers/block/zram/zram_drv.c

2845

if (zram->comp_algs[prio] != default_compressor)

drivers/block/zram/zram_drv.c

2846

kfree(zram->comp_algs[prio]);

drivers/block/zram/zram_drv.c

2847

zram->comp_algs[prio] = NULL;

drivers/block/zram/zram_drv.c

2850

zram_comp_params_reset(zram);

drivers/block/zram/zram_drv.c

2853

static void zram_reset_device(struct zram *zram)

drivers/block/zram/zram_drv.c

2855

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

2857

zram->limit_pages = 0;

drivers/block/zram/zram_drv.c

2859

set_capacity_and_notify(zram->disk, 0);

drivers/block/zram/zram_drv.c

2860

part_stat_set_all(zram->disk->part0, 0);

drivers/block/zram/zram_drv.c

2863

zram_meta_free(zram, zram->disksize);

drivers/block/zram/zram_drv.c

2864

zram->disksize = 0;

drivers/block/zram/zram_drv.c

2865

zram_destroy_comps(zram);

drivers/block/zram/zram_drv.c

2866

memset(&zram->stats, 0, sizeof(zram->stats));

drivers/block/zram/zram_drv.c

2867

reset_bdev(zram);

drivers/block/zram/zram_drv.c

2869

comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor);

drivers/block/zram/zram_drv.c

287

release_pp_slot(zram, pps);

drivers/block/zram/zram_drv.c

2877

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

2885

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

2886

if (init_done(zram)) {

drivers/block/zram/zram_drv.c

2892

if (!zram_meta_alloc(zram, disksize))

drivers/block/zram/zram_drv.c

2896

if (!zram->comp_algs[prio])

drivers/block/zram/zram_drv.c

2899

comp = zcomp_create(zram->comp_algs[prio],

drivers/block/zram/zram_drv.c

2900

&zram->params[prio]);

drivers/block/zram/zram_drv.c

2903

zram->comp_algs[prio]);

drivers/block/zram/zram_drv.c

2908

zram->comps[prio] = comp;

drivers/block/zram/zram_drv.c

2909

zram->num_active_comps++;

drivers/block/zram/zram_drv.c

2911

zram->disksize = disksize;

drivers/block/zram/zram_drv.c

2912

set_capacity_and_notify(zram->disk, zram->disksize >> SECTOR_SHIFT);

drivers/block/zram/zram_drv.c

2917

zram_destroy_comps(zram);

drivers/block/zram/zram_drv.c

2918

zram_meta_free(zram, disksize);

drivers/block/zram/zram_drv.c

2927

struct zram *zram;

drivers/block/zram/zram_drv.c

2937

zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

2938

disk = zram->disk;

drivers/block/zram/zram_drv.c

294

static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl,

drivers/block/zram/zram_drv.c

2942

if (disk_openers(disk) || zram->claim) {

drivers/block/zram/zram_drv.c

2948

zram->claim = true;

drivers/block/zram/zram_drv.c

2953

zram_reset_device(zram);

drivers/block/zram/zram_drv.c

2956

zram->claim = false;

drivers/block/zram/zram_drv.c

2964

struct zram *zram = disk->private_data;

drivers/block/zram/zram_drv.c

2969

if (zram->claim)

drivers/block/zram/zram_drv.c

3068

struct zram *zram;

drivers/block/zram/zram_drv.c

307

bid = get_slot_size(zram, pps->index) / PP_BUCKET_SIZE_RANGE;

drivers/block/zram/zram_drv.c

3071

zram = kzalloc_obj(struct zram);

drivers/block/zram/zram_drv.c

3072

if (!zram)

drivers/block/zram/zram_drv.c

3075

ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL);

drivers/block/zram/zram_drv.c

3080

init_rwsem(&zram->dev_lock);

drivers/block/zram/zram_drv.c

3082

zram->wb_batch_size = 32;

drivers/block/zram/zram_drv.c

3083

zram->compressed_wb = false;

drivers/block/zram/zram_drv.c

3087

zram->disk = blk_alloc_disk(&lim, NUMA_NO_NODE);

drivers/block/zram/zram_drv.c

3088

if (IS_ERR(zram->disk)) {

drivers/block/zram/zram_drv.c

3091

ret = PTR_ERR(zram->disk);

drivers/block/zram/zram_drv.c

3095

zram->disk->major = zram_major;

drivers/block/zram/zram_drv.c

3096

zram->disk->first_minor = device_id;

drivers/block/zram/zram_drv.c

3097

zram->disk->minors = 1;

drivers/block/zram/zram_drv.c

3098

zram->disk->flags |= GENHD_FL_NO_PART;

drivers/block/zram/zram_drv.c

3099

zram->disk->fops = &zram_devops;

drivers/block/zram/zram_drv.c

310

set_slot_flag(zram, pps->index, ZRAM_PP_SLOT);

drivers/block/zram/zram_drv.c

3100

zram->disk->private_data = zram;

drivers/block/zram/zram_drv.c

3101

snprintf(zram->disk->disk_name, 16, "zram%d", device_id);

drivers/block/zram/zram_drv.c

3102

zram_comp_params_reset(zram);

drivers/block/zram/zram_drv.c

3103

comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor);

drivers/block/zram/zram_drv.c

3106

set_capacity(zram->disk, 0);

drivers/block/zram/zram_drv.c

3107

ret = device_add_disk(NULL, zram->disk, zram_disk_groups);

drivers/block/zram/zram_drv.c

3111

zram_debugfs_register(zram);

drivers/block/zram/zram_drv.c

3112

pr_info("Added device: %s\n", zram->disk->disk_name);

drivers/block/zram/zram_drv.c

3116

put_disk(zram->disk);

drivers/block/zram/zram_drv.c

3120

kfree(zram);

drivers/block/zram/zram_drv.c

3124

static int zram_remove(struct zram *zram)

drivers/block/zram/zram_drv.c

3128

mutex_lock(&zram->disk->open_mutex);

drivers/block/zram/zram_drv.c

3129

if (disk_openers(zram->disk)) {

drivers/block/zram/zram_drv.c

3130

mutex_unlock(&zram->disk->open_mutex);

drivers/block/zram/zram_drv.c

3134

claimed = zram->claim;

drivers/block/zram/zram_drv.c

3136

zram->claim = true;

drivers/block/zram/zram_drv.c

3137

mutex_unlock(&zram->disk->open_mutex);

drivers/block/zram/zram_drv.c

3139

zram_debugfs_unregister(zram);

drivers/block/zram/zram_drv.c

3149

sync_blockdev(zram->disk->part0);

drivers/block/zram/zram_drv.c

3150

zram_reset_device(zram);

drivers/block/zram/zram_drv.c

3153

pr_info("Removed device: %s\n", zram->disk->disk_name);

drivers/block/zram/zram_drv.c

3155

del_gendisk(zram->disk);

drivers/block/zram/zram_drv.c

3158

WARN_ON_ONCE(claimed && zram->claim);

drivers/block/zram/zram_drv.c

3165

zram_reset_device(zram);

drivers/block/zram/zram_drv.c

3167

put_disk(zram->disk);

drivers/block/zram/zram_drv.c

3168

kfree(zram);

drivers/block/zram/zram_drv.c

3203

struct zram *zram;

drivers/block/zram/zram_drv.c

3215

zram = idr_find(&zram_index_idr, dev_id);

drivers/block/zram/zram_drv.c

3216

if (zram) {

drivers/block/zram/zram_drv.c

3217

ret = zram_remove(zram);

drivers/block/zram/zram_drv.c

366

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

368

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

369

val = init_done(zram);

drivers/block/zram/zram_drv.c

377

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

379

return sysfs_emit(buf, "%llu\n", zram->disksize);

drivers/block/zram/zram_drv.c

388

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

394

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

395

zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;

drivers/block/zram/zram_drv.c

406

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

412

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

413

if (init_done(zram)) {

drivers/block/zram/zram_drv.c

414

atomic_long_set(&zram->stats.max_used_pages,

drivers/block/zram/zram_drv.c

415

zs_get_total_pages(zram->mem_pool));

drivers/block/zram/zram_drv.c

425

static void mark_idle(struct zram *zram, ktime_t cutoff)

drivers/block/zram/zram_drv.c

428

unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;

drivers/block/zram/zram_drv.c

439

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

440

if (!slot_allocated(zram, index) ||

drivers/block/zram/zram_drv.c

441

test_slot_flag(zram, index, ZRAM_WB) ||

drivers/block/zram/zram_drv.c

442

test_slot_flag(zram, index, ZRAM_SAME)) {

drivers/block/zram/zram_drv.c

443

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

449

ktime_after(cutoff, zram->table[index].attr.ac_time);

drivers/block/zram/zram_drv.c

452

set_slot_flag(zram, index, ZRAM_IDLE);

drivers/block/zram/zram_drv.c

454

clear_slot_flag(zram, index, ZRAM_IDLE);

drivers/block/zram/zram_drv.c

455

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

462

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

480

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

481

if (!init_done(zram))

drivers/block/zram/zram_drv.c

488

mark_idle(zram, cutoff);

drivers/block/zram/zram_drv.c

495

static int read_from_zspool_raw(struct zram *zram, struct page *page,

drivers/block/zram/zram_drv.c

497

static int read_from_zspool(struct zram *zram, struct page *page, u32 index);

drivers/block/zram/zram_drv.c

521

struct zram *zram;

drivers/block/zram/zram_drv.c

539

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

542

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

545

FOUR_K((u64)atomic64_read(&zram->stats.bd_count)),

drivers/block/zram/zram_drv.c

546

FOUR_K((u64)atomic64_read(&zram->stats.bd_reads)),

drivers/block/zram/zram_drv.c

547

FOUR_K((u64)atomic64_read(&zram->stats.bd_writes)));

drivers/block/zram/zram_drv.c

556

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

562

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

563

if (init_done(zram)) {

drivers/block/zram/zram_drv.c

567

zram->compressed_wb = val;

drivers/block/zram/zram_drv.c

577

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

579

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

58

static void slot_free(struct zram *zram, u32 index);

drivers/block/zram/zram_drv.c

580

val = zram->compressed_wb;

drivers/block/zram/zram_drv.c

589

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

59

#define slot_dep_map(zram, index) (&(zram)->table[(index)].dep_map)

drivers/block/zram/zram_drv.c

595

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

596

zram->wb_limit_enable = val;

drivers/block/zram/zram_drv.c

606

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

608

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

609

val = zram->wb_limit_enable;

drivers/block/zram/zram_drv.c

61

static void slot_lock_init(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

618

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

634

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

635

zram->bd_wb_limit = val;

drivers/block/zram/zram_drv.c

644

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

646

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

647

val = zram->bd_wb_limit;

drivers/block/zram/zram_drv.c

65

lockdep_init_map(slot_dep_map(zram, index), "zram->table[index].lock",

drivers/block/zram/zram_drv.c

656

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

665

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

666

zram->wb_batch_size = val;

drivers/block/zram/zram_drv.c

676

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

678

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

679

val = zram->wb_batch_size;

drivers/block/zram/zram_drv.c

684

static void reset_bdev(struct zram *zram)

drivers/block/zram/zram_drv.c

686

if (!zram->backing_dev)

drivers/block/zram/zram_drv.c

690

filp_close(zram->backing_dev, NULL);

drivers/block/zram/zram_drv.c

691

zram->backing_dev = NULL;

drivers/block/zram/zram_drv.c

692

zram->bdev = NULL;

drivers/block/zram/zram_drv.c

693

zram->disk->fops = &zram_devops;

drivers/block/zram/zram_drv.c

694

kvfree(zram->bitmap);

drivers/block/zram/zram_drv.c

695

zram->bitmap = NULL;

drivers/block/zram/zram_drv.c

702

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

706

guard(rwsem_read)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

707

file = zram->backing_dev;

drivers/block/zram/zram_drv.c

734

struct zram *zram = dev_to_zram(dev);

drivers/block/zram/zram_drv.c

740

guard(rwsem_write)(&zram->dev_lock);

drivers/block/zram/zram_drv.c

741

if (init_done(zram)) {

drivers/block/zram/zram_drv.c

782

reset_bdev(zram);

drivers/block/zram/zram_drv.c

784

zram->bdev = I_BDEV(inode);

drivers/block/zram/zram_drv.c

785

zram->backing_dev = backing_dev;

drivers/block/zram/zram_drv.c

786

zram->bitmap = bitmap;

drivers/block/zram/zram_drv.c

787

zram->nr_pages = nr_pages;

drivers/block/zram/zram_drv.c

804

static unsigned long zram_reserve_bdev_block(struct zram *zram)

drivers/block/zram/zram_drv.c

808

blk_idx = find_next_zero_bit(zram->bitmap, zram->nr_pages, 0);

drivers/block/zram/zram_drv.c

809

if (blk_idx == zram->nr_pages)

drivers/block/zram/zram_drv.c

81

static __must_check bool slot_trylock(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

812

set_bit(blk_idx, zram->bitmap);

drivers/block/zram/zram_drv.c

813

atomic64_inc(&zram->stats.bd_count);

drivers/block/zram/zram_drv.c

817

static void zram_release_bdev_block(struct zram *zram, unsigned long blk_idx)

drivers/block/zram/zram_drv.c

821

was_set = test_and_clear_bit(blk_idx, zram->bitmap);

drivers/block/zram/zram_drv.c

823

atomic64_dec(&zram->stats.bd_count);

drivers/block/zram/zram_drv.c

83

unsigned long *lock = &zram->table[index].__lock;

drivers/block/zram/zram_drv.c

853

static struct zram_wb_ctl *init_wb_ctl(struct zram *zram)

drivers/block/zram/zram_drv.c

86

mutex_acquire(slot_dep_map(zram, index), 0, 1, _RET_IP_);

drivers/block/zram/zram_drv.c

868

for (i = 0; i < zram->wb_batch_size; i++) {

drivers/block/zram/zram_drv.c

87

lock_acquired(slot_dep_map(zram, index), _RET_IP_);

drivers/block/zram/zram_drv.c

902

static void zram_account_writeback_rollback(struct zram *zram)

drivers/block/zram/zram_drv.c

904

lockdep_assert_held_write(&zram->dev_lock);

drivers/block/zram/zram_drv.c

906

if (zram->wb_limit_enable)

drivers/block/zram/zram_drv.c

907

zram->bd_wb_limit += 1UL << (PAGE_SHIFT - 12);

drivers/block/zram/zram_drv.c

910

static void zram_account_writeback_submit(struct zram *zram)

drivers/block/zram/zram_drv.c

912

lockdep_assert_held_write(&zram->dev_lock);

drivers/block/zram/zram_drv.c

914

if (zram->wb_limit_enable && zram->bd_wb_limit > 0)

drivers/block/zram/zram_drv.c

915

zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12);

drivers/block/zram/zram_drv.c

918

static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req)

drivers/block/zram/zram_drv.c

929

zram_account_writeback_rollback(zram);

drivers/block/zram/zram_drv.c

930

zram_release_bdev_block(zram, req->blk_idx);

drivers/block/zram/zram_drv.c

934

atomic64_inc(&zram->stats.bd_writes);

drivers/block/zram/zram_drv.c

935

slot_lock(zram, index);

drivers/block/zram/zram_drv.c

94

static void slot_lock(struct zram *zram, u32 index)

drivers/block/zram/zram_drv.c

943

if (!test_slot_flag(zram, index, ZRAM_PP_SLOT)) {

drivers/block/zram/zram_drv.c

944

zram_release_bdev_block(zram, req->blk_idx);

drivers/block/zram/zram_drv.c

948

clear_slot_flag(zram, index, ZRAM_IDLE);

drivers/block/zram/zram_drv.c

949

if (test_slot_flag(zram, index, ZRAM_HUGE))

drivers/block/zram/zram_drv.c

950

atomic64_dec(&zram->stats.huge_pages);

drivers/block/zram/zram_drv.c

951

atomic64_sub(get_slot_size(zram, index), &zram->stats.compr_data_size);

drivers/block/zram/zram_drv.c

952

zs_free(zram->mem_pool, get_slot_handle(zram, index));

drivers/block/zram/zram_drv.c

953

set_slot_handle(zram, index, req->blk_idx);

drivers/block/zram/zram_drv.c

954

set_slot_flag(zram, index, ZRAM_WB);

drivers/block/zram/zram_drv.c

957

slot_unlock(zram, index);

drivers/block/zram/zram_drv.c

96

unsigned long *lock = &zram->table[index].__lock;

drivers/block/zram/zram_drv.c

974

static void zram_submit_wb_request(struct zram *zram,

drivers/block/zram/zram_drv.c

98

mutex_acquire(slot_dep_map(zram, index), 0, 0, _RET_IP_);

drivers/block/zram/zram_drv.c

982

zram_account_writeback_submit(zram);

drivers/block/zram/zram_drv.c

988

static int zram_complete_done_reqs(struct zram *zram,