Symbol: zram
drivers/block/zram/zram_drv.c
101
lock_acquired(slot_dep_map(zram, index), _RET_IP_);
drivers/block/zram/zram_drv.c
1011
err = zram_writeback_complete(zram, req);
drivers/block/zram/zram_drv.c
1016
release_pp_slot(zram, req->pps);
drivers/block/zram/zram_drv.c
1036
static int zram_writeback_slots(struct zram *zram,
drivers/block/zram/zram_drv.c
104
static void slot_unlock(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
1047
if (zram->wb_limit_enable && !zram->bd_wb_limit) {
drivers/block/zram/zram_drv.c
106
unsigned long *lock = &zram->table[index].__lock;
drivers/block/zram/zram_drv.c
1060
err = zram_complete_done_reqs(zram, wb_ctl);
drivers/block/zram/zram_drv.c
1074
blk_idx = zram_reserve_bdev_block(zram);
drivers/block/zram/zram_drv.c
108
mutex_release(slot_dep_map(zram, index), _RET_IP_);
drivers/block/zram/zram_drv.c
1082
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
1089
if (!test_slot_flag(zram, index, ZRAM_PP_SLOT))
drivers/block/zram/zram_drv.c
1091
if (zram->compressed_wb)
drivers/block/zram/zram_drv.c
1092
err = read_from_zspool_raw(zram, req->page, index);
drivers/block/zram/zram_drv.c
1094
err = read_from_zspool(zram, req->page, index);
drivers/block/zram/zram_drv.c
1097
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
1107
bio_init(&req->bio, zram->bdev, &req->bio_vec, 1, REQ_OP_WRITE);
drivers/block/zram/zram_drv.c
1112
zram_submit_wb_request(zram, wb_ctl, req);
drivers/block/zram/zram_drv.c
1119
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
112
static inline bool init_done(struct zram *zram)
drivers/block/zram/zram_drv.c
1120
release_pp_slot(zram, pps);
drivers/block/zram/zram_drv.c
1131
zram_release_bdev_block(zram, blk_idx);
drivers/block/zram/zram_drv.c
1135
err = zram_complete_done_reqs(zram, wb_ctl);
drivers/block/zram/zram_drv.c
114
return zram->disksize;
drivers/block/zram/zram_drv.c
117
static inline struct zram *dev_to_zram(struct device *dev)
drivers/block/zram/zram_drv.c
119
return (struct zram *)dev_to_disk(dev)->private_data;
drivers/block/zram/zram_drv.c
1206
static void scan_slots_for_writeback(struct zram *zram, u32 mode,
drivers/block/zram/zram_drv.c
1215
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
1216
if (!slot_allocated(zram, index))
drivers/block/zram/zram_drv.c
1219
if (test_slot_flag(zram, index, ZRAM_WB) ||
drivers/block/zram/zram_drv.c
122
static unsigned long get_slot_handle(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
1220
test_slot_flag(zram, index, ZRAM_SAME))
drivers/block/zram/zram_drv.c
1224
!test_slot_flag(zram, index, ZRAM_IDLE))
drivers/block/zram/zram_drv.c
1227
!test_slot_flag(zram, index, ZRAM_HUGE))
drivers/block/zram/zram_drv.c
1230
!test_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE))
drivers/block/zram/zram_drv.c
1233
ok = place_pp_slot(zram, ctl, index);
drivers/block/zram/zram_drv.c
1235
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
124
return zram->table[index].handle;
drivers/block/zram/zram_drv.c
1246
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
1247
u64 nr_pages = zram->disksize >> PAGE_SHIFT;
drivers/block/zram/zram_drv.c
1255
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
1256
if (!init_done(zram))
drivers/block/zram/zram_drv.c
1259
if (!zram->backing_dev)
drivers/block/zram/zram_drv.c
1266
wb_ctl = init_wb_ctl(zram);
drivers/block/zram/zram_drv.c
127
static void set_slot_handle(struct zram *zram, u32 index, unsigned long handle)
drivers/block/zram/zram_drv.c
129
zram->table[index].handle = handle;
drivers/block/zram/zram_drv.c
1296
scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);
drivers/block/zram/zram_drv.c
1307
scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);
drivers/block/zram/zram_drv.c
1318
scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);
drivers/block/zram/zram_drv.c
132
static bool test_slot_flag(struct zram *zram, u32 index,
drivers/block/zram/zram_drv.c
1329
scan_slots_for_writeback(zram, mode, lo, hi, pp_ctl);
drivers/block/zram/zram_drv.c
1334
err = zram_writeback_slots(zram, pp_ctl, wb_ctl);
drivers/block/zram/zram_drv.c
1339
release_pp_ctl(zram, pp_ctl);
drivers/block/zram/zram_drv.c
1345
static int decompress_bdev_page(struct zram *zram, struct page *page, u32 index)
drivers/block/zram/zram_drv.c
135
return zram->table[index].attr.flags & BIT(flag);
drivers/block/zram/zram_drv.c
1352
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
1354
if (!test_slot_flag(zram, index, ZRAM_WB)) {
drivers/block/zram/zram_drv.c
1355
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
1361
if (test_slot_flag(zram, index, ZRAM_HUGE)) {
drivers/block/zram/zram_drv.c
1362
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
1366
size = get_slot_size(zram, index);
drivers/block/zram/zram_drv.c
1367
prio = get_slot_comp_priority(zram, index);
drivers/block/zram/zram_drv.c
1369
zstrm = zcomp_stream_get(zram->comps[prio]);
drivers/block/zram/zram_drv.c
1371
ret = zcomp_decompress(zram->comps[prio], zstrm, src, size,
drivers/block/zram/zram_drv.c
1377
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
138
static void set_slot_flag(struct zram *zram, u32 index,
drivers/block/zram/zram_drv.c
1386
struct zram *zram = req->zram;
drivers/block/zram/zram_drv.c
1390
ret = decompress_bdev_page(zram, page, index);
drivers/block/zram/zram_drv.c
1403
struct zram *zram = req->zram;
drivers/block/zram/zram_drv.c
141
zram->table[index].attr.flags |= BIT(flag);
drivers/block/zram/zram_drv.c
1421
if (zram->compressed_wb == false) {
drivers/block/zram/zram_drv.c
1437
static int read_from_bdev_async(struct zram *zram, struct page *page,
drivers/block/zram/zram_drv.c
144
static void clear_slot_flag(struct zram *zram, u32 index,
drivers/block/zram/zram_drv.c
1448
bio = bio_alloc(zram->bdev, 1, parent->bi_opf, GFP_NOIO);
drivers/block/zram/zram_drv.c
1454
req->zram = zram;
drivers/block/zram/zram_drv.c
147
zram->table[index].attr.flags &= ~BIT(flag);
drivers/block/zram/zram_drv.c
1477
bio_init(&bio, req->zram->bdev, &bv, 1, REQ_OP_READ);
drivers/block/zram/zram_drv.c
1488
static int read_from_bdev_sync(struct zram *zram, struct page *page, u32 index,
drivers/block/zram/zram_drv.c
1494
req.zram = zram;
drivers/block/zram/zram_drv.c
150
static size_t get_slot_size(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
1502
if (req.error || zram->compressed_wb == false)
drivers/block/zram/zram_drv.c
1505
return decompress_bdev_page(zram, page, index);
drivers/block/zram/zram_drv.c
1508
static int read_from_bdev(struct zram *zram, struct page *page, u32 index,
drivers/block/zram/zram_drv.c
1511
atomic64_inc(&zram->stats.bd_reads);
drivers/block/zram/zram_drv.c
1515
return read_from_bdev_sync(zram, page, index, blk_idx);
drivers/block/zram/zram_drv.c
1517
return read_from_bdev_async(zram, page, index, blk_idx, parent);
drivers/block/zram/zram_drv.c
152
return zram->table[index].attr.flags & (BIT(ZRAM_FLAG_SHIFT) - 1);
drivers/block/zram/zram_drv.c
1520
static inline void reset_bdev(struct zram *zram) {};
drivers/block/zram/zram_drv.c
1521
static int read_from_bdev(struct zram *zram, struct page *page, u32 index,
drivers/block/zram/zram_drv.c
1527
static void zram_release_bdev_block(struct zram *zram, unsigned long blk_idx)
drivers/block/zram/zram_drv.c
155
static void set_slot_size(struct zram *zram, u32 index, size_t size)
drivers/block/zram/zram_drv.c
1551
struct zram *zram = file->private_data;
drivers/block/zram/zram_drv.c
1552
unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
drivers/block/zram/zram_drv.c
1558
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
1559
if (!init_done(zram)) {
drivers/block/zram/zram_drv.c
1567
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
1568
if (!slot_allocated(zram, index))
drivers/block/zram/zram_drv.c
157
unsigned long flags = zram->table[index].attr.flags >> ZRAM_FLAG_SHIFT;
drivers/block/zram/zram_drv.c
1573
index, zram->table[index].attr.ac_time, 0,
drivers/block/zram/zram_drv.c
1574
test_slot_flag(zram, index, ZRAM_SAME) ? 's' : '.',
drivers/block/zram/zram_drv.c
1575
test_slot_flag(zram, index, ZRAM_WB) ? 'w' : '.',
drivers/block/zram/zram_drv.c
1576
test_slot_flag(zram, index, ZRAM_HUGE) ? 'h' : '.',
drivers/block/zram/zram_drv.c
1577
test_slot_flag(zram, index, ZRAM_IDLE) ? 'i' : '.',
drivers/block/zram/zram_drv.c
1578
get_slot_comp_priority(zram, index) ? 'r' : '.',
drivers/block/zram/zram_drv.c
1579
test_slot_flag(zram, index,
drivers/block/zram/zram_drv.c
1583
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
1589
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
159
zram->table[index].attr.flags = (flags << ZRAM_FLAG_SHIFT) | size;
drivers/block/zram/zram_drv.c
1606
static void zram_debugfs_register(struct zram *zram)
drivers/block/zram/zram_drv.c
1611
zram->debugfs_dir = debugfs_create_dir(zram->disk->disk_name,
drivers/block/zram/zram_drv.c
1613
debugfs_create_file("block_state", 0400, zram->debugfs_dir,
drivers/block/zram/zram_drv.c
1614
zram, &proc_zram_block_state_op);
drivers/block/zram/zram_drv.c
1617
static void zram_debugfs_unregister(struct zram *zram)
drivers/block/zram/zram_drv.c
1619
debugfs_remove_recursive(zram->debugfs_dir);
drivers/block/zram/zram_drv.c
162
static inline bool slot_allocated(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
1624
static void zram_debugfs_register(struct zram *zram) {};
drivers/block/zram/zram_drv.c
1625
static void zram_debugfs_unregister(struct zram *zram) {};
drivers/block/zram/zram_drv.c
1629
static int lookup_algo_priority(struct zram *zram, const char *algo,
drivers/block/zram/zram_drv.c
1635
if (!zram->comp_algs[prio])
drivers/block/zram/zram_drv.c
1638
if (!strcmp(zram->comp_algs[prio], algo))
drivers/block/zram/zram_drv.c
164
return get_slot_size(zram, index) ||
drivers/block/zram/zram_drv.c
1646
static int validate_algo_priority(struct zram *zram, const char *algo, u32 prio)
drivers/block/zram/zram_drv.c
165
test_slot_flag(zram, index, ZRAM_SAME) ||
drivers/block/zram/zram_drv.c
1651
if (!zram->comp_algs[prio])
drivers/block/zram/zram_drv.c
1654
if (strcmp(zram->comp_algs[prio], algo))
drivers/block/zram/zram_drv.c
1659
static void comp_algorithm_set(struct zram *zram, u32 prio, const char *alg)
drivers/block/zram/zram_drv.c
166
test_slot_flag(zram, index, ZRAM_WB);
drivers/block/zram/zram_drv.c
1661
zram->comp_algs[prio] = alg;
drivers/block/zram/zram_drv.c
1664
static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf)
drivers/block/zram/zram_drv.c
1677
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
1678
if (init_done(zram)) {
drivers/block/zram/zram_drv.c
1683
comp_algorithm_set(zram, prio, alg);
drivers/block/zram/zram_drv.c
1687
static void comp_params_reset(struct zram *zram, u32 prio)
drivers/block/zram/zram_drv.c
1689
struct zcomp_params *params = &zram->params[prio];
drivers/block/zram/zram_drv.c
169
static inline void set_slot_comp_priority(struct zram *zram, u32 index,
drivers/block/zram/zram_drv.c
1698
static int comp_params_store(struct zram *zram, u32 prio, s32 level,
drivers/block/zram/zram_drv.c
1704
comp_params_reset(zram, prio);
drivers/block/zram/zram_drv.c
1708
&zram->params[prio].dict,
drivers/block/zram/zram_drv.c
1716
zram->params[prio].dict_sz = sz;
drivers/block/zram/zram_drv.c
1717
zram->params[prio].level = level;
drivers/block/zram/zram_drv.c
1718
zram->params[prio].deflate.winbits = deflate_params->winbits;
drivers/block/zram/zram_drv.c
1730
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
177
zram->table[index].attr.flags &= ~(ZRAM_COMP_PRIORITY_MASK <<
drivers/block/zram/zram_drv.c
1776
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
1777
if (init_done(zram))
drivers/block/zram/zram_drv.c
1786
ret = validate_algo_priority(zram, algo, prio);
drivers/block/zram/zram_drv.c
179
zram->table[index].attr.flags |= (prio << ZRAM_COMP_PRIORITY_BIT1);
drivers/block/zram/zram_drv.c
1792
prio = lookup_algo_priority(zram, algo, ZRAM_PRIMARY_COMP);
drivers/block/zram/zram_drv.c
1797
ret = comp_params_store(zram, prio, level, dict_path, &deflate_params);
drivers/block/zram/zram_drv.c
1805
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
1808
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
1809
sz = zcomp_available_show(zram->comp_algs[ZRAM_PRIMARY_COMP], buf, 0);
drivers/block/zram/zram_drv.c
1818
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
182
static inline u32 get_slot_comp_priority(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
1821
ret = __comp_algorithm_store(zram, ZRAM_PRIMARY_COMP, buf);
drivers/block/zram/zram_drv.c
1830
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
1834
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
1836
if (!zram->comp_algs[prio])
drivers/block/zram/zram_drv.c
184
u32 prio = zram->table[index].attr.flags >> ZRAM_COMP_PRIORITY_BIT1;
drivers/block/zram/zram_drv.c
1840
sz += zcomp_available_show(zram->comp_algs[prio], buf, sz);
drivers/block/zram/zram_drv.c
1850
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
1882
ret = __comp_algorithm_store(zram, prio, alg);
drivers/block/zram/zram_drv.c
189
static void mark_slot_accessed(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
1890
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
1892
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
1893
if (!init_done(zram))
drivers/block/zram/zram_drv.c
1896
zs_compact(zram->mem_pool);
drivers/block/zram/zram_drv.c
1904
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
1907
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
191
clear_slot_flag(zram, index, ZRAM_IDLE);
drivers/block/zram/zram_drv.c
1910
(u64)atomic64_read(&zram->stats.failed_reads),
drivers/block/zram/zram_drv.c
1911
(u64)atomic64_read(&zram->stats.failed_writes),
drivers/block/zram/zram_drv.c
1912
(u64)atomic64_read(&zram->stats.notify_free));
drivers/block/zram/zram_drv.c
192
clear_slot_flag(zram, index, ZRAM_PP_SLOT);
drivers/block/zram/zram_drv.c
1920
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
1928
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
1929
if (init_done(zram)) {
drivers/block/zram/zram_drv.c
1930
mem_used = zs_get_total_pages(zram->mem_pool);
drivers/block/zram/zram_drv.c
1931
zs_pool_stats(zram->mem_pool, &pool_stats);
drivers/block/zram/zram_drv.c
1934
orig_size = atomic64_read(&zram->stats.pages_stored);
drivers/block/zram/zram_drv.c
1935
max_used = atomic_long_read(&zram->stats.max_used_pages);
drivers/block/zram/zram_drv.c
194
zram->table[index].attr.ac_time = (u32)ktime_get_boottime_seconds();
drivers/block/zram/zram_drv.c
1940
(u64)atomic64_read(&zram->stats.compr_data_size),
drivers/block/zram/zram_drv.c
1942
zram->limit_pages << PAGE_SHIFT,
drivers/block/zram/zram_drv.c
1944
(u64)atomic64_read(&zram->stats.same_pages),
drivers/block/zram/zram_drv.c
1946
(u64)atomic64_read(&zram->stats.huge_pages),
drivers/block/zram/zram_drv.c
1947
(u64)atomic64_read(&zram->stats.huge_pages_since));
drivers/block/zram/zram_drv.c
1956
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
1959
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
1963
(u64)atomic64_read(&zram->stats.miss_free));
drivers/block/zram/zram_drv.c
1968
static void zram_meta_free(struct zram *zram, u64 disksize)
drivers/block/zram/zram_drv.c
1973
if (!zram->table)
drivers/block/zram/zram_drv.c
1978
slot_free(zram, index);
drivers/block/zram/zram_drv.c
198
static inline void update_used_max(struct zram *zram, const unsigned long pages)
drivers/block/zram/zram_drv.c
1980
zs_destroy_pool(zram->mem_pool);
drivers/block/zram/zram_drv.c
1981
vfree(zram->table);
drivers/block/zram/zram_drv.c
1982
zram->table = NULL;
drivers/block/zram/zram_drv.c
1985
static bool zram_meta_alloc(struct zram *zram, u64 disksize)
drivers/block/zram/zram_drv.c
1990
zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table)));
drivers/block/zram/zram_drv.c
1991
if (!zram->table)
drivers/block/zram/zram_drv.c
1994
zram->mem_pool = zs_create_pool(zram->disk->disk_name);
drivers/block/zram/zram_drv.c
1995
if (!zram->mem_pool) {
drivers/block/zram/zram_drv.c
1996
vfree(zram->table);
drivers/block/zram/zram_drv.c
1997
zram->table = NULL;
drivers/block/zram/zram_drv.c
200
unsigned long cur_max = atomic_long_read(&zram->stats.max_used_pages);
drivers/block/zram/zram_drv.c
2002
huge_class_size = zs_huge_class_size(zram->mem_pool);
drivers/block/zram/zram_drv.c
2005
slot_lock_init(zram, index);
drivers/block/zram/zram_drv.c
2010
static void slot_free(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
2015
zram->table[index].attr.ac_time = 0;
drivers/block/zram/zram_drv.c
2018
clear_slot_flag(zram, index, ZRAM_IDLE);
drivers/block/zram/zram_drv.c
2019
clear_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE);
drivers/block/zram/zram_drv.c
2020
clear_slot_flag(zram, index, ZRAM_PP_SLOT);
drivers/block/zram/zram_drv.c
2021
set_slot_comp_priority(zram, index, 0);
drivers/block/zram/zram_drv.c
2023
if (test_slot_flag(zram, index, ZRAM_HUGE)) {
drivers/block/zram/zram_drv.c
2028
if (!test_slot_flag(zram, index, ZRAM_WB))
drivers/block/zram/zram_drv.c
2029
atomic64_dec(&zram->stats.huge_pages);
drivers/block/zram/zram_drv.c
2030
clear_slot_flag(zram, index, ZRAM_HUGE);
drivers/block/zram/zram_drv.c
2033
if (test_slot_flag(zram, index, ZRAM_WB)) {
drivers/block/zram/zram_drv.c
2034
clear_slot_flag(zram, index, ZRAM_WB);
drivers/block/zram/zram_drv.c
2035
zram_release_bdev_block(zram, get_slot_handle(zram, index));
drivers/block/zram/zram_drv.c
2043
if (test_slot_flag(zram, index, ZRAM_SAME)) {
drivers/block/zram/zram_drv.c
2044
clear_slot_flag(zram, index, ZRAM_SAME);
drivers/block/zram/zram_drv.c
2045
atomic64_dec(&zram->stats.same_pages);
drivers/block/zram/zram_drv.c
2049
handle = get_slot_handle(zram, index);
drivers/block/zram/zram_drv.c
205
} while (!atomic_long_try_cmpxchg(&zram->stats.max_used_pages,
drivers/block/zram/zram_drv.c
2053
zs_free(zram->mem_pool, handle);
drivers/block/zram/zram_drv.c
2055
atomic64_sub(get_slot_size(zram, index),
drivers/block/zram/zram_drv.c
2056
&zram->stats.compr_data_size);
drivers/block/zram/zram_drv.c
2058
atomic64_dec(&zram->stats.pages_stored);
drivers/block/zram/zram_drv.c
2059
set_slot_handle(zram, index, 0);
drivers/block/zram/zram_drv.c
2060
set_slot_size(zram, index, 0);
drivers/block/zram/zram_drv.c
2063
static int read_same_filled_page(struct zram *zram, struct page *page,
drivers/block/zram/zram_drv.c
2069
zram_fill_page(mem, PAGE_SIZE, get_slot_handle(zram, index));
drivers/block/zram/zram_drv.c
2074
static int read_incompressible_page(struct zram *zram, struct page *page,
drivers/block/zram/zram_drv.c
2080
handle = get_slot_handle(zram, index);
drivers/block/zram/zram_drv.c
2081
src = zs_obj_read_begin(zram->mem_pool, handle, PAGE_SIZE, NULL);
drivers/block/zram/zram_drv.c
2085
zs_obj_read_end(zram->mem_pool, handle, PAGE_SIZE, src);
drivers/block/zram/zram_drv.c
209
static bool zram_can_store_page(struct zram *zram)
drivers/block/zram/zram_drv.c
2090
static int read_compressed_page(struct zram *zram, struct page *page, u32 index)
drivers/block/zram/zram_drv.c
2098
handle = get_slot_handle(zram, index);
drivers/block/zram/zram_drv.c
2099
size = get_slot_size(zram, index);
drivers/block/zram/zram_drv.c
2100
prio = get_slot_comp_priority(zram, index);
drivers/block/zram/zram_drv.c
2102
zstrm = zcomp_stream_get(zram->comps[prio]);
drivers/block/zram/zram_drv.c
2103
src = zs_obj_read_begin(zram->mem_pool, handle, size,
drivers/block/zram/zram_drv.c
2106
ret = zcomp_decompress(zram->comps[prio], zstrm, src, size, dst);
drivers/block/zram/zram_drv.c
2108
zs_obj_read_end(zram->mem_pool, handle, size, src);
drivers/block/zram/zram_drv.c
2115
static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index)
drivers/block/zram/zram_drv.c
2122
handle = get_slot_handle(zram, index);
drivers/block/zram/zram_drv.c
2123
size = get_slot_size(zram, index);
drivers/block/zram/zram_drv.c
213
alloced_pages = zs_get_total_pages(zram->mem_pool);
drivers/block/zram/zram_drv.c
2130
zstrm = zcomp_stream_get(zram->comps[ZRAM_PRIMARY_COMP]);
drivers/block/zram/zram_drv.c
2131
src = zs_obj_read_begin(zram->mem_pool, handle, size,
drivers/block/zram/zram_drv.c
2134
zs_obj_read_end(zram->mem_pool, handle, size, src);
drivers/block/zram/zram_drv.c
214
update_used_max(zram, alloced_pages);
drivers/block/zram/zram_drv.c
2147
static int read_from_zspool(struct zram *zram, struct page *page, u32 index)
drivers/block/zram/zram_drv.c
2149
if (test_slot_flag(zram, index, ZRAM_SAME) ||
drivers/block/zram/zram_drv.c
2150
!get_slot_handle(zram, index))
drivers/block/zram/zram_drv.c
2151
return read_same_filled_page(zram, page, index);
drivers/block/zram/zram_drv.c
2153
if (!test_slot_flag(zram, index, ZRAM_HUGE))
drivers/block/zram/zram_drv.c
2154
return read_compressed_page(zram, page, index);
drivers/block/zram/zram_drv.c
2156
return read_incompressible_page(zram, page, index);
drivers/block/zram/zram_drv.c
2159
static int zram_read_page(struct zram *zram, struct page *page, u32 index,
drivers/block/zram/zram_drv.c
216
return !zram->limit_pages || alloced_pages <= zram->limit_pages;
drivers/block/zram/zram_drv.c
2164
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
2165
if (!test_slot_flag(zram, index, ZRAM_WB)) {
drivers/block/zram/zram_drv.c
2167
ret = read_from_zspool(zram, page, index);
drivers/block/zram/zram_drv.c
2168
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
2170
unsigned long blk_idx = get_slot_handle(zram, index);
drivers/block/zram/zram_drv.c
2176
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
2177
ret = read_from_bdev(zram, page, index, blk_idx, parent);
drivers/block/zram/zram_drv.c
2191
static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec,
drivers/block/zram/zram_drv.c
2199
ret = zram_read_page(zram, page, index, NULL);
drivers/block/zram/zram_drv.c
2206
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
drivers/block/zram/zram_drv.c
2210
return zram_bvec_read_partial(zram, bvec, index, offset);
drivers/block/zram/zram_drv.c
2211
return zram_read_page(zram, bvec->bv_page, index, bio);
drivers/block/zram/zram_drv.c
2214
static int write_same_filled_page(struct zram *zram, unsigned long fill,
drivers/block/zram/zram_drv.c
2217
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
2218
slot_free(zram, index);
drivers/block/zram/zram_drv.c
2219
set_slot_flag(zram, index, ZRAM_SAME);
drivers/block/zram/zram_drv.c
2220
set_slot_handle(zram, index, fill);
drivers/block/zram/zram_drv.c
2221
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
2223
atomic64_inc(&zram->stats.same_pages);
drivers/block/zram/zram_drv.c
2224
atomic64_inc(&zram->stats.pages_stored);
drivers/block/zram/zram_drv.c
2229
static int write_incompressible_page(struct zram *zram, struct page *page,
drivers/block/zram/zram_drv.c
2240
handle = zs_malloc(zram->mem_pool, PAGE_SIZE,
drivers/block/zram/zram_drv.c
2246
if (!zram_can_store_page(zram)) {
drivers/block/zram/zram_drv.c
2247
zs_free(zram->mem_pool, handle);
drivers/block/zram/zram_drv.c
2252
zs_obj_write(zram->mem_pool, handle, src, PAGE_SIZE);
drivers/block/zram/zram_drv.c
2255
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
2256
slot_free(zram, index);
drivers/block/zram/zram_drv.c
2257
set_slot_flag(zram, index, ZRAM_HUGE);
drivers/block/zram/zram_drv.c
2258
set_slot_handle(zram, index, handle);
drivers/block/zram/zram_drv.c
2259
set_slot_size(zram, index, PAGE_SIZE);
drivers/block/zram/zram_drv.c
2260
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
2262
atomic64_add(PAGE_SIZE, &zram->stats.compr_data_size);
drivers/block/zram/zram_drv.c
2263
atomic64_inc(&zram->stats.huge_pages);
drivers/block/zram/zram_drv.c
2264
atomic64_inc(&zram->stats.huge_pages_since);
drivers/block/zram/zram_drv.c
2265
atomic64_inc(&zram->stats.pages_stored);
drivers/block/zram/zram_drv.c
2270
static int zram_write_page(struct zram *zram, struct page *page, u32 index)
drivers/block/zram/zram_drv.c
2284
return write_same_filled_page(zram, element, index);
drivers/block/zram/zram_drv.c
2286
zstrm = zcomp_stream_get(zram->comps[ZRAM_PRIMARY_COMP]);
drivers/block/zram/zram_drv.c
2288
ret = zcomp_compress(zram->comps[ZRAM_PRIMARY_COMP], zstrm,
drivers/block/zram/zram_drv.c
2300
return write_incompressible_page(zram, page, index);
drivers/block/zram/zram_drv.c
2303
handle = zs_malloc(zram->mem_pool, comp_len,
drivers/block/zram/zram_drv.c
2311
if (!zram_can_store_page(zram)) {
drivers/block/zram/zram_drv.c
2313
zs_free(zram->mem_pool, handle);
drivers/block/zram/zram_drv.c
2317
zs_obj_write(zram->mem_pool, handle, zstrm->buffer, comp_len);
drivers/block/zram/zram_drv.c
2320
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
2321
slot_free(zram, index);
drivers/block/zram/zram_drv.c
2322
set_slot_handle(zram, index, handle);
drivers/block/zram/zram_drv.c
2323
set_slot_size(zram, index, comp_len);
drivers/block/zram/zram_drv.c
2324
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
2327
atomic64_inc(&zram->stats.pages_stored);
drivers/block/zram/zram_drv.c
2328
atomic64_add(comp_len, &zram->stats.compr_data_size);
drivers/block/zram/zram_drv.c
2336
static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec,
drivers/block/zram/zram_drv.c
2345
ret = zram_read_page(zram, page, index, NULL);
drivers/block/zram/zram_drv.c
2348
ret = zram_write_page(zram, page, index);
drivers/block/zram/zram_drv.c
2354
static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec,
drivers/block/zram/zram_drv.c
2358
return zram_bvec_write_partial(zram, bvec, index, offset);
drivers/block/zram/zram_drv.c
2359
return zram_write_page(zram, bvec->bv_page, index);
drivers/block/zram/zram_drv.c
2366
static bool highest_priority_algorithm(struct zram *zram, u32 prio)
drivers/block/zram/zram_drv.c
2371
if (zram->comp_algs[p])
drivers/block/zram/zram_drv.c
2378
static void scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio,
drivers/block/zram/zram_drv.c
2381
unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
drivers/block/zram/zram_drv.c
2387
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
2388
if (!slot_allocated(zram, index))
drivers/block/zram/zram_drv.c
2392
!test_slot_flag(zram, index, ZRAM_IDLE))
drivers/block/zram/zram_drv.c
2396
!test_slot_flag(zram, index, ZRAM_HUGE))
drivers/block/zram/zram_drv.c
2399
if (test_slot_flag(zram, index, ZRAM_WB) ||
drivers/block/zram/zram_drv.c
2400
test_slot_flag(zram, index, ZRAM_SAME) ||
drivers/block/zram/zram_drv.c
2401
test_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE))
drivers/block/zram/zram_drv.c
2405
if (get_slot_comp_priority(zram, index) >= prio)
drivers/block/zram/zram_drv.c
2408
ok = place_pp_slot(zram, ctl, index);
drivers/block/zram/zram_drv.c
2410
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
2423
static int recompress_slot(struct zram *zram, u32 index, struct page *page,
drivers/block/zram/zram_drv.c
2436
handle_old = get_slot_handle(zram, index);
drivers/block/zram/zram_drv.c
2440
comp_len_old = get_slot_size(zram, index);
drivers/block/zram/zram_drv.c
2447
ret = read_from_zspool(zram, page, index);
drivers/block/zram/zram_drv.c
2456
clear_slot_flag(zram, index, ZRAM_IDLE);
drivers/block/zram/zram_drv.c
2458
zstrm = zcomp_stream_get(zram->comps[prio]);
drivers/block/zram/zram_drv.c
2460
ret = zcomp_compress(zram->comps[prio], zstrm, src, &comp_len_new);
drivers/block/zram/zram_drv.c
2477
class_index_old = zs_lookup_class_index(zram->mem_pool, comp_len_old);
drivers/block/zram/zram_drv.c
2478
class_index_new = zs_lookup_class_index(zram->mem_pool, comp_len_new);
drivers/block/zram/zram_drv.c
2491
if (highest_priority_algorithm(zram, prio))
drivers/block/zram/zram_drv.c
2492
set_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE);
drivers/block/zram/zram_drv.c
2506
handle_new = zs_malloc(zram->mem_pool, comp_len_new,
drivers/block/zram/zram_drv.c
2515
zs_obj_write(zram->mem_pool, handle_new, zstrm->buffer, comp_len_new);
drivers/block/zram/zram_drv.c
2518
slot_free(zram, index);
drivers/block/zram/zram_drv.c
2519
set_slot_handle(zram, index, handle_new);
drivers/block/zram/zram_drv.c
2520
set_slot_size(zram, index, comp_len_new);
drivers/block/zram/zram_drv.c
2521
set_slot_comp_priority(zram, index, prio);
drivers/block/zram/zram_drv.c
2523
atomic64_add(comp_len_new, &zram->stats.compr_data_size);
drivers/block/zram/zram_drv.c
2524
atomic64_inc(&zram->stats.pages_stored);
drivers/block/zram/zram_drv.c
2533
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
2602
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
2603
if (!init_done(zram))
drivers/block/zram/zram_drv.c
2612
ret = validate_algo_priority(zram, algo, prio);
drivers/block/zram/zram_drv.c
2618
prio = lookup_algo_priority(zram, algo, ZRAM_SECONDARY_COMP);
drivers/block/zram/zram_drv.c
2623
if (!zram->comps[prio])
drivers/block/zram/zram_drv.c
263
static void release_pp_slot(struct zram *zram, struct zram_pp_slot *pps)
drivers/block/zram/zram_drv.c
2638
scan_slots_for_recompress(zram, mode, prio, ctl);
drivers/block/zram/zram_drv.c
2647
slot_lock(zram, pps->index);
drivers/block/zram/zram_drv.c
2648
if (!test_slot_flag(zram, pps->index, ZRAM_PP_SLOT))
drivers/block/zram/zram_drv.c
2651
err = recompress_slot(zram, pps->index, page,
drivers/block/zram/zram_drv.c
2654
slot_unlock(zram, pps->index);
drivers/block/zram/zram_drv.c
2655
release_pp_slot(zram, pps);
drivers/block/zram/zram_drv.c
2668
release_pp_ctl(zram, ctl);
drivers/block/zram/zram_drv.c
267
slot_lock(zram, pps->index);
drivers/block/zram/zram_drv.c
2673
static void zram_bio_discard(struct zram *zram, struct bio *bio)
drivers/block/zram/zram_drv.c
268
clear_slot_flag(zram, pps->index, ZRAM_PP_SLOT);
drivers/block/zram/zram_drv.c
269
slot_unlock(zram, pps->index);
drivers/block/zram/zram_drv.c
2699
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
2700
slot_free(zram, index);
drivers/block/zram/zram_drv.c
2701
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
2702
atomic64_inc(&zram->stats.notify_free);
drivers/block/zram/zram_drv.c
2711
static void zram_bio_read(struct zram *zram, struct bio *bio)
drivers/block/zram/zram_drv.c
2724
if (zram_bvec_read(zram, &bv, index, offset, bio) < 0) {
drivers/block/zram/zram_drv.c
2725
atomic64_inc(&zram->stats.failed_reads);
drivers/block/zram/zram_drv.c
2731
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
2732
mark_slot_accessed(zram, index);
drivers/block/zram/zram_drv.c
2733
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
274
static void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl)
drivers/block/zram/zram_drv.c
2742
static void zram_bio_write(struct zram *zram, struct bio *bio)
drivers/block/zram/zram_drv.c
2755
if (zram_bvec_write(zram, &bv, index, offset) < 0) {
drivers/block/zram/zram_drv.c
2756
atomic64_inc(&zram->stats.failed_writes);
drivers/block/zram/zram_drv.c
2761
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
2762
mark_slot_accessed(zram, index);
drivers/block/zram/zram_drv.c
2763
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
2777
struct zram *zram = bio->bi_bdev->bd_disk->private_data;
drivers/block/zram/zram_drv.c
2781
zram_bio_read(zram, bio);
drivers/block/zram/zram_drv.c
2784
zram_bio_write(zram, bio);
drivers/block/zram/zram_drv.c
2788
zram_bio_discard(zram, bio);
drivers/block/zram/zram_drv.c
2799
struct zram *zram;
drivers/block/zram/zram_drv.c
2801
zram = bdev->bd_disk->private_data;
drivers/block/zram/zram_drv.c
2803
atomic64_inc(&zram->stats.notify_free);
drivers/block/zram/zram_drv.c
2804
if (!slot_trylock(zram, index)) {
drivers/block/zram/zram_drv.c
2805
atomic64_inc(&zram->stats.miss_free);
drivers/block/zram/zram_drv.c
2809
slot_free(zram, index);
drivers/block/zram/zram_drv.c
2810
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
2813
static void zram_comp_params_reset(struct zram *zram)
drivers/block/zram/zram_drv.c
2818
comp_params_reset(zram, prio);
drivers/block/zram/zram_drv.c
2822
static void zram_destroy_comps(struct zram *zram)
drivers/block/zram/zram_drv.c
2827
struct zcomp *comp = zram->comps[prio];
drivers/block/zram/zram_drv.c
2829
zram->comps[prio] = NULL;
drivers/block/zram/zram_drv.c
2836
zram->comp_algs[prio] = NULL;
drivers/block/zram/zram_drv.c
2838
zram_comp_params_reset(zram);
drivers/block/zram/zram_drv.c
2841
static void zram_reset_device(struct zram *zram)
drivers/block/zram/zram_drv.c
2843
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
2845
zram->limit_pages = 0;
drivers/block/zram/zram_drv.c
2847
set_capacity_and_notify(zram->disk, 0);
drivers/block/zram/zram_drv.c
2848
part_stat_set_all(zram->disk->part0, 0);
drivers/block/zram/zram_drv.c
2851
zram_meta_free(zram, zram->disksize);
drivers/block/zram/zram_drv.c
2852
zram->disksize = 0;
drivers/block/zram/zram_drv.c
2853
zram_destroy_comps(zram);
drivers/block/zram/zram_drv.c
2854
memset(&zram->stats, 0, sizeof(zram->stats));
drivers/block/zram/zram_drv.c
2855
reset_bdev(zram);
drivers/block/zram/zram_drv.c
2857
comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor);
drivers/block/zram/zram_drv.c
2865
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
2873
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
2874
if (init_done(zram)) {
drivers/block/zram/zram_drv.c
288
release_pp_slot(zram, pps);
drivers/block/zram/zram_drv.c
2880
if (!zram_meta_alloc(zram, disksize))
drivers/block/zram/zram_drv.c
2884
if (!zram->comp_algs[prio])
drivers/block/zram/zram_drv.c
2887
comp = zcomp_create(zram->comp_algs[prio],
drivers/block/zram/zram_drv.c
2888
&zram->params[prio]);
drivers/block/zram/zram_drv.c
2891
zram->comp_algs[prio]);
drivers/block/zram/zram_drv.c
2896
zram->comps[prio] = comp;
drivers/block/zram/zram_drv.c
2898
zram->disksize = disksize;
drivers/block/zram/zram_drv.c
2899
set_capacity_and_notify(zram->disk, zram->disksize >> SECTOR_SHIFT);
drivers/block/zram/zram_drv.c
2904
zram_destroy_comps(zram);
drivers/block/zram/zram_drv.c
2905
zram_meta_free(zram, disksize);
drivers/block/zram/zram_drv.c
2914
struct zram *zram;
drivers/block/zram/zram_drv.c
2924
zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
2925
disk = zram->disk;
drivers/block/zram/zram_drv.c
2929
if (disk_openers(disk) || zram->claim) {
drivers/block/zram/zram_drv.c
2935
zram->claim = true;
drivers/block/zram/zram_drv.c
2940
zram_reset_device(zram);
drivers/block/zram/zram_drv.c
2943
zram->claim = false;
drivers/block/zram/zram_drv.c
295
static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl,
drivers/block/zram/zram_drv.c
2951
struct zram *zram = disk->private_data;
drivers/block/zram/zram_drv.c
2956
if (zram->claim)
drivers/block/zram/zram_drv.c
3055
struct zram *zram;
drivers/block/zram/zram_drv.c
3058
zram = kzalloc_obj(struct zram);
drivers/block/zram/zram_drv.c
3059
if (!zram)
drivers/block/zram/zram_drv.c
3062
ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL);
drivers/block/zram/zram_drv.c
3067
init_rwsem(&zram->dev_lock);
drivers/block/zram/zram_drv.c
3069
zram->wb_batch_size = 32;
drivers/block/zram/zram_drv.c
3070
zram->compressed_wb = false;
drivers/block/zram/zram_drv.c
3074
zram->disk = blk_alloc_disk(&lim, NUMA_NO_NODE);
drivers/block/zram/zram_drv.c
3075
if (IS_ERR(zram->disk)) {
drivers/block/zram/zram_drv.c
3078
ret = PTR_ERR(zram->disk);
drivers/block/zram/zram_drv.c
308
bid = get_slot_size(zram, pps->index) / PP_BUCKET_SIZE_RANGE;
drivers/block/zram/zram_drv.c
3082
zram->disk->major = zram_major;
drivers/block/zram/zram_drv.c
3083
zram->disk->first_minor = device_id;
drivers/block/zram/zram_drv.c
3084
zram->disk->minors = 1;
drivers/block/zram/zram_drv.c
3085
zram->disk->flags |= GENHD_FL_NO_PART;
drivers/block/zram/zram_drv.c
3086
zram->disk->fops = &zram_devops;
drivers/block/zram/zram_drv.c
3087
zram->disk->private_data = zram;
drivers/block/zram/zram_drv.c
3088
snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
drivers/block/zram/zram_drv.c
3089
zram_comp_params_reset(zram);
drivers/block/zram/zram_drv.c
3090
comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor);
drivers/block/zram/zram_drv.c
3093
set_capacity(zram->disk, 0);
drivers/block/zram/zram_drv.c
3094
ret = device_add_disk(NULL, zram->disk, zram_disk_groups);
drivers/block/zram/zram_drv.c
3098
zram_debugfs_register(zram);
drivers/block/zram/zram_drv.c
3099
pr_info("Added device: %s\n", zram->disk->disk_name);
drivers/block/zram/zram_drv.c
3103
put_disk(zram->disk);
drivers/block/zram/zram_drv.c
3107
kfree(zram);
drivers/block/zram/zram_drv.c
311
set_slot_flag(zram, pps->index, ZRAM_PP_SLOT);
drivers/block/zram/zram_drv.c
3111
static int zram_remove(struct zram *zram)
drivers/block/zram/zram_drv.c
3115
mutex_lock(&zram->disk->open_mutex);
drivers/block/zram/zram_drv.c
3116
if (disk_openers(zram->disk)) {
drivers/block/zram/zram_drv.c
3117
mutex_unlock(&zram->disk->open_mutex);
drivers/block/zram/zram_drv.c
3121
claimed = zram->claim;
drivers/block/zram/zram_drv.c
3123
zram->claim = true;
drivers/block/zram/zram_drv.c
3124
mutex_unlock(&zram->disk->open_mutex);
drivers/block/zram/zram_drv.c
3126
zram_debugfs_unregister(zram);
drivers/block/zram/zram_drv.c
3136
sync_blockdev(zram->disk->part0);
drivers/block/zram/zram_drv.c
3137
zram_reset_device(zram);
drivers/block/zram/zram_drv.c
3140
pr_info("Removed device: %s\n", zram->disk->disk_name);
drivers/block/zram/zram_drv.c
3142
del_gendisk(zram->disk);
drivers/block/zram/zram_drv.c
3145
WARN_ON_ONCE(claimed && zram->claim);
drivers/block/zram/zram_drv.c
3152
zram_reset_device(zram);
drivers/block/zram/zram_drv.c
3154
put_disk(zram->disk);
drivers/block/zram/zram_drv.c
3155
kfree(zram);
drivers/block/zram/zram_drv.c
3190
struct zram *zram;
drivers/block/zram/zram_drv.c
3202
zram = idr_find(&zram_index_idr, dev_id);
drivers/block/zram/zram_drv.c
3203
if (zram) {
drivers/block/zram/zram_drv.c
3204
ret = zram_remove(zram);
drivers/block/zram/zram_drv.c
367
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
369
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
370
val = init_done(zram);
drivers/block/zram/zram_drv.c
378
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
380
return sysfs_emit(buf, "%llu\n", zram->disksize);
drivers/block/zram/zram_drv.c
389
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
395
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
396
zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
drivers/block/zram/zram_drv.c
407
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
413
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
414
if (init_done(zram)) {
drivers/block/zram/zram_drv.c
415
atomic_long_set(&zram->stats.max_used_pages,
drivers/block/zram/zram_drv.c
416
zs_get_total_pages(zram->mem_pool));
drivers/block/zram/zram_drv.c
426
static void mark_idle(struct zram *zram, ktime_t cutoff)
drivers/block/zram/zram_drv.c
429
unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
drivers/block/zram/zram_drv.c
440
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
441
if (!slot_allocated(zram, index) ||
drivers/block/zram/zram_drv.c
442
test_slot_flag(zram, index, ZRAM_WB) ||
drivers/block/zram/zram_drv.c
443
test_slot_flag(zram, index, ZRAM_SAME)) {
drivers/block/zram/zram_drv.c
444
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
450
ktime_after(cutoff, zram->table[index].attr.ac_time);
drivers/block/zram/zram_drv.c
453
set_slot_flag(zram, index, ZRAM_IDLE);
drivers/block/zram/zram_drv.c
455
clear_slot_flag(zram, index, ZRAM_IDLE);
drivers/block/zram/zram_drv.c
456
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
463
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
481
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
482
if (!init_done(zram))
drivers/block/zram/zram_drv.c
489
mark_idle(zram, cutoff);
drivers/block/zram/zram_drv.c
496
static int read_from_zspool_raw(struct zram *zram, struct page *page,
drivers/block/zram/zram_drv.c
498
static int read_from_zspool(struct zram *zram, struct page *page, u32 index);
drivers/block/zram/zram_drv.c
523
struct zram *zram;
drivers/block/zram/zram_drv.c
541
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
544
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
547
FOUR_K((u64)atomic64_read(&zram->stats.bd_count)),
drivers/block/zram/zram_drv.c
548
FOUR_K((u64)atomic64_read(&zram->stats.bd_reads)),
drivers/block/zram/zram_drv.c
549
FOUR_K((u64)atomic64_read(&zram->stats.bd_writes)));
drivers/block/zram/zram_drv.c
558
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
564
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
565
if (init_done(zram)) {
drivers/block/zram/zram_drv.c
569
zram->compressed_wb = val;
drivers/block/zram/zram_drv.c
579
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
581
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
582
val = zram->compressed_wb;
drivers/block/zram/zram_drv.c
59
static void slot_free(struct zram *zram, u32 index);
drivers/block/zram/zram_drv.c
591
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
597
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
598
zram->wb_limit_enable = val;
drivers/block/zram/zram_drv.c
60
#define slot_dep_map(zram, index) (&(zram)->table[(index)].dep_map)
drivers/block/zram/zram_drv.c
608
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
610
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
611
val = zram->wb_limit_enable;
drivers/block/zram/zram_drv.c
62
static void slot_lock_init(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
620
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
636
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
637
zram->bd_wb_limit = val;
drivers/block/zram/zram_drv.c
646
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
648
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
649
val = zram->bd_wb_limit;
drivers/block/zram/zram_drv.c
658
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
66
lockdep_init_map(slot_dep_map(zram, index), "zram->table[index].lock",
drivers/block/zram/zram_drv.c
667
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
668
zram->wb_batch_size = val;
drivers/block/zram/zram_drv.c
678
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
680
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
681
val = zram->wb_batch_size;
drivers/block/zram/zram_drv.c
686
static void reset_bdev(struct zram *zram)
drivers/block/zram/zram_drv.c
688
if (!zram->backing_dev)
drivers/block/zram/zram_drv.c
692
filp_close(zram->backing_dev, NULL);
drivers/block/zram/zram_drv.c
693
zram->backing_dev = NULL;
drivers/block/zram/zram_drv.c
694
zram->bdev = NULL;
drivers/block/zram/zram_drv.c
695
zram->disk->fops = &zram_devops;
drivers/block/zram/zram_drv.c
696
kvfree(zram->bitmap);
drivers/block/zram/zram_drv.c
697
zram->bitmap = NULL;
drivers/block/zram/zram_drv.c
704
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
708
guard(rwsem_read)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
709
file = zram->backing_dev;
drivers/block/zram/zram_drv.c
736
struct zram *zram = dev_to_zram(dev);
drivers/block/zram/zram_drv.c
742
guard(rwsem_write)(&zram->dev_lock);
drivers/block/zram/zram_drv.c
743
if (init_done(zram)) {
drivers/block/zram/zram_drv.c
784
reset_bdev(zram);
drivers/block/zram/zram_drv.c
786
zram->bdev = I_BDEV(inode);
drivers/block/zram/zram_drv.c
787
zram->backing_dev = backing_dev;
drivers/block/zram/zram_drv.c
788
zram->bitmap = bitmap;
drivers/block/zram/zram_drv.c
789
zram->nr_pages = nr_pages;
drivers/block/zram/zram_drv.c
806
static unsigned long zram_reserve_bdev_block(struct zram *zram)
drivers/block/zram/zram_drv.c
810
blk_idx = find_next_zero_bit(zram->bitmap, zram->nr_pages, 0);
drivers/block/zram/zram_drv.c
811
if (blk_idx == zram->nr_pages)
drivers/block/zram/zram_drv.c
814
set_bit(blk_idx, zram->bitmap);
drivers/block/zram/zram_drv.c
815
atomic64_inc(&zram->stats.bd_count);
drivers/block/zram/zram_drv.c
819
static void zram_release_bdev_block(struct zram *zram, unsigned long blk_idx)
drivers/block/zram/zram_drv.c
82
static __must_check bool slot_trylock(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
823
was_set = test_and_clear_bit(blk_idx, zram->bitmap);
drivers/block/zram/zram_drv.c
825
atomic64_dec(&zram->stats.bd_count);
drivers/block/zram/zram_drv.c
84
unsigned long *lock = &zram->table[index].__lock;
drivers/block/zram/zram_drv.c
855
static struct zram_wb_ctl *init_wb_ctl(struct zram *zram)
drivers/block/zram/zram_drv.c
87
mutex_acquire(slot_dep_map(zram, index), 0, 1, _RET_IP_);
drivers/block/zram/zram_drv.c
870
for (i = 0; i < zram->wb_batch_size; i++) {
drivers/block/zram/zram_drv.c
88
lock_acquired(slot_dep_map(zram, index), _RET_IP_);
drivers/block/zram/zram_drv.c
904
static void zram_account_writeback_rollback(struct zram *zram)
drivers/block/zram/zram_drv.c
906
lockdep_assert_held_write(&zram->dev_lock);
drivers/block/zram/zram_drv.c
908
if (zram->wb_limit_enable)
drivers/block/zram/zram_drv.c
909
zram->bd_wb_limit += 1UL << (PAGE_SHIFT - 12);
drivers/block/zram/zram_drv.c
912
static void zram_account_writeback_submit(struct zram *zram)
drivers/block/zram/zram_drv.c
914
lockdep_assert_held_write(&zram->dev_lock);
drivers/block/zram/zram_drv.c
916
if (zram->wb_limit_enable && zram->bd_wb_limit > 0)
drivers/block/zram/zram_drv.c
917
zram->bd_wb_limit -= 1UL << (PAGE_SHIFT - 12);
drivers/block/zram/zram_drv.c
920
static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req)
drivers/block/zram/zram_drv.c
931
zram_account_writeback_rollback(zram);
drivers/block/zram/zram_drv.c
932
zram_release_bdev_block(zram, req->blk_idx);
drivers/block/zram/zram_drv.c
936
atomic64_inc(&zram->stats.bd_writes);
drivers/block/zram/zram_drv.c
937
slot_lock(zram, index);
drivers/block/zram/zram_drv.c
945
if (!test_slot_flag(zram, index, ZRAM_PP_SLOT)) {
drivers/block/zram/zram_drv.c
946
zram_release_bdev_block(zram, req->blk_idx);
drivers/block/zram/zram_drv.c
95
static void slot_lock(struct zram *zram, u32 index)
drivers/block/zram/zram_drv.c
950
clear_slot_flag(zram, index, ZRAM_IDLE);
drivers/block/zram/zram_drv.c
951
if (test_slot_flag(zram, index, ZRAM_HUGE))
drivers/block/zram/zram_drv.c
952
atomic64_dec(&zram->stats.huge_pages);
drivers/block/zram/zram_drv.c
953
atomic64_sub(get_slot_size(zram, index), &zram->stats.compr_data_size);
drivers/block/zram/zram_drv.c
954
zs_free(zram->mem_pool, get_slot_handle(zram, index));
drivers/block/zram/zram_drv.c
955
set_slot_handle(zram, index, req->blk_idx);
drivers/block/zram/zram_drv.c
956
set_slot_flag(zram, index, ZRAM_WB);
drivers/block/zram/zram_drv.c
959
slot_unlock(zram, index);
drivers/block/zram/zram_drv.c
97
unsigned long *lock = &zram->table[index].__lock;
drivers/block/zram/zram_drv.c
978
static void zram_submit_wb_request(struct zram *zram,
drivers/block/zram/zram_drv.c
986
zram_account_writeback_submit(zram);
drivers/block/zram/zram_drv.c
99
mutex_acquire(slot_dep_map(zram, index), 0, 0, _RET_IP_);
drivers/block/zram/zram_drv.c
992
static int zram_complete_done_reqs(struct zram *zram,