// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/blkdev.h>

#include "../dm-core.h"
#include "pcache_internal.h"
#include "cache_dev.h"
#include "backing_dev.h"
#include "cache.h"
#include "dm_pcache.h"

static struct kmem_cache *backing_req_cache;
static struct kmem_cache *backing_bvec_cache;

static void backing_dev_exit(struct pcache_backing_dev *backing_dev)
{
        mempool_exit(&backing_dev->req_pool);
        mempool_exit(&backing_dev->bvec_pool);
}

static void req_submit_fn(struct work_struct *work);
static void req_complete_fn(struct work_struct *work);
static int backing_dev_init(struct dm_pcache *pcache)
{
        struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
        int ret;

        ret = mempool_init_slab_pool(&backing_dev->req_pool, 128, backing_req_cache);
        if (ret)
                goto err;

        ret = mempool_init_slab_pool(&backing_dev->bvec_pool, 128, backing_bvec_cache);
        if (ret)
                goto req_pool_exit;

        INIT_LIST_HEAD(&backing_dev->submit_list);
        INIT_LIST_HEAD(&backing_dev->complete_list);
        spin_lock_init(&backing_dev->submit_lock);
        spin_lock_init(&backing_dev->complete_lock);
        INIT_WORK(&backing_dev->req_submit_work, req_submit_fn);
        INIT_WORK(&backing_dev->req_complete_work, req_complete_fn);
        atomic_set(&backing_dev->inflight_reqs, 0);
        init_waitqueue_head(&backing_dev->inflight_wq);

        return 0;

req_pool_exit:
        mempool_exit(&backing_dev->req_pool);
err:
        return ret;
}

int backing_dev_start(struct dm_pcache *pcache)
{
        struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
        int ret;

        ret = backing_dev_init(pcache);
        if (ret)
                return ret;

        backing_dev->dev_size = bdev_nr_sectors(backing_dev->dm_dev->bdev);

        return 0;
}

void backing_dev_stop(struct dm_pcache *pcache)
{
        struct pcache_backing_dev *backing_dev = &pcache->backing_dev;

        /*
         * There should not be any new request comming, just wait
         * inflight requests done.
         */
        wait_event(backing_dev->inflight_wq,
                        atomic_read(&backing_dev->inflight_reqs) == 0);

        flush_work(&backing_dev->req_submit_work);
        flush_work(&backing_dev->req_complete_work);

        backing_dev_exit(backing_dev);
}

/* pcache_backing_dev_req functions */
void backing_dev_req_end(struct pcache_backing_dev_req *backing_req)
{
        struct pcache_backing_dev *backing_dev = backing_req->backing_dev;

        if (backing_req->end_req)
                backing_req->end_req(backing_req, backing_req->ret);

        switch (backing_req->type) {
        case BACKING_DEV_REQ_TYPE_REQ:
                if (backing_req->req.upper_req)
                        pcache_req_put(backing_req->req.upper_req, backing_req->ret);
                break;
        case BACKING_DEV_REQ_TYPE_KMEM:
                if (backing_req->kmem.bvecs != backing_req->kmem.inline_bvecs)
                        mempool_free(backing_req->kmem.bvecs, &backing_dev->bvec_pool);
                break;
        default:
                BUG();
        }

        mempool_free(backing_req, &backing_dev->req_pool);

        if (atomic_dec_and_test(&backing_dev->inflight_reqs))
                wake_up(&backing_dev->inflight_wq);
}

static void req_complete_fn(struct work_struct *work)
{
        struct pcache_backing_dev *backing_dev = container_of(work, struct pcache_backing_dev, req_complete_work);
        struct pcache_backing_dev_req *backing_req;
        LIST_HEAD(tmp_list);

        spin_lock_irq(&backing_dev->complete_lock);
        list_splice_init(&backing_dev->complete_list, &tmp_list);
        spin_unlock_irq(&backing_dev->complete_lock);

        while (!list_empty(&tmp_list)) {
                backing_req = list_first_entry(&tmp_list,
                                            struct pcache_backing_dev_req, node);
                list_del_init(&backing_req->node);
                backing_dev_req_end(backing_req);
        }
}

static void backing_dev_bio_end(struct bio *bio)
{
        struct pcache_backing_dev_req *backing_req = bio->bi_private;
        struct pcache_backing_dev *backing_dev = backing_req->backing_dev;
        unsigned long flags;

        backing_req->ret = blk_status_to_errno(bio->bi_status);

        spin_lock_irqsave(&backing_dev->complete_lock, flags);
        list_move_tail(&backing_req->node, &backing_dev->complete_list);
        queue_work(BACKING_DEV_TO_PCACHE(backing_dev)->task_wq, &backing_dev->req_complete_work);
        spin_unlock_irqrestore(&backing_dev->complete_lock, flags);
}

static void req_submit_fn(struct work_struct *work)
{
        struct pcache_backing_dev *backing_dev = container_of(work, struct pcache_backing_dev, req_submit_work);
        struct pcache_backing_dev_req *backing_req;
        LIST_HEAD(tmp_list);

        spin_lock(&backing_dev->submit_lock);
        list_splice_init(&backing_dev->submit_list, &tmp_list);
        spin_unlock(&backing_dev->submit_lock);

        while (!list_empty(&tmp_list)) {
                backing_req = list_first_entry(&tmp_list,
                                            struct pcache_backing_dev_req, node);
                list_del_init(&backing_req->node);
                submit_bio_noacct(&backing_req->bio);
        }
}

void backing_dev_req_submit(struct pcache_backing_dev_req *backing_req, bool direct)
{
        struct pcache_backing_dev *backing_dev = backing_req->backing_dev;

        if (direct) {
                submit_bio_noacct(&backing_req->bio);
                return;
        }

        spin_lock(&backing_dev->submit_lock);
        list_add_tail(&backing_req->node, &backing_dev->submit_list);
        queue_work(BACKING_DEV_TO_PCACHE(backing_dev)->task_wq, &backing_dev->req_submit_work);
        spin_unlock(&backing_dev->submit_lock);
}

static void bio_map(struct bio *bio, void *base, size_t size)
{
        struct page *page;
        unsigned int offset;
        unsigned int len;

        if (!is_vmalloc_addr(base)) {
                page = virt_to_page(base);
                offset = offset_in_page(base);

                BUG_ON(!bio_add_page(bio, page, size, offset));
                return;
        }

        flush_kernel_vmap_range(base, size);
        while (size) {
                page = vmalloc_to_page(base);
                offset = offset_in_page(base);
                len = min_t(size_t, PAGE_SIZE - offset, size);

                BUG_ON(!bio_add_page(bio, page, len, offset));
                size -= len;
                base += len;
        }
}

static struct pcache_backing_dev_req *req_type_req_alloc(struct pcache_backing_dev *backing_dev,
                                                        struct pcache_backing_dev_req_opts *opts)
{
        struct pcache_request *pcache_req = opts->req.upper_req;
        struct pcache_backing_dev_req *backing_req;
        struct bio *orig = pcache_req->bio;

        backing_req = mempool_alloc(&backing_dev->req_pool, opts->gfp_mask);
        if (!backing_req)
                return NULL;

        memset(backing_req, 0, sizeof(struct pcache_backing_dev_req));

        bio_init_clone(backing_dev->dm_dev->bdev, &backing_req->bio, orig, opts->gfp_mask);

        backing_req->type = BACKING_DEV_REQ_TYPE_REQ;
        backing_req->backing_dev = backing_dev;
        atomic_inc(&backing_dev->inflight_reqs);

        return backing_req;
}

static struct pcache_backing_dev_req *kmem_type_req_alloc(struct pcache_backing_dev *backing_dev,
                                                struct pcache_backing_dev_req_opts *opts)
{
        struct pcache_backing_dev_req *backing_req;
        u32 n_vecs = bio_add_max_vecs(opts->kmem.data, opts->kmem.len);

        backing_req = mempool_alloc(&backing_dev->req_pool, opts->gfp_mask);
        if (!backing_req)
                return NULL;

        memset(backing_req, 0, sizeof(struct pcache_backing_dev_req));

        if (n_vecs > BACKING_DEV_REQ_INLINE_BVECS) {
                backing_req->kmem.bvecs = mempool_alloc(&backing_dev->bvec_pool, opts->gfp_mask);
                if (!backing_req->kmem.bvecs)
                        goto free_backing_req;
        } else {
                backing_req->kmem.bvecs = backing_req->kmem.inline_bvecs;
        }

        backing_req->kmem.n_vecs = n_vecs;
        backing_req->type = BACKING_DEV_REQ_TYPE_KMEM;
        backing_req->backing_dev = backing_dev;
        atomic_inc(&backing_dev->inflight_reqs);

        return backing_req;

free_backing_req:
        mempool_free(backing_req, &backing_dev->req_pool);
        return NULL;
}

struct pcache_backing_dev_req *backing_dev_req_alloc(struct pcache_backing_dev *backing_dev,
                                                struct pcache_backing_dev_req_opts *opts)
{
        if (opts->type == BACKING_DEV_REQ_TYPE_REQ)
                return req_type_req_alloc(backing_dev, opts);

        if (opts->type == BACKING_DEV_REQ_TYPE_KMEM)
                return kmem_type_req_alloc(backing_dev, opts);

        BUG();
}

static void req_type_req_init(struct pcache_backing_dev_req *backing_req,
                        struct pcache_backing_dev_req_opts *opts)
{
        struct pcache_request *pcache_req = opts->req.upper_req;
        struct bio *clone;
        u32 off = opts->req.req_off;
        u32 len = opts->req.len;

        clone = &backing_req->bio;
        BUG_ON(off & SECTOR_MASK);
        BUG_ON(len & SECTOR_MASK);
        bio_trim(clone, off >> SECTOR_SHIFT, len >> SECTOR_SHIFT);

        clone->bi_iter.bi_sector = (pcache_req->off + off) >> SECTOR_SHIFT;
        clone->bi_private = backing_req;
        clone->bi_end_io = backing_dev_bio_end;

        INIT_LIST_HEAD(&backing_req->node);
        backing_req->end_req     = opts->end_fn;

        pcache_req_get(pcache_req);
        backing_req->req.upper_req      = pcache_req;
        backing_req->req.bio_off        = off;
}

static void kmem_type_req_init(struct pcache_backing_dev_req *backing_req,
                        struct pcache_backing_dev_req_opts *opts)
{
        struct pcache_backing_dev *backing_dev = backing_req->backing_dev;
        struct bio *backing_bio;

        bio_init(&backing_req->bio, backing_dev->dm_dev->bdev, backing_req->kmem.bvecs,
                        backing_req->kmem.n_vecs, opts->kmem.opf);

        backing_bio = &backing_req->bio;
        bio_map(backing_bio, opts->kmem.data, opts->kmem.len);

        backing_bio->bi_iter.bi_sector = (opts->kmem.backing_off) >> SECTOR_SHIFT;
        backing_bio->bi_private = backing_req;
        backing_bio->bi_end_io = backing_dev_bio_end;

        INIT_LIST_HEAD(&backing_req->node);
        backing_req->end_req    = opts->end_fn;
        backing_req->priv_data  = opts->priv_data;
}

void backing_dev_req_init(struct pcache_backing_dev_req *backing_req,
                        struct pcache_backing_dev_req_opts *opts)
{
        if (opts->type == BACKING_DEV_REQ_TYPE_REQ)
                return req_type_req_init(backing_req, opts);

        if (opts->type == BACKING_DEV_REQ_TYPE_KMEM)
                return kmem_type_req_init(backing_req, opts);

        BUG();
}

struct pcache_backing_dev_req *backing_dev_req_create(struct pcache_backing_dev *backing_dev,
                                                struct pcache_backing_dev_req_opts *opts)
{
        struct pcache_backing_dev_req *backing_req;

        backing_req = backing_dev_req_alloc(backing_dev, opts);
        if (!backing_req)
                return NULL;

        backing_dev_req_init(backing_req, opts);

        return backing_req;
}

void backing_dev_flush(struct pcache_backing_dev *backing_dev)
{
        blkdev_issue_flush(backing_dev->dm_dev->bdev);
}

int pcache_backing_init(void)
{
        u32 max_bvecs = (PCACHE_CACHE_SUBTREE_SIZE >> PAGE_SHIFT) + 1;
        int ret;

        backing_req_cache = KMEM_CACHE(pcache_backing_dev_req, 0);
        if (!backing_req_cache) {
                ret = -ENOMEM;
                goto err;
        }

        backing_bvec_cache = kmem_cache_create("pcache-bvec-slab",
                                        max_bvecs * sizeof(struct bio_vec),
                                        0, 0, NULL);
        if (!backing_bvec_cache) {
                ret = -ENOMEM;
                goto destroy_req_cache;
        }

        return 0;
destroy_req_cache:
        kmem_cache_destroy(backing_req_cache);
err:
        return ret;
}

void pcache_backing_exit(void)
{
        kmem_cache_destroy(backing_bvec_cache);
        kmem_cache_destroy(backing_req_cache);
}